Commit ac693d84 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Update the Metadata generation

Changelog:
* Add `fiphobo:` prefix to all metadata saved by our server, that should not be sent back directly
* When sending the metadata, all entries starting with `fiphobo:` are ignored
* remove the bucket_name entry from all set metadata functions throughout the server and add the bucketname where all other metdata is set (put handler)
parent dc31d3c1
Pipeline #1761 passed with stages
in 7 minutes and 18 seconds
......@@ -164,7 +164,7 @@ assignment operator deleted
[Go to Top](#fiphoboserver-storage-Phobos_file)
### <a name='fiphoboserver-storage-Phobos_file-set_meta_data' /> public void fiphoboserver::storage::Phobos_file::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) override
### <a name='fiphoboserver-storage-Phobos_file-set_meta_data' /> public void fiphoboserver::storage::Phobos_file::set_meta_data (std::map< std::string, std::string > meta_data) override
set the metadata that an object that is added to the database should get
......@@ -175,7 +175,6 @@ set the metadata that an object that is added to the database should get
| Type | Name | Description |
| ---- | ---- | ---- |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
......
......@@ -42,7 +42,7 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob
## Metadata functions
### <a name='fiphoboserver-storage-Storage-set_meta_data' /> public void fiphoboserver::storage::Storage::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name)=0
### <a name='fiphoboserver-storage-Storage-set_meta_data' /> public void fiphoboserver::storage::Storage::set_meta_data (std::map< std::string, std::string > meta_data)=0
set the metadata that an object that is added to the database should get
......@@ -53,7 +53,6 @@ set the metadata that an object that is added to the database should get
| Type | Name | Description |
| ---- | ---- | ---- |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
......
......@@ -233,7 +233,7 @@ assignment operator deleted
[Go to Top](#fiphoboserver-stream-Fifo)
### <a name='fiphoboserver-stream-Fifo-set_storage_meta_data' /> public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const override
### <a name='fiphoboserver-stream-Fifo-set_storage_meta_data' /> public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::map< std::string, std::string > meta_data) const override
setting the metadata that the created object should get
......@@ -243,8 +243,7 @@ setting the metadata that the created object should get
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
......
......@@ -287,7 +287,7 @@ default destructor
[Go to Top](#fiphoboserver-stream-Stream)
### <a name='fiphoboserver-stream-Stream-set_storage_meta_data' /> public void fiphoboserver::stream::Stream::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const =0
### <a name='fiphoboserver-stream-Stream-set_storage_meta_data' /> public void fiphoboserver::stream::Stream::set_storage_meta_data (std::map< std::string, std::string > meta_data) const =0
setting the metadata that the created object should get
......@@ -297,8 +297,7 @@ setting the metadata that the created object should get
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
......
......@@ -59,7 +59,7 @@ void GetRequestHandler::onRequest(
// Check if metadata for file exists
meta_data = m_stream->get_meta_data(m_s3_header.get_key());
content_length = meta_data["content_length"];
content_length = meta_data["fiphobo:content_length"];
/* Tell stream to coordinate with backend to prepare for GET operation
*/
......@@ -95,8 +95,7 @@ void GetRequestHandler::onRequest(
std::for_each(
meta_data.begin(), meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
if (pair.first != "creation_time" && pair.first != "bucket"
&& pair.first != "content_length") {
if (pair.first.rfind("fiphobo:", 0) != 0) {
response.header("x-amz-meta-" + pair.first, pair.second);
}
});
......
......@@ -77,7 +77,7 @@ void GetmdRequestHandler::onRequest(
std::for_each(
m_meta_data.begin(), m_meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
if (pair.first != "creation_time" && pair.first != "bucket") {
if (pair.first.rfind("fiphobo:", 0) != 0) {
response.header("x-amz-meta-" + pair.first, pair.second);
}
});
......
......@@ -59,11 +59,12 @@ void PutRequestHandler::onRequest(
* Add timestamp and content length to meta data and send it to backend
* through the stream
*/
auto metadata = m_s3_header.get_meta_data();
metadata["creation_time"] = std::to_string(std::time(nullptr));
metadata["content_length"] =
auto metadata = m_s3_header.get_meta_data();
metadata["fiphobo:creation_time"] = std::to_string(std::time(nullptr));
metadata["fiphobo:content_length"] =
std::to_string(m_s3_header.get_body_length());
m_stream->set_storage_meta_data(metadata, m_s3_header.get_bucket());
metadata["fiphobo:bucket"] = m_s3_header.get_bucket();
m_stream->set_storage_meta_data(metadata);
/*
* Tell stream to coordinate with backend to prepare for PUT operation
......
......@@ -88,21 +88,9 @@ ssize_t Phobos_file::db_get()
return rc;
}
void Phobos_file::set_bucket_name(std::string bucket_name)
{
int rc =
pho_attr_set(&m_descriptor.xd_attrs, "bucket", bucket_name.c_str());
if (rc != 0) {
throw PhobosException(
"Phobos_file::set_bucket_name", "pho_attr_set", rc);
}
}
/* Pass s3 meta data to Phobos */
void Phobos_file::set_meta_data(
std::map<std::string, std::string> meta_data, std::string bucket_name)
void Phobos_file::set_meta_data(std::map<std::string, std::string> meta_data)
{
set_bucket_name(bucket_name);
std::for_each(
meta_data.begin(), meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
......
......@@ -42,9 +42,7 @@ class Phobos_file : public Storage {
Phobos_file& operator=(const Phobos_file&) = delete;
/// @copydoc Storage::set_meta_data
void set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) override;
void set_meta_data(std::map<std::string, std::string> meta_data) override;
/// @copydoc Storage::get_meta_data
std::map<std::string, std::string> get_meta_data(
std::string object_id) override;
......
......@@ -38,12 +38,9 @@ class Storage {
///
/// @param meta_data a map of all key:value pairs that should be
/// added to the data
/// @param bucket_name the name of the S3 bucket - currently also
/// handled as normal metadata
///
virtual void set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) = 0;
std::map<std::string, std::string> meta_data) = 0;
///
/// @brief get the metadata associated to the current object as a
/// map of key:value pairs
......
......@@ -26,9 +26,9 @@ Fifo::~Fifo()
}
/* Simply pass along to the backend */
void Fifo::set_storage_meta_data(
std::map<std::string, std::string> meta_data, std::string bucket_name) const
std::map<std::string, std::string> meta_data) const
{
m_storage->set_meta_data(meta_data, bucket_name);
m_storage->set_meta_data(meta_data);
}
std::map<std::string, std::string> Fifo::get_meta_data(
......
......@@ -60,8 +60,7 @@ class Fifo : public Stream {
/// Just passes the data along to the backend
///
void set_storage_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) const override;
std::map<std::string, std::string> meta_data) const override;
/// @copydoc Stream::get_meta_data
std::map<std::string, std::string> get_meta_data(
std::string object_id) const override;
......
......@@ -50,15 +50,12 @@ class Stream {
///
/// @param meta_data a map of all key:value pairs that should be
/// added to the data
/// @param bucket_name the name of the S3 bucket - currently also
/// handled as normal metadata
///
/// @warning this function is responsible for getting metadata to
/// the storage implementation
///
virtual void set_storage_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) const = 0;
std::map<std::string, std::string> meta_data) const = 0;
///
/// @brief get the metadata associated to the current object as a
/// map of key:value pairs
......
......@@ -66,7 +66,6 @@ class Server_test_case(unittest.TestCase):
key,
meta_data=put_meta_data)
put_meta_data['content_length'] = '20662'
get_meta_data = self.client.get_md(self.bucket_name, key)
self.assertTrue(put_meta_data == get_meta_data)
......
......@@ -42,9 +42,8 @@ SCENARIO("stream file ownership", "[basic]")
}
}
std::string bucket_name = "bucket_" + random_string(10);
std::string key = "id_" + random_string(10);
std::string dummy_key = "id_" + random_string(10);
std::string key = "id_" + random_string(10);
std::string dummy_key = "id_" + random_string(10);
std::string buffer;
std::map<std::string, std::string> meta_data;
......@@ -61,7 +60,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
for (int i = 0; i < 5; i++) {
meta_data[key + std::to_string(i)] = std::to_string(i);
}
fifo.set_storage_meta_data(meta_data, bucket_name);
fifo.set_storage_meta_data(meta_data);
fifo.start_put(buffer.size(), key);
ssize_t rc = fifo.put(buffer.data(), buffer.size());
......@@ -77,7 +76,6 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
fifo.get_meta_data(key);
THEN("the retreived meta data matches the input meta data")
{
REQUIRE(meta_data2["bucket"] == bucket_name);
std::for_each(
meta_data.begin(), meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
......@@ -111,7 +109,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
meta_data[key + std::to_string(i)] = std::to_string(i);
}
fifo.set_storage_meta_data(meta_data, bucket_name);
fifo.set_storage_meta_data(meta_data);
fifo.start_put(buffer.size(), dummy_key);
ssize_t rc = fifo.put(buffer.data(), buffer.size());
fifo.finish_io();
......
......@@ -64,9 +64,8 @@ SCENARIO("Phobos_file construction", "[basic]")
}
}
std::string bucket_name = "bucket_" + random_string(10);
std::string object_id = "id_" + random_string(10);
std::string dummy_key = "id_" + random_string(10);
std::string object_id = "id_" + random_string(10);
std::string dummy_key = "id_" + random_string(10);
std::map<std::string, std::string> meta_data;
SCENARIO("Phobos_file storage operations", "[I/O]")
......@@ -87,7 +86,7 @@ SCENARIO("Phobos_file storage operations", "[I/O]")
for (int i = 0; i < 5; i++) {
meta_data[object_id + std::to_string(i)] = std::to_string(i);
}
store.set_meta_data(meta_data, bucket_name);
store.set_meta_data(meta_data);
store.prepare_put(fd, object_id);
REQUIRE(0 < store.get_fd());
......@@ -104,8 +103,6 @@ SCENARIO("Phobos_file storage operations", "[I/O]")
store.get_meta_data(object_id);
THEN("the retreived meta data matches the input meta data")
{
REQUIRE(meta_data2["bucket"] == bucket_name);
std::for_each(
meta_data.begin(), meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
......
......@@ -102,11 +102,8 @@ ssize_t Disk::db_get()
}
/* Pass s3 meta data to Phobos */
void Disk::set_meta_data(
std::map<std::string, std::string> meta_data, std::string bucket_name)
void Disk::set_meta_data(std::map<std::string, std::string> meta_data)
{
m_meta_data["bucket"] = bucket_name;
std::for_each(
meta_data.begin(), meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
......
......@@ -12,9 +12,7 @@ class Disk : public Storage {
Disk(const Disk&) = delete;
Disk& operator=(const Disk&) = delete;
void set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) override;
void set_meta_data(std::map<std::string, std::string> meta_data) override;
std::map<std::string, std::string> get_meta_data(
std::string object_id) override;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment