Update the Metadata generation

Changelog:
* Add `fiphobo:` prefix to all metadata saved by our server, that should not be sent back directly
* When sending the metadata, all entries starting with `fiphobo:` are ignored
* remove the bucket_name entry from all set metadata functions throughout the server and add the bucketname where all other metdata is set (put handler)

doc/Markdown/fiphoboserver/storage/Phobos_file.md

@@ -164,7 +164,7 @@ assignment operator deleted
 
 [Go to Top](#fiphoboserver-storage-Phobos_file)
 
-### <a name='fiphoboserver-storage-Phobos_file-set_meta_data' /> public void fiphoboserver::storage::Phobos_file::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) override
+### <a name='fiphoboserver-storage-Phobos_file-set_meta_data' /> public void fiphoboserver::storage::Phobos_file::set_meta_data (std::map< std::string, std::string > meta_data) override
 
 set the metadata that an object that is added to the database should get 
 
@@ -175,7 +175,6 @@ set the metadata that an object that is added to the database should get
 | Type | Name | Description | 
 | ---- | ---- | ---- |
 | std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data  |
-| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata  |
 
 
 

doc/Markdown/fiphoboserver/storage/Storage.md

@@ -42,7 +42,7 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob
 
 
 ## Metadata functions
-### <a name='fiphoboserver-storage-Storage-set_meta_data' /> public void fiphoboserver::storage::Storage::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name)=0
+### <a name='fiphoboserver-storage-Storage-set_meta_data' /> public void fiphoboserver::storage::Storage::set_meta_data (std::map< std::string, std::string > meta_data)=0
 
 set the metadata that an object that is added to the database should get 
 
@@ -53,7 +53,6 @@ set the metadata that an object that is added to the database should get
 | Type | Name | Description | 
 | ---- | ---- | ---- |
 | std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data  |
-| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata  |
 
 
 

doc/Markdown/fiphoboserver/stream/Fifo.md

@@ -233,7 +233,7 @@ assignment operator deleted
 
 [Go to Top](#fiphoboserver-stream-Fifo)
 
-### <a name='fiphoboserver-stream-Fifo-set_storage_meta_data' /> public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const override
+### <a name='fiphoboserver-stream-Fifo-set_storage_meta_data' /> public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::map< std::string, std::string > meta_data) const override
 
 setting the metadata that the created object should get 
 
@@ -243,8 +243,7 @@ setting the metadata that the created object should get
 #### Parameters: 
 | Type | Name | Description | 
 | ---- | ---- | ---- |
-| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data  |
-| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
+| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
 
 
 

doc/Markdown/fiphoboserver/stream/Stream.md

@@ -287,7 +287,7 @@ default destructor
 
 [Go to Top](#fiphoboserver-stream-Stream)
 
-### <a name='fiphoboserver-stream-Stream-set_storage_meta_data' /> public void fiphoboserver::stream::Stream::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const =0
+### <a name='fiphoboserver-stream-Stream-set_storage_meta_data' /> public void fiphoboserver::stream::Stream::set_storage_meta_data (std::map< std::string, std::string > meta_data) const =0
 
 setting the metadata that the created object should get 
 
@@ -297,8 +297,7 @@ setting the metadata that the created object should get
 #### Parameters: 
 | Type | Name | Description | 
 | ---- | ---- | ---- |
-| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data  |
-| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata |
+| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data |
 
 
 

src/server/get_request_handler.cc

@@ -59,7 +59,7 @@ void GetRequestHandler::onRequest(
 
         // Check if metadata for file exists
         meta_data      = m_stream->get_meta_data(m_s3_header.get_key());
-        content_length = meta_data["content_length"];
+        content_length = meta_data["fiphobo:content_length"];
 
         /* Tell stream to coordinate with backend to prepare for GET operation
          */
@@ -95,8 +95,7 @@ void GetRequestHandler::onRequest(
     std::for_each(
         meta_data.begin(), meta_data.end(),
         [&](std::pair<std::string, std::string> pair) {
-            if (pair.first != "creation_time" && pair.first != "bucket"
-                && pair.first != "content_length") {
+            if (pair.first.rfind("fiphobo:", 0) != 0) {
                 response.header("x-amz-meta-" + pair.first, pair.second);
             }
         });

src/server/getmd_request_handler.cc

@@ -77,7 +77,7 @@ void GetmdRequestHandler::onRequest(
     std::for_each(
         m_meta_data.begin(), m_meta_data.end(),
         [&](std::pair<std::string, std::string> pair) {
-            if (pair.first != "creation_time" && pair.first != "bucket") {
+            if (pair.first.rfind("fiphobo:", 0) != 0) {
                 response.header("x-amz-meta-" + pair.first, pair.second);
             }
         });

src/server/put_request_handler.cc

@@ -59,11 +59,12 @@ void PutRequestHandler::onRequest(
          * Add timestamp and content length to meta data and send it to backend
          * through the stream
          */
-        auto metadata             = m_s3_header.get_meta_data();
-        metadata["creation_time"] = std::to_string(std::time(nullptr));
-        metadata["content_length"] =
+        auto metadata                     = m_s3_header.get_meta_data();
+        metadata["fiphobo:creation_time"] = std::to_string(std::time(nullptr));
+        metadata["fiphobo:content_length"] =
             std::to_string(m_s3_header.get_body_length());
-        m_stream->set_storage_meta_data(metadata, m_s3_header.get_bucket());
+        metadata["fiphobo:bucket"] = m_s3_header.get_bucket();
+        m_stream->set_storage_meta_data(metadata);
 
         /*
          * Tell stream to coordinate with backend to prepare for PUT operation

src/storage/phobos_file.cc

@@ -88,21 +88,9 @@ ssize_t Phobos_file::db_get()
     return rc;
 }
 
-void Phobos_file::set_bucket_name(std::string bucket_name)
-{
-    int rc =
-        pho_attr_set(&m_descriptor.xd_attrs, "bucket", bucket_name.c_str());
-    if (rc != 0) {
-        throw PhobosException(
-            "Phobos_file::set_bucket_name", "pho_attr_set", rc);
-    }
-}
-
 /* Pass s3 meta data to Phobos */
-void Phobos_file::set_meta_data(
-    std::map<std::string, std::string> meta_data, std::string bucket_name)
+void Phobos_file::set_meta_data(std::map<std::string, std::string> meta_data)
 {
-    set_bucket_name(bucket_name);
     std::for_each(
         meta_data.begin(), meta_data.end(),
         [&](std::pair<std::string, std::string> pair) {

src/storage/phobos_file.h

@@ -42,9 +42,7 @@ class Phobos_file : public Storage {
     Phobos_file& operator=(const Phobos_file&) = delete;
 
     /// @copydoc Storage::set_meta_data
-    void set_meta_data(
-        std::map<std::string, std::string> meta_data,
-        std::string bucket_name) override;
+    void set_meta_data(std::map<std::string, std::string> meta_data) override;
     /// @copydoc Storage::get_meta_data
     std::map<std::string, std::string> get_meta_data(
         std::string object_id) override;

src/storage/storage.h

@@ -38,12 +38,9 @@ class Storage {
     ///
     /// @param meta_data    a map of all key:value pairs that should be
     ///                     added to the data
-    /// @param bucket_name  the name of the S3 bucket - currently also
-    ///                     handled as normal metadata
     ///
     virtual void set_meta_data(
-        std::map<std::string, std::string> meta_data,
-        std::string bucket_name) = 0;
+        std::map<std::string, std::string> meta_data) = 0;
     ///
     /// @brief  get the metadata associated to the current object as a
     ///         map of key:value pairs

src/stream/fifo.cc

@@ -26,9 +26,9 @@ Fifo::~Fifo()
 }
 /* Simply pass along to the backend */
 void Fifo::set_storage_meta_data(
-    std::map<std::string, std::string> meta_data, std::string bucket_name) const
+    std::map<std::string, std::string> meta_data) const
 {
-    m_storage->set_meta_data(meta_data, bucket_name);
+    m_storage->set_meta_data(meta_data);
 }
 
 std::map<std::string, std::string> Fifo::get_meta_data(

src/stream/fifo.h

@@ -60,8 +60,7 @@ class Fifo : public Stream {
     /// Just passes the data along to the backend
     ///
     void set_storage_meta_data(
-        std::map<std::string, std::string> meta_data,
-        std::string bucket_name) const override;
+        std::map<std::string, std::string> meta_data) const override;
     /// @copydoc Stream::get_meta_data
     std::map<std::string, std::string> get_meta_data(
         std::string object_id) const override;

src/stream/stream.h

@@ -50,15 +50,12 @@ class Stream {
     ///
     /// @param meta_data    a map of all key:value pairs that should be
     ///                     added to the data
-    /// @param bucket_name  the name of the S3 bucket - currently also
-    ///                     handled as normal metadata
     ///
     /// @warning    this function is responsible for getting metadata to
     ///             the storage implementation
     ///
     virtual void set_storage_meta_data(
-        std::map<std::string, std::string> meta_data,
-        std::string bucket_name) const = 0;
+        std::map<std::string, std::string> meta_data) const = 0;
     ///
     /// @brief  get the metadata associated to the current object as a
     ///         map of key:value pairs

test/integration_tests/client.py

@@ -66,7 +66,6 @@ class Server_test_case(unittest.TestCase):
                         key,
                         meta_data=put_meta_data)
 
-        put_meta_data['content_length'] = '20662'
         get_meta_data = self.client.get_md(self.bucket_name, key)
 
         self.assertTrue(put_meta_data == get_meta_data)

test/unit_tests/fifo.cc

@@ -42,9 +42,8 @@ SCENARIO("stream file ownership", "[basic]")
     }
 }
 
-std::string bucket_name = "bucket_" + random_string(10);
-std::string key         = "id_" + random_string(10);
-std::string dummy_key   = "id_" + random_string(10);
+std::string key       = "id_" + random_string(10);
+std::string dummy_key = "id_" + random_string(10);
 std::string buffer;
 std::map<std::string, std::string> meta_data;
 
@@ -61,7 +60,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
             for (int i = 0; i < 5; i++) {
                 meta_data[key + std::to_string(i)] = std::to_string(i);
             }
-            fifo.set_storage_meta_data(meta_data, bucket_name);
+            fifo.set_storage_meta_data(meta_data);
 
             fifo.start_put(buffer.size(), key);
             ssize_t rc = fifo.put(buffer.data(), buffer.size());
@@ -77,7 +76,6 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
                 fifo.get_meta_data(key);
             THEN("the retreived meta data matches the input meta data")
             {
-                REQUIRE(meta_data2["bucket"] == bucket_name);
                 std::for_each(
                     meta_data.begin(), meta_data.end(),
                     [&](std::pair<std::string, std::string> pair) {
@@ -111,7 +109,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
                 meta_data[key + std::to_string(i)] = std::to_string(i);
             }
 
-            fifo.set_storage_meta_data(meta_data, bucket_name);
+            fifo.set_storage_meta_data(meta_data);
             fifo.start_put(buffer.size(), dummy_key);
             ssize_t rc = fifo.put(buffer.data(), buffer.size());
             fifo.finish_io();

test/unit_tests/phobos_file.cc

@@ -64,9 +64,8 @@ SCENARIO("Phobos_file construction", "[basic]")
     }
 }
 
-std::string bucket_name = "bucket_" + random_string(10);
-std::string object_id   = "id_" + random_string(10);
-std::string dummy_key   = "id_" + random_string(10);
+std::string object_id = "id_" + random_string(10);
+std::string dummy_key = "id_" + random_string(10);
 std::map<std::string, std::string> meta_data;
 
 SCENARIO("Phobos_file storage operations", "[I/O]")
@@ -87,7 +86,7 @@ SCENARIO("Phobos_file storage operations", "[I/O]")
             for (int i = 0; i < 5; i++) {
                 meta_data[object_id + std::to_string(i)] = std::to_string(i);
             }
-            store.set_meta_data(meta_data, bucket_name);
+            store.set_meta_data(meta_data);
 
             store.prepare_put(fd, object_id);
             REQUIRE(0 < store.get_fd());
@@ -104,8 +103,6 @@ SCENARIO("Phobos_file storage operations", "[I/O]")
                 store.get_meta_data(object_id);
             THEN("the retreived meta data matches the input meta data")
             {
-                REQUIRE(meta_data2["bucket"] == bucket_name);
-
                 std::for_each(
                     meta_data.begin(), meta_data.end(),
                     [&](std::pair<std::string, std::string> pair) {

test/unit_tests/storage/disk.cc

@@ -102,11 +102,8 @@ ssize_t Disk::db_get()
 }
 
 /* Pass s3 meta data to Phobos */
-void Disk::set_meta_data(
-    std::map<std::string, std::string> meta_data, std::string bucket_name)
+void Disk::set_meta_data(std::map<std::string, std::string> meta_data)
 {
-    m_meta_data["bucket"] = bucket_name;
-
     std::for_each(
         meta_data.begin(), meta_data.end(),
         [&](std::pair<std::string, std::string> pair) {

test/unit_tests/storage/disk.h

@@ -12,9 +12,7 @@ class Disk : public Storage {
     Disk(const Disk&)       = delete;
     Disk& operator=(const Disk&) = delete;
 
-    void set_meta_data(
-        std::map<std::string, std::string> meta_data,
-        std::string bucket_name) override;
+    void set_meta_data(std::map<std::string, std::string> meta_data) override;
     std::map<std::string, std::string> get_meta_data(
         std::string object_id) override;