Commit 2c31093f authored by Ciarán Ó Rourke's avatar Ciarán Ó Rourke
Browse files

Object Listing

Support the list objects functionality of s3. Bucket only GET requests
get a response with a list of the objects in the bucket

Change log:
* add list object functions to the phobos_cpp_wrapper
* add listing objects to storage functionality
* add bucket object listing wrapper to stream
* empty list objects function is disk storage backend to satisfy storage class
* string_to_map function to aid returning object_list in storage
* remove Catch2 header file
* oid_to_key function to parse away bucket from object name
* s3_utilities: function to create xml response for list objects
* bucket only GET request calls list objects
* get_request_handler: respond with object_list for bucket only requests
* add object listing to unit tests
* integration test for object listing
* unit test for empty bucket
* integration test for empty bucket
* Use bucket and key in unit tests to reflect actual usage
* generate random bucket name for testing so number of objects in
    returned list can be measured
* replace assertTrue with assertEqual in integration tests where appropriate
* number integration tests to ensure order (alphanumerical) of execution
* instruct integration tests to stop on first error
* free object meta data struct
* fix development flags not being used during debug build
* use debug build for CI
* use release build for CentOS7
* appease development flags
* disable LeakSanitization for CI (does not work on Gitlab-CI)
* source relevant env list in CI before_script
* export LD_LIBRARY_PATH in CI configure step
* pass bucket name to xml parser to avoid attempting to access an object of an empty list
* update documentation
parent fd49d2a5
Pipeline #2406 passed with stages
in 18 minutes and 11 seconds
......@@ -4,7 +4,8 @@ image: ciaranorourke/deimos:centos8
- &start_phobos
/home/superdeimos/start-db.sh
- &configure
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${DEPS_DIR}/phobos/lib"
export LD_LIBRARY_PATH
&& export ASAN_OPTIONS=detect_leaks=0
&& mkdir build
&& pushd build
&& (
......@@ -17,9 +18,10 @@ image: ciaranorourke/deimos:centos8
-DCMAKE_C_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include"
-DCMAKE_CXX_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include -isystem ${DEPS_DIR}/spdlog/include"
-DCMAKE_EXE_LINKER_FLAGS="-L ${DEPS_DIR}/phobos/lib"
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
-DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}/phobos;${DEPS_DIR}/proxygen;${DEPS_DIR}/folly;${DEPS_DIR}/fmt;${DEPS_DIR}/fizz;${DEPS_DIR}/wangle;${DEPS_DIR}/gflags"
-DCMAKE_INSTALL_PREFIX="${DEPS_DIR}" \
-DCMAKE_INSTALL_PREFIX="${DEPS_DIR}"
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}"
......@@ -41,8 +43,10 @@ image: ciaranorourke/deimos:centos8
DEIMOS_AUTHORISATION: "ON"
DEIMOS_BUILD_DOCUMENTATION: "OFF"
CMAKE_EXPORT_COMPILE_COMMANDS: "OFF"
CMAKE_BUILD_TYPE: Debug
CC: gcc
CXX: g++
ENV_LIST: /home/superdeimos/centos8_envs.list
.default_job: &default_job
tags:
......@@ -55,6 +59,7 @@ image: ciaranorourke/deimos:centos8
cmake
clang clang-tools-extra
- pip3 install boto3
- source ${ENV_LIST}
script:
- *start_phobos
- *configure
......@@ -140,6 +145,10 @@ Centos7:
<<: *default_job
stage: test
image: ciaranorourke/deimos:centos7
variables:
<<: *default_cmake_variables
ENV_LIST: /home/superdeimos/centos7_envs.list
CMAKE_BUILD_TYPE: Release
before_script:
- yum update -y
- yum install -y epel-release
......@@ -148,4 +157,4 @@ Centos7:
python3
- pip3 install boto3
- ln -s /usr/bin/cmake3 /usr/bin/cmake
- source /home/superdeimos/envs.list
- source ${ENV_LIST}
......@@ -12,13 +12,13 @@ project(
option(
DEIMOS_BUILD_TESTS
"build Deimos tests. Depends on Catch2."
"Build Deimos tests. Depends on Catch2."
OFF
)
option(
DEIMOS_BUILD_DOCUMENTATION
"build Deimos Documentation. Depends on Doxygen."
"Build Deimos Documentation. Depends on Doxygen."
OFF
)
......@@ -63,7 +63,7 @@ set(
option(
DEIMOS_AUTHORISATION
"build Deimos with AWS authorisation enabled. Default is enabled"
"Build Deimos with AWS authorisation enabled. Default is enabled"
ON
)
......
......@@ -2,7 +2,7 @@ add_library(development_flags INTERFACE)
option(DEIMOS_ENABLE_SANITIZERS "Enable address and undefined behavior sanitizers." ON)
if(DEIMOS_CXX_COMPILER_ID MATCHES "Clang|GNU")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
target_compile_options(
development_flags
INTERFACE
......@@ -11,7 +11,7 @@ if(DEIMOS_CXX_COMPILER_ID MATCHES "Clang|GNU")
-Wswitch-enum
-Wimplicit-fallthrough
-Werror
-0g
-Og
)
if(DEIMOS_ENABLE_SANITIZERS)
......@@ -19,7 +19,7 @@ if(DEIMOS_CXX_COMPILER_ID MATCHES "Clang|GNU")
development_flags
INTERFACE
-fsanitize=address,undefined
-fno-sanitize=recover=all
-fno-sanitize-recover=all
-fno-omit-frame-pointer
)
target_link_libraries(
......@@ -28,7 +28,7 @@ if(DEIMOS_CXX_COMPILER_ID MATCHES "Clang|GNU")
-fsanitize=address,undefined
)
endif(DEIMOS_ENABLE_SANITIZERS)
endif()
endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
target_compile_options(development_flags INTERFACE -ferror-limit=1)
......
......@@ -20,6 +20,7 @@ Inherits from RequestHandler.
| [onEgressPaused](#deimos-GetRequestHandler-onEgressPaused) | called when the queue is full. |
| [onEgressResumed](#deimos-GetRequestHandler-onEgressResumed) | called when the queue is not longer full. |
| [GetRequestHandler](#deimos-GetRequestHandler-GetRequestHandler) | Constructor for stream class initialization. |
| [list_objects](#deimos-GetRequestHandler-list_objects) | function called in the case of a list_objects request |
......@@ -226,6 +227,26 @@ Constructor for stream class initialization.
* inline
[Go to Top](#deimos-GetRequestHandler)
### <a name='deimos-GetRequestHandler-list_objects' /> public void deimos::GetRequestHandler::list_objects () noexcept
function called in the case of a list_objects request
Objects for a given bucket are added to the response
[Go to Top](#deimos-GetRequestHandler)
[deimos-stream-Stream]:./stream/Stream.md
......
......@@ -28,6 +28,7 @@ namespace for S3 specific utilities
| [openssl_sha256](#deimos-s3_utilities-openssl_sha256-1) | shortcut for openssl_sha256(void*, unsigned int) |
| [openssl_hmac](#deimos-s3_utilities-openssl_hmac) | create a key hashed message |
| [openssl_hmac_hex](#deimos-s3_utilities-openssl_hmac_hex) | create a key hashed message |
| [list_objects_xml_response](#deimos-s3_utilities-list_objects_xml_response) | create an xml response for the list_objects request |
......@@ -51,6 +52,10 @@ enum to store all the different states a [deimos::s3_utilities::S3_authorisation
#### Qualifiers:
* strong
[Go to Top](#deimos-s3_utilities)
## Functions
......@@ -236,6 +241,37 @@ create a key hashed message
[Go to Top](#deimos-s3_utilities)
### <a name='deimos-s3_utilities-list_objects_xml_response' /> public std::string deimos::s3_utilities::list_objects_xml_response (const std::vector< std::map< std::string, std::string >> &object_list, const std::string &bucket_name)
create an xml response for the list_objects request
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| const std::vector< std::map< std::string, std::string >> & | object_list | a vector of object meta data maps |
| const std::string & | bucket_name | name of the bucket containing the objects |
#### Returns:
| Type | Description |
| ---- | ---- |
| std::string | xml response |
[Go to Top](#deimos-s3_utilities)
[deimos-s3_utilities-S3_authorisation]:./S3_authorisation.md
......
......@@ -29,6 +29,7 @@ Inherits from [deimos::storage::Storage][deimos-storage-Storage].
| [operator=](#deimos-storage-Phobos_file-operator=-1) | assignment operator deleted |
| [set_meta_data](#deimos-storage-Phobos_file-set_meta_data) | set the metadata that an object that is added to the database should get |
| [get_meta_data](#deimos-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs |
| [get_object_list](#deimos-storage-Phobos_file-get_object_list) | get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs |
| [db_put](#deimos-storage-Phobos_file-db_put) | Puts data to the databse. |
| [db_get](#deimos-storage-Phobos_file-db_get) | Gets data from the databse. |
| [prepare_put](#deimos-storage-Phobos_file-prepare_put) | Starts a put operation to the database. |
......@@ -44,7 +45,7 @@ Inherits from [deimos::storage::Storage][deimos-storage-Storage].
## Private Attributes
### <a name='deimos-storage-Phobos_file-m_descriptor' /> private deimos::storage::Phobos_file::m_descriptor = {0}
### <a name='deimos-storage-Phobos_file-m_descriptor' /> private deimos::storage::Phobos_file::m_descriptor
struct to contain information for phobos
......@@ -221,6 +222,40 @@ get the metadata associated to the current object as a map of key:value pairs
#### Qualifiers:
* virtual
[Go to Top](#deimos-storage-Phobos_file)
### <a name='deimos-storage-Phobos_file-get_object_list' /> public std::vector< std::map< std::string, std::string > > deimos::storage::Phobos_file::get_object_list (std::string bucket_name) override
get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::string | bucket_name | The name of the requested bucket |
#### Returns:
| Type | Description |
| ---- | ---- |
| std::vector< std::map< std::string, std::string > > | a vector of maps of the key:value metadata pairs |
#### Qualifiers:
* virtual
......
......@@ -17,6 +17,7 @@ Is inherited by [deimos::storage::Phobos_file][deimos-storage-Phobos_file].
| ---- | ---- |
| [set_meta_data](#deimos-storage-Storage-set_meta_data) | set the metadata that an object that is added to the database should get |
| [get_meta_data](#deimos-storage-Storage-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs |
| [get_object_list](#deimos-storage-Storage-get_object_list) | get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs |
## PUT functions
......@@ -99,6 +100,40 @@ get the metadata associated to the current object as a map of key:value pairs
#### Qualifiers:
* virtual
[Go to Top](#deimos-storage-Storage)
### <a name='deimos-storage-Storage-get_object_list' /> public std::vector< std::map< std::string, std::string > > deimos::storage::Storage::get_object_list (std::string bucket_name)=0
get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::string | bucket_name | The name of the requested bucket |
#### Returns:
| Type | Description |
| ---- | ---- |
| std::vector< std::map< std::string, std::string > > | a vector of maps of the key:value metadata pairs |
#### Qualifiers:
* virtual
......
......@@ -35,6 +35,7 @@ Inherits from [deimos::stream::Stream][deimos-stream-Stream].
| [operator=](#deimos-stream-Fifo-operator=-1) | assignment operator deleted |
| [set_storage_meta_data](#deimos-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get |
| [get_meta_data](#deimos-stream-Fifo-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs |
| [get_bucket_object_list](#deimos-stream-Fifo-get_bucket_object_list) | get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs |
| [start_put](#deimos-stream-Fifo-start_put) | start a put operation |
| [put](#deimos-stream-Fifo-put) | add a chunk of data to the object |
| [start_get](#deimos-stream-Fifo-start_get) | start a gett operation |
......@@ -298,6 +299,41 @@ get the metadata associated to the current object as a map of key:value pairs
#### Qualifiers:
* const
* virtual
[Go to Top](#deimos-stream-Fifo)
### <a name='deimos-stream-Fifo-get_bucket_object_list' /> public std::vector< std::map< std::string, std::string > > deimos::stream::Fifo::get_bucket_object_list (std::string bucket_name) const override
get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::string | bucket_name | The name of the requested bucket |
#### Returns:
| Type | Description |
| ---- | ---- |
| std::vector< std::map< std::string, std::string > > | a vector of maps of the key:value metadata pairs |
#### Qualifiers:
* const
* virtual
......
......@@ -34,6 +34,7 @@ Is inherited by [deimos::stream::Fifo][deimos-stream-Fifo].
| [~Stream](#deimos-stream-Stream-~Stream) | default destructor |
| [set_storage_meta_data](#deimos-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get |
| [get_meta_data](#deimos-stream-Stream-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs |
| [get_bucket_object_list](#deimos-stream-Stream-get_bucket_object_list) | get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs |
......@@ -347,6 +348,41 @@ get the metadata associated to the current object as a map of key:value pairs
#### Qualifiers:
* const
* virtual
[Go to Top](#deimos-stream-Stream)
### <a name='deimos-stream-Stream-get_bucket_object_list' /> public std::vector< std::map< std::string, std::string > > deimos::stream::Stream::get_bucket_object_list (std::string bucket_name) const =0
get the metadata associated to all objects of a particular bucket as a vector of maps of key:value pairs
#### Parameters:
| Type | Name | Description |
| ---- | ---- | ---- |
| std::string | bucket_name | The name of the requested bucket |
#### Returns:
| Type | Description |
| ---- | ---- |
| std::vector< std::map< std::string, std::string > > | a vector of maps of the key:value metadata pairs |
#### Qualifiers:
* const
* virtual
......
This diff is collapsed.
......@@ -24,10 +24,10 @@ target_link_system_libraries(deimos PUBLIC spdlog::spdlog)
target_link_libraries(deimos PUBLIC server)
if(CMAKE_BUILD_TYPE MATCHES DEBUG)
if(CMAKE_BUILD_TYPE MATCHES Debug)
include(../cmake/development_flags.cmake)
target_link_libraries(deimos PUBLIC development_flags)
endif(CMAKE_BUILD_TYPE MATCHES DEBUG)
endif(CMAKE_BUILD_TYPE MATCHES Debug)
add_custom_command(
......
......@@ -52,6 +52,38 @@ void GetRequestHandler::onRequest(
}
#endif
if (m_s3_header.is_bucket_only_request()) {
spdlog::info("bucket only request");
try {
spdlog::info("Retrieving objects");
std::vector<std::map<std::string, std::string>> object_list =
m_stream->get_bucket_object_list(m_s3_header.get_bucket());
spdlog::debug("{} objects in bucket", object_list.size());
proxygen::ResponseBuilder(downstream_)
.status(200, "Ok")
.body(s3_utilities::list_objects_xml_response(
object_list, m_s3_header.get_bucket()))
.sendWithEOM();
spdlog::info("object list send complete");
return;
}
catch (const DeimosException& ex) {
spdlog::error("caught exception: {}", ex.what());
s3_utilities::s3_error_info error =
s3_utilities::create_s3_error(ex.get_inner_error());
proxygen::ResponseBuilder(downstream_)
.status(error.https_error_code, error.https_error_identifier)
.body(error.get_xml(m_s3_header.get_key()))
.sendWithEOM();
return;
}
}
std::map<std::string, std::string> meta_data;
std::string content_length;
......
......@@ -96,16 +96,24 @@ class GetRequestHandler : public proxygen::RequestHandler {
{
}
///
/// @brief function called in the case of a list_objects request
///
/// Objects for a given bucket are added to the response
///
void list_objects() noexcept;
private:
void read_file(folly::EventBase* evb);
bool check_for_completion();
std::unique_ptr<stream::Stream> m_stream;
s3_utilities::S3_header m_s3_header;
bool m_read_file_scheduled{false};
std::atomic<bool> m_paused{false};
bool m_finished{false};
bool m_stream_completed{false};
bool m_finished{false};
};
} // namespace deimos
......@@ -14,6 +14,7 @@
#include <openssl/hmac.h>
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <sstream>
......@@ -151,6 +152,32 @@ std::string openssl_hmac_hex(const std::string key, const std::string msg)
return (ss.str());
}
std::string list_objects_xml_response(
const std::vector<std::map<std::string, std::string>>& object_list,
const std::string& bucket_name)
{
std::stringstream xml_stream;
xml_stream << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
xml_stream << "<ListBucketResult>\n";
xml_stream << "\t<Name>" << bucket_name << "</Name>\n";
xml_stream << "\t<KeyCount>" << object_list.size() << "</KeyCount>\n";
std::for_each(
object_list.begin(), object_list.end(),
[&](std::map<std::string, std::string> meta_data) {
xml_stream << "\t<Contents>\n";
xml_stream << "\t\t<Key>" << meta_data["key"] << "</Key>\n";
xml_stream << "\t\t<LastModified>"
<< meta_data["deimos:creation_time"]
<< "</LastModified>\n";
xml_stream << "\t\t<Size>" << meta_data["deimos:content_length"]
<< "</Size>\n";
xml_stream << "\t</Contents>\n";
});
xml_stream << "</ListBucketResult>\n";
return xml_stream.str();
}
} // namespace s3_utilities
} // namespace deimos
......@@ -10,7 +10,9 @@
#pragma once
#include <map>
#include <string>
#include <vector>
namespace deimos {
///
......@@ -133,5 +135,17 @@ std::string openssl_hmac(const std::string key, const std::string msg);
///
std::string openssl_hmac_hex(const std::string key, const std::string msg);
///
/// @brief create an xml response for the list_objects request
///
/// @param object_list a vector of object meta data maps
/// @param bucket_name name of the bucket containing the objects
///
/// @return xml response
///
std::string list_objects_xml_response(
const std::vector<std::map<std::string, std::string>>& object_list,
const std::string& bucket_name);
} // namespace s3_utilities
} // namespace deimos
......@@ -23,15 +23,15 @@ class UnsupportedRequestHandler : public proxygen::RequestHandler {
void onRequest(
std::unique_ptr<proxygen::HTTPMessage> headers) noexcept override;
void onBody(std::unique_ptr<folly::IOBuf> body) noexcept override {}
void onBody(std::unique_ptr<folly::IOBuf> /*body*/) noexcept override {}
void onEOM() noexcept override {}
void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override {}
void onUpgrade(proxygen::UpgradeProtocol /*proto*/) noexcept override {}
void requestComplete() noexcept override {}
void onError(proxygen::ProxygenError err) noexcept override {}
void onError(proxygen::ProxygenError /*err*/) noexcept override {}
void onEgressPaused() noexcept override {}
......
......@@ -42,3 +42,19 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer)
{
return pho_xfer_desc_destroy(xfer);
}
int phobos_store_object_list_cpp(
const char* pattern,
const char** metadata,
int n_metadata,
struct object_info** objs,
int* n_objs)
{
return phobos_store_object_list(
pattern, metadata, n_metadata, objs, n_objs);
}
void phobos_store_object_list_free_cpp(struct object_info* objs, int n_objs)
{
return phobos_store_object_list_free(objs, n_objs);
}
......@@ -29,9 +29,9 @@ enum rsc_family {
};
static const char* const rsc_family_names[] = {
[PHO_RSC_DISK] = "disk",
[PHO_RSC_TAPE] = "tape",
[PHO_RSC_DIR] = "dir",
"disk",
"tape",
"dir",
};
static inline const char* rsc_family2str(enum rsc_family family)
......@@ -126,6 +126,11 @@ struct pho_xfer_desc {