From 2836db9f3b542fe36530e03333fb3ee7ffe02a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Wed, 5 Feb 2020 12:23:07 +0000 Subject: [PATCH 01/62] Fix for FIFO related race conditions Change Log: * Changed PUT functionality to match GET logic * All threading now handled by server * Moved asynchronous calls to backend to server source to prevent FIFO open from blocking * Introduced file_closed boolean to prevent extra read of FIFO during GET request --- src/backend/backend.cc | 74 +++++++++++++++++++------------ src/backend/backend.h | 59 ++++++++++++------------ src/fifo/FiPhoExceptions.h | 2 +- src/fifo/fifo_get.cc | 24 +++++++--- src/fifo/fifo_ops.cc | 19 +++----- src/fifo/fifo_ops.h | 14 +++--- src/fifo/fifo_put.cc | 16 ++++++- src/server/get_request_handler.cc | 31 ++++++++----- src/server/get_request_handler.h | 2 + src/server/put_request_handler.cc | 21 +++++---- src/server/put_request_handler.h | 1 + src/server/server.cc | 6 +-- 12 files changed, 158 insertions(+), 111 deletions(-) diff --git a/src/backend/backend.cc b/src/backend/backend.cc index b5622b0..14cabea 100644 --- a/src/backend/backend.cc +++ b/src/backend/backend.cc @@ -1,21 +1,42 @@ #include "backend.h" +#include #include #include +#include #include #include -#include -#include #include -#include #include #include #include +#include #include "../fifo/FiPhoExceptions.h" namespace fiphoboserver { +void DbInterface::openFile(std::string fileName, int flags) +{ +#ifdef DEBUG + std::cout << "Backend::openFile" << std::endl; +#endif + file_descriptor = open(fileName.c_str(), flags); + if (file_descriptor < 0) { + std::stringstream ss; + ss << "open with filename " << fileName.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } +} + +void DbInterface::closeFile() +{ + if (file_descriptor > 0) { + close(file_descriptor); + } +} + PhobosDb::PhobosDb() { #ifdef DEBUG @@ -35,12 +56,14 @@ PhobosDb::~PhobosDb() pho_xfer_desc_destroy_cpp(&descriptor); } -void PhobosDb::db_put(ssize_t size, std::string objectID, int fd) +int PhobosDb::db_put(ssize_t size, std::string objectID, std::string fileName) { #ifdef DEBUG std::cout << "PhobosDb::db_put" << std::endl; +#endif + openFile(fileName, O_RDONLY); +#ifdef DEBUG std::cout << "PhobosDb: Object size = " << size << std::endl; - std::cout << "PhobosDb: fifo_descriptor = " << fd << std::endl; #endif char* unconstedObjectID = new char[objectID.length() + 1]; @@ -48,46 +71,39 @@ void PhobosDb::db_put(ssize_t size, std::string objectID, int fd) descriptor.xd_op = PHO_XFER_OP_PUT; descriptor.xd_objid = unconstedObjectID; - descriptor.xd_fd = fd; + descriptor.xd_fd = file_descriptor; descriptor.xd_size = size; #ifdef DEBUG - std::cout << "Starting phobos_put_cpp" << std::endl; + std::cout << "PhobosDb: Starting phobos_put_cpp" << std::endl; #endif - db_result = std::async([&]() { - std::cout << "Phobos Thread id " << std::this_thread::get_id() - << std::endl; - return phobos_put_cpp(&descriptor, 1, NULL, NULL); - }); + int rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); -#ifdef DEBUG - std::cout << "Main Thread id " << std::this_thread::get_id() - << std::endl; -#endif - // TODOOOO - // delete [] unconstedObjectID; + closeFile(); + + return rc; } -void PhobosDb::db_get(std::string objectID, int fd) +int PhobosDb::db_get(std::string objectID, std::string fileName) { #ifdef DEBUG std::cout << "PhobosDb::db_get" << std::endl; - std::cout << "PhobosDb: fifo_descriptor = " << fd << std::endl; #endif + openFile(fileName, O_WRONLY); descriptor.xd_objid = new char[objectID.length() + 1]; strcpy(descriptor.xd_objid, objectID.c_str()); - descriptor.xd_op = PHO_XFER_OP_GET; - descriptor.xd_fd = fd; + descriptor.xd_op = PHO_XFER_OP_GET; + descriptor.xd_fd = file_descriptor; #ifdef DEBUG - std::cout << "Starting phobos_get_cpp" << std::endl; + std::cout << "PhobosDB: Starting phobos_get_cpp" << std::endl; #endif - db_result = std::async([&]() { - std::cout << "Phobos Thread id " << std::this_thread::get_id() - << std::endl; - return phobos_get_cpp(&descriptor, 1, NULL, NULL); - }); + int rc = phobos_get_cpp(&descriptor, 1, NULL, NULL); + + closeFile(); + + return rc; } void PhobosDb::setBucketName(std::string bucketName) @@ -98,7 +114,7 @@ void PhobosDb::setBucketName(std::string bucketName) int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucketName.c_str()); if (rc) { throw fiphoexceptions::PhobosException("pho_attr_set", rc); - } + } } void PhobosDb::setMetaData( diff --git a/src/backend/backend.h b/src/backend/backend.h index b0ab68d..7e24d52 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -1,9 +1,9 @@ #pragma once +#include #include -#include #include -#include +#include extern "C" { #include "phobos_cpp_wrapper/phobos_cpp_wrapper.h" @@ -15,35 +15,36 @@ namespace fiphoboserver { // one file. Then this structure is stupid. Maybe with a vector of descriptors? class DbInterface { - public: - /* Interface functions */ - virtual void db_put(ssize_t size, std::string objectID, int fd) = 0; - virtual void db_get(std::string, int fd) = 0; - /* - * virtual ssize_t db_getmd(); - */ - std::future get_db_result() { return std::move(db_result); } - - protected: - std::future db_result; + public: + /* Interface functions */ + virtual int db_put( + ssize_t size, std::string objectID, std::string fileName) = 0; + virtual int db_get(std::string objectID, std::string fileName) = 0; + /* + * virtual ssize_t db_getmd(); + */ + void openFile(std::string fileName, int flags); + void closeFile(); + + protected: + int file_descriptor = -1; }; class PhobosDb : public DbInterface { - public: - PhobosDb(); - ~PhobosDb(); - void db_put(ssize_t size, std::string objectID, int fd); - void db_get(std::string, int fd); - /* - * ssize_t db_getmd(); - */ - - void setBucketName(std::string bucketName); - void setMetaData( - std::vector> metaData); - - private: - struct pho_xfer_desc descriptor = {0}; + public: + PhobosDb(); + ~PhobosDb(); + int db_put(ssize_t size, std::string objectID, std::string fileName); + int db_get(std::string, std::string fileName); + /* + * ssize_t db_getmd(); + */ + + void setBucketName(std::string bucketName); + void setMetaData(std::vector> metaData); + + private: + struct pho_xfer_desc descriptor = {0}; }; -} // namespace fiphoboserver +} // namespace fiphoboserver diff --git a/src/fifo/FiPhoExceptions.h b/src/fifo/FiPhoExceptions.h index 09533f2..f2d6c10 100644 --- a/src/fifo/FiPhoExceptions.h +++ b/src/fifo/FiPhoExceptions.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include namespace fiphoexceptions { diff --git a/src/fifo/fifo_get.cc b/src/fifo/fifo_get.cc index 2aa19a6..34503a2 100644 --- a/src/fifo/fifo_get.cc +++ b/src/fifo/fifo_get.cc @@ -2,20 +2,34 @@ #include #include +#include +#include #include #include -#include #include #include #include -#include namespace fiphoboserver { +void FIFOGet::openFIFO() +{ +#ifdef DEBUG + std::cout << "FIFOGet::openFIFO" << std::endl; +#endif + fifo_descriptor = open(fifoName.c_str(), O_RDONLY); + if (fifo_descriptor < 0) { + std::stringstream ss; + ss << "open with filename " << fifoName.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } +} + ssize_t FIFOGet::getDataFromFIFO(void* buf, size_t count) { #ifdef DEBUG - std::cout << "Phobos_Layer::getDataFromFIFO" << std::endl; + std::cout << "FIFOGet::getDataFromFIFO" << std::endl; #endif ssize_t rc = read(fifo_descriptor, buf, count); if (rc < 0) { @@ -24,10 +38,6 @@ ssize_t FIFOGet::getDataFromFIFO(void* buf, size_t count) #endif throw fiphoexceptions::FIFOException("read", rc); } -#ifdef DEBUG - std::cout << "Count = " << rc << std::endl; -#endif - return rc; } diff --git a/src/fifo/fifo_ops.cc b/src/fifo/fifo_ops.cc index 9ef10c8..e29a61e 100644 --- a/src/fifo/fifo_ops.cc +++ b/src/fifo/fifo_ops.cc @@ -1,8 +1,8 @@ #include "fifo_ops.h" -#include #include #include +#include namespace fiphoboserver { @@ -11,6 +11,9 @@ namespace fiphoboserver { FIFO::FIFO() { +#ifdef DEBUG + std::cout << "FIFO::FIFO" << std::endl; +#endif startFIFO(); } @@ -33,9 +36,6 @@ void FIFO::startFIFO() std::cout << "FIFO::startFIFO" << std::endl; #endif fifoName = std::tmpnam(nullptr); -#ifdef DEBUG - std::cout << "FIFO name: " << fifoName << std::endl; -#endif int rc = mkfifo(fifoName.c_str(), 0777); if (rc < 0) { @@ -44,15 +44,6 @@ void FIFO::startFIFO() return; } } - - fifo_descriptor = open(fifoName.c_str(), O_RDWR); - if (fifo_descriptor < 0) { - std::stringstream ss; - ss << "open with filename " << fifoName.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); - return; - } } -} // namespace fiphoboserver - +} // namespace fiphoboserver diff --git a/src/fifo/fifo_ops.h b/src/fifo/fifo_ops.h index 213f0b1..c148477 100644 --- a/src/fifo/fifo_ops.h +++ b/src/fifo/fifo_ops.h @@ -1,8 +1,8 @@ #pragma once -#include -#include #include "FiPhoExceptions.h" +#include +#include namespace fiphoboserver { @@ -10,7 +10,8 @@ class FIFO { public: FIFO(); void setBucketName(std::string bucketName); - int get_fifo_descriptor() { return fifo_descriptor; } + std::string getFifoName() { return fifoName; } + virtual void openFIFO() = 0; ~FIFO(); protected: @@ -22,6 +23,7 @@ class FIFO { class FIFOPut : public FIFO { public: void setMetaData(std::vector> metaData); + void openFIFO(); void addDataToFIFO(std::unique_ptr buffer); void finishPut(); @@ -30,13 +32,13 @@ class FIFOPut : public FIFO { bool putRunning = false; }; -class FIFOGet: public FIFO { +class FIFOGet : public FIFO { public: + void openFIFO(); ssize_t getDataFromFIFO(void* buf, size_t count); /* * finishGet(); */ }; -} // namespace fiphoboserver - +} // namespace fiphoboserver diff --git a/src/fifo/fifo_put.cc b/src/fifo/fifo_put.cc index 3cd271e..c8e4772 100644 --- a/src/fifo/fifo_put.cc +++ b/src/fifo/fifo_put.cc @@ -2,15 +2,29 @@ #include #include +#include #include #include -#include #include #include #include namespace fiphoboserver { +void FIFOPut::openFIFO() +{ +#ifdef DEBUG + std::cout << "FIFOPut::openFIFO" << std::endl; +#endif + fifo_descriptor = open(fifoName.c_str(), O_WRONLY); + if (fifo_descriptor < 0) { + std::stringstream ss; + ss << "open with filename " << fifoName.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } +} + void FIFOPut::addDataToFIFO(std::unique_ptr buffer) { #ifdef DEBUG diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index bd39607..b7140f5 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -1,8 +1,7 @@ /* * Copyright (c) Facebook, Inc. and its affiliates. * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the + * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. */ @@ -25,9 +24,6 @@ namespace fiphoboserver { void GetRequestHandler::onRequest( std::unique_ptr headers) noexcept { -#ifdef DEBUG - std::cout << "Entered GetRequestHandler::onRequest" << std::endl; -#endif if (headers->getMethod() != proxygen::HTTPMethod::GET) { proxygen::ResponseBuilder(downstream_) .status(400, "Bad method") @@ -49,7 +45,6 @@ void GetRequestHandler::onRequest( s3_header.setHeaders(std::move(headers)); - try { #ifdef DEBUG @@ -60,9 +55,15 @@ void GetRequestHandler::onRequest( backend.setBucketName(s3_header.getBucket()); #ifdef DEBUG - std::cout << "Send get to phobos layer" << std::endl; + std::cout << "Creating backend GET thread" << std::endl; #endif - backend.db_get(s3_header.getKey(), fifo.get_fifo_descriptor()); + db_result = std::async( + [this, key = s3_header.getKey(), fifoName = fifo.getFifoName()]() { + return backend.db_get(key, fifoName); + }); + + fifo.openFIFO(); + file_closed_ = false; } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -91,11 +92,17 @@ void GetRequestHandler::onRequest( void GetRequestHandler::readFile(folly::EventBase* evb) { +#ifdef DEBUG + std::cout << "GetRequestHandler::readFile" << std::endl; +#endif folly::IOBufQueue buf; - while (!paused_) { + while (!file_closed_ && !paused_) { // read 4k-ish chunks and foward each one to the client auto data = buf.preallocate(4000, 4000); auto rc = fifo.getDataFromFIFO(data.first, data.second); +#ifdef DEBUG + std::cout << "getDataFromFIFO: rc = " << rc << std::endl; +#endif if (rc < 0) { // error VLOG(4) << "Read error=" << rc; @@ -107,6 +114,7 @@ void GetRequestHandler::readFile(folly::EventBase* evb) } else if (rc == 0) { // done + file_closed_ = true; VLOG(4) << "Read EOF"; evb->runInEventBaseThread([this] { proxygen::ResponseBuilder(downstream_).sendWithEOM(); @@ -160,7 +168,10 @@ void GetRequestHandler::onBody(std::unique_ptr /*body*/) noexcept // ignore, only support GET } -void GetRequestHandler::onEOM() noexcept {} +void GetRequestHandler::onEOM() noexcept +{ + db_result.wait(); +} void GetRequestHandler::onUpgrade( proxygen::UpgradeProtocol /*protocol*/) noexcept diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 6356054..848562d 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -51,6 +51,8 @@ class GetRequestHandler : public proxygen::RequestHandler { bool readFileScheduled_{false}; std::atomic paused_{false}; bool finished_{false}; + bool file_closed_{false}; + std::future db_result; }; } // namespace fiphoboserver diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 3ac0be7..7d9ea18 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -38,7 +38,7 @@ void PutRequestHandler::onRequest( #endif #ifdef DEBUG - std::cout << "Starting Create Bucket queries..." << std::endl; + std::cout << "Creating Bucket queries..." << std::endl; #endif s3_header.setHeaders(std::move(headers)); if (s3_header.isCreateBucketRequest()) { @@ -51,7 +51,6 @@ void PutRequestHandler::onRequest( try { - #ifdef DEBUG std::cout << "Bucket: " << s3_header.getBucket() << '\n'; std::cout << "Key: " << s3_header.getKey() << '\n'; @@ -68,14 +67,14 @@ void PutRequestHandler::onRequest( backend.setMetaData(s3_header.getMetaData()); #ifdef DEBUG - std::cout << "Send put to phobos layer asynchronously" << std::endl; -#endif - backend.db_put( - s3_header.getBodyLength(), s3_header.getKey(), fifo.get_fifo_descriptor()); -#ifdef DEBUG - std::cout << "Past asynchronous put call" << std::endl; + std::cout << "Creating backend PUT thread" << std::endl; #endif - + db_result = std::async([this]() { + return backend.db_put( + s3_header.getBodyLength(), s3_header.getKey(), + fifo.getFifoName()); + }); + fifo.openFIFO(); } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -102,7 +101,7 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept if (body) { body->coalesce(); size_t totalBodyLength = body->computeChainDataLength(); - std::cout << "Body Data Length " << totalBodyLength << '\n'; + std::cout << "Body Data Length = " << totalBodyLength << '\n'; } #endif try { @@ -125,7 +124,7 @@ void PutRequestHandler::onEOM() noexcept #ifdef DEBUG std::cout << "Finish IO..." << std::endl; #endif - backend.get_db_result().wait(); + db_result.wait(); fifo.finishPut(); } catch (fiphoexceptions::FIFOException& fifoExcp) { diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 1f35237..8cdac8f 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -40,6 +40,7 @@ class PutRequestHandler : public proxygen::RequestHandler { S3_header s3_header; PhobosDb backend; FIFOPut fifo; + std::future db_result; }; } // namespace fiphoboserver diff --git a/src/server/server.cc b/src/server/server.cc index a5a257e..2d9b48d 100644 --- a/src/server/server.cc +++ b/src/server/server.cc @@ -7,7 +7,7 @@ */ #include -#include +#include #include #include #include @@ -51,13 +51,13 @@ class HandlerFactory : public proxygen::RequestHandlerFactory { #endif if (headers->getMethod() == proxygen::HTTPMethod::GET) { #ifdef DEBUG - std::cout << "Chose GET" << std::endl; + std::cout << "GET" << std::endl; #endif return new fiphoboserver::GetRequestHandler; } else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { #ifdef DEBUG - std::cout << "Chose PUT" << std::endl; + std::cout << "PUT" << std::endl; #endif return new fiphoboserver::PutRequestHandler; } -- GitLab From 98ca36030b15dfc44bfa495cbae28c8eab131835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Thu, 6 Feb 2020 22:57:17 +0000 Subject: [PATCH 02/62] Added .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d171c66 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.swp +todo.txt -- GitLab From 9a8740af04a5f95e81f4192eefb84f9aebe2565a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Fri, 7 Feb 2020 17:18:46 +0000 Subject: [PATCH 03/62] Modularise project and implement abstract interfaces Project consists of three components: server, stream, and storage. Each need not know the implementation details of the others. Stream and Storage are presented as base classes with Fifo and Phobos_file serving as implementations. Inter-component communication is reworked such that the stream acts as an exclusive interface between the server and storage. Change Log: * Rename backend to storage * Rename fifo to stream * PUT logic functions like server->stream->storage * GET logic functions like storage->stream->server * Each stage represented by a class * Communication between stages handled by class ownership * Ownership is server server.main.cc} | 23 +-- src/server/unsupported_request_handler.cc | 2 +- src/storage/CMakeLists.txt | 16 ++ .../phobos_cpp_wrapper/CMakeLists.txt | 0 .../phobos_cpp_wrapper/phobos_cpp_wrapper.c | 0 .../phobos_cpp_wrapper/phobos_cpp_wrapper.h | 0 src/storage/phobos_file.cc | 133 ++++++++++++++++ src/storage/phobos_file.h | 29 ++++ src/storage/storage.h | 25 +++ src/{fifo => stream}/CMakeLists.txt | 12 +- src/{fifo => stream}/FiPhoExceptions.h | 2 +- src/stream/fifo.cc | 142 ++++++++++++++++++ src/stream/fifo.h | 31 ++++ src/stream/stream.h | 44 ++++++ 27 files changed, 483 insertions(+), 544 deletions(-) delete mode 100644 src/backend/CMakeLists.txt delete mode 100644 src/backend/backend.cc delete mode 100644 src/backend/backend.h delete mode 100644 src/fifo/fifo_get.cc delete mode 100644 src/fifo/fifo_ops.cc delete mode 100644 src/fifo/fifo_ops.h delete mode 100644 src/fifo/fifo_put.cc rename src/server/{server.cc => server.main.cc} (84%) create mode 100644 src/storage/CMakeLists.txt rename src/{backend => storage}/phobos_cpp_wrapper/CMakeLists.txt (100%) rename src/{backend => storage}/phobos_cpp_wrapper/phobos_cpp_wrapper.c (100%) rename src/{backend => storage}/phobos_cpp_wrapper/phobos_cpp_wrapper.h (100%) create mode 100644 src/storage/phobos_file.cc create mode 100644 src/storage/phobos_file.h create mode 100644 src/storage/storage.h rename src/{fifo => stream}/CMakeLists.txt (61%) rename src/{fifo => stream}/FiPhoExceptions.h (100%) create mode 100644 src/stream/fifo.cc create mode 100644 src/stream/fifo.h create mode 100644 src/stream/stream.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3857ed9..282155f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,6 @@ if(DEBUG) add_compile_definitions(DEBUG) endif(DEBUG) -add_subdirectory(backend) -add_subdirectory(fifo) +add_subdirectory(storage) +add_subdirectory(stream) add_subdirectory(server) diff --git a/src/backend/CMakeLists.txt b/src/backend/CMakeLists.txt deleted file mode 100644 index 5012486..0000000 --- a/src/backend/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -include_directories(${PHOBOS_INCLUDE_DIRECTORY}) -include_directories(/usr/include/glib-2.0) -include_directories(/usr/lib64/glib-2.0/include) - -add_subdirectory(phobos_cpp_wrapper) - -add_library( - backend - backend.cc -) - -target_compile_features(backend PUBLIC cxx_std_14) - -target_link_libraries(backend PUBLIC phobos_store) -target_link_libraries(backend PUBLIC phobos_cpp_wrapper) -target_link_libraries(backend PUBLIC proxygen::proxygen) diff --git a/src/backend/backend.cc b/src/backend/backend.cc deleted file mode 100644 index 14cabea..0000000 --- a/src/backend/backend.cc +++ /dev/null @@ -1,137 +0,0 @@ -#include "backend.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../fifo/FiPhoExceptions.h" - -namespace fiphoboserver { - -void DbInterface::openFile(std::string fileName, int flags) -{ -#ifdef DEBUG - std::cout << "Backend::openFile" << std::endl; -#endif - file_descriptor = open(fileName.c_str(), flags); - if (file_descriptor < 0) { - std::stringstream ss; - ss << "open with filename " << fileName.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); - return; - } -} - -void DbInterface::closeFile() -{ - if (file_descriptor > 0) { - close(file_descriptor); - } -} - -PhobosDb::PhobosDb() -{ -#ifdef DEBUG - std::cout << "PhobosDb::PhobosDb" << std::endl; -#endif - memset(&descriptor, 0, sizeof(descriptor)); - descriptor.xd_attrs = {0}; -} - -PhobosDb::~PhobosDb() -{ -#ifdef DEBUG - std::cout << "PhobosDb::~PhobosDb" << std::endl; -#endif - // pho_attrs_free(&descriptor.xd_attrs); // TODO: Is that done in - // // desc_destroy?? - pho_xfer_desc_destroy_cpp(&descriptor); -} - -int PhobosDb::db_put(ssize_t size, std::string objectID, std::string fileName) -{ -#ifdef DEBUG - std::cout << "PhobosDb::db_put" << std::endl; -#endif - openFile(fileName, O_RDONLY); -#ifdef DEBUG - std::cout << "PhobosDb: Object size = " << size << std::endl; -#endif - - char* unconstedObjectID = new char[objectID.length() + 1]; - strcpy(unconstedObjectID, objectID.c_str()); - - descriptor.xd_op = PHO_XFER_OP_PUT; - descriptor.xd_objid = unconstedObjectID; - descriptor.xd_fd = file_descriptor; - descriptor.xd_size = size; - -#ifdef DEBUG - std::cout << "PhobosDb: Starting phobos_put_cpp" << std::endl; -#endif - int rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); - - closeFile(); - - return rc; -} - -int PhobosDb::db_get(std::string objectID, std::string fileName) -{ -#ifdef DEBUG - std::cout << "PhobosDb::db_get" << std::endl; -#endif - openFile(fileName, O_WRONLY); - descriptor.xd_objid = new char[objectID.length() + 1]; - strcpy(descriptor.xd_objid, objectID.c_str()); - - descriptor.xd_op = PHO_XFER_OP_GET; - descriptor.xd_fd = file_descriptor; - -#ifdef DEBUG - std::cout << "PhobosDB: Starting phobos_get_cpp" << std::endl; -#endif - int rc = phobos_get_cpp(&descriptor, 1, NULL, NULL); - - closeFile(); - - return rc; -} - -void PhobosDb::setBucketName(std::string bucketName) -{ -#ifdef DEBUG - std::cout << "PhobosDb::setBucketName" << std::endl; -#endif - int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucketName.c_str()); - if (rc) { - throw fiphoexceptions::PhobosException("pho_attr_set", rc); - } -} - -void PhobosDb::setMetaData( - std::vector> metaData) -{ -#ifdef DEBUG - std::cout << "PhobosDb::setMetaData" << std::endl; -#endif - std::for_each( - metaData.begin(), metaData.end(), - [&](std::pair pair) { - int rc = pho_attr_set( - &descriptor.xd_attrs, pair.first.c_str(), pair.second.c_str()); - if (rc) { - throw fiphoexceptions::PhobosException("pho_attr_set", rc); - } - }); -} - -} // namespace fiphoboserver diff --git a/src/backend/backend.h b/src/backend/backend.h deleted file mode 100644 index 7e24d52..0000000 --- a/src/backend/backend.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -extern "C" { -#include "phobos_cpp_wrapper/phobos_cpp_wrapper.h" -} - -namespace fiphoboserver { - -// TODO: Remember the m-versions: PUT and GET functions can work on more than -// one file. Then this structure is stupid. Maybe with a vector of descriptors? - -class DbInterface { - public: - /* Interface functions */ - virtual int db_put( - ssize_t size, std::string objectID, std::string fileName) = 0; - virtual int db_get(std::string objectID, std::string fileName) = 0; - /* - * virtual ssize_t db_getmd(); - */ - void openFile(std::string fileName, int flags); - void closeFile(); - - protected: - int file_descriptor = -1; -}; - -class PhobosDb : public DbInterface { - public: - PhobosDb(); - ~PhobosDb(); - int db_put(ssize_t size, std::string objectID, std::string fileName); - int db_get(std::string, std::string fileName); - /* - * ssize_t db_getmd(); - */ - - void setBucketName(std::string bucketName); - void setMetaData(std::vector> metaData); - - private: - struct pho_xfer_desc descriptor = {0}; -}; - -} // namespace fiphoboserver diff --git a/src/fifo/fifo_get.cc b/src/fifo/fifo_get.cc deleted file mode 100644 index 34503a2..0000000 --- a/src/fifo/fifo_get.cc +++ /dev/null @@ -1,44 +0,0 @@ -#include "fifo_ops.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fiphoboserver { - -void FIFOGet::openFIFO() -{ -#ifdef DEBUG - std::cout << "FIFOGet::openFIFO" << std::endl; -#endif - fifo_descriptor = open(fifoName.c_str(), O_RDONLY); - if (fifo_descriptor < 0) { - std::stringstream ss; - ss << "open with filename " << fifoName.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); - return; - } -} - -ssize_t FIFOGet::getDataFromFIFO(void* buf, size_t count) -{ -#ifdef DEBUG - std::cout << "FIFOGet::getDataFromFIFO" << std::endl; -#endif - ssize_t rc = read(fifo_descriptor, buf, count); - if (rc < 0) { -#ifdef DEBUG - std::cout << "getDataFromFIFO read failed" << std::endl; -#endif - throw fiphoexceptions::FIFOException("read", rc); - } - return rc; -} - -} // namespace fiphoboserver diff --git a/src/fifo/fifo_ops.cc b/src/fifo/fifo_ops.cc deleted file mode 100644 index e29a61e..0000000 --- a/src/fifo/fifo_ops.cc +++ /dev/null @@ -1,49 +0,0 @@ -#include "fifo_ops.h" - -#include -#include -#include - -namespace fiphoboserver { - -// TODO: Remember the m-versions: PUT and GET functions can work on more than -// one file. Then this structure is stupid. Maybe with a vector of descriptors? - -FIFO::FIFO() -{ -#ifdef DEBUG - std::cout << "FIFO::FIFO" << std::endl; -#endif - startFIFO(); -} - -FIFO::~FIFO() -{ -#ifdef DEBUG - std::cout << "FIFO::~FIFO" << std::endl; -#endif - if (access(fifoName.c_str(), F_OK) != -1) { - remove(fifoName.c_str()); - } - if (fifo_descriptor > 0) { - close(fifo_descriptor); - } -} - -void FIFO::startFIFO() -{ -#ifdef DEBUG - std::cout << "FIFO::startFIFO" << std::endl; -#endif - fifoName = std::tmpnam(nullptr); - - int rc = mkfifo(fifoName.c_str(), 0777); - if (rc < 0) { - if (errno != EEXIST) { - throw fiphoexceptions::FIFOException("mkfifo", errno); - return; - } - } -} - -} // namespace fiphoboserver diff --git a/src/fifo/fifo_ops.h b/src/fifo/fifo_ops.h deleted file mode 100644 index c148477..0000000 --- a/src/fifo/fifo_ops.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "FiPhoExceptions.h" -#include -#include - -namespace fiphoboserver { - -class FIFO { - public: - FIFO(); - void setBucketName(std::string bucketName); - std::string getFifoName() { return fifoName; } - virtual void openFIFO() = 0; - ~FIFO(); - - protected: - void startFIFO(); - std::string fifoName; - int fifo_descriptor = -1; -}; - -class FIFOPut : public FIFO { - public: - void setMetaData(std::vector> metaData); - void openFIFO(); - void addDataToFIFO(std::unique_ptr buffer); - void finishPut(); - - private: - std::future put_result; - bool putRunning = false; -}; - -class FIFOGet : public FIFO { - public: - void openFIFO(); - ssize_t getDataFromFIFO(void* buf, size_t count); - /* - * finishGet(); - */ -}; - -} // namespace fiphoboserver diff --git a/src/fifo/fifo_put.cc b/src/fifo/fifo_put.cc deleted file mode 100644 index c8e4772..0000000 --- a/src/fifo/fifo_put.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "fifo_ops.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fiphoboserver { - -void FIFOPut::openFIFO() -{ -#ifdef DEBUG - std::cout << "FIFOPut::openFIFO" << std::endl; -#endif - fifo_descriptor = open(fifoName.c_str(), O_WRONLY); - if (fifo_descriptor < 0) { - std::stringstream ss; - ss << "open with filename " << fifoName.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); - return; - } -} - -void FIFOPut::addDataToFIFO(std::unique_ptr buffer) -{ -#ifdef DEBUG - std::cout << "FIFOPut::addDataToFIFO" << std::endl; -#endif - if (putRunning) { - int ioRC = put_result.get(); - if (ioRC < 0) { - throw fiphoexceptions::FIFOException("write", ioRC); - } - } - putRunning = true; - - if (buffer) { - put_result = - std::async([fd = fifo_descriptor, buffer = std::move(buffer)] { - return write(fd, buffer->data(), buffer->length()); - }); - // io_result.wait(); - } -} - -void FIFOPut::finishPut() -{ -#ifdef DEBUG - std::cout << "FIFOPut::finishPUT" << std::endl; -#endif - int ioRC = put_result.get(); - if (ioRC < 0) { - throw fiphoexceptions::FIFOException("write", ioRC); - } - - putRunning = false; -} - -} // namespace fiphoboserver diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 5203e6f..00a88a7 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories(/usr/lib64/glib-2.0/include) add_executable( fiphoboserver - server.cc + server.main.cc get_request_handler.cc put_request_handler.cc unsupported_request_handler.cc @@ -24,5 +24,5 @@ target_compile_features(fiphoboserver PUBLIC cxx_std_14) target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen) target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver) -target_link_libraries(fiphoboserver PUBLIC backend) -target_link_libraries(fiphoboserver PUBLIC fifops) +target_link_libraries(fiphoboserver PUBLIC stream) +target_link_libraries(fiphoboserver PUBLIC storage) diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index b7140f5..2a4072a 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -17,7 +17,7 @@ #include #include "../S3_header.h" -#include "../fifo/FiPhoExceptions.h" +#include "../stream/FiPhoExceptions.h" namespace fiphoboserver { @@ -32,38 +32,16 @@ void GetRequestHandler::onRequest( return; } -#ifdef DEBUG - std::cout << "Got the following GET request: \n"; - std::string path = headers->getPath(); - std::cout << path << '\n'; - - headers->getHeaders().forEach( - [&](const std::string& header, const std::string& val) { - std::cout << header << ": " << val << '\n'; - }); -#endif - + /* Creating Bucket queries */ s3_header.setHeaders(std::move(headers)); - try { -#ifdef DEBUG - std::cout << "Bucket: " << s3_header.getBucket() << '\n'; - std::cout << "Key: " << s3_header.getKey() << '\n'; - std::cout << std::endl; -#endif - backend.setBucketName(s3_header.getBucket()); - -#ifdef DEBUG - std::cout << "Creating backend GET thread" << std::endl; -#endif - db_result = std::async( - [this, key = s3_header.getKey(), fifoName = fifo.getFifoName()]() { - return backend.db_get(key, fifoName); - }); + /* Send meta data to backend through the stream */ + stream->set_storage_meta_data(s3_header.getMetaData(), s3_header.getBucket()); + /* Tell stream to coordinate with backend to prepare for GET operation */ + stream->start_get(s3_header.getKey()); - fifo.openFIFO(); - file_closed_ = false; + file_closed_ = false; // TODO: Better way of communicating this } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -84,6 +62,7 @@ void GetRequestHandler::onRequest( proxygen::ResponseBuilder(downstream_).status(200, "Ok").send(); + /* Initiating a read from the stream and creating a body to send */ readFileScheduled_ = true; folly::getCPUExecutor()->add(std::bind( &GetRequestHandler::readFile, this, @@ -92,17 +71,18 @@ void GetRequestHandler::onRequest( void GetRequestHandler::readFile(folly::EventBase* evb) { -#ifdef DEBUG - std::cout << "GetRequestHandler::readFile" << std::endl; -#endif folly::IOBufQueue buf; while (!file_closed_ && !paused_) { - // read 4k-ish chunks and foward each one to the client + /* read 4k-ish chunks and foward each one to the client */ auto data = buf.preallocate(4000, 4000); - auto rc = fifo.getDataFromFIFO(data.first, data.second); -#ifdef DEBUG - std::cout << "getDataFromFIFO: rc = " << rc << std::endl; -#endif + /* + * rc is set to error code or bytes read + * This informs the server how to proceed + * Note: zero will only be returned if the + * file is empty AND the file is closed + * for writing + */ + auto rc = stream->get(data.first, data.second); if (rc < 0) { // error VLOG(4) << "Read error=" << rc; @@ -130,7 +110,7 @@ void GetRequestHandler::readFile(folly::EventBase* evb) }); } } - // Notify the request thread that we terminated the readFile loop + /* Notify the request thread that we terminated the readFile loop */ evb->runInEventBaseThread([this] { readFileScheduled_ = false; if (!checkForCompletion() && !paused_) { @@ -142,7 +122,7 @@ void GetRequestHandler::readFile(folly::EventBase* evb) void GetRequestHandler::onEgressPaused() noexcept { - // This will terminate readFile soon + /* This will terminate readFile soon */ VLOG(4) << "GetRequestHandler pause"; paused_ = true; } @@ -151,7 +131,7 @@ void GetRequestHandler::onEgressResumed() noexcept { VLOG(4) << "GetRequestHandler resumed"; paused_ = false; - // If readFileScheduled_, it will reschedule itself + /* If readFileScheduled_, it will reschedule itself */ if (!readFileScheduled_) { readFileScheduled_ = true; folly::getCPUExecutor()->add(std::bind( @@ -170,7 +150,7 @@ void GetRequestHandler::onBody(std::unique_ptr /*body*/) noexcept void GetRequestHandler::onEOM() noexcept { - db_result.wait(); + } void GetRequestHandler::onUpgrade( diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 848562d..7f11d4b 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -13,8 +13,7 @@ #include #include "../S3_header.h" -#include "../backend/backend.h" -#include "../fifo/fifo_ops.h" +#include "../stream/stream.h" namespace proxygen { class ResponseHandler; @@ -41,18 +40,19 @@ class GetRequestHandler : public proxygen::RequestHandler { void onEgressResumed() noexcept override; + GetRequestHandler(std::unique_ptr input_stream): + stream(std::move(input_stream)) {} + private: void readFile(folly::EventBase* evb); bool checkForCompletion(); + std::unique_ptr stream; S3_header s3_header; - PhobosDb backend; - FIFOGet fifo; bool readFileScheduled_{false}; std::atomic paused_{false}; bool finished_{false}; bool file_closed_{false}; - std::future db_result; }; } // namespace fiphoboserver diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 7d9ea18..a495882 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -14,7 +14,7 @@ #include #include "../S3_header.h" -#include "../fifo/FiPhoExceptions.h" +#include "../stream/FiPhoExceptions.h" namespace fiphoboserver { @@ -26,55 +26,22 @@ void PutRequestHandler::onRequest( return; } -#ifdef DEBUG - std::cout << "Got the following PUT request: \n"; - std::string path = headers->getPath(); - std::cout << path << '\n'; - - headers->getHeaders().forEach( - [&](const std::string& header, const std::string& val) { - std::cout << header << ": " << val << '\n'; - }); -#endif - -#ifdef DEBUG - std::cout << "Creating Bucket queries..." << std::endl; -#endif + /* Creating Bucket queries */ s3_header.setHeaders(std::move(headers)); if (s3_header.isCreateBucketRequest()) { - // Ignore, since we don't really have buckets. - // Or do we want a list of the actual buckets to give an error, when a - // requested bucket doesn't exist? + /* + * Ignore, since we don't really have buckets. + * Or do we want a list of the actual buckets to give an error, when a + * requested bucket doesn't exist? + */ proxygen::ResponseBuilder(downstream_).status(200, "Ok").sendWithEOM(); return; } - - try { -#ifdef DEBUG - std::cout << "Bucket: " << s3_header.getBucket() << '\n'; - std::cout << "Key: " << s3_header.getKey() << '\n'; - auto metaData = s3_header.getMetaData(); - std::cout << "Metadata: " << '\n'; - std::for_each( - metaData.begin(), metaData.end(), - [&](std::pair pair) { - std::cout << pair.first << " : " << pair.second << '\n'; - }); - std::cout << std::endl; -#endif - backend.setBucketName(s3_header.getBucket()); - backend.setMetaData(s3_header.getMetaData()); - -#ifdef DEBUG - std::cout << "Creating backend PUT thread" << std::endl; -#endif - db_result = std::async([this]() { - return backend.db_put( - s3_header.getBodyLength(), s3_header.getKey(), - fifo.getFifoName()); - }); - fifo.openFIFO(); + /* Send meta data to backend through the stream */ + stream->set_storage_meta_data(s3_header.getMetaData(), s3_header.getBucket()); + /* Tell stream to coordinate with backend to prepare for PUT operation */ + stream->start_put(s3_header.getBodyLength(), s3_header.getKey()); } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -96,16 +63,9 @@ void PutRequestHandler::onRequest( void PutRequestHandler::onBody(std::unique_ptr body) noexcept { -#ifdef DEBUG - std::cout << "Adding data to FIFO... " << std::endl; - if (body) { - body->coalesce(); - size_t totalBodyLength = body->computeChainDataLength(); - std::cout << "Body Data Length = " << totalBodyLength << '\n'; - } -#endif try { - fifo.addDataToFIFO(std::move(body)); + /* Hand message body over to stream for PUT operation */ + stream->put(std::move(body)); } catch (fiphoexceptions::FIFOException& fifoExcp) { proxygen::ResponseBuilder(downstream_) @@ -121,11 +81,8 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept void PutRequestHandler::onEOM() noexcept { try { -#ifdef DEBUG - std::cout << "Finish IO..." << std::endl; -#endif - db_result.wait(); - fifo.finishPut(); + /* Tell stream it's time to clean up */ + stream->finish_put(); } catch (fiphoexceptions::FIFOException& fifoExcp) { proxygen::ResponseBuilder(downstream_) diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 8cdac8f..ddd2807 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -12,8 +12,7 @@ #include #include "../S3_header.h" -#include "../backend/backend.h" -#include "../fifo/fifo_ops.h" +#include "../stream/stream.h" namespace proxygen { class ResponseHandler; @@ -36,11 +35,12 @@ class PutRequestHandler : public proxygen::RequestHandler { void onError(proxygen::ProxygenError err) noexcept override; + PutRequestHandler(std::unique_ptr input_stream): + stream(std::move(input_stream)) {} + private: S3_header s3_header; - PhobosDb backend; - FIFOPut fifo; - std::future db_result; + std::unique_ptr stream; }; } // namespace fiphoboserver diff --git a/src/server/server.cc b/src/server/server.main.cc similarity index 84% rename from src/server/server.cc rename to src/server/server.main.cc index 2d9b48d..c052a9a 100644 --- a/src/server/server.cc +++ b/src/server/server.main.cc @@ -20,6 +20,8 @@ #include "put_request_handler.h" #include "unsupported_request_handler.h" +#include "../stream/fifo.h" +#include "../storage/phobos_file.h" using folly::SocketAddress; @@ -46,25 +48,13 @@ class HandlerFactory : public proxygen::RequestHandlerFactory { proxygen::RequestHandler*, proxygen::HTTPMessage* headers) noexcept override { -#ifdef DEBUG - std::cout << "Choosing method: "; -#endif if (headers->getMethod() == proxygen::HTTPMethod::GET) { -#ifdef DEBUG - std::cout << "GET" << std::endl; -#endif - return new fiphoboserver::GetRequestHandler; + return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); } else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { -#ifdef DEBUG - std::cout << "PUT" << std::endl; -#endif - return new fiphoboserver::PutRequestHandler; + return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); } else { -#ifdef DEBUG - std::cout << "Unsupported Request" << std::endl; -#endif return new fiphoboserver::UnsupportedRequestHandler; } } @@ -102,10 +92,7 @@ int main(int argc, char* argv[]) proxygen::HTTPServer server(std::move(options)); server.bind(IPs); - // Start HTTPServer mainloop in a separate thread -#ifdef DEBUG - std::cout << "main: Starting HTTPServer thread" << std::endl; -#endif + /* Start HTTPServer mainloop in a separate thread */ std::thread t([&]() { server.start(); }); t.join(); diff --git a/src/server/unsupported_request_handler.cc b/src/server/unsupported_request_handler.cc index 33e1154..279a2bd 100644 --- a/src/server/unsupported_request_handler.cc +++ b/src/server/unsupported_request_handler.cc @@ -25,7 +25,7 @@ void UnsupportedRequestHandler::onRequest( ss << "The method " << method << " is not supported!"; proxygen::ResponseBuilder(downstream_) - .status(200, "Ok") + .status(200, "Ok") // TODO: This is NOT "Ok" .body(ss.str()) .sendWithEOM(); return; diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt new file mode 100644 index 0000000..4fba3af --- /dev/null +++ b/src/storage/CMakeLists.txt @@ -0,0 +1,16 @@ +include_directories(${PHOBOS_INCLUDE_DIRECTORY}) +include_directories(/usr/include/glib-2.0) +include_directories(/usr/lib64/glib-2.0/include) + +add_subdirectory(phobos_cpp_wrapper) + +add_library( + storage + phobos_file.cc +) + +target_compile_features(storage PUBLIC cxx_std_14) + +target_link_libraries(storage PUBLIC phobos_store) +target_link_libraries(storage PUBLIC phobos_cpp_wrapper) +target_link_libraries(storage PUBLIC proxygen::proxygen) diff --git a/src/backend/phobos_cpp_wrapper/CMakeLists.txt b/src/storage/phobos_cpp_wrapper/CMakeLists.txt similarity index 100% rename from src/backend/phobos_cpp_wrapper/CMakeLists.txt rename to src/storage/phobos_cpp_wrapper/CMakeLists.txt diff --git a/src/backend/phobos_cpp_wrapper/phobos_cpp_wrapper.c b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c similarity index 100% rename from src/backend/phobos_cpp_wrapper/phobos_cpp_wrapper.c rename to src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c diff --git a/src/backend/phobos_cpp_wrapper/phobos_cpp_wrapper.h b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h similarity index 100% rename from src/backend/phobos_cpp_wrapper/phobos_cpp_wrapper.h rename to src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc new file mode 100644 index 0000000..a1f0af2 --- /dev/null +++ b/src/storage/phobos_file.cc @@ -0,0 +1,133 @@ +#include "phobos_file.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../stream/FiPhoExceptions.h" + +namespace fiphoboserver { + +/* Just need to allocate Phobos' descriptor struct */ +Phobos_file::Phobos_file() +{ + memset(&descriptor, 0, sizeof(descriptor)); + descriptor.xd_attrs = {0}; +} + +/* Destroy Phobos' descriptor struct */ +Phobos_file::~Phobos_file() +{ + // pho_attrs_free(&descriptor.xd_attrs); + // // TODO: Is that done in + // desc_destroy?? + pho_xfer_desc_destroy_cpp(&descriptor); +} + +/* + * Open the file for reading + * fill the descriptor with relevant data + */ +void Phobos_file::prepare_put(std::string file_name, std::string object_id) +{ + descriptor.xd_fd = open(file_name.c_str(), O_RDONLY); + if (descriptor.xd_fd < 0) { + std::stringstream ss; + ss << "open with filename " << file_name.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + descriptor.xd_objid = unconsted_object_id; + descriptor.xd_op = PHO_XFER_OP_PUT; +} + +/* Tell Phobos to start reading from the file */ +ssize_t Phobos_file::db_put(size_t size) +{ + descriptor.xd_size = size; + ssize_t rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); + close_file(); + return rc; +} + +/* + * Open the file for writing + * fill the descriptor with relevant data + */ +void Phobos_file::prepare_get(std::string file_name, std::string object_id) +{ + descriptor.xd_fd = open(file_name.c_str(), O_WRONLY); + if (descriptor.xd_fd < 0) { + std::stringstream ss; + ss << "open with filename " << file_name.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + descriptor.xd_objid = unconsted_object_id; + descriptor.xd_op = PHO_XFER_OP_GET; +} + +/* Tell Phobos to write the object to file */ +ssize_t Phobos_file::db_get() +{ + ssize_t rc = phobos_get_cpp(&descriptor, 1, NULL, NULL); + /* + * Need to close file here so the stream knows + * there will be no more data to read + */ + close_file(); + return rc; +} + +void Phobos_file::set_bucket_name(std::string bucket_name) +{ + int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucket_name.c_str()); + if (rc) { + throw fiphoexceptions::PhobosException("pho_attr_set", rc); + } +} + +/* Pass s3 meta data to Phobos */ +void Phobos_file::set_meta_data( + std::vector> meta_data, std::string bucket_name) +{ + set_bucket_name(bucket_name); + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + int rc = pho_attr_set( + &descriptor.xd_attrs, pair.first.c_str(), pair.second.c_str()); + if (rc) { + throw fiphoexceptions::PhobosException("pho_attr_set", rc); + } + }); +} + +// TODO +void Phobos_file::get_meta_data() const +{ +} + +/* Close the file */ +void Phobos_file::close_file() +{ + if (descriptor.xd_fd > 0) { + close(descriptor.xd_fd); + } +} + +} // namespace fiphoboserver diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h new file mode 100644 index 0000000..0df2880 --- /dev/null +++ b/src/storage/phobos_file.h @@ -0,0 +1,29 @@ +#include "storage.h" + +extern "C" { +#include "phobos_cpp_wrapper/phobos_cpp_wrapper.h" +} + +namespace fiphoboserver { + +class Phobos_file : public Storage { + public: + Phobos_file(); + ~Phobos_file(); + + void set_meta_data(std::vector> meta_data, std::string bucket_name) override; + void get_meta_data() const override; + + ssize_t db_put(size_t size) override; + ssize_t db_get() override; + + void prepare_put(std::string file_name, std::string object_id) override; + void prepare_get(std::string file_name, std::string object_id) override; + private: + /* Implementation specific variables */ + struct pho_xfer_desc descriptor = {0}; + void set_bucket_name(std::string bucket_name); + void close_file(); +}; + +} // namespace fiphoboserver diff --git a/src/storage/storage.h b/src/storage/storage.h new file mode 100644 index 0000000..4992c7a --- /dev/null +++ b/src/storage/storage.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include +#include + +namespace fiphoboserver { + +class Storage { + public: + Storage() {}; + virtual ~Storage() {}; + + virtual void set_meta_data(std::vector> meta_data, std::string bucket_name) = 0; + virtual void get_meta_data() const = 0; + + virtual void prepare_put(std::string, std::string) = 0; + virtual ssize_t db_put(size_t size) = 0; + + virtual void prepare_get(std::string, std::string) = 0; + virtual ssize_t db_get() = 0; +}; + +} // namespace fiphoboserver diff --git a/src/fifo/CMakeLists.txt b/src/stream/CMakeLists.txt similarity index 61% rename from src/fifo/CMakeLists.txt rename to src/stream/CMakeLists.txt index ea3afcf..a297ff2 100644 --- a/src/fifo/CMakeLists.txt +++ b/src/stream/CMakeLists.txt @@ -1,12 +1,10 @@ add_library( - fifops - fifo_ops.cc - fifo_put.cc - fifo_get.cc + stream + fifo.cc ) if(CUSTOM_OUTPUT_DIRECTORY) - set_target_properties(fifops + set_target_properties(stream PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" @@ -14,6 +12,6 @@ if(CUSTOM_OUTPUT_DIRECTORY) ) endif(CUSTOM_OUTPUT_DIRECTORY) -target_compile_features(fifops PUBLIC cxx_std_14) +target_compile_features(stream PUBLIC cxx_std_14) -target_link_libraries(fifops PUBLIC proxygen::proxygen) +target_link_libraries(stream PUBLIC proxygen::proxygen) diff --git a/src/fifo/FiPhoExceptions.h b/src/stream/FiPhoExceptions.h similarity index 100% rename from src/fifo/FiPhoExceptions.h rename to src/stream/FiPhoExceptions.h index f2d6c10..09533f2 100644 --- a/src/fifo/FiPhoExceptions.h +++ b/src/stream/FiPhoExceptions.h @@ -1,8 +1,8 @@ #pragma once -#include #include #include +#include namespace fiphoexceptions { diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc new file mode 100644 index 0000000..21664b4 --- /dev/null +++ b/src/stream/fifo.cc @@ -0,0 +1,142 @@ +#include "fifo.h" +#include "FiPhoExceptions.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fiphoboserver { + +// TODO: Remember the m-versions: PUT and GET functions can work on more than +// one file. Then this structure is stupid. Maybe with a vector of descriptors? + +/* We need to pass the storage implementation to the base constructor */ +Fifo::Fifo(std::unique_ptr input_storage) : Stream(std::move(input_storage)) +{ + create_fifo(); +} + +Fifo::~Fifo() +{ + if (access(fifo_name.c_str(), F_OK) != -1) { + remove(fifo_name.c_str()); + } + if (fifo_descriptor > 0) { + close(fifo_descriptor); + } +} +/* Simply pass along to the backend */ +void Fifo::set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const +{ + storage->set_meta_data(meta_data, bucket_name); +} + +// TODO +void Fifo::get_meta_data() const +{ +} + +/* + * Generate a randomly named FIFO + * Opening here would block so we don't + */ +void Fifo::create_fifo() +{ + fifo_name = std::tmpnam(nullptr); + int rc = mkfifo(fifo_name.c_str(), 0777); + if (rc < 0) { + if (errno != EEXIST) { + throw fiphoexceptions::FIFOException("mkfifo", errno); + return; + } + } +} + +/* + * Tell the backend that it must be ready for a PUT operation + * Opening the FIFO here will not block as the backend will + * open on the other end + */ +void Fifo::start_put(ssize_t size, std::string object_id) +{ + /* Start backend storage solution asynchronously */ + db_result = std::async([this, object_id, size]() { + storage->prepare_put(fifo_name, object_id); + return storage->db_put(size); + }); + + /* Open fifo on the writing end */ + fifo_descriptor = open(fifo_name.c_str(), O_WRONLY); + if (fifo_descriptor < 0) { + std::stringstream ss; + ss << "open with filename " << fifo_name.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } +} + +/* + * Repeatedly called to write buffer to FIFO until + * the server's message body is depleted + */ +void Fifo::put(std::unique_ptr buffer) const +{ + ssize_t rc = write(fifo_descriptor, buffer->data(), buffer->length()); + if (rc < 0) { + throw fiphoexceptions::FIFOException("write", rc); + } +} + +void Fifo::finish_put() +{ +} + +/* + * Tell the backend that it must be ready for a GET operation + * Opening the FIFO here will not block as the backend will + * open on the other end + */ +void Fifo::start_get(std::string object_id) +{ + /* Start backend storage solution asynchronously */ + db_result = std::async([this, object_id]() { + storage->prepare_get(fifo_name, object_id); + return storage->db_get(); + }); + + /* Open fifo on the reading end */ + fifo_descriptor = open(fifo_name.c_str(), O_RDONLY); + if (fifo_descriptor < 0) { + std::stringstream ss; + ss << "open with filename " << fifo_name.c_str(); + throw fiphoexceptions::FIFOException(ss.str(), errno); + return; + } +} + +/* + * Repeatedly called to read from FIFO into buffer + * until the FIFO is empty and closed on the + * writing end + * Returns the number of bytes read + */ +ssize_t Fifo::get(void* buf, size_t count) const +{ + ssize_t rc = read(fifo_descriptor, buf, count); + if (rc < 0) { + throw fiphoexceptions::FIFOException("read", rc); + } + return rc; +} + +void Fifo::finish_get() +{ +} + +} // namespace fiphoboserver + diff --git a/src/stream/fifo.h b/src/stream/fifo.h new file mode 100644 index 0000000..aa599fc --- /dev/null +++ b/src/stream/fifo.h @@ -0,0 +1,31 @@ +#include "stream.h" + +namespace fiphoboserver { + +class Fifo : public Stream { + public: + Fifo(std::unique_ptr input_storage); + ~Fifo(); + + void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; + void get_meta_data() const override; + + void start_put(ssize_t size, std::string object_id) override; + void put(std::unique_ptr buffer) const override; + void finish_put() override; + + void start_get(std::string object_id) override; + ssize_t get(void *buf, size_t count) const override; + void finish_get() override; + + /* Implementation specific details */ + int get_fifo_descriptor() const { return fifo_descriptor; } + std::string get_fifo_name() const { return fifo_name; } + + private: + void create_fifo(); + std::string fifo_name; + int fifo_descriptor = -1; +}; + +} // namespace fiphoboserver diff --git a/src/stream/stream.h b/src/stream/stream.h new file mode 100644 index 0000000..42e117b --- /dev/null +++ b/src/stream/stream.h @@ -0,0 +1,44 @@ +#pragma once + +#include "../storage/storage.h" + +#include +#include +#include +#include +#include +#include + +namespace fiphoboserver { + +class Stream { + public: + Stream() {}; + /* + * Derived class must be initialised with a + * storage implementation + */ + Stream(std::unique_ptr input_storage): + storage(std::move(input_storage)) {} + + virtual ~Stream() {}; + + /* Responsible for getting meta data to the storage implementation */ + virtual void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const = 0; + virtual void get_meta_data() const = 0; + + virtual void start_put(ssize_t size, std::string object_id) = 0; + virtual void put(std::unique_ptr buffer) const = 0; + virtual void finish_put() = 0; + + virtual void start_get(std::string object_id) = 0; + virtual ssize_t get(void *buf, size_t count) const = 0; + virtual void finish_get() = 0; + protected: + /* Abstract class that requires implementation */ + std::unique_ptr storage; + /* This may be used to wait on the asynchronous backend */ + std::future db_result; +}; + +} // namespace fiphoboserver -- GitLab From 08fdc907c065b61d765d6e96a4efb24dbe002c0c Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 11 Feb 2020 15:18:23 +0000 Subject: [PATCH 04/62] update CMake Lists to do all that was formerly done by the superbuild - which really is not that much --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39e0cb6..c836455 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,7 @@ project(fiphoboserver) +set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +find_package(proxygen REQUIRED) + add_subdirectory(src) -- GitLab From 64d9fe72920a5a58c7e6d09df58701d9fe050ed7 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 12 Feb 2020 15:16:38 +0000 Subject: [PATCH 05/62] fix CmakeLists --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39e0cb6..54e2e61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,9 @@ +cmake_minimum_required(VERSION 3.0) + project(fiphoboserver) +set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +find_package(proxygen REQUIRED) + add_subdirectory(src) -- GitLab From 04b65547b64423601c1681010ab187f217913538 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 19 Feb 2020 10:22:23 +0000 Subject: [PATCH 06/62] Remove all the classes again and replace with less. Could be done in one and a lot faster (const char* instead of string) but I like this now --- src/fiphoboserver_exception.h | 34 +++++++++++++++++++ src/{ => server}/S3_header.h | 0 src/server/get_request_handler.cc | 55 +++++++++++++++++-------------- src/server/get_request_handler.h | 6 ++-- src/server/put_request_handler.cc | 23 +++++-------- src/server/put_request_handler.h | 6 ++-- src/server/server.main.cc | 4 +-- src/storage/phobos_exception.h | 24 ++++++++++++++ src/storage/phobos_file.cc | 32 +++++++++++------- src/storage/phobos_file.h | 2 ++ src/storage/storage.h | 2 ++ src/stream/FiPhoExceptions.h | 50 ---------------------------- src/stream/fifo.cc | 31 +++++++++-------- src/stream/fifo.h | 4 ++- src/stream/stream.h | 6 ++-- 15 files changed, 156 insertions(+), 123 deletions(-) create mode 100644 src/fiphoboserver_exception.h rename src/{ => server}/S3_header.h (100%) create mode 100644 src/storage/phobos_exception.h delete mode 100644 src/stream/FiPhoExceptions.h diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h new file mode 100644 index 0000000..c2189bb --- /dev/null +++ b/src/fiphoboserver_exception.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +namespace fiphoboserver { + +class FIPhoboServerException : public std::exception { +public: + FIPhoboServerException() : message("") { } + + FIPhoboServerException(const char* message) : message(message) { } + + virtual const char* what() const noexcept override + { + return message.c_str(); + } + +protected: + std::string message; +}; + +class IOException : public FIPhoboServerException { +public: + IOException(const char* caller, const char* functionName, int returnValue) + { + std::stringstream ss; + ss << "[" << caller << "]: IO Exception occurred: " << functionName + << ": " << std::strerror(returnValue) << " (" << returnValue << ")"; + message = ss.str(); + } +}; + +} // namespace fiphoboserver diff --git a/src/S3_header.h b/src/server/S3_header.h similarity index 100% rename from src/S3_header.h rename to src/server/S3_header.h diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index 2a4072a..c8a9f03 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -16,8 +16,7 @@ #include #include -#include "../S3_header.h" -#include "../stream/FiPhoExceptions.h" +#include "../fiphoboserver_exception.h" namespace fiphoboserver { @@ -52,7 +51,8 @@ void GetRequestHandler::onRequest( .sendWithEOM(); return; } - catch (const fiphoexceptions::PhobosException& ex) { + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught exception: " << ex.what() << '\n'; proxygen::ResponseBuilder(downstream_) .status(409, "Conflict") .body(ex.what()) @@ -82,33 +82,40 @@ void GetRequestHandler::readFile(folly::EventBase* evb) * file is empty AND the file is closed * for writing */ - auto rc = stream->get(data.first, data.second); - if (rc < 0) { - // error - VLOG(4) << "Read error=" << rc; + try { + auto rc = stream->get(data.first, data.second); + if (rc < 0) { + // should not happen as an exception should have been thrown before + throw FIPhoboServerException("stream->get returned an error"); + } + else if (rc == 0) { + // done + file_closed_ = true; + stream->finish_get(); + VLOG(4) << "Read EOF"; + evb->runInEventBaseThread([this] { + proxygen::ResponseBuilder(downstream_).sendWithEOM(); + }); + break; + } + else { + buf.postallocate(rc); + evb->runInEventBaseThread([this, body = buf.move()]() mutable { + proxygen::ResponseBuilder(downstream_) + .body(std::move(body)) + .send(); + }); + } + } + catch(const FIPhoboServerException& ex) + { + std::cerr << "Read error=" << ex.what() << '\n'; evb->runInEventBaseThread([this] { LOG(ERROR) << "Error reading file"; downstream_->sendAbort(); }); break; } - else if (rc == 0) { - // done - file_closed_ = true; - VLOG(4) << "Read EOF"; - evb->runInEventBaseThread([this] { - proxygen::ResponseBuilder(downstream_).sendWithEOM(); - }); - break; - } - else { - buf.postallocate(rc); - evb->runInEventBaseThread([this, body = buf.move()]() mutable { - proxygen::ResponseBuilder(downstream_) - .body(std::move(body)) - .send(); - }); - } } /* Notify the request thread that we terminated the readFile loop */ evb->runInEventBaseThread([this] { diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 7f11d4b..7f67791 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -12,7 +12,7 @@ #include #include -#include "../S3_header.h" +#include "S3_header.h" #include "../stream/stream.h" namespace proxygen { @@ -40,14 +40,14 @@ class GetRequestHandler : public proxygen::RequestHandler { void onEgressResumed() noexcept override; - GetRequestHandler(std::unique_ptr input_stream): + GetRequestHandler(std::unique_ptr input_stream): stream(std::move(input_stream)) {} private: void readFile(folly::EventBase* evb); bool checkForCompletion(); - std::unique_ptr stream; + std::unique_ptr stream; S3_header s3_header; bool readFileScheduled_{false}; std::atomic paused_{false}; diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index a495882..2cb95e0 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -13,8 +13,7 @@ #include #include -#include "../S3_header.h" -#include "../stream/FiPhoExceptions.h" +#include "../fiphoboserver_exception.h" namespace fiphoboserver { @@ -52,7 +51,8 @@ void PutRequestHandler::onRequest( .sendWithEOM(); return; } - catch (const fiphoexceptions::PhobosException& ex) { + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught exception: " << ex.what() << '\n'; proxygen::ResponseBuilder(downstream_) .status(409, "Conflict") .body(ex.what()) @@ -67,10 +67,11 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept /* Hand message body over to stream for PUT operation */ stream->put(std::move(body)); } - catch (fiphoexceptions::FIFOException& fifoExcp) { + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught an exception in put: " << ex.what() << '\n'; proxygen::ResponseBuilder(downstream_) .status(409, "Conflict") - .body(fifoExcp.what()) + .body(ex.what()) .sendWithEOM(); return; } @@ -84,17 +85,11 @@ void PutRequestHandler::onEOM() noexcept /* Tell stream it's time to clean up */ stream->finish_put(); } - catch (fiphoexceptions::FIFOException& fifoExcp) { + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught an exception in finish put: " << ex.what() << '\n'; proxygen::ResponseBuilder(downstream_) .status(409, "Conflict") - .body(fifoExcp.what()) - .sendWithEOM(); - return; - } - catch (fiphoexceptions::PhobosException& phoExcp) { - proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(phoExcp.what()) + .body(ex.what()) .sendWithEOM(); return; } diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index ddd2807..7f0166c 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -11,7 +11,7 @@ #include #include -#include "../S3_header.h" +#include "S3_header.h" #include "../stream/stream.h" namespace proxygen { @@ -35,12 +35,12 @@ class PutRequestHandler : public proxygen::RequestHandler { void onError(proxygen::ProxygenError err) noexcept override; - PutRequestHandler(std::unique_ptr input_stream): + PutRequestHandler(std::unique_ptr input_stream): stream(std::move(input_stream)) {} private: S3_header s3_header; - std::unique_ptr stream; + std::unique_ptr stream; }; } // namespace fiphoboserver diff --git a/src/server/server.main.cc b/src/server/server.main.cc index c052a9a..49c54da 100644 --- a/src/server/server.main.cc +++ b/src/server/server.main.cc @@ -49,10 +49,10 @@ class HandlerFactory : public proxygen::RequestHandlerFactory { proxygen::HTTPMessage* headers) noexcept override { if (headers->getMethod() == proxygen::HTTPMethod::GET) { - return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); + return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); } else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { - return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); + return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); } else { return new fiphoboserver::UnsupportedRequestHandler; diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h new file mode 100644 index 0000000..129941d --- /dev/null +++ b/src/storage/phobos_exception.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +#include "../fiphoboserver_exception.h" + +namespace fiphoboserver { +namespace storage { + +class PhobosException : public FIPhoboServerException { +public: + PhobosException(const char* caller, const char* functionName, int returnValue) + { + std::stringstream ss; + ss << "[" << caller << "]: Phobos returned an error: " << functionName + << ": " << std::strerror(-returnValue) << " (" << -returnValue << ")"; + message = ss.str(); + } +}; + +} // namespace storage +} // namespace fiphoboserver diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index a1f0af2..4bb9e93 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -12,9 +12,11 @@ #include #include -#include "../stream/FiPhoExceptions.h" +#include "../fiphoboserver_exception.h" +#include "phobos_exception.h" namespace fiphoboserver { +namespace storage { /* Just need to allocate Phobos' descriptor struct */ Phobos_file::Phobos_file() @@ -26,9 +28,9 @@ Phobos_file::Phobos_file() /* Destroy Phobos' descriptor struct */ Phobos_file::~Phobos_file() { + // TODO: See issue #23 // pho_attrs_free(&descriptor.xd_attrs); - // // TODO: Is that done in - // desc_destroy?? + pho_xfer_desc_destroy_cpp(&descriptor); } @@ -40,9 +42,7 @@ void Phobos_file::prepare_put(std::string file_name, std::string object_id) { descriptor.xd_fd = open(file_name.c_str(), O_RDONLY); if (descriptor.xd_fd < 0) { - std::stringstream ss; - ss << "open with filename " << file_name.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); + throw IOException("Phobos_file::prepare_put", file_name.c_str(), errno); return; } char* unconsted_object_id = new char[object_id.length() + 1]; @@ -58,6 +58,11 @@ ssize_t Phobos_file::db_put(size_t size) descriptor.xd_size = size; ssize_t rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); close_file(); + + if(rc) + { + throw PhobosException("Phobos_file::db_put", "phobos_put", rc); + } return rc; } @@ -69,9 +74,7 @@ void Phobos_file::prepare_get(std::string file_name, std::string object_id) { descriptor.xd_fd = open(file_name.c_str(), O_WRONLY); if (descriptor.xd_fd < 0) { - std::stringstream ss; - ss << "open with filename " << file_name.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); + throw IOException("Phobos_file::prepare_get", file_name.c_str(), errno); return; } char* unconsted_object_id = new char[object_id.length() + 1]; @@ -90,6 +93,12 @@ ssize_t Phobos_file::db_get() * there will be no more data to read */ close_file(); + + if(rc) + { + throw PhobosException("Phobos_file::db_get", "phobos_get", rc); + } + return rc; } @@ -97,7 +106,7 @@ void Phobos_file::set_bucket_name(std::string bucket_name) { int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucket_name.c_str()); if (rc) { - throw fiphoexceptions::PhobosException("pho_attr_set", rc); + throw PhobosException("Phobos_file::set_bucket_name", "pho_attr_set", rc); } } @@ -112,7 +121,7 @@ void Phobos_file::set_meta_data( int rc = pho_attr_set( &descriptor.xd_attrs, pair.first.c_str(), pair.second.c_str()); if (rc) { - throw fiphoexceptions::PhobosException("pho_attr_set", rc); + throw PhobosException("Phobos_file::set_meta_data", "pho_attr_set", rc); } }); } @@ -130,4 +139,5 @@ void Phobos_file::close_file() } } +} // namespace storage } // namespace fiphoboserver diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 0df2880..4acc53f 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -5,6 +5,7 @@ extern "C" { } namespace fiphoboserver { +namespace storage { class Phobos_file : public Storage { public: @@ -26,4 +27,5 @@ class Phobos_file : public Storage { void close_file(); }; +} // namespace storage } // namespace fiphoboserver diff --git a/src/storage/storage.h b/src/storage/storage.h index 4992c7a..92f5608 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -6,6 +6,7 @@ #include namespace fiphoboserver { +namespace storage { class Storage { public: @@ -22,4 +23,5 @@ class Storage { virtual ssize_t db_get() = 0; }; +} // namespace storage } // namespace fiphoboserver diff --git a/src/stream/FiPhoExceptions.h b/src/stream/FiPhoExceptions.h deleted file mode 100644 index 09533f2..0000000 --- a/src/stream/FiPhoExceptions.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace fiphoexceptions { - -class PhobosException : std::exception { - public: - PhobosException(std::string callingPhobosFunction, int rc) : - function(callingPhobosFunction), - rc(rc) - { - } - - const char* what() const noexcept override - { - std::stringstream ss; - ss << "Phobos had an error in function " << function << ": " - << std::strerror(rc); - return ss.str().c_str(); - } - - private: - int rc; - std::string function; -}; - - -class FIFOException : std::exception { - public: - FIFOException(std::string caller, int value) : caller(caller), value(value) - { - } - - const char* what() const noexcept override - { - std::stringstream ss; - ss << "FIFO IO had an error in function " << caller << ": " - << std::strerror(errno); - return ss.str().c_str(); - } - - private: - int value; - std::string caller; -}; - -} // namespace fiphoexceptions diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index 21664b4..cd42c92 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -1,5 +1,6 @@ #include "fifo.h" -#include "FiPhoExceptions.h" + +#include "../fiphoboserver_exception.h" #include #include @@ -11,12 +12,10 @@ #include namespace fiphoboserver { - -// TODO: Remember the m-versions: PUT and GET functions can work on more than -// one file. Then this structure is stupid. Maybe with a vector of descriptors? +namespace stream { /* We need to pass the storage implementation to the base constructor */ -Fifo::Fifo(std::unique_ptr input_storage) : Stream(std::move(input_storage)) +Fifo::Fifo(std::unique_ptr input_storage) : Stream(std::move(input_storage)) { create_fifo(); } @@ -32,7 +31,7 @@ Fifo::~Fifo() } /* Simply pass along to the backend */ void Fifo::set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const -{ +{ storage->set_meta_data(meta_data, bucket_name); } @@ -51,7 +50,7 @@ void Fifo::create_fifo() int rc = mkfifo(fifo_name.c_str(), 0777); if (rc < 0) { if (errno != EEXIST) { - throw fiphoexceptions::FIFOException("mkfifo", errno); + throw IOException("Fifo::create_fifo", "mkfifo", errno); return; } } @@ -74,8 +73,8 @@ void Fifo::start_put(ssize_t size, std::string object_id) fifo_descriptor = open(fifo_name.c_str(), O_WRONLY); if (fifo_descriptor < 0) { std::stringstream ss; - ss << "open with filename " << fifo_name.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); + ss << "open (" << fifo_name.c_str() << ")"; + throw IOException("Fifo::start_put", ss.str().c_str(), errno); return; } } @@ -88,12 +87,15 @@ void Fifo::put(std::unique_ptr buffer) const { ssize_t rc = write(fifo_descriptor, buffer->data(), buffer->length()); if (rc < 0) { - throw fiphoexceptions::FIFOException("write", rc); + throw IOException("Fifo::put", "write", rc); } } +// TODO see git issue #26 void Fifo::finish_put() { + // will throw the inner exception if one occurred + db_result.get(); } /* @@ -113,8 +115,8 @@ void Fifo::start_get(std::string object_id) fifo_descriptor = open(fifo_name.c_str(), O_RDONLY); if (fifo_descriptor < 0) { std::stringstream ss; - ss << "open with filename " << fifo_name.c_str(); - throw fiphoexceptions::FIFOException(ss.str(), errno); + ss << "open (" << fifo_name.c_str() << ")"; + throw IOException("Fifo::start_get", ss.str().c_str(), errno); return; } } @@ -129,14 +131,17 @@ ssize_t Fifo::get(void* buf, size_t count) const { ssize_t rc = read(fifo_descriptor, buf, count); if (rc < 0) { - throw fiphoexceptions::FIFOException("read", rc); + throw IOException("Fifo::get", "read", rc); } return rc; } +// TODO see git issue #26 void Fifo::finish_get() { + db_result.get(); } +} // namespace stream } // namespace fiphoboserver diff --git a/src/stream/fifo.h b/src/stream/fifo.h index aa599fc..e56e0bf 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -1,10 +1,11 @@ #include "stream.h" namespace fiphoboserver { +namespace stream { class Fifo : public Stream { public: - Fifo(std::unique_ptr input_storage); + Fifo(std::unique_ptr input_storage); ~Fifo(); void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; @@ -28,4 +29,5 @@ class Fifo : public Stream { int fifo_descriptor = -1; }; +} // namespace stream } // namespace fiphoboserver diff --git a/src/stream/stream.h b/src/stream/stream.h index 42e117b..9632e93 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -10,6 +10,7 @@ #include namespace fiphoboserver { +namespace stream { class Stream { public: @@ -18,7 +19,7 @@ class Stream { * Derived class must be initialised with a * storage implementation */ - Stream(std::unique_ptr input_storage): + Stream(std::unique_ptr input_storage): storage(std::move(input_storage)) {} virtual ~Stream() {}; @@ -36,9 +37,10 @@ class Stream { virtual void finish_get() = 0; protected: /* Abstract class that requires implementation */ - std::unique_ptr storage; + std::unique_ptr storage; /* This may be used to wait on the asynchronous backend */ std::future db_result; }; +} // namespace stream } // namespace fiphoboserver -- GitLab From 7792d2dd65d5d8f3a3e9a619d774c78178613b75 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 19 Feb 2020 16:18:16 +0000 Subject: [PATCH 07/62] Start with doxygen documentation: Server is documented, Storage base class, buidl files are added but not tested --- src/CMakeLists.txt | 2 + src/doc/CMakeLists.txt | 32 ++++++++++++ src/doc/Doxyfile.in | 53 ++++++++++++++++++++ src/server/S3_header.h | 46 +++++++++++++++++ src/server/get_request_handler.h | 51 +++++++++++++++++++ src/server/put_request_handler.h | 25 +++++++++ src/server/unsupported_request_handler.h | 4 ++ src/storage/storage.h | 64 ++++++++++++++++++++++-- 8 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 src/doc/CMakeLists.txt create mode 100644 src/doc/Doxyfile.in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 282155f..106b756 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,3 +5,5 @@ endif(DEBUG) add_subdirectory(storage) add_subdirectory(stream) add_subdirectory(server) + +add_subdirectory(doc) \ No newline at end of file diff --git a/src/doc/CMakeLists.txt b/src/doc/CMakeLists.txt new file mode 100644 index 0000000..c6e517f --- /dev/null +++ b/src/doc/CMakeLists.txt @@ -0,0 +1,32 @@ +find_package(Doxygen REQUIRED) + +# Write a Doxyfile including the main doc/Doxyfile. +# This sets the @INCLUDE_PATH variable to the project root. +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + @ONLY +) + +set(INCLUDE_DOXYFILE "@INCLUDE = ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile") + +# Custom command to run doxygen on build/doc/Doxyfile in the build/doc dir. +# Running doxygen is tied to the html/index.html file. +# TODO: The main dependency is currently doc/Doxyfile, but there should be +# a depencency on *every* file being documented! +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/html/index.html + COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in + DEPENDS ${EXSEISDAT_DOCUMENTED_EXAMPLES} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building the documentation..." +) + +# Target to wire up building the documentation to the ALL target. +add_custom_target( + doc ALL + DEPENDS + ${CMAKE_CURRENT_BINARY_DIR}/html/index.html +) + diff --git a/src/doc/Doxyfile.in b/src/doc/Doxyfile.in new file mode 100644 index 0000000..aae5e06 --- /dev/null +++ b/src/doc/Doxyfile.in @@ -0,0 +1,53 @@ +# Doxyfile 1.8.13 + +# +# This is a template file used for configuration in doc/CMakeLists.txt. +# + +PROJECT_NAME = FiPhoboServer + +# Extraction options +EXTRACT_PRIVATE = NO +EXTRACT_PACKAGE = YES +EXTRACT_STATIC = YES +EXTRACT_LOCAL_METHODS = YES +EXTRACT_ANON_NSPACES = YES +EXCLUDE_SYMBOLS = detail + +# Generate case sensitive filenames +CASE_SENSE_NAMES = YES + +# Warning options +# Warn on everything possible, and fail on warning +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = NO +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = YES +WARN_AS_ERROR = YES + +# Input directories +INPUT = \ + @CMAKE_SOURCE_DIR@/src/server \ + @CMAKE_SOURCE_DIR@/src/storage \ + @CMAKE_SOURCE_DIR@/src/stream + +RECURSIVE = YES + +# HTML output options +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +GENERATE_TREEVIEW = YES + +# Latex font size +FORMULA_FONTSIZE = 10 + +# Macro options +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +PREDEFINED = __cplusplus +SEARCH_INCLUDES = YES + +GENERATE_XML = YES +XML_NS_MEMB_FILE_SCOPE = YES diff --git a/src/server/S3_header.h b/src/server/S3_header.h index 520408d..c9a0f99 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -3,13 +3,29 @@ #include namespace fiphoboserver { + +/// +/// @brief class to extract the S3 specific information from proxygens +/// HTTPMessage headers +/// class S3_header { public: + + /// + /// @brief sets the internal header instance to the given header object + /// + /// @param newHeaders headers that should be read out in this instance + /// void setHeaders(std::unique_ptr newHeaders) { headers = std::move(newHeaders); } + /// + /// @brief extracts the name of the S3 bucket requested in the HTTP message + /// + /// @returns name of the bucket + /// std::string getBucket() { if (!headers) { @@ -21,6 +37,11 @@ class S3_header { return bucketName; } + /// + /// @brief extracts the name of the key id requested in the HTTP message + /// + /// @returns name of the key + /// std::string getKey() { if (!headers) { @@ -38,6 +59,15 @@ class S3_header { return fileKey; } + /// + /// @brief checks if the message we got is one for just creating a bucket + /// + /// This is convenient since we don't really support buckets themselves and + /// can easily stop working if this returns true + /// + /// @returns true if these headers belong to a create bucket request, + /// false for all other requests + /// bool isCreateBucketRequest() { if (!headers) { @@ -47,6 +77,13 @@ class S3_header { return false; } + /// + /// @brief gets the total length of all body chunks that is expected + /// + /// This reads out the header "Content-Length" + /// + /// @returns the length of the total body to be expected + /// size_t getBodyLength() { std::string contentLength = @@ -54,6 +91,15 @@ class S3_header { return static_cast(std::stoi(contentLength)); } + /// + /// @brief gets the S3 metadata stored in the HTTP headers + /// + /// In S3 arbitrary metadata can be defined. This has the form + /// x-amx-meta-KEY: VALUE + /// for for example a KEY:VALUE pair. + /// + /// @returns a vector consisting of all key value pairs found in the header + /// std::vector> getMetaData() { std::vector> metadata; diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 7f67791..2707556 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -21,25 +21,76 @@ class ResponseHandler; namespace fiphoboserver { +/// +/// @brief proxygen class implementation for handling GET requests +/// class GetRequestHandler : public proxygen::RequestHandler { public: + + /// + /// @brief first function to be called when a new request comes in + /// + /// @param headers headers of the HTTP message this handler was created for + /// void onRequest( std::unique_ptr headers) noexcept override; + /// + /// @brief function called on every body chunk belonging to this message + /// + /// This is not used in this case because GET requests don't usually have a + /// body + /// + /// @param body buffer for the body + /// void onBody(std::unique_ptr body) noexcept override; + /// + /// @brief function called when the incoming message is finished + /// void onEOM() noexcept override; + /// + /// @brief function called on upgrade + /// + /// Not supported in our case! + /// void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + /// + /// @brief function ... + /// + /// @todo find out what this does? + /// void requestComplete() noexcept override; + /// + /// @brief function called when an error occurred + /// + /// Not supported in our case! + /// void onError(proxygen::ProxygenError err) noexcept override; + /// + /// @brief called when the queue is full. + /// + /// Contents are copies from a proxygen example + /// void onEgressPaused() noexcept override; + /// + /// @brief called when the queue is not longer full. + /// + /// Contents are copies from a proxygen example + /// void onEgressResumed() noexcept override; + /// + /// @brief Constructor for stream class initialization + /// + /// @param input_stream stream::Stream class instance to initialize the + /// server + /// GetRequestHandler(std::unique_ptr input_stream): stream(std::move(input_stream)) {} diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 7f0166c..1366f52 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -20,21 +20,46 @@ class ResponseHandler; namespace fiphoboserver { +/// +/// @brief proxygen class implementation for handling PUT requests +/// class PutRequestHandler : public proxygen::RequestHandler { public: + + /// + /// @copydoc GetRequestHandler::onRequest + /// void onRequest( std::unique_ptr headers) noexcept override; + /// + /// @copydoc GetRequestHandler::onBody + /// void onBody(std::unique_ptr body) noexcept override; + /// + /// @copydoc GetRequestHandler::onEOM + /// void onEOM() noexcept override; + /// + /// @copydoc GetRequestHandler::onUpgrade + /// void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + /// + /// @copydoc GetRequestHandler::requestComplete + /// void requestComplete() noexcept override; + /// + /// @copydoc GetRequestHandler::onError + /// void onError(proxygen::ProxygenError err) noexcept override; + /// + /// @copydoc GetRequestHandler::GetRequestHandler + /// PutRequestHandler(std::unique_ptr input_stream): stream(std::move(input_stream)) {} diff --git a/src/server/unsupported_request_handler.h b/src/server/unsupported_request_handler.h index 78ba4ef..5bc15a4 100644 --- a/src/server/unsupported_request_handler.h +++ b/src/server/unsupported_request_handler.h @@ -17,6 +17,10 @@ class ResponseHandler; namespace fiphoboserver { +/// +/// @brief proxygen class implementation for handling all requests that we don't +/// support +/// class UnsupportedRequestHandler : public proxygen::RequestHandler { public: void onRequest( diff --git a/src/storage/storage.h b/src/storage/storage.h index 92f5608..aa9a313 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -6,20 +6,78 @@ #include namespace fiphoboserver { +/// +/// @brief namespace for storage classes that belong to / inherit from from +/// fiphoboserver::storage::Storage +/// namespace storage { +/// +/// @brief virtual storage class to be implemented by backend storage +/// +/// This class is a prototype for every class that handles to internal database +/// that should be accessed by the FiPhoboServer. +/// This class represents exactly one object file in the internal database +/// class Storage { public: + + /// + /// @brief default constructor + /// Storage() {}; + /// + /// @brief default destructor + /// virtual ~Storage() {}; - virtual void set_meta_data(std::vector> meta_data, std::string bucket_name) = 0; + /// + /// @brief set the metadata an object that is added to the database + /// should get + /// + /// @param meta_data a vector of all key:value pairs that should be + /// added to the data + /// @param bucket_name the name of the S3 bucket - currently also + /// handled as normal metadata + /// + virtual void set_meta_data( + std::vector> meta_data, + std::string bucket_name) = 0; + /// + /// @brief get the metadata associated to the current object as a + /// vector of key:value pairs + /// virtual void get_meta_data() const = 0; - virtual void prepare_put(std::string, std::string) = 0; + /// + /// @brief Starts a put operation to the database + /// + /// @param file_name the filename the data that will be added to the + /// database shouldbe read from + /// @param object_id the internal storage id the data should get + /// + virtual void prepare_put(std::string file_name, std::string object_id) = 0; + /// + /// @brief Puts data to the databse + /// + /// @param size the number of bytes to add to the file in the database + /// virtual ssize_t db_put(size_t size) = 0; - virtual void prepare_get(std::string, std::string) = 0; + /// + /// @brief Starts a get operation to the database + /// + /// @param file_name the filename the data from the database should + /// be written to + /// @param object_id the internal storage id from the data that + /// should be retrieved + /// + virtual void prepare_get(std::string file_name, std::string object_id) = 0; + /// + /// @brief Gets data from the databse + /// + /// @returns the number of bytes read from the database + /// virtual ssize_t db_get() = 0; }; -- GitLab From 4ee6eda34ac38acc0d35b9d62fb265fd81e2cc14 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 19 Feb 2020 16:28:13 +0000 Subject: [PATCH 08/62] move doc directory to upper level --- CMakeLists.txt | 1 + {src/doc => doc}/CMakeLists.txt | 0 {src/doc => doc}/Doxyfile.in | 0 src/CMakeLists.txt | 1 - 4 files changed, 1 insertion(+), 1 deletion(-) rename {src/doc => doc}/CMakeLists.txt (100%) rename {src/doc => doc}/Doxyfile.in (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54e2e61..fff1380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,3 +7,4 @@ set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) find_package(proxygen REQUIRED) add_subdirectory(src) +add_subdirectory(doc) diff --git a/src/doc/CMakeLists.txt b/doc/CMakeLists.txt similarity index 100% rename from src/doc/CMakeLists.txt rename to doc/CMakeLists.txt diff --git a/src/doc/Doxyfile.in b/doc/Doxyfile.in similarity index 100% rename from src/doc/Doxyfile.in rename to doc/Doxyfile.in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 106b756..41d5c07 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,4 +6,3 @@ add_subdirectory(storage) add_subdirectory(stream) add_subdirectory(server) -add_subdirectory(doc) \ No newline at end of file -- GitLab From 17cf71beac3a908eb9f959b439aa43dae0951f5b Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 19 Feb 2020 17:51:45 +0000 Subject: [PATCH 09/62] Document most of the other classes --- src/fiphoboserver_exception.h | 31 +++++++++++ src/server/get_request_handler.h | 2 +- src/storage/phobos_exception.h | 10 ++++ src/storage/phobos_file.h | 13 +++++ src/storage/storage.h | 21 +++++++- src/stream/fifo.h | 38 ++++++++++++- src/stream/stream.h | 93 +++++++++++++++++++++++++++++--- 7 files changed, 198 insertions(+), 10 deletions(-) diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index c2189bb..321b9c2 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -5,23 +5,54 @@ namespace fiphoboserver { +/// +/// @brief exception class for all user defined exceptions in FiPhboServer +/// class FIPhoboServerException : public std::exception { public: + + /// + /// @brief default constructor + /// FIPhoboServerException() : message("") { } + /// + /// @brief constructor for a message with information on the exception + /// + /// @param message message explaining what went wrong such that this + /// exception was thrown. + /// FIPhoboServerException(const char* message) : message(message) { } + /// + /// @brief get information on this exception + /// + /// @return A message explaining what this exception is about + /// virtual const char* what() const noexcept override { return message.c_str(); } protected: + /// + /// @brief the internal message + /// std::string message; }; +/// +/// @brief exceptions specifically for errors in I/O +/// class IOException : public FIPhoboServerException { public: + /// + /// @brief constructor + /// + /// @param caller the function that threw this exception + /// @param functionName the io function that returned an error + /// @param returnValue the return value of the io function + /// IOException(const char* caller, const char* functionName, int returnValue) { std::stringstream ss; diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 2707556..c2f2121 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -88,7 +88,7 @@ class GetRequestHandler : public proxygen::RequestHandler { /// /// @brief Constructor for stream class initialization /// - /// @param input_stream stream::Stream class instance to initialize the + /// @param input_stream @ref stream::Stream class instance to initialize the /// server /// GetRequestHandler(std::unique_ptr input_stream): diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index 129941d..9066611 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -9,8 +9,18 @@ namespace fiphoboserver { namespace storage { +/// +/// @brief exceptions specifically for the phobos backend library +/// class PhobosException : public FIPhoboServerException { public: + /// + /// @brief constructor + /// + /// @param caller the function that threw this exception + /// @param functionName the phobos function that returned an error + /// @param returnValue the return value of the phobos function + /// PhobosException(const char* caller, const char* functionName, int returnValue) { std::stringstream ss; diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 4acc53f..054bd46 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -7,18 +7,31 @@ extern "C" { namespace fiphoboserver { namespace storage { +/// @copydoc Storage +/// +/// @todo maybe not only copy from interface but add specific phobos +/// information? +/// class Phobos_file : public Storage { public: + /// @copydoc Storage::Storage Phobos_file(); + /// @copydoc Storage::~Storage ~Phobos_file(); + /// @copydoc Storage::set_meta_data void set_meta_data(std::vector> meta_data, std::string bucket_name) override; + /// @copydoc Storage::get_meta_data void get_meta_data() const override; + /// @copydoc Storage::db_put ssize_t db_put(size_t size) override; + /// @copydoc Storage::db_get ssize_t db_get() override; + /// @copydoc Storage::prepare_put void prepare_put(std::string file_name, std::string object_id) override; + /// @copydoc Storage::prepare_get void prepare_get(std::string file_name, std::string object_id) override; private: /* Implementation specific variables */ diff --git a/src/storage/storage.h b/src/storage/storage.h index aa9a313..1d35e6c 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -8,7 +8,7 @@ namespace fiphoboserver { /// /// @brief namespace for storage classes that belong to / inherit from from -/// fiphoboserver::storage::Storage +/// @ref fiphoboserver::storage::Storage /// namespace storage { @@ -31,6 +31,10 @@ class Storage { /// virtual ~Storage() {}; +/// @name Metadata functions +/// +/// @{ + /// /// @brief set the metadata an object that is added to the database /// should get @@ -49,6 +53,12 @@ class Storage { /// virtual void get_meta_data() const = 0; +/// @} +/// +/// @name PUT functions +/// +/// @{ + /// /// @brief Starts a put operation to the database /// @@ -64,6 +74,12 @@ class Storage { /// virtual ssize_t db_put(size_t size) = 0; +/// @} +/// +/// @name GET functions +/// +/// @{ + /// /// @brief Starts a get operation to the database /// @@ -79,6 +95,9 @@ class Storage { /// @returns the number of bytes read from the database /// virtual ssize_t db_get() = 0; + +/// @} + }; } // namespace storage diff --git a/src/stream/fifo.h b/src/stream/fifo.h index e56e0bf..acc0110 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -3,26 +3,62 @@ namespace fiphoboserver { namespace stream { +/// +/// @copydoc Stream +/// +/// @todo maybe not only copy from interface but add specific fifo +/// information? +/// class Fifo : public Stream { public: + /// @copydoc Stream::Stream Fifo(std::unique_ptr input_storage); + /// @copydoc Stream::~Stream ~Fifo(); + /// @copydoc Stream::set_storage_meta_data void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; + /// @copydoc Stream::set_storage_meta_data void get_meta_data() const override; + /// @copydoc Stream::set_storage_meta_data void start_put(ssize_t size, std::string object_id) override; + /// @copydoc Stream::set_storage_meta_data void put(std::unique_ptr buffer) const override; + /// @copydoc Stream::set_storage_meta_data void finish_put() override; + /// @copydoc Stream::set_storage_meta_data void start_get(std::string object_id) override; + /// @copydoc Stream::set_storage_meta_data ssize_t get(void *buf, size_t count) const override; + /// @copydoc Stream::set_storage_meta_data void finish_get() override; - /* Implementation specific details */ + +/// @name Implementation specific details +/// +/// @{ + + /// + /// @brief get the descriptor of the fifo + /// + /// @returns the descriptor of the fifo + /// + /// @todo Do we use this? should this really be public? + /// int get_fifo_descriptor() const { return fifo_descriptor; } + /// + /// @brief get the name of the fifo + /// + /// @returns the name of the fifo + /// + /// @todo Do we use this? should this really be public? + /// std::string get_fifo_name() const { return fifo_name; } +/// @} + private: void create_fifo(); std::string fifo_name; diff --git a/src/stream/stream.h b/src/stream/stream.h index 9632e93..6c043b3 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -10,35 +10,114 @@ #include namespace fiphoboserver { +/// +/// @brief namespace for stream classes that belong to / inherit from from +/// @ref fiphoboserver::stream::Stream +/// namespace stream { +/// +/// @brief virtual Stream class to be implemented for streaming chunks of data +/// between the server and a backend @ref storage::Storage class +/// class Stream { public: + /// + /// @brief default constructor + /// Stream() {}; - /* - * Derived class must be initialised with a - * storage implementation - */ + /// + /// @brief constructor with storage implementation + /// + /// @warning every class that inherits this interface must have an + /// internal @ref storage::Storage implementation and + /// initialize this here. + /// Stream(std::unique_ptr input_storage): storage(std::move(input_storage)) {} + /// + /// @brief default destructor + /// virtual ~Stream() {}; - /* Responsible for getting meta data to the storage implementation */ + /// + /// @brief setting the metadata that the created object should get + /// + /// @warning this function is responsible for getting metadata to + /// the storage implementation + /// virtual void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const = 0; + /// + /// @brief get the metadata from a stored object + /// virtual void get_meta_data() const = 0; +/// @name PUT functions +/// +/// @{ + + /// + /// @brief start a put operation + /// + /// @param size amount of bytes that shall be added to the + /// internal storage + /// @param object_id the identifier the object should get + /// virtual void start_put(ssize_t size, std::string object_id) = 0; + /// + /// @brief add a chunk of data to the object + /// + /// @param buffer the chunked data to add + /// virtual void put(std::unique_ptr buffer) const = 0; + /// + /// @brief end a put operation + /// virtual void finish_put() = 0; +/// @} +/// +/// @name PUT functions +/// +/// @{ + + /// + /// @brief start a gett operation + /// + /// @param object_id the identifier of the object to retrieve + /// virtual void start_get(std::string object_id) = 0; + /// + /// @brief gets a chunk of data of the object + /// + /// @param buf a buffer for the data to be filled in this function + /// @param count the size of the buffer / chunk + /// + /// @returns the number of bytes the buffer has been filled with + /// virtual ssize_t get(void *buf, size_t count) const = 0; + /// + /// @brief end a get operation + /// virtual void finish_get() = 0; + +/// @} + protected: - /* Abstract class that requires implementation */ + /// + /// @brief an implementation of @ref storage::Storage that will be + /// used as a database for the objects + /// std::unique_ptr storage; - /* This may be used to wait on the asynchronous backend */ + /// + /// @brief a future object that can be used to get the results from the + /// asynchronous backend + /// + /// @todo should this really be a protected member in the interface? + /// Isn't it the decision of the implementation to make this + /// asynchronous? + /// std::future db_result; }; -- GitLab From 4bd9dfd8302eccc83eb294c1db01670f77cc1918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Thu, 20 Feb 2020 11:34:41 +0000 Subject: [PATCH 10/62] Add basic test Change log: * sub libraries are combined into libfiphoboserver * libfiphoboserver is used by the build executable and the tests * test move only class design --- CMakeLists.txt | 16 ++++++ README.md | 7 +++ src/CMakeLists.txt | 14 +++++ src/fiphoboserver_exception.h | 1 + src/server/CMakeLists.txt | 16 ++---- src/server/server.main.cc | 100 ---------------------------------- src/storage/CMakeLists.txt | 8 +-- src/storage/phobos_file.cc | 5 ++ src/storage/phobos_file.h | 7 +++ src/storage/storage.h | 1 - src/stream/fifo.h | 5 ++ test/CMakeLists.txt | 22 +++++++- 12 files changed, 86 insertions(+), 116 deletions(-) delete mode 100644 src/server/server.main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 54e2e61..80d70d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,4 +6,20 @@ set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) find_package(proxygen REQUIRED) +# +# Provide options to build or skip building the +# programs/libraries in the given subdirectories +# +option( + FIPHOBOSERVER_BUILD_TESTS + "build FiPhoboServer tests. Depends on Catch2." + ON +) + add_subdirectory(src) + +# Build tests +if(FIPHOBOSERVER_BUILD_TESTS) + add_subdirectory(test) + enable_testing() +endif(FIPHOBOSERVER_BUILD_TESTS) diff --git a/README.md b/README.md index b4322f9..1595e79 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # FIPhoboServer +To use the Phobos raid1 layout which is compatible with FIFOs, the following environment variables must be set; + +``` +export PHOBOS_STORE_layout=raid1 +export PHOBOS_LAYOUT_RAID1_repl_count={1,2,3,...} +``` + Build with ``` diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 282155f..4ffe6b2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,3 +5,17 @@ endif(DEBUG) add_subdirectory(storage) add_subdirectory(stream) add_subdirectory(server) + +add_library(libfiphoboserver main.cc) + +target_link_libraries(libfiphoboserver PUBLIC stream) +target_link_libraries(libfiphoboserver PUBLIC storage) +target_link_libraries(libfiphoboserver PUBLIC server) + +add_executable( + fiphoboserver + + main.cc + ) + +target_link_libraries(fiphoboserver PUBLIC libfiphoboserver) diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index c2189bb..f01f47b 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace fiphoboserver { diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 00a88a7..14ad367 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -2,16 +2,15 @@ include_directories(${PHOBOS_INCLUDE_DIRECTORY}) include_directories(/usr/include/glib-2.0) include_directories(/usr/lib64/glib-2.0/include) -add_executable( - fiphoboserver - server.main.cc +add_library( + server get_request_handler.cc put_request_handler.cc unsupported_request_handler.cc ) if(CUSTOM_OUTPUT_DIRECTORY) - set_target_properties( fiphoboserver + set_target_properties( server PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" @@ -19,10 +18,7 @@ if(CUSTOM_OUTPUT_DIRECTORY) ) endif(CUSTOM_OUTPUT_DIRECTORY) -target_compile_features(fiphoboserver PUBLIC cxx_std_14) +target_compile_features(server PUBLIC cxx_std_14) - -target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen) -target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver) -target_link_libraries(fiphoboserver PUBLIC stream) -target_link_libraries(fiphoboserver PUBLIC storage) +target_link_libraries(server PUBLIC proxygen::proxygen) +target_link_libraries(server PUBLIC proxygen::proxygenhttpserver) diff --git a/src/server/server.main.cc b/src/server/server.main.cc deleted file mode 100644 index 49c54da..0000000 --- a/src/server/server.main.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "get_request_handler.h" -#include "put_request_handler.h" -#include "unsupported_request_handler.h" - -#include "../stream/fifo.h" -#include "../storage/phobos_file.h" - -using folly::SocketAddress; - -using Protocol = proxygen::HTTPServer::Protocol; - -DEFINE_int32(http_port, 11000, "Port to listen on with HTTP protocol"); -DEFINE_int32(h2_port, 11002, "Port to listen on with HTTP/2 protocol"); -DEFINE_string(ip, "localhost", "IP/Hostname to bind to"); -DEFINE_int32( - threads, - 0, - "Number of threads to listen on. Numbers <= 0 " - "will use the number of cores on this machine."); - -namespace { - -class HandlerFactory : public proxygen::RequestHandlerFactory { - public: - void onServerStart(folly::EventBase* /*evb*/) noexcept override {} - - void onServerStop() noexcept override {} - - proxygen::RequestHandler* onRequest( - proxygen::RequestHandler*, - proxygen::HTTPMessage* headers) noexcept override - { - if (headers->getMethod() == proxygen::HTTPMethod::GET) { - return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); - } - else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { - return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); - } - else { - return new fiphoboserver::UnsupportedRequestHandler; - } - } -}; -} // namespace - -int main(int argc, char* argv[]) -{ - folly::init(&argc, &argv, true); - - std::vector IPs = { - {SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP}, - {SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2}, - }; - - if (FLAGS_threads <= 0) { - FLAGS_threads = sysconf(_SC_NPROCESSORS_ONLN); - CHECK_GT(FLAGS_threads, 0); - } - - proxygen::HTTPServerOptions options; - options.threads = static_cast(FLAGS_threads); - options.idleTimeout = std::chrono::milliseconds(60000); - options.shutdownOn = {SIGINT, SIGTERM}; - options.enableContentCompression = false; - options.handlerFactories = - proxygen::RequestHandlerChain().addThen().build(); - options.h2cEnabled = true; - - auto diskIOThreadPool = std::make_shared( - FLAGS_threads, - std::make_shared("FIFOIOThread")); - folly::setCPUExecutor(diskIOThreadPool); - - proxygen::HTTPServer server(std::move(options)); - server.bind(IPs); - - /* Start HTTPServer mainloop in a separate thread */ - std::thread t([&]() { server.start(); }); - - t.join(); - return 0; -} diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 4fba3af..2e9e38e 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -1,7 +1,3 @@ -include_directories(${PHOBOS_INCLUDE_DIRECTORY}) -include_directories(/usr/include/glib-2.0) -include_directories(/usr/lib64/glib-2.0/include) - add_subdirectory(phobos_cpp_wrapper) add_library( @@ -9,6 +5,10 @@ add_library( phobos_file.cc ) +target_include_directories(storage PUBLIC ${PHOBOS_INCLUDE_DIRECTORY}) +target_include_directories(storage PUBLIC /usr/include/glib-2.0) +target_include_directories(storage PUBLIC /usr/lib64/glib-2.0/include) + target_compile_features(storage PUBLIC cxx_std_14) target_link_libraries(storage PUBLIC phobos_store) diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index 4bb9e93..6db9222 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -131,6 +131,11 @@ void Phobos_file::get_meta_data() const { } +int Phobos_file::get_fd() const +{ + return descriptor.xd_fd; +} + /* Close the file */ void Phobos_file::close_file() { diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 4acc53f..2cf5fed 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -12,6 +12,11 @@ class Phobos_file : public Storage { Phobos_file(); ~Phobos_file(); + Phobos_file(Phobos_file&&) = default; + Phobos_file& operator= (Phobos_file&&) = default; + Phobos_file(const Phobos_file&) = delete; + Phobos_file& operator= (const Phobos_file&) = delete; + void set_meta_data(std::vector> meta_data, std::string bucket_name) override; void get_meta_data() const override; @@ -20,6 +25,8 @@ class Phobos_file : public Storage { void prepare_put(std::string file_name, std::string object_id) override; void prepare_get(std::string file_name, std::string object_id) override; + + int get_fd() const; private: /* Implementation specific variables */ struct pho_xfer_desc descriptor = {0}; diff --git a/src/storage/storage.h b/src/storage/storage.h index 92f5608..e5ee624 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace fiphoboserver { namespace storage { diff --git a/src/stream/fifo.h b/src/stream/fifo.h index e56e0bf..4523e65 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -8,6 +8,11 @@ class Fifo : public Stream { Fifo(std::unique_ptr input_storage); ~Fifo(); + Fifo(Fifo&&) = default; + Fifo& operator= (Fifo&&) = default; + Fifo(const Fifo&) = delete; + Fifo& operator= (const Fifo&) = delete; + void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; void get_meta_data() const override; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 017ba83..2d4aefc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,9 +1,12 @@ +include_directories(/usr/include/glib-2.0) +include_directories(/usr/lib64/glib-2.0/include) + # # Define the options # set( - PHOBOSERVER_TEST_DATA_DIR "data" + FIPHOBOSERVER_TEST_DATA_DIR "data" CACHE PATH "The directory to read/write test data to/from." ) @@ -19,6 +22,23 @@ find_package(Catch2 REQUIRED) # Setup the test executables # +add_executable(phobos_file phobos_file.cc) + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${FIPHOBOSERVER_TEST_DATA_DIR}) + +target_link_libraries(phobos_file PUBLIC libfiphoboserver) +target_link_libraries(phobos_file PUBLIC Catch2::Catch2) + +# +# Create directory for data I/O +# +add_custom_target(make_data_directory) +add_custom_target( + TARGET make_data_directory PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${FIPHOBOSERVER_TEST_DATA_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) include(CTest) include(Catch) +catch_discover_tests(phobos_file) -- GitLab From 7c2199aeedd87ec14360d9a195766ba9ff3655d5 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 20 Feb 2020 11:40:56 +0000 Subject: [PATCH 11/62] Add some undocumented members and Markdown output of the documentation --- README.md | 2 + doc/Doxyfile.in | 5 +- .../HandlerFactory.md | 55 +++ .../index.md | 13 + .../fiphoboserver/FIPhoboServerException.md | 99 +++++ .../fiphoboserver/GetRequestHandler.md | 311 ++++++++++++++++ doc/Markdown/fiphoboserver/IOException.md | 46 +++ .../fiphoboserver/PutRequestHandler.md | 199 +++++++++++ doc/Markdown/fiphoboserver/S3_header.md | 187 ++++++++++ .../UnsupportedRequestHandler.md | 116 ++++++ doc/Markdown/fiphoboserver/index.md | 39 ++ .../fiphoboserver/storage/PhobosException.md | 46 +++ .../fiphoboserver/storage/Phobos_file.md | 244 +++++++++++++ doc/Markdown/fiphoboserver/storage/Storage.md | 211 +++++++++++ doc/Markdown/fiphoboserver/storage/index.md | 22 ++ doc/Markdown/fiphoboserver/stream/Fifo.md | 338 ++++++++++++++++++ doc/Markdown/fiphoboserver/stream/Stream.md | 296 +++++++++++++++ doc/Markdown/fiphoboserver/stream/index.md | 20 ++ doc/Markdown/pho_xfer_desc.md | 166 +++++++++ doc/Markdown/proxygen/index.md | 6 + doc/Markdown/tags.md | 47 +++ doc/Markdown/todo.md | 10 + src/stream/fifo.h | 14 +- src/stream/stream.h | 9 + 24 files changed, 2490 insertions(+), 11 deletions(-) create mode 100644 doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md create mode 100644 doc/Markdown/anonymous_namespace{server.main.cc}/index.md create mode 100644 doc/Markdown/fiphoboserver/FIPhoboServerException.md create mode 100644 doc/Markdown/fiphoboserver/GetRequestHandler.md create mode 100644 doc/Markdown/fiphoboserver/IOException.md create mode 100644 doc/Markdown/fiphoboserver/PutRequestHandler.md create mode 100644 doc/Markdown/fiphoboserver/S3_header.md create mode 100644 doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md create mode 100644 doc/Markdown/fiphoboserver/index.md create mode 100644 doc/Markdown/fiphoboserver/storage/PhobosException.md create mode 100644 doc/Markdown/fiphoboserver/storage/Phobos_file.md create mode 100644 doc/Markdown/fiphoboserver/storage/Storage.md create mode 100644 doc/Markdown/fiphoboserver/storage/index.md create mode 100644 doc/Markdown/fiphoboserver/stream/Fifo.md create mode 100644 doc/Markdown/fiphoboserver/stream/Stream.md create mode 100644 doc/Markdown/fiphoboserver/stream/index.md create mode 100644 doc/Markdown/pho_xfer_desc.md create mode 100644 doc/Markdown/proxygen/index.md create mode 100644 doc/Markdown/tags.md create mode 100644 doc/Markdown/todo.md diff --git a/README.md b/README.md index b4322f9..d6c6b80 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,5 @@ make from any directory you want and hope for the best! ;) Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there + +The Doxygen documentation can be found [here][doc/Markdown/fiphoboserver] diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index aae5e06..30282a7 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -27,10 +27,7 @@ WARN_NO_PARAMDOC = YES WARN_AS_ERROR = YES # Input directories -INPUT = \ - @CMAKE_SOURCE_DIR@/src/server \ - @CMAKE_SOURCE_DIR@/src/storage \ - @CMAKE_SOURCE_DIR@/src/stream +INPUT = @CMAKE_SOURCE_DIR@/src RECURSIVE = YES diff --git a/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md b/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md new file mode 100644 index 0000000..d36e730 --- /dev/null +++ b/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md @@ -0,0 +1,55 @@ +# public anonymous_namespace{server.main.cc}::HandlerFactory + + + + +## Inheritance: +Inherits from RequestHandlerFactory. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onServerStart](#anonymous_namespace{server.main.cc}-HandlerFactory-onServerStart) | | +| [onServerStop](#anonymous_namespace{server.main.cc}-HandlerFactory-onServerStop) | | +| [onRequest](#anonymous_namespace{server.main.cc}-HandlerFactory-onRequest) | | + + + +## Public Functions +### public void anonymous_namespace{server.main.cc}::HandlerFactory::onServerStart (folly::EventBase *) noexcept override + + + + + + + + +### public void anonymous_namespace{server.main.cc}::HandlerFactory::onServerStop () noexcept override + + + + + + + + +### public proxygen::RequestHandler * anonymous_namespace{server.main.cc}::HandlerFactory::onRequest (proxygen::RequestHandler *, proxygen::HTTPMessage *headers) noexcept override + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::HTTPMessage * | headers | | + +#### Returns: +| Type | Description | +| ---- | ---- | +| proxygen::RequestHandler * | | + + + + + diff --git a/doc/Markdown/anonymous_namespace{server.main.cc}/index.md b/doc/Markdown/anonymous_namespace{server.main.cc}/index.md new file mode 100644 index 0000000..bd28c13 --- /dev/null +++ b/doc/Markdown/anonymous_namespace{server.main.cc}/index.md @@ -0,0 +1,13 @@ +# anonymous_namespace{server.main.cc} + + + + +## Classes +| Name | Description | +| ---- | ---- | +| [HandlerFactory](./doc/build/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md) | | + + + +## Classes diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md new file mode 100644 index 0000000..c3a5bb7 --- /dev/null +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -0,0 +1,99 @@ +# public fiphoboserver::FIPhoboServerException + +exception class for all user defined exceptions in FiPhboServer + + + +## Inheritance: +Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphoboserver::storage::PhobosException][fiphoboserver-storage-PhobosException]. + +## Protected Attributes +| Name | Description | +| ---- | ---- | +| [message](#fiphoboserver-FIPhoboServerException-message) | the internal message + | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException) | default constructor + | +| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException-1) | constructor for a message with information on the exception + | +| [what](#fiphoboserver-FIPhoboServerException-what) | get information on this exception + | + + + +## Protected Attributes +### protected fiphoboserver::FIPhoboServerException::message + +the internal message + + + + + + + +## Public Functions +### public fiphoboserver::FIPhoboServerException::FIPhoboServerException () + +default constructor + + + + + + + +### public fiphoboserver::FIPhoboServerException::FIPhoboServerException (const char *message) + +constructor for a message with information on the exception + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | const char * | message | message explaining what went wrong such that this exception was thrown. + | + + + + + + + + + + + + +### public const char * fiphoboserver::FIPhoboServerException::what () const noexcept override + +get information on this exception + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| const char * | A message explaining what this exception is about + | + + + + + + + + + + + + + +[fiphoboserver-IOException]:./doc/build/Markdown/fiphoboserver/IOException.md +[fiphoboserver-storage-PhobosException]:./doc/build/Markdown/fiphoboserver/storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md new file mode 100644 index 0000000..e8f681c --- /dev/null +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -0,0 +1,311 @@ +# public fiphoboserver::GetRequestHandler + +proxygen class implementation for handling GET requests + + + +## Inheritance: +Inherits from RequestHandler. + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [Stream>](#fiphoboserver-GetRequestHandler-stream) | | +| [s3_header](#fiphoboserver-GetRequestHandler-s3_header) | | +| [readFileScheduled_](#fiphoboserver-GetRequestHandler-readFileScheduled_) | | +| [paused_](#fiphoboserver-GetRequestHandler-paused_) | | +| [finished_](#fiphoboserver-GetRequestHandler-finished_) | | +| [file_closed_](#fiphoboserver-GetRequestHandler-file_closed_) | | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onRequest](#fiphoboserver-GetRequestHandler-onRequest) | first function to be called when a new request comes in + | +| [onBody](#fiphoboserver-GetRequestHandler-onBody) | function called on every body chunk belonging to this message + | +| [onEOM](#fiphoboserver-GetRequestHandler-onEOM) | function called when the incoming message is finished + | +| [onUpgrade](#fiphoboserver-GetRequestHandler-onUpgrade) | function called on upgrade + | +| [requestComplete](#fiphoboserver-GetRequestHandler-requestComplete) | function ... + | +| [onError](#fiphoboserver-GetRequestHandler-onError) | function called when an error occurred + | +| [onEgressPaused](#fiphoboserver-GetRequestHandler-onEgressPaused) | called when the queue is full. + | +| [onEgressResumed](#fiphoboserver-GetRequestHandler-onEgressResumed) | called when the queue is not longer full. + | +| [GetRequestHandler](#fiphoboserver-GetRequestHandler-GetRequestHandler) | Constructor for stream class initialization. + | + + +## Private Functions +| Name | Description | +| ---- | ---- | +| [readFile](#fiphoboserver-GetRequestHandler-readFile) | | +| [checkForCompletion](#fiphoboserver-GetRequestHandler-checkForCompletion) | | + + + +## Private Attributes +### private std::unique_ptr + + + + + + + + +### private fiphoboserver::GetRequestHandler::s3_header + + + + + + + + +### private fiphoboserver::GetRequestHandler::readFileScheduled_ + + + + + + + + +### private fiphoboserver::GetRequestHandler::paused_ + + + + + + + + +### private fiphoboserver::GetRequestHandler::finished_ + + + + + + + + +### private fiphoboserver::GetRequestHandler::file_closed_ + + + + + + + + +## Public Functions +### public void fiphoboserver::GetRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override + +first function to be called when a new request comes in + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for + | + + + + + + + + + + + + +### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override + +function called on every body chunk belonging to this message + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< folly::IOBuf > | body | buffer for the body + | + + + + + + + +This is not used in this case because GET requests don't usually have a body + + + + + +### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override + +function called when the incoming message is finished + + + + + + + +### public void fiphoboserver::GetRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override + +function called on upgrade + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::UpgradeProtocol | proto | | + + + + + + + +Not supported in our case! + + + +### public void fiphoboserver::GetRequestHandler::requestComplete () noexcept override + +function ... + + + + + + + + + +> **Todo:** find out what this does? + + + + + + +### public void fiphoboserver::GetRequestHandler::onError (proxygen::ProxygenError err) noexcept override + +function called when an error occurred + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::ProxygenError | err | | + + + + + + + +Not supported in our case! + + + +### public void fiphoboserver::GetRequestHandler::onEgressPaused () noexcept override + +called when the queue is full. + + + + + + + + + +Contents are copies from a proxygen example + + + +### public void fiphoboserver::GetRequestHandler::onEgressResumed () noexcept override + +called when the queue is not longer full. + + + + + + + + + +Contents are copies from a proxygen example + + + +### public fiphoboserver::GetRequestHandler::GetRequestHandler (std::unique_ptr< stream::Stream > input_stream) + +Constructor for stream class initialization. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server + | + + + + + + + + + + + + +## Private Functions +### private void fiphoboserver::GetRequestHandler::readFile (folly::EventBase *evb) + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | folly::EventBase * | evb | | + + + + + +### private bool fiphoboserver::GetRequestHandler::checkForCompletion () + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | | + + + + + +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-S3_header]:./doc/build/Markdown/fiphoboserver/S3_header.md diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md new file mode 100644 index 0000000..c7f79ab --- /dev/null +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -0,0 +1,46 @@ +# public fiphoboserver::IOException + +exceptions specifically for errors in I/O + + + +## Inheritance: +Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [IOException](#fiphoboserver-IOException-IOException) | constructor + | + + + +## Public Functions +### public fiphoboserver::IOException::IOException (const char *caller, const char *functionName, int returnValue) + +constructor + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | const char * | caller | the function that threw this exception + | +| | const char * | functionName | the io function that returned an error + | +| | int | returnValue | the return value of the io function + | + + + + + + + + + + + + +[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md new file mode 100644 index 0000000..9275a12 --- /dev/null +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -0,0 +1,199 @@ +# public fiphoboserver::PutRequestHandler + +proxygen class implementation for handling PUT requests + + + +## Inheritance: +Inherits from RequestHandler. + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [s3_header](#fiphoboserver-PutRequestHandler-s3_header) | | +| [Stream>](#fiphoboserver-PutRequestHandler-stream) | | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onRequest](#fiphoboserver-PutRequestHandler-onRequest) | first function to be called when a new request comes in + | +| [onBody](#fiphoboserver-PutRequestHandler-onBody) | function called on every body chunk belonging to this message + | +| [onEOM](#fiphoboserver-PutRequestHandler-onEOM) | function called when the incoming message is finished + | +| [onUpgrade](#fiphoboserver-PutRequestHandler-onUpgrade) | function called on upgrade + | +| [requestComplete](#fiphoboserver-PutRequestHandler-requestComplete) | function ... + | +| [onError](#fiphoboserver-PutRequestHandler-onError) | function called when an error occurred + | +| [PutRequestHandler](#fiphoboserver-PutRequestHandler-PutRequestHandler) | Constructor for stream class initialization. + | + + + +## Private Attributes +### private fiphoboserver::PutRequestHandler::s3_header + + + + + + + + +### private std::unique_ptr + + + + + + + + +## Public Functions +### public void fiphoboserver::PutRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override + +first function to be called when a new request comes in + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for + | + + + + + + + + + + + + +### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override + +function called on every body chunk belonging to this message + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< folly::IOBuf > | body | buffer for the body + | + + + + + + + +This is not used in this case because GET requests don't usually have a body + + + + + +### public void fiphoboserver::PutRequestHandler::onEOM () noexcept override + +function called when the incoming message is finished + + + + + + + +### public void fiphoboserver::PutRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override + +function called on upgrade + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::UpgradeProtocol | proto | | + + + + + + + +Not supported in our case! + + + +### public void fiphoboserver::PutRequestHandler::requestComplete () noexcept override + +function ... + + + + + + + + + +> **Todo:** find out what this does? + + + + + + +### public void fiphoboserver::PutRequestHandler::onError (proxygen::ProxygenError err) noexcept override + +function called when an error occurred + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::ProxygenError | err | | + + + + + + + +Not supported in our case! + + + +### public fiphoboserver::PutRequestHandler::PutRequestHandler (std::unique_ptr< stream::Stream > input_stream) + +Constructor for stream class initialization. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server + | + + + + + + + + + + + + +[fiphoboserver-S3_header]:./doc/build/Markdown/fiphoboserver/S3_header.md +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md new file mode 100644 index 0000000..a0d699f --- /dev/null +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -0,0 +1,187 @@ +# public fiphoboserver::S3_header + +class to extract the S3 specific information from proxygens HTTPMessage headers + + + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [headers](#fiphoboserver-S3_header-headers) | | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [setHeaders](#fiphoboserver-S3_header-setHeaders) | sets the internal header instance to the given header object + | +| [getBucket](#fiphoboserver-S3_header-getBucket) | extracts the name of the S3 bucket requested in the HTTP message + | +| [getKey](#fiphoboserver-S3_header-getKey) | extracts the name of the key id requested in the HTTP message + | +| [isCreateBucketRequest](#fiphoboserver-S3_header-isCreateBucketRequest) | checks if the message we got is one for just creating a bucket + | +| [getBodyLength](#fiphoboserver-S3_header-getBodyLength) | gets the total length of all body chunks that is expected + | +| [getMetaData](#fiphoboserver-S3_header-getMetaData) | gets the S3 metadata stored in the HTTP headers + | + + + +## Private Attributes +### private fiphoboserver::S3_header::headers + + + + + + + + +## Public Functions +### public void fiphoboserver::S3_header::setHeaders (std::unique_ptr< proxygen::HTTPMessage > newHeaders) + +sets the internal header instance to the given header object + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance + | + + + + + + + + + + + + +### public std::string fiphoboserver::S3_header::getBucket () + +extracts the name of the S3 bucket requested in the HTTP message + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the bucket + | + + + + + + + + + + + + + +### public std::string fiphoboserver::S3_header::getKey () + +extracts the name of the key id requested in the HTTP message + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the key + | + + + + + + + + + + + + + +### public bool fiphoboserver::S3_header::isCreateBucketRequest () + +checks if the message we got is one for just creating a bucket + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if these headers belong to a create bucket request, false for all other requests + | + + + + + + + +This is convenient since we don't really support buckets themselves and can easily stop working if this returns true + + + + + + +### public size_t fiphoboserver::S3_header::getBodyLength () + +gets the total length of all body chunks that is expected + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| size_t | the length of the total body to be expected + | + + + + + + + +This reads out the header "Content-Length" + + + + + + +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::getMetaData () + +gets the S3 metadata stored in the HTTP headers + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::vector< std::pair< std::string, std::string > > | a vector consisting of all key value pairs found in the header + | + + + + + + + +In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE for for example a KEY:VALUE pair. + + + + + + diff --git a/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md new file mode 100644 index 0000000..0c63dcc --- /dev/null +++ b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md @@ -0,0 +1,116 @@ +# public fiphoboserver::UnsupportedRequestHandler + +proxygen class implementation for handling all requests that we don't support + + + +## Inheritance: +Inherits from RequestHandler. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onRequest](#fiphoboserver-UnsupportedRequestHandler-onRequest) | | +| [onBody](#fiphoboserver-UnsupportedRequestHandler-onBody) | | +| [onEOM](#fiphoboserver-UnsupportedRequestHandler-onEOM) | | +| [onUpgrade](#fiphoboserver-UnsupportedRequestHandler-onUpgrade) | | +| [requestComplete](#fiphoboserver-UnsupportedRequestHandler-requestComplete) | | +| [onError](#fiphoboserver-UnsupportedRequestHandler-onError) | | +| [onEgressPaused](#fiphoboserver-UnsupportedRequestHandler-onEgressPaused) | | +| [onEgressResumed](#fiphoboserver-UnsupportedRequestHandler-onEgressResumed) | | + + + +## Public Functions +### public void fiphoboserver::UnsupportedRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< proxygen::HTTPMessage > | headers | | + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< folly::IOBuf > | body | | + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onEOM () noexcept override + + + + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::UpgradeProtocol | proto | | + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::requestComplete () noexcept override + + + + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onError (proxygen::ProxygenError err) noexcept override + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | proxygen::ProxygenError | err | | + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onEgressPaused () noexcept override + + + + + + + + +### public void fiphoboserver::UnsupportedRequestHandler::onEgressResumed () noexcept override + + + + + + + + diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md new file mode 100644 index 0000000..281011b --- /dev/null +++ b/doc/Markdown/fiphoboserver/index.md @@ -0,0 +1,39 @@ +# fiphoboserver + + + + +## Classes +| Name | Description | +| ---- | ---- | +| [FIPhoboServerException](./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer + | +| [GetRequestHandler](./doc/build/Markdown/fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests + | +| [IOException](./doc/build/Markdown/fiphoboserver/IOException.md) | exceptions specifically for errors in I/O + | +| [PutRequestHandler](./doc/build/Markdown/fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests + | +| [S3_header](./doc/build/Markdown/fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers + | +| [UnsupportedRequestHandler](./doc/build/Markdown/fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support + | + + +## Namespaces +| Name | Description | +| ---- | ---- | +| [storage](./doc/build/Markdown/fiphoboserver/storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] + | +| [stream](./doc/build/Markdown/fiphoboserver/stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] + | + + + +## Classes +## Namespaces +[fiphoboserver-IOException]:./doc/build/Markdown/fiphoboserver/IOException.md +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-PhobosException]:./doc/build/Markdown/fiphoboserver/storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md new file mode 100644 index 0000000..65288ae --- /dev/null +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -0,0 +1,46 @@ +# public fiphoboserver::storage::PhobosException + +exceptions specifically for the phobos backend library + + + +## Inheritance: +Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [PhobosException](#fiphoboserver-storage-PhobosException-PhobosException) | constructor + | + + + +## Public Functions +### public fiphoboserver::storage::PhobosException::PhobosException (const char *caller, const char *functionName, int returnValue) + +constructor + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | const char * | caller | the function that threw this exception + | +| | const char * | functionName | the phobos function that returned an error + | +| | int | returnValue | the return value of the phobos function + | + + + + + + + + + + + + +[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md new file mode 100644 index 0000000..41ba10c --- /dev/null +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -0,0 +1,244 @@ +# public fiphoboserver::storage::Phobos_file + +default constructor + + +> **Todo:** maybe not only copy from interface but add specific phobos information? + + + + + + +## Inheritance: +Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [descriptor](#fiphoboserver-storage-Phobos_file-descriptor) | | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file) | default constructor + | +| [~Phobos_file](#fiphoboserver-storage-Phobos_file-~Phobos_file) | default destructor + | +| [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata an object that is added to the database should get + | +| [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs + | +| [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. + | +| [db_get](#fiphoboserver-storage-Phobos_file-db_get) | Gets data from the databse. + | +| [prepare_put](#fiphoboserver-storage-Phobos_file-prepare_put) | Starts a put operation to the database. + | +| [prepare_get](#fiphoboserver-storage-Phobos_file-prepare_get) | Starts a get operation to the database. + | + + +## Private Functions +| Name | Description | +| ---- | ---- | +| [set_bucket_name](#fiphoboserver-storage-Phobos_file-set_bucket_name) | | +| [close_file](#fiphoboserver-storage-Phobos_file-close_file) | | + + + +## Private Attributes +### private fiphoboserver::storage::Phobos_file::descriptor + + + + + + + + +## Public Functions +### public fiphoboserver::storage::Phobos_file::Phobos_file () + +default constructor + + + + + + + +### public fiphoboserver::storage::Phobos_file::~Phobos_file () + +default destructor + + + + + + + +### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override + +set the metadata an object that is added to the database should get + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data + | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata + | + + + + + + + + + + + + +### public void fiphoboserver::storage::Phobos_file::get_meta_data () const override + +get the metadata associated to the current object as a vector of key:value pairs + + + + + + + +### public ssize_t fiphoboserver::storage::Phobos_file::db_put (size_t size) override + +Puts data to the databse. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | size_t | size | the number of bytes to add to the file in the database + | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | | + + + + + + + + + + + + +### public ssize_t fiphoboserver::storage::Phobos_file::db_get () override + +Gets data from the databse. + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the number of bytes read from the database + | + + + + + + + + + + + + + +### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override + +Starts a put operation to the database. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | file_name | the filename the data that will be added to the database shouldbe read from + | +| | std::string | object_id | the internal storage id the data should get + | + + + + + + + + + + + + +### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override + +Starts a get operation to the database. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | file_name | the filename the data from the database should be written to + | +| | std::string | object_id | the internal storage id from the data that should be retrieved + | + + + + + + + + + + + + +## Private Functions +### private void fiphoboserver::storage::Phobos_file::set_bucket_name (std::string bucket_name) + + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | bucket_name | | + + + + + +### private void fiphoboserver::storage::Phobos_file::close_file () + + + + + + + + +[pho_xfer_desc]:./doc/build/Markdown/pho_xfer_desc.md#pho_xfer_desc +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md new file mode 100644 index 0000000..eb46221 --- /dev/null +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -0,0 +1,211 @@ +# public fiphoboserver::storage::Storage + +virtual storage class to be implemented by backend storage + + +This class is a prototype for every class that handles to internal database that should be accessed by the FiPhoboServer. This class represents exactly one object file in the internal database + + + +## Inheritance: +Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phobos_file]. + +## Metadata functions +| Name | Description | +| ---- | ---- | +| [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata an object that is added to the database should get + | +| [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs + | + + +## PUT functions +| Name | Description | +| ---- | ---- | +| [prepare_put](#fiphoboserver-storage-Storage-prepare_put) | Starts a put operation to the database. + | +| [db_put](#fiphoboserver-storage-Storage-db_put) | Puts data to the databse. + | + + +## GET functions +| Name | Description | +| ---- | ---- | +| [prepare_get](#fiphoboserver-storage-Storage-prepare_get) | Starts a get operation to the database. + | +| [db_get](#fiphoboserver-storage-Storage-db_get) | Gets data from the databse. + | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [Storage](#fiphoboserver-storage-Storage-Storage) | default constructor + | +| [~Storage](#fiphoboserver-storage-Storage-~Storage) | default destructor + | + + + +## Metadata functions +### public void fiphoboserver::storage::Storage::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name)=0 + +set the metadata an object that is added to the database should get + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data + | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata + | + + + + + + + + + + + + +### public void fiphoboserver::storage::Storage::get_meta_data () const =0 + +get the metadata associated to the current object as a vector of key:value pairs + + + + + + + +## PUT functions +### public void fiphoboserver::storage::Storage::prepare_put (std::string file_name, std::string object_id)=0 + +Starts a put operation to the database. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | file_name | the filename the data that will be added to the database shouldbe read from + | +| | std::string | object_id | the internal storage id the data should get + | + + + + + + + + + + + + +### public ssize_t fiphoboserver::storage::Storage::db_put (size_t size)=0 + +Puts data to the databse. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | size_t | size | the number of bytes to add to the file in the database + | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | | + + + + + + + + + + + + +## GET functions +### public void fiphoboserver::storage::Storage::prepare_get (std::string file_name, std::string object_id)=0 + +Starts a get operation to the database. + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | file_name | the filename the data from the database should be written to + | +| | std::string | object_id | the internal storage id from the data that should be retrieved + | + + + + + + + + + + + + +### public ssize_t fiphoboserver::storage::Storage::db_get ()=0 + +Gets data from the databse. + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the number of bytes read from the database + | + + + + + + + + + + + + + +## Public Functions +### public fiphoboserver::storage::Storage::Storage () + +default constructor + + + + + + + +### public fiphoboserver::storage::Storage::~Storage () + +default destructor + + + + + + + +[fiphoboserver-storage-Phobos_file]:./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md new file mode 100644 index 0000000..b8da99f --- /dev/null +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -0,0 +1,22 @@ +# fiphoboserver::storage + +namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] + + + +## Classes +| Name | Description | +| ---- | ---- | +| [Phobos_file](./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md) | default constructor + | +| [PhobosException](./doc/build/Markdown/fiphoboserver/storage/PhobosException.md) | exceptions specifically for the phobos backend library + | +| [Storage](./doc/build/Markdown/fiphoboserver/storage/Storage.md) | virtual storage class to be implemented by backend storage + | + + + +## Classes +[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-storage-Phobos_file]:./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md new file mode 100644 index 0000000..870ddef --- /dev/null +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -0,0 +1,338 @@ +# public fiphoboserver::stream::Fifo + +default constructor + + +> **Todo:** maybe not only copy from interface but add specific fifo information? + + + + + + +## Inheritance: +Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. + +## Implementation specific details +| Name | Description | +| ---- | ---- | +| [get_fifo_descriptor](#fiphoboserver-stream-Fifo-get_fifo_descriptor) | get the descriptor of the fifo + | +| [get_fifo_name](#fiphoboserver-stream-Fifo-get_fifo_name) | get the name of the fifo + | + + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [fifo_name](#fiphoboserver-stream-Fifo-fifo_name) | | +| [fifo_descriptor](#fiphoboserver-stream-Fifo-fifo_descriptor) | | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [Fifo](#fiphoboserver-stream-Fifo-Fifo) | default constructor + | +| [~Fifo](#fiphoboserver-stream-Fifo-~Fifo) | default destructor + | +| [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get + | +| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata from a stored object + | +| [start_put](#fiphoboserver-stream-Fifo-start_put) | start a put operation + | +| [put](#fiphoboserver-stream-Fifo-put) | add a chunk of data to the object + | +| [finish_put](#fiphoboserver-stream-Fifo-finish_put) | end a put operation + | +| [start_get](#fiphoboserver-stream-Fifo-start_get) | start a gett operation + | +| [get](#fiphoboserver-stream-Fifo-get) | gets a chunk of data of the object + | +| [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation + | + + +## Private Functions +| Name | Description | +| ---- | ---- | +| [create_fifo](#fiphoboserver-stream-Fifo-create_fifo) | | + + + +## Implementation specific details +### public int fiphoboserver::stream::Fifo::get_fifo_descriptor () const + +get the descriptor of the fifo + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| int | the descriptor of the fifo + | + + + + + + + + + +> **Todo:** Do we use this? should this really be public? + + + + + + +### public std::string fiphoboserver::stream::Fifo::get_fifo_name () const + +get the name of the fifo + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the name of the fifo + | + + + + + + + + + +> **Todo:** Do we use this? should this really be public? + + + + + + +## Private Attributes +### private fiphoboserver::stream::Fifo::fifo_name + + + + + + + + +### private fiphoboserver::stream::Fifo::fifo_descriptor + + + + + + + + +## Public Functions +### public fiphoboserver::stream::Fifo::Fifo (std::unique_ptr< storage::Storage > input_storage) + +default constructor + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | | + + + + + +### public fiphoboserver::stream::Fifo::~Fifo () + +default destructor + + + + + + + +### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override + +setting the metadata that the created object should get + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data + | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata + | + + + + + + + + +> **warning:** this function is responsible for getting metadata to the storage implementation + + + + + + + +### public void fiphoboserver::stream::Fifo::get_meta_data () const override + +get the metadata from a stored object + + + + + + + +### public void fiphoboserver::stream::Fifo::start_put (ssize_t size, std::string object_id) override + +start a put operation + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | ssize_t | size | amount of bytes that shall be added to the internal storage + | +| | std::string | object_id | the identifier the object should get + | + + + + + + + + + + + + +### public void fiphoboserver::stream::Fifo::put (std::unique_ptr< folly::IOBuf > buffer) const override + +add a chunk of data to the object + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add + | + + + + + + + + + + + + +### public void fiphoboserver::stream::Fifo::finish_put () override + +end a put operation + + + + + + + +### public void fiphoboserver::stream::Fifo::start_get (std::string object_id) override + +start a gett operation + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | object_id | the identifier of the object to retrieve + | + + + + + + + + + + + + +### public ssize_t fiphoboserver::stream::Fifo::get (void *buf, size_t count) const override + +gets a chunk of data of the object + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | void * | buf | a buffer for the data to be filled in this function + | +| | size_t | count | the size of the buffer / chunk + | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the number of bytes the buffer has been filled with + | + + + + + + + + + + + + + + +### public void fiphoboserver::stream::Fifo::finish_get () override + +end a get operation + + + + + + + +## Private Functions +### private void fiphoboserver::stream::Fifo::create_fifo () + + + + + + + + +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md new file mode 100644 index 0000000..6ed6e12 --- /dev/null +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -0,0 +1,296 @@ +# public fiphoboserver::stream::Stream + +virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class + + + +## Inheritance: +Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. + +## PUT functions +| Name | Description | +| ---- | ---- | +| [start_put](#fiphoboserver-stream-Stream-start_put) | start a put operation + | +| [put](#fiphoboserver-stream-Stream-put) | add a chunk of data to the object + | +| [finish_put](#fiphoboserver-stream-Stream-finish_put) | end a put operation + | +| [start_get](#fiphoboserver-stream-Stream-start_get) | start a gett operation + | +| [get](#fiphoboserver-stream-Stream-get) | gets a chunk of data of the object + | +| [finish_get](#fiphoboserver-stream-Stream-finish_get) | end a get operation + | + + +## Protected Attributes +| Name | Description | +| ---- | ---- | +| [Storage>](#fiphoboserver-stream-Stream-storage) | an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects + | +| [db_result](#fiphoboserver-stream-Stream-db_result) | a future object that can be used to get the results from the asynchronous backend + | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [Stream](#fiphoboserver-stream-Stream-Stream) | default constructor + | +| [Stream](#fiphoboserver-stream-Stream-Stream-1) | constructor with storage implementation + | +| [~Stream](#fiphoboserver-stream-Stream-~Stream) | default destructor + | +| [set_storage_meta_data](#fiphoboserver-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get + | +| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata from a stored object + | + + + +## PUT functions +### public void fiphoboserver::stream::Stream::start_put (ssize_t size, std::string object_id)=0 + +start a put operation + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | ssize_t | size | amount of bytes that shall be added to the internal storage + | +| | std::string | object_id | the identifier the object should get + | + + + + + + + + + + + + +### public void fiphoboserver::stream::Stream::put (std::unique_ptr< folly::IOBuf > buffer) const =0 + +add a chunk of data to the object + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add + | + + + + + + + + + + + + +### public void fiphoboserver::stream::Stream::finish_put ()=0 + +end a put operation + + + + + + + +### public void fiphoboserver::stream::Stream::start_get (std::string object_id)=0 + +start a gett operation + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::string | object_id | the identifier of the object to retrieve + | + + + + + + + + + + + + +### public ssize_t fiphoboserver::stream::Stream::get (void *buf, size_t count) const =0 + +gets a chunk of data of the object + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | void * | buf | a buffer for the data to be filled in this function + | +| | size_t | count | the size of the buffer / chunk + | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the number of bytes the buffer has been filled with + | + + + + + + + + + + + + + + +### public void fiphoboserver::stream::Stream::finish_get ()=0 + +end a get operation + + + + + + + +## Protected Attributes +### protected std::unique_ptr + +an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects + + + + + + + +### protected fiphoboserver::stream::Stream::db_result + +a future object that can be used to get the results from the asynchronous backend + + + + + + + + + +> **Todo:** should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? + + + + + + +## Public Functions +### public fiphoboserver::stream::Stream::Stream () + +default constructor + + + + + + + +### public fiphoboserver::stream::Stream::Stream (std::unique_ptr< storage::Storage > input_storage) + +constructor with storage implementation + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | pointer to a implementation of the [storage::Storage][fiphoboserver-storage-Storage] class that will be used to store the data + | + + + + + + + + +> **warning:** every class that inherits this interface must have an internal [storage::Storage][fiphoboserver-storage-Storage] implementation and initialize this here. + + + + + + + +### public fiphoboserver::stream::Stream::~Stream () + +default destructor + + + + + + + +### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const =0 + +setting the metadata that the created object should get + + + +#### Parameters: +| Direction | Type | Name | Description | +| ---- | ---- | ---- | ---- | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data + | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata + | + + + + + + + + +> **warning:** this function is responsible for getting metadata to the storage implementation + + + + + + + +### public void fiphoboserver::stream::Stream::get_meta_data () const =0 + +get the metadata from a stored object + + + + + + + +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Fifo]:./doc/build/Markdown/fiphoboserver/stream/Fifo.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md new file mode 100644 index 0000000..4cf6054 --- /dev/null +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -0,0 +1,20 @@ +# fiphoboserver::stream + +namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] + + + +## Classes +| Name | Description | +| ---- | ---- | +| [Fifo](./doc/build/Markdown/fiphoboserver/stream/Fifo.md) | default constructor + | +| [Stream](./doc/build/Markdown/fiphoboserver/stream/Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class + | + + + +## Classes +[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Fifo]:./doc/build/Markdown/fiphoboserver/stream/Fifo.md diff --git a/doc/Markdown/pho_xfer_desc.md b/doc/Markdown/pho_xfer_desc.md new file mode 100644 index 0000000..83a103d --- /dev/null +++ b/doc/Markdown/pho_xfer_desc.md @@ -0,0 +1,166 @@ +# public pho_xfer_desc + + + +GET / PUT parameter. The source/destination semantics of the fields vary depending on the nature of the operation. See below:phobos_get() + +phobos_put() + + + + + + + +## Public Attributes +| Name | Description | +| ---- | ---- | +| [xd_objid](#pho_xfer_desc-xd_objid) | | +| [xd_op](#pho_xfer_desc-xd_op) | | +| [xd_fd](#pho_xfer_desc-xd_fd) | | +| [xd_size](#pho_xfer_desc-xd_size) | | +| [xd_layout_name](#pho_xfer_desc-xd_layout_name) | | +| [xd_attrs](#pho_xfer_desc-xd_attrs) | | +| [xd_flags](#pho_xfer_desc-xd_flags) | | +| [xd_tags](#pho_xfer_desc-xd_tags) | | +| [xd_rc](#pho_xfer_desc-xd_rc) | | + + + +## Public Attributes +### public pho_xfer_desc::xd_objid + + + + + + + + + + +Object id to read or write + + + +### public pho_xfer_desc::xd_op + + + + + + + + + + +Operation to perform (GET, GETMD or PUT) + + + +### public pho_xfer_desc::xd_fd + + + + + + + + + + +positive fd if xd_id_open + + + +### public pho_xfer_desc::xd_size + + + + + + + + + + +Amount of data to write (for the GET operation, the size read is equal to the size of the retrieved object) + + + +### public pho_xfer_desc::xd_layout_name + + + + + + + + + + +Name of the layout module to use (for put). + + + +### public pho_xfer_desc::xd_attrs + + + + + + + + + + +User defined attribute to get / put + + + +### public pho_xfer_desc::xd_flags + + + + + + + + + + +See enum pho_xfer_flags doc + + + +### public pho_xfer_desc::xd_tags + + + + + + + + + + +Tags to select a media to write + + + +### public pho_xfer_desc::xd_rc + + + + + + + + + + +Outcome of this xfer + + + +[tags]:./doc/build/Markdown/tags.md#tags diff --git a/doc/Markdown/proxygen/index.md b/doc/Markdown/proxygen/index.md new file mode 100644 index 0000000..46e621a --- /dev/null +++ b/doc/Markdown/proxygen/index.md @@ -0,0 +1,6 @@ +# proxygen + + + + + diff --git a/doc/Markdown/tags.md b/doc/Markdown/tags.md new file mode 100644 index 0000000..4ee7e44 --- /dev/null +++ b/doc/Markdown/tags.md @@ -0,0 +1,47 @@ +# public tags + + + +A simple array of tags (strings) + + + +## Public Attributes +| Name | Description | +| ---- | ---- | +| [tags](#tags-tags) | | +| [n_tags](#tags-n_tags) | | + + + +## Public Attributes +### public tags::tags + + + + + + + + + + +The array of tags + + + +### public tags::n_tags + + + + + + + + + + +Number of tags + + + diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md new file mode 100644 index 0000000..0c44059 --- /dev/null +++ b/doc/Markdown/todo.md @@ -0,0 +1,10 @@ +# Todo List + + + + + + + + + diff --git a/src/stream/fifo.h b/src/stream/fifo.h index acc0110..ff0c8b5 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -18,21 +18,21 @@ class Fifo : public Stream { /// @copydoc Stream::set_storage_meta_data void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::get_meta_data void get_meta_data() const override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::start_put void start_put(ssize_t size, std::string object_id) override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::put void put(std::unique_ptr buffer) const override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::finish_put void finish_put() override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::start_get void start_get(std::string object_id) override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::get ssize_t get(void *buf, size_t count) const override; - /// @copydoc Stream::set_storage_meta_data + /// @copydoc Stream::finish_get void finish_get() override; diff --git a/src/stream/stream.h b/src/stream/stream.h index 6c043b3..1932cbd 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -28,6 +28,10 @@ class Stream { Stream() {}; /// /// @brief constructor with storage implementation + /// + /// @param input_storage pointer to a implementation of the @ref + /// storage::Storage class that will be used to + /// store the data /// /// @warning every class that inherits this interface must have an /// internal @ref storage::Storage implementation and @@ -44,6 +48,11 @@ class Stream { /// /// @brief setting the metadata that the created object should get /// + /// @param meta_data a vector of all key:value pairs that should be + /// added to the data + /// @param bucket_name the name of the S3 bucket - currently also + /// handled as normal metadata + /// /// @warning this function is responsible for getting metadata to /// the storage implementation /// -- GitLab From d1c616ee26e9644dba6ce9161b45454c5afec828 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 20 Feb 2020 11:43:51 +0000 Subject: [PATCH 12/62] Fix link to documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6c6b80..b585f3b 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,4 @@ from any directory you want and hope for the best! ;) Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there -The Doxygen documentation can be found [here][doc/Markdown/fiphoboserver] +The Doxygen documentation can be found [here](doc/Markdown/fiphoboserver) -- GitLab From 73aa911101e592e9c492943a5a849e7a2e25dcfe Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 20 Feb 2020 13:32:26 +0000 Subject: [PATCH 13/62] Fix Markdown a bit --- .../index.md | 2 +- .../fiphoboserver/FIPhoboServerException.md | 22 ++---- .../fiphoboserver/GetRequestHandler.md | 44 ++++------- doc/Markdown/fiphoboserver/IOException.md | 14 ++-- .../fiphoboserver/PutRequestHandler.md | 38 ++++----- doc/Markdown/fiphoboserver/S3_header.md | 36 +++------ doc/Markdown/fiphoboserver/index.md | 34 ++++---- .../fiphoboserver/storage/PhobosException.md | 14 ++-- .../fiphoboserver/storage/Phobos_file.md | 52 +++++-------- doc/Markdown/fiphoboserver/storage/Storage.md | 50 ++++-------- doc/Markdown/fiphoboserver/storage/index.md | 15 ++-- doc/Markdown/fiphoboserver/stream/Fifo.md | 73 ++++++------------ doc/Markdown/fiphoboserver/stream/Stream.md | 77 +++++++------------ doc/Markdown/fiphoboserver/stream/index.md | 12 ++- doc/Markdown/pho_xfer_desc.md | 2 +- 15 files changed, 173 insertions(+), 312 deletions(-) diff --git a/doc/Markdown/anonymous_namespace{server.main.cc}/index.md b/doc/Markdown/anonymous_namespace{server.main.cc}/index.md index bd28c13..d27f463 100644 --- a/doc/Markdown/anonymous_namespace{server.main.cc}/index.md +++ b/doc/Markdown/anonymous_namespace{server.main.cc}/index.md @@ -6,7 +6,7 @@ ## Classes | Name | Description | | ---- | ---- | -| [HandlerFactory](./doc/build/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md) | | +| [HandlerFactory](doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md) | | diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index c3a5bb7..f9a002e 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -10,19 +10,15 @@ Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphob ## Protected Attributes | Name | Description | | ---- | ---- | -| [message](#fiphoboserver-FIPhoboServerException-message) | the internal message - | +| [message](#fiphoboserver-FIPhoboServerException-message) | the internal message | ## Public Functions | Name | Description | | ---- | ---- | -| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException) | default constructor - | -| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException-1) | constructor for a message with information on the exception - | -| [what](#fiphoboserver-FIPhoboServerException-what) | get information on this exception - | +| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException) | default constructor | +| [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException-1) | constructor for a message with information on the exception | +| [what](#fiphoboserver-FIPhoboServerException-what) | get information on this exception | @@ -57,8 +53,7 @@ constructor for a message with information on the exception #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | const char * | message | message explaining what went wrong such that this exception was thrown. - | +| | const char * | message | message explaining what went wrong such that this exception was thrown. | @@ -80,8 +75,7 @@ get information on this exception #### Returns: | Type | Description | | ---- | ---- | -| const char * | A message explaining what this exception is about - | +| const char * | A message explaining what this exception is about | @@ -95,5 +89,5 @@ get information on this exception -[fiphoboserver-IOException]:./doc/build/Markdown/fiphoboserver/IOException.md -[fiphoboserver-storage-PhobosException]:./doc/build/Markdown/fiphoboserver/storage/PhobosException.md +[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md +[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index e8f681c..efca0ff 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -10,7 +10,7 @@ Inherits from RequestHandler. ## Private Attributes | Name | Description | | ---- | ---- | -| [Stream>](#fiphoboserver-GetRequestHandler-stream) | | +| [Stream >](#fiphoboserver-GetRequestHandler-stream) | | | [s3_header](#fiphoboserver-GetRequestHandler-s3_header) | | | [readFileScheduled_](#fiphoboserver-GetRequestHandler-readFileScheduled_) | | | [paused_](#fiphoboserver-GetRequestHandler-paused_) | | @@ -21,24 +21,15 @@ Inherits from RequestHandler. ## Public Functions | Name | Description | | ---- | ---- | -| [onRequest](#fiphoboserver-GetRequestHandler-onRequest) | first function to be called when a new request comes in - | -| [onBody](#fiphoboserver-GetRequestHandler-onBody) | function called on every body chunk belonging to this message - | -| [onEOM](#fiphoboserver-GetRequestHandler-onEOM) | function called when the incoming message is finished - | -| [onUpgrade](#fiphoboserver-GetRequestHandler-onUpgrade) | function called on upgrade - | -| [requestComplete](#fiphoboserver-GetRequestHandler-requestComplete) | function ... - | -| [onError](#fiphoboserver-GetRequestHandler-onError) | function called when an error occurred - | -| [onEgressPaused](#fiphoboserver-GetRequestHandler-onEgressPaused) | called when the queue is full. - | -| [onEgressResumed](#fiphoboserver-GetRequestHandler-onEgressResumed) | called when the queue is not longer full. - | -| [GetRequestHandler](#fiphoboserver-GetRequestHandler-GetRequestHandler) | Constructor for stream class initialization. - | +| [onRequest](#fiphoboserver-GetRequestHandler-onRequest) | first function to be called when a new request comes in | +| [onBody](#fiphoboserver-GetRequestHandler-onBody) | function called on every body chunk belonging to this message | +| [onEOM](#fiphoboserver-GetRequestHandler-onEOM) | function called when the incoming message is finished | +| [onUpgrade](#fiphoboserver-GetRequestHandler-onUpgrade) | function called on upgrade | +| [requestComplete](#fiphoboserver-GetRequestHandler-requestComplete) | function ... | +| [onError](#fiphoboserver-GetRequestHandler-onError) | function called when an error occurred | +| [onEgressPaused](#fiphoboserver-GetRequestHandler-onEgressPaused) | called when the queue is full. | +| [onEgressResumed](#fiphoboserver-GetRequestHandler-onEgressResumed) | called when the queue is not longer full. | +| [GetRequestHandler](#fiphoboserver-GetRequestHandler-GetRequestHandler) | Constructor for stream class initialization. | ## Private Functions @@ -50,7 +41,7 @@ Inherits from RequestHandler. ## Private Attributes -### private std::unique_ptr +### private std::unique_ptr< stream::Stream > @@ -114,8 +105,7 @@ first function to be called when a new request comes in #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for - | +| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | @@ -137,8 +127,7 @@ function called on every body chunk belonging to this message #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | body | buffer for the body - | +| | std::unique_ptr< folly::IOBuf > | body | buffer for the body | @@ -264,8 +253,7 @@ Constructor for stream class initialization. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server - | +| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | @@ -307,5 +295,5 @@ Constructor for stream class initialization. -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-S3_header]:./doc/build/Markdown/fiphoboserver/S3_header.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index c7f79ab..371e426 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -10,8 +10,7 @@ Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServe ## Public Functions | Name | Description | | ---- | ---- | -| [IOException](#fiphoboserver-IOException-IOException) | constructor - | +| [IOException](#fiphoboserver-IOException-IOException) | constructor | @@ -25,12 +24,9 @@ constructor #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | const char * | caller | the function that threw this exception - | -| | const char * | functionName | the io function that returned an error - | -| | int | returnValue | the return value of the io function - | +| | const char * | caller | the function that threw this exception | +| | const char * | functionName | the io function that returned an error | +| | int | returnValue | the return value of the io function | @@ -43,4 +39,4 @@ constructor -[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 9275a12..5bb6c6f 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -11,26 +11,19 @@ Inherits from RequestHandler. | Name | Description | | ---- | ---- | | [s3_header](#fiphoboserver-PutRequestHandler-s3_header) | | -| [Stream>](#fiphoboserver-PutRequestHandler-stream) | | +| [Stream >](#fiphoboserver-PutRequestHandler-stream) | | ## Public Functions | Name | Description | | ---- | ---- | -| [onRequest](#fiphoboserver-PutRequestHandler-onRequest) | first function to be called when a new request comes in - | -| [onBody](#fiphoboserver-PutRequestHandler-onBody) | function called on every body chunk belonging to this message - | -| [onEOM](#fiphoboserver-PutRequestHandler-onEOM) | function called when the incoming message is finished - | -| [onUpgrade](#fiphoboserver-PutRequestHandler-onUpgrade) | function called on upgrade - | -| [requestComplete](#fiphoboserver-PutRequestHandler-requestComplete) | function ... - | -| [onError](#fiphoboserver-PutRequestHandler-onError) | function called when an error occurred - | -| [PutRequestHandler](#fiphoboserver-PutRequestHandler-PutRequestHandler) | Constructor for stream class initialization. - | +| [onRequest](#fiphoboserver-PutRequestHandler-onRequest) | first function to be called when a new request comes in | +| [onBody](#fiphoboserver-PutRequestHandler-onBody) | function called on every body chunk belonging to this message | +| [onEOM](#fiphoboserver-PutRequestHandler-onEOM) | function called when the incoming message is finished | +| [onUpgrade](#fiphoboserver-PutRequestHandler-onUpgrade) | function called on upgrade | +| [requestComplete](#fiphoboserver-PutRequestHandler-requestComplete) | function ... | +| [onError](#fiphoboserver-PutRequestHandler-onError) | function called when an error occurred | +| [PutRequestHandler](#fiphoboserver-PutRequestHandler-PutRequestHandler) | Constructor for stream class initialization. | @@ -44,7 +37,7 @@ Inherits from RequestHandler. -### private std::unique_ptr +### private std::unique_ptr< stream::Stream > @@ -63,8 +56,7 @@ first function to be called when a new request comes in #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for - | +| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | @@ -86,8 +78,7 @@ function called on every body chunk belonging to this message #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | body | buffer for the body - | +| | std::unique_ptr< folly::IOBuf > | body | buffer for the body | @@ -181,8 +172,7 @@ Constructor for stream class initialization. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server - | +| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | @@ -195,5 +185,5 @@ Constructor for stream class initialization. -[fiphoboserver-S3_header]:./doc/build/Markdown/fiphoboserver/S3_header.md -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index a0d699f..1934b71 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -13,18 +13,12 @@ class to extract the S3 specific information from proxygens HTTPMessage headers ## Public Functions | Name | Description | | ---- | ---- | -| [setHeaders](#fiphoboserver-S3_header-setHeaders) | sets the internal header instance to the given header object - | -| [getBucket](#fiphoboserver-S3_header-getBucket) | extracts the name of the S3 bucket requested in the HTTP message - | -| [getKey](#fiphoboserver-S3_header-getKey) | extracts the name of the key id requested in the HTTP message - | -| [isCreateBucketRequest](#fiphoboserver-S3_header-isCreateBucketRequest) | checks if the message we got is one for just creating a bucket - | -| [getBodyLength](#fiphoboserver-S3_header-getBodyLength) | gets the total length of all body chunks that is expected - | -| [getMetaData](#fiphoboserver-S3_header-getMetaData) | gets the S3 metadata stored in the HTTP headers - | +| [setHeaders](#fiphoboserver-S3_header-setHeaders) | sets the internal header instance to the given header object | +| [getBucket](#fiphoboserver-S3_header-getBucket) | extracts the name of the S3 bucket requested in the HTTP message | +| [getKey](#fiphoboserver-S3_header-getKey) | extracts the name of the key id requested in the HTTP message | +| [isCreateBucketRequest](#fiphoboserver-S3_header-isCreateBucketRequest) | checks if the message we got is one for just creating a bucket | +| [getBodyLength](#fiphoboserver-S3_header-getBodyLength) | gets the total length of all body chunks that is expected | +| [getMetaData](#fiphoboserver-S3_header-getMetaData) | gets the S3 metadata stored in the HTTP headers | @@ -48,8 +42,7 @@ sets the internal header instance to the given header object #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance - | +| | std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance | @@ -71,8 +64,7 @@ extracts the name of the S3 bucket requested in the HTTP message #### Returns: | Type | Description | | ---- | ---- | -| std::string | name of the bucket - | +| std::string | name of the bucket | @@ -95,8 +87,7 @@ extracts the name of the key id requested in the HTTP message #### Returns: | Type | Description | | ---- | ---- | -| std::string | name of the key - | +| std::string | name of the key | @@ -119,8 +110,7 @@ checks if the message we got is one for just creating a bucket #### Returns: | Type | Description | | ---- | ---- | -| bool | true if these headers belong to a create bucket request, false for all other requests - | +| bool | true if these headers belong to a create bucket request, false for all other requests | @@ -144,8 +134,7 @@ gets the total length of all body chunks that is expected #### Returns: | Type | Description | | ---- | ---- | -| size_t | the length of the total body to be expected - | +| size_t | the length of the total body to be expected | @@ -169,8 +158,7 @@ gets the S3 metadata stored in the HTTP headers #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector consisting of all key value pairs found in the header - | +| std::vector< std::pair< std::string, std::string > > | a vector consisting of all key value pairs found in the header | diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 281011b..1db3d31 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -6,34 +6,26 @@ ## Classes | Name | Description | | ---- | ---- | -| [FIPhoboServerException](./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer - | -| [GetRequestHandler](./doc/build/Markdown/fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests - | -| [IOException](./doc/build/Markdown/fiphoboserver/IOException.md) | exceptions specifically for errors in I/O - | -| [PutRequestHandler](./doc/build/Markdown/fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests - | -| [S3_header](./doc/build/Markdown/fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers - | -| [UnsupportedRequestHandler](./doc/build/Markdown/fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support - | +| [FIPhoboServerException](doc/Markdown/fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | +| [GetRequestHandler](doc/Markdown/fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests | +| [IOException](doc/Markdown/fiphoboserver/IOException.md) | exceptions specifically for errors in I/O | +| [PutRequestHandler](doc/Markdown/fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests | +| [S3_header](doc/Markdown/fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | +| [UnsupportedRequestHandler](doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | ## Namespaces | Name | Description | | ---- | ---- | -| [storage](./doc/build/Markdown/fiphoboserver/storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] - | -| [stream](./doc/build/Markdown/fiphoboserver/stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] - | +| [storage](doc/Markdown/fiphoboserver/storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | +| [stream](doc/Markdown/fiphoboserver/stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | ## Classes ## Namespaces -[fiphoboserver-IOException]:./doc/build/Markdown/fiphoboserver/IOException.md -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-storage-PhobosException]:./doc/build/Markdown/fiphoboserver/storage/PhobosException.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index 65288ae..f577ee7 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -10,8 +10,7 @@ Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServe ## Public Functions | Name | Description | | ---- | ---- | -| [PhobosException](#fiphoboserver-storage-PhobosException-PhobosException) | constructor - | +| [PhobosException](#fiphoboserver-storage-PhobosException-PhobosException) | constructor | @@ -25,12 +24,9 @@ constructor #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | const char * | caller | the function that threw this exception - | -| | const char * | functionName | the phobos function that returned an error - | -| | int | returnValue | the return value of the phobos function - | +| | const char * | caller | the function that threw this exception | +| | const char * | functionName | the phobos function that returned an error | +| | int | returnValue | the return value of the phobos function | @@ -43,4 +39,4 @@ constructor -[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 41ba10c..01736e0 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -22,22 +22,14 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. ## Public Functions | Name | Description | | ---- | ---- | -| [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file) | default constructor - | -| [~Phobos_file](#fiphoboserver-storage-Phobos_file-~Phobos_file) | default destructor - | -| [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata an object that is added to the database should get - | -| [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs - | -| [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. - | -| [db_get](#fiphoboserver-storage-Phobos_file-db_get) | Gets data from the databse. - | -| [prepare_put](#fiphoboserver-storage-Phobos_file-prepare_put) | Starts a put operation to the database. - | -| [prepare_get](#fiphoboserver-storage-Phobos_file-prepare_get) | Starts a get operation to the database. - | +| [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file) | default constructor | +| [~Phobos_file](#fiphoboserver-storage-Phobos_file-~Phobos_file) | default destructor | +| [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata an object that is added to the database should get | +| [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | +| [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. | +| [db_get](#fiphoboserver-storage-Phobos_file-db_get) | Gets data from the databse. | +| [prepare_put](#fiphoboserver-storage-Phobos_file-prepare_put) | Starts a put operation to the database. | +| [prepare_get](#fiphoboserver-storage-Phobos_file-prepare_get) | Starts a get operation to the database. | ## Private Functions @@ -88,10 +80,8 @@ set the metadata an object that is added to the database should get #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data - | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata - | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -123,8 +113,7 @@ Puts data to the databse. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | size_t | size | the number of bytes to add to the file in the database - | +| | size_t | size | the number of bytes to add to the file in the database | #### Returns: | Type | Description | @@ -151,8 +140,7 @@ Gets data from the databse. #### Returns: | Type | Description | | ---- | ---- | -| ssize_t | the number of bytes read from the database - | +| ssize_t | the number of bytes read from the database | @@ -175,10 +163,8 @@ Starts a put operation to the database. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data that will be added to the database shouldbe read from - | -| | std::string | object_id | the internal storage id the data should get - | +| | std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| | std::string | object_id | the internal storage id the data should get | @@ -200,10 +186,8 @@ Starts a get operation to the database. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data from the database should be written to - | -| | std::string | object_id | the internal storage id from the data that should be retrieved - | +| | std::string | file_name | the filename the data from the database should be written to | +| | std::string | object_id | the internal storage id from the data that should be retrieved | @@ -240,5 +224,5 @@ Starts a get operation to the database. -[pho_xfer_desc]:./doc/build/Markdown/pho_xfer_desc.md#pho_xfer_desc -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[pho_xfer_desc]:doc/Markdown/pho_xfer_desc.md#pho_xfer_desc diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index eb46221..f3c680f 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -13,37 +13,29 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob ## Metadata functions | Name | Description | | ---- | ---- | -| [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata an object that is added to the database should get - | -| [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs - | +| [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata an object that is added to the database should get | +| [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | ## PUT functions | Name | Description | | ---- | ---- | -| [prepare_put](#fiphoboserver-storage-Storage-prepare_put) | Starts a put operation to the database. - | -| [db_put](#fiphoboserver-storage-Storage-db_put) | Puts data to the databse. - | +| [prepare_put](#fiphoboserver-storage-Storage-prepare_put) | Starts a put operation to the database. | +| [db_put](#fiphoboserver-storage-Storage-db_put) | Puts data to the databse. | ## GET functions | Name | Description | | ---- | ---- | -| [prepare_get](#fiphoboserver-storage-Storage-prepare_get) | Starts a get operation to the database. - | -| [db_get](#fiphoboserver-storage-Storage-db_get) | Gets data from the databse. - | +| [prepare_get](#fiphoboserver-storage-Storage-prepare_get) | Starts a get operation to the database. | +| [db_get](#fiphoboserver-storage-Storage-db_get) | Gets data from the databse. | ## Public Functions | Name | Description | | ---- | ---- | -| [Storage](#fiphoboserver-storage-Storage-Storage) | default constructor - | -| [~Storage](#fiphoboserver-storage-Storage-~Storage) | default destructor - | +| [Storage](#fiphoboserver-storage-Storage-Storage) | default constructor | +| [~Storage](#fiphoboserver-storage-Storage-~Storage) | default destructor | @@ -57,10 +49,8 @@ set the metadata an object that is added to the database should get #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data - | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata - | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -93,10 +83,8 @@ Starts a put operation to the database. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data that will be added to the database shouldbe read from - | -| | std::string | object_id | the internal storage id the data should get - | +| | std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| | std::string | object_id | the internal storage id the data should get | @@ -118,8 +106,7 @@ Puts data to the databse. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | size_t | size | the number of bytes to add to the file in the database - | +| | size_t | size | the number of bytes to add to the file in the database | #### Returns: | Type | Description | @@ -147,10 +134,8 @@ Starts a get operation to the database. #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data from the database should be written to - | -| | std::string | object_id | the internal storage id from the data that should be retrieved - | +| | std::string | file_name | the filename the data from the database should be written to | +| | std::string | object_id | the internal storage id from the data that should be retrieved | @@ -172,8 +157,7 @@ Gets data from the databse. #### Returns: | Type | Description | | ---- | ---- | -| ssize_t | the number of bytes read from the database - | +| ssize_t | the number of bytes read from the database | @@ -208,4 +192,4 @@ default destructor -[fiphoboserver-storage-Phobos_file]:./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md +[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index b8da99f..f2563e5 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -7,16 +7,13 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: ## Classes | Name | Description | | ---- | ---- | -| [Phobos_file](./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md) | default constructor - | -| [PhobosException](./doc/build/Markdown/fiphoboserver/storage/PhobosException.md) | exceptions specifically for the phobos backend library - | -| [Storage](./doc/build/Markdown/fiphoboserver/storage/Storage.md) | virtual storage class to be implemented by backend storage - | +| [Phobos_file](doc/Markdown/fiphoboserver/storage/Phobos_file.md) | default constructor | +| [PhobosException](doc/Markdown/fiphoboserver/storage/PhobosException.md) | exceptions specifically for the phobos backend library | +| [Storage](doc/Markdown/fiphoboserver/storage/Storage.md) | virtual storage class to be implemented by backend storage | ## Classes -[fiphoboserver-FIPhoboServerException]:./doc/build/Markdown/fiphoboserver/FIPhoboServerException.md -[fiphoboserver-storage-Phobos_file]:./doc/build/Markdown/fiphoboserver/storage/Phobos_file.md -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 870ddef..4e3048f 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -16,10 +16,8 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. ## Implementation specific details | Name | Description | | ---- | ---- | -| [get_fifo_descriptor](#fiphoboserver-stream-Fifo-get_fifo_descriptor) | get the descriptor of the fifo - | -| [get_fifo_name](#fiphoboserver-stream-Fifo-get_fifo_name) | get the name of the fifo - | +| [get_fifo_descriptor](#fiphoboserver-stream-Fifo-get_fifo_descriptor) | get the descriptor of the fifo | +| [get_fifo_name](#fiphoboserver-stream-Fifo-get_fifo_name) | get the name of the fifo | ## Private Attributes @@ -32,26 +30,16 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. ## Public Functions | Name | Description | | ---- | ---- | -| [Fifo](#fiphoboserver-stream-Fifo-Fifo) | default constructor - | -| [~Fifo](#fiphoboserver-stream-Fifo-~Fifo) | default destructor - | -| [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get - | -| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata from a stored object - | -| [start_put](#fiphoboserver-stream-Fifo-start_put) | start a put operation - | -| [put](#fiphoboserver-stream-Fifo-put) | add a chunk of data to the object - | -| [finish_put](#fiphoboserver-stream-Fifo-finish_put) | end a put operation - | -| [start_get](#fiphoboserver-stream-Fifo-start_get) | start a gett operation - | -| [get](#fiphoboserver-stream-Fifo-get) | gets a chunk of data of the object - | -| [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation - | +| [Fifo](#fiphoboserver-stream-Fifo-Fifo) | default constructor | +| [~Fifo](#fiphoboserver-stream-Fifo-~Fifo) | default destructor | +| [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get | +| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata from a stored object | +| [start_put](#fiphoboserver-stream-Fifo-start_put) | start a put operation | +| [put](#fiphoboserver-stream-Fifo-put) | add a chunk of data to the object | +| [finish_put](#fiphoboserver-stream-Fifo-finish_put) | end a put operation | +| [start_get](#fiphoboserver-stream-Fifo-start_get) | start a gett operation | +| [get](#fiphoboserver-stream-Fifo-get) | gets a chunk of data of the object | +| [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation | ## Private Functions @@ -71,8 +59,7 @@ get the descriptor of the fifo #### Returns: | Type | Description | | ---- | ---- | -| int | the descriptor of the fifo - | +| int | the descriptor of the fifo | @@ -98,8 +85,7 @@ get the name of the fifo #### Returns: | Type | Description | | ---- | ---- | -| std::string | the name of the fifo - | +| std::string | the name of the fifo | @@ -170,10 +156,8 @@ setting the metadata that the created object should get #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data - | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata - | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -209,10 +193,8 @@ start a put operation #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | ssize_t | size | amount of bytes that shall be added to the internal storage - | -| | std::string | object_id | the identifier the object should get - | +| | ssize_t | size | amount of bytes that shall be added to the internal storage | +| | std::string | object_id | the identifier the object should get | @@ -234,8 +216,7 @@ add a chunk of data to the object #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add - | +| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | @@ -267,8 +248,7 @@ start a gett operation #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | object_id | the identifier of the object to retrieve - | +| | std::string | object_id | the identifier of the object to retrieve | @@ -290,16 +270,13 @@ gets a chunk of data of the object #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | void * | buf | a buffer for the data to be filled in this function - | -| | size_t | count | the size of the buffer / chunk - | +| | void * | buf | a buffer for the data to be filled in this function | +| | size_t | count | the size of the buffer / chunk | #### Returns: | Type | Description | | ---- | ---- | -| ssize_t | the number of bytes the buffer has been filled with - | +| ssize_t | the number of bytes the buffer has been filled with | @@ -334,5 +311,5 @@ end a get operation -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 6ed6e12..967e0f5 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -10,42 +10,29 @@ Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. ## PUT functions | Name | Description | | ---- | ---- | -| [start_put](#fiphoboserver-stream-Stream-start_put) | start a put operation - | -| [put](#fiphoboserver-stream-Stream-put) | add a chunk of data to the object - | -| [finish_put](#fiphoboserver-stream-Stream-finish_put) | end a put operation - | -| [start_get](#fiphoboserver-stream-Stream-start_get) | start a gett operation - | -| [get](#fiphoboserver-stream-Stream-get) | gets a chunk of data of the object - | -| [finish_get](#fiphoboserver-stream-Stream-finish_get) | end a get operation - | +| [start_put](#fiphoboserver-stream-Stream-start_put) | start a put operation | +| [put](#fiphoboserver-stream-Stream-put) | add a chunk of data to the object | +| [finish_put](#fiphoboserver-stream-Stream-finish_put) | end a put operation | +| [start_get](#fiphoboserver-stream-Stream-start_get) | start a gett operation | +| [get](#fiphoboserver-stream-Stream-get) | gets a chunk of data of the object | +| [finish_get](#fiphoboserver-stream-Stream-finish_get) | end a get operation | ## Protected Attributes | Name | Description | | ---- | ---- | -| [Storage>](#fiphoboserver-stream-Stream-storage) | an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects - | -| [db_result](#fiphoboserver-stream-Stream-db_result) | a future object that can be used to get the results from the asynchronous backend - | +| [Storage >](#fiphoboserver-stream-Stream-storage) | an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects | +| [db_result](#fiphoboserver-stream-Stream-db_result) | a future object that can be used to get the results from the asynchronous backend | ## Public Functions | Name | Description | | ---- | ---- | -| [Stream](#fiphoboserver-stream-Stream-Stream) | default constructor - | -| [Stream](#fiphoboserver-stream-Stream-Stream-1) | constructor with storage implementation - | -| [~Stream](#fiphoboserver-stream-Stream-~Stream) | default destructor - | -| [set_storage_meta_data](#fiphoboserver-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get - | -| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata from a stored object - | +| [Stream](#fiphoboserver-stream-Stream-Stream) | default constructor | +| [Stream](#fiphoboserver-stream-Stream-Stream-1) | constructor with storage implementation | +| [~Stream](#fiphoboserver-stream-Stream-~Stream) | default destructor | +| [set_storage_meta_data](#fiphoboserver-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get | +| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata from a stored object | @@ -59,10 +46,8 @@ start a put operation #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | ssize_t | size | amount of bytes that shall be added to the internal storage - | -| | std::string | object_id | the identifier the object should get - | +| | ssize_t | size | amount of bytes that shall be added to the internal storage | +| | std::string | object_id | the identifier the object should get | @@ -84,8 +69,7 @@ add a chunk of data to the object #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add - | +| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | @@ -117,8 +101,7 @@ start a gett operation #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::string | object_id | the identifier of the object to retrieve - | +| | std::string | object_id | the identifier of the object to retrieve | @@ -140,16 +123,13 @@ gets a chunk of data of the object #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | void * | buf | a buffer for the data to be filled in this function - | -| | size_t | count | the size of the buffer / chunk - | +| | void * | buf | a buffer for the data to be filled in this function | +| | size_t | count | the size of the buffer / chunk | #### Returns: | Type | Description | | ---- | ---- | -| ssize_t | the number of bytes the buffer has been filled with - | +| ssize_t | the number of bytes the buffer has been filled with | @@ -175,7 +155,7 @@ end a get operation ## Protected Attributes -### protected std::unique_ptr +### protected std::unique_ptr< storage::Storage > an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects @@ -224,8 +204,7 @@ constructor with storage implementation #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | pointer to a implementation of the [storage::Storage][fiphoboserver-storage-Storage] class that will be used to store the data - | +| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | pointer to a implementation of the [storage::Storage][fiphoboserver-storage-Storage] class that will be used to store the data | @@ -261,10 +240,8 @@ setting the metadata that the created object should get #### Parameters: | Direction | Type | Name | Description | | ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data - | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata - | +| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -291,6 +268,6 @@ get the metadata from a stored object -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-stream-Fifo]:./doc/build/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 4cf6054..b738daa 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -7,14 +7,12 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: ## Classes | Name | Description | | ---- | ---- | -| [Fifo](./doc/build/Markdown/fiphoboserver/stream/Fifo.md) | default constructor - | -| [Stream](./doc/build/Markdown/fiphoboserver/stream/Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class - | +| [Fifo](doc/Markdown/fiphoboserver/stream/Fifo.md) | default constructor | +| [Stream](doc/Markdown/fiphoboserver/stream/Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class | ## Classes -[fiphoboserver-stream-Stream]:./doc/build/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-storage-Storage]:./doc/build/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-stream-Fifo]:./doc/build/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md diff --git a/doc/Markdown/pho_xfer_desc.md b/doc/Markdown/pho_xfer_desc.md index 83a103d..d9dc002 100644 --- a/doc/Markdown/pho_xfer_desc.md +++ b/doc/Markdown/pho_xfer_desc.md @@ -163,4 +163,4 @@ Outcome of this xfer -[tags]:./doc/build/Markdown/tags.md#tags +[tags]:doc/Markdown/tags.md#tags -- GitLab From 9cd6f6d83138489e381a30c20430a719072dbd38 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 20 Feb 2020 14:11:38 +0000 Subject: [PATCH 14/62] Clean up documentation and unused forward declarations in code --- doc/Doxyfile.in | 3 + .../HandlerFactory.md | 55 ------ .../index.md | 13 -- .../fiphoboserver/FIPhoboServerException.md | 2 +- .../fiphoboserver/GetRequestHandler.md | 2 +- .../fiphoboserver/PutRequestHandler.md | 2 +- doc/Markdown/fiphoboserver/index.md | 2 +- .../fiphoboserver/storage/Phobos_file.md | 1 - doc/Markdown/fiphoboserver/stream/Fifo.md | 2 +- doc/Markdown/fiphoboserver/stream/index.md | 2 +- doc/Markdown/pho_xfer_desc.md | 166 ------------------ doc/Markdown/proxygen/index.md | 6 - doc/Markdown/tags.md | 47 ----- src/server/get_request_handler.h | 4 - src/server/put_request_handler.h | 4 - src/server/unsupported_request_handler.h | 5 - 16 files changed, 9 insertions(+), 307 deletions(-) delete mode 100644 doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md delete mode 100644 doc/Markdown/anonymous_namespace{server.main.cc}/index.md delete mode 100644 doc/Markdown/pho_xfer_desc.md delete mode 100644 doc/Markdown/proxygen/index.md delete mode 100644 doc/Markdown/tags.md diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 30282a7..044c0d7 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -29,6 +29,9 @@ WARN_AS_ERROR = YES # Input directories INPUT = @CMAKE_SOURCE_DIR@/src +EXCLUDE = @CMAKE_SOURCE_DIR@/src/server/server.main.cc +EXCLUDE_PATTERNS = */phobos_cpp_wrapper/* + RECURSIVE = YES # HTML output options diff --git a/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md b/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md deleted file mode 100644 index d36e730..0000000 --- a/doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md +++ /dev/null @@ -1,55 +0,0 @@ -# public anonymous_namespace{server.main.cc}::HandlerFactory - - - - -## Inheritance: -Inherits from RequestHandlerFactory. - -## Public Functions -| Name | Description | -| ---- | ---- | -| [onServerStart](#anonymous_namespace{server.main.cc}-HandlerFactory-onServerStart) | | -| [onServerStop](#anonymous_namespace{server.main.cc}-HandlerFactory-onServerStop) | | -| [onRequest](#anonymous_namespace{server.main.cc}-HandlerFactory-onRequest) | | - - - -## Public Functions -### public void anonymous_namespace{server.main.cc}::HandlerFactory::onServerStart (folly::EventBase *) noexcept override - - - - - - - - -### public void anonymous_namespace{server.main.cc}::HandlerFactory::onServerStop () noexcept override - - - - - - - - -### public proxygen::RequestHandler * anonymous_namespace{server.main.cc}::HandlerFactory::onRequest (proxygen::RequestHandler *, proxygen::HTTPMessage *headers) noexcept override - - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::HTTPMessage * | headers | | - -#### Returns: -| Type | Description | -| ---- | ---- | -| proxygen::RequestHandler * | | - - - - - diff --git a/doc/Markdown/anonymous_namespace{server.main.cc}/index.md b/doc/Markdown/anonymous_namespace{server.main.cc}/index.md deleted file mode 100644 index d27f463..0000000 --- a/doc/Markdown/anonymous_namespace{server.main.cc}/index.md +++ /dev/null @@ -1,13 +0,0 @@ -# anonymous_namespace{server.main.cc} - - - - -## Classes -| Name | Description | -| ---- | ---- | -| [HandlerFactory](doc/Markdown/anonymous_namespace{server.main.cc}/HandlerFactory.md) | | - - - -## Classes diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index f9a002e..b162b1b 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -89,5 +89,5 @@ get information on this exception -[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md [fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md +[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index efca0ff..03e2c7c 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -295,5 +295,5 @@ Constructor for stream class initialization. -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 5bb6c6f..9a89a63 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -185,5 +185,5 @@ Constructor for stream class initialization. -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 1db3d31..b05abb2 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -27,5 +27,5 @@ [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md [fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md [fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 01736e0..6af6d5b 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -225,4 +225,3 @@ Starts a get operation to the database. [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md -[pho_xfer_desc]:doc/Markdown/pho_xfer_desc.md#pho_xfer_desc diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 4e3048f..c0c71cd 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -311,5 +311,5 @@ end a get operation -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index b738daa..9978a91 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -13,6 +13,6 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: ## Classes -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md diff --git a/doc/Markdown/pho_xfer_desc.md b/doc/Markdown/pho_xfer_desc.md deleted file mode 100644 index d9dc002..0000000 --- a/doc/Markdown/pho_xfer_desc.md +++ /dev/null @@ -1,166 +0,0 @@ -# public pho_xfer_desc - - - -GET / PUT parameter. The source/destination semantics of the fields vary depending on the nature of the operation. See below:phobos_get() - -phobos_put() - - - - - - - -## Public Attributes -| Name | Description | -| ---- | ---- | -| [xd_objid](#pho_xfer_desc-xd_objid) | | -| [xd_op](#pho_xfer_desc-xd_op) | | -| [xd_fd](#pho_xfer_desc-xd_fd) | | -| [xd_size](#pho_xfer_desc-xd_size) | | -| [xd_layout_name](#pho_xfer_desc-xd_layout_name) | | -| [xd_attrs](#pho_xfer_desc-xd_attrs) | | -| [xd_flags](#pho_xfer_desc-xd_flags) | | -| [xd_tags](#pho_xfer_desc-xd_tags) | | -| [xd_rc](#pho_xfer_desc-xd_rc) | | - - - -## Public Attributes -### public pho_xfer_desc::xd_objid - - - - - - - - - - -Object id to read or write - - - -### public pho_xfer_desc::xd_op - - - - - - - - - - -Operation to perform (GET, GETMD or PUT) - - - -### public pho_xfer_desc::xd_fd - - - - - - - - - - -positive fd if xd_id_open - - - -### public pho_xfer_desc::xd_size - - - - - - - - - - -Amount of data to write (for the GET operation, the size read is equal to the size of the retrieved object) - - - -### public pho_xfer_desc::xd_layout_name - - - - - - - - - - -Name of the layout module to use (for put). - - - -### public pho_xfer_desc::xd_attrs - - - - - - - - - - -User defined attribute to get / put - - - -### public pho_xfer_desc::xd_flags - - - - - - - - - - -See enum pho_xfer_flags doc - - - -### public pho_xfer_desc::xd_tags - - - - - - - - - - -Tags to select a media to write - - - -### public pho_xfer_desc::xd_rc - - - - - - - - - - -Outcome of this xfer - - - -[tags]:doc/Markdown/tags.md#tags diff --git a/doc/Markdown/proxygen/index.md b/doc/Markdown/proxygen/index.md deleted file mode 100644 index 46e621a..0000000 --- a/doc/Markdown/proxygen/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# proxygen - - - - - diff --git a/doc/Markdown/tags.md b/doc/Markdown/tags.md deleted file mode 100644 index 4ee7e44..0000000 --- a/doc/Markdown/tags.md +++ /dev/null @@ -1,47 +0,0 @@ -# public tags - - - -A simple array of tags (strings) - - - -## Public Attributes -| Name | Description | -| ---- | ---- | -| [tags](#tags-tags) | | -| [n_tags](#tags-n_tags) | | - - - -## Public Attributes -### public tags::tags - - - - - - - - - - -The array of tags - - - -### public tags::n_tags - - - - - - - - - - -Number of tags - - - diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index c2f2121..2043cb4 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -15,10 +15,6 @@ #include "S3_header.h" #include "../stream/stream.h" -namespace proxygen { -class ResponseHandler; -} - namespace fiphoboserver { /// diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 1366f52..e86c2a5 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -14,10 +14,6 @@ #include "S3_header.h" #include "../stream/stream.h" -namespace proxygen { -class ResponseHandler; -} - namespace fiphoboserver { /// diff --git a/src/server/unsupported_request_handler.h b/src/server/unsupported_request_handler.h index 5bc15a4..ae3efc3 100644 --- a/src/server/unsupported_request_handler.h +++ b/src/server/unsupported_request_handler.h @@ -10,11 +10,6 @@ #include - -namespace proxygen { -class ResponseHandler; -} - namespace fiphoboserver { /// -- GitLab From 74549931ceb652078636062193b9bc9da44ca864 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 20 Feb 2020 16:32:45 +0000 Subject: [PATCH 15/62] Update documentation --- .../fiphoboserver/FIPhoboServerException.md | 32 +++- .../fiphoboserver/GetRequestHandler.md | 180 +++++++----------- doc/Markdown/fiphoboserver/IOException.md | 17 +- .../fiphoboserver/PutRequestHandler.md | 100 ++++++---- doc/Markdown/fiphoboserver/S3_header.md | 64 +++++-- .../UnsupportedRequestHandler.md | 106 ----------- doc/Markdown/fiphoboserver/index.md | 8 +- .../fiphoboserver/storage/PhobosException.md | 17 +- .../fiphoboserver/storage/Phobos_file.md | 118 ++++++------ doc/Markdown/fiphoboserver/storage/Storage.md | 81 ++++++-- doc/Markdown/fiphoboserver/storage/index.md | 5 +- doc/Markdown/fiphoboserver/stream/Fifo.md | 152 +++++++++------ doc/Markdown/fiphoboserver/stream/Stream.md | 119 +++++++++--- doc/Markdown/fiphoboserver/stream/index.md | 3 +- 14 files changed, 545 insertions(+), 457 deletions(-) diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index b162b1b..31e9f2d 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -33,6 +33,8 @@ the internal message +[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) + ## Public Functions ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException () @@ -44,6 +46,13 @@ default constructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) + ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException (const char *message) constructor for a message with information on the exception @@ -51,9 +60,11 @@ constructor for a message with information on the exception #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | const char * | message | message explaining what went wrong such that this exception was thrown. | +| Type | Name | Description | +| ---- | ---- | ---- | +| const char * | message | message explaining what went wrong such that this exception was thrown. | + + @@ -64,8 +75,13 @@ constructor for a message with information on the exception +#### Qualifiers: +* inline +* virtual +[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) + ### public const char * fiphoboserver::FIPhoboServerException::what () const noexcept override get information on this exception @@ -89,5 +105,13 @@ get information on this exception -[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md +#### Qualifiers: +* const +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) + [fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md +[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index 03e2c7c..c5a1d18 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -7,17 +7,6 @@ proxygen class implementation for handling GET requests ## Inheritance: Inherits from RequestHandler. -## Private Attributes -| Name | Description | -| ---- | ---- | -| [Stream >](#fiphoboserver-GetRequestHandler-stream) | | -| [s3_header](#fiphoboserver-GetRequestHandler-s3_header) | | -| [readFileScheduled_](#fiphoboserver-GetRequestHandler-readFileScheduled_) | | -| [paused_](#fiphoboserver-GetRequestHandler-paused_) | | -| [finished_](#fiphoboserver-GetRequestHandler-finished_) | | -| [file_closed_](#fiphoboserver-GetRequestHandler-file_closed_) | | - - ## Public Functions | Name | Description | | ---- | ---- | @@ -32,25 +21,23 @@ Inherits from RequestHandler. | [GetRequestHandler](#fiphoboserver-GetRequestHandler-GetRequestHandler) | Constructor for stream class initialization. | -## Private Functions -| Name | Description | -| ---- | ---- | -| [readFile](#fiphoboserver-GetRequestHandler-readFile) | | -| [checkForCompletion](#fiphoboserver-GetRequestHandler-checkForCompletion) | | - +## Public Functions +### public void fiphoboserver::GetRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override -## Private Attributes -### private std::unique_ptr< stream::Stream > +first function to be called when a new request comes in +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | -### private fiphoboserver::GetRequestHandler::s3_header @@ -58,58 +45,67 @@ Inherits from RequestHandler. +#### Qualifiers: +* virtual -### private fiphoboserver::GetRequestHandler::readFileScheduled_ +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override +function called on every body chunk belonging to this message +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< folly::IOBuf > | body | buffer for the body | -### private fiphoboserver::GetRequestHandler::paused_ +This is not used in this case because GET requests don't usually have a body -### private fiphoboserver::GetRequestHandler::finished_ +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override +function called when the incoming message is finished -### private fiphoboserver::GetRequestHandler::file_closed_ +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) -## Public Functions -### public void fiphoboserver::GetRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override +### public void fiphoboserver::GetRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override -first function to be called when a new request comes in +function called on upgrade #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | - - - - +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::UpgradeProtocol | proto | | @@ -117,50 +113,51 @@ first function to be called when a new request comes in +Not supported in our case! -### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override -function called on every body chunk belonging to this message +#### Qualifiers: +* virtual -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | body | buffer for the body | +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +### public void fiphoboserver::GetRequestHandler::requestComplete () noexcept override +function ... -This is not used in this case because GET requests don't usually have a body +> **Todo:** find out what this does? -### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override -function called when the incoming message is finished +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) -### public void fiphoboserver::GetRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override +### public void fiphoboserver::GetRequestHandler::onError (proxygen::ProxygenError err) noexcept override -function called on upgrade +function called when an error occurred #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::UpgradeProtocol | proto | | +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::ProxygenError | err | | @@ -172,49 +169,37 @@ Not supported in our case! -### public void fiphoboserver::GetRequestHandler::requestComplete () noexcept override - -function ... - - - - - - - - - -> **Todo:** find out what this does? - +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +### public void fiphoboserver::GetRequestHandler::onEgressPaused () noexcept override +called when the queue is full. -### public void fiphoboserver::GetRequestHandler::onError (proxygen::ProxygenError err) noexcept override -function called when an error occurred -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::ProxygenError | err | | +Contents are copies from a proxygen example -Not supported in our case! +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) -### public void fiphoboserver::GetRequestHandler::onEgressPaused () noexcept override +### public void fiphoboserver::GetRequestHandler::onEgressResumed () noexcept override -called when the queue is full. +called when the queue is not longer full. @@ -228,21 +213,11 @@ Contents are copies from a proxygen example -### public void fiphoboserver::GetRequestHandler::onEgressResumed () noexcept override - -called when the queue is not longer full. - - - - - - - - - -Contents are copies from a proxygen example +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) ### public fiphoboserver::GetRequestHandler::GetRequestHandler (std::unique_ptr< stream::Stream > input_stream) @@ -251,49 +226,26 @@ Constructor for stream class initialization. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | - - - - - - +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | -## Private Functions -### private void fiphoboserver::GetRequestHandler::readFile (folly::EventBase *evb) -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | folly::EventBase * | evb | | - - - - - -### private bool fiphoboserver::GetRequestHandler::checkForCompletion () - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| bool | | +#### Qualifiers: +* inline +* virtual +[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) -[fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index 371e426..4fb2f44 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -22,11 +22,11 @@ constructor #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | const char * | caller | the function that threw this exception | -| | const char * | functionName | the io function that returned an error | -| | int | returnValue | the return value of the io function | +| Type | Name | Description | +| ---- | ---- | ---- | +| const char * | caller | the function that threw this exception | +| const char * | functionName | the io function that returned an error | +| int | returnValue | the return value of the io function | @@ -39,4 +39,11 @@ constructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/IOException.md) + [fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 9a89a63..fe5f57f 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -7,13 +7,6 @@ proxygen class implementation for handling PUT requests ## Inheritance: Inherits from RequestHandler. -## Private Attributes -| Name | Description | -| ---- | ---- | -| [s3_header](#fiphoboserver-PutRequestHandler-s3_header) | | -| [Stream >](#fiphoboserver-PutRequestHandler-stream) | | - - ## Public Functions | Name | Description | | ---- | ---- | @@ -27,25 +20,6 @@ Inherits from RequestHandler. -## Private Attributes -### private fiphoboserver::PutRequestHandler::s3_header - - - - - - - - -### private std::unique_ptr< stream::Stream > - - - - - - - - ## Public Functions ### public void fiphoboserver::PutRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override @@ -54,9 +28,10 @@ first function to be called when a new request comes in #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | + @@ -68,6 +43,11 @@ first function to be called when a new request comes in +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -76,9 +56,9 @@ function called on every body chunk belonging to this message #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | body | buffer for the body | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< folly::IOBuf > | body | buffer for the body | @@ -92,6 +72,12 @@ This is not used in this case because GET requests don't usually have a body +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) + ### public void fiphoboserver::PutRequestHandler::onEOM () noexcept override function called when the incoming message is finished @@ -102,6 +88,12 @@ function called when the incoming message is finished +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) + ### public void fiphoboserver::PutRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override function called on upgrade @@ -109,9 +101,9 @@ function called on upgrade #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::UpgradeProtocol | proto | | +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::UpgradeProtocol | proto | | @@ -123,6 +115,12 @@ Not supported in our case! +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) + ### public void fiphoboserver::PutRequestHandler::requestComplete () noexcept override function ... @@ -142,6 +140,12 @@ function ... +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) + ### public void fiphoboserver::PutRequestHandler::onError (proxygen::ProxygenError err) noexcept override function called when an error occurred @@ -149,9 +153,9 @@ function called when an error occurred #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::ProxygenError | err | | +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::ProxygenError | err | | @@ -163,6 +167,12 @@ Not supported in our case! +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) + ### public fiphoboserver::PutRequestHandler::PutRequestHandler (std::unique_ptr< stream::Stream > input_stream) Constructor for stream class initialization. @@ -170,9 +180,12 @@ Constructor for stream class initialization. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | + + + @@ -182,8 +195,11 @@ Constructor for stream class initialization. +#### Qualifiers: +* inline +* virtual +[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) -[fiphoboserver-S3_header]:doc/Markdown/fiphoboserver/S3_header.md [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index 1934b71..84e1da1 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -4,12 +4,6 @@ class to extract the S3 specific information from proxygens HTTPMessage headers -## Private Attributes -| Name | Description | -| ---- | ---- | -| [headers](#fiphoboserver-S3_header-headers) | | - - ## Public Functions | Name | Description | | ---- | ---- | @@ -22,16 +16,6 @@ class to extract the S3 specific information from proxygens HTTPMessage headers -## Private Attributes -### private fiphoboserver::S3_header::headers - - - - - - - - ## Public Functions ### public void fiphoboserver::S3_header::setHeaders (std::unique_ptr< proxygen::HTTPMessage > newHeaders) @@ -40,9 +24,12 @@ sets the internal header instance to the given header object #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance | + + + @@ -52,8 +39,12 @@ sets the internal header instance to the given header object +#### Qualifiers: +* inline +* virtual +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) ### public std::string fiphoboserver::S3_header::getBucket () @@ -78,6 +69,13 @@ extracts the name of the S3 bucket requested in the HTTP message +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) + ### public std::string fiphoboserver::S3_header::getKey () extracts the name of the key id requested in the HTTP message @@ -101,6 +99,13 @@ extracts the name of the key id requested in the HTTP message +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) + ### public bool fiphoboserver::S3_header::isCreateBucketRequest () checks if the message we got is one for just creating a bucket @@ -125,6 +130,13 @@ This is convenient since we don't really support buckets themselves and can easi +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) + ### public size_t fiphoboserver::S3_header::getBodyLength () gets the total length of all body chunks that is expected @@ -149,6 +161,13 @@ This reads out the header "Content-Length" +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) + ### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::getMetaData () gets the S3 metadata stored in the HTTP headers @@ -173,3 +192,10 @@ In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) + diff --git a/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md index 0c63dcc..c54d3ab 100644 --- a/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md +++ b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md @@ -7,110 +7,4 @@ proxygen class implementation for handling all requests that we don't support ## Inheritance: Inherits from RequestHandler. -## Public Functions -| Name | Description | -| ---- | ---- | -| [onRequest](#fiphoboserver-UnsupportedRequestHandler-onRequest) | | -| [onBody](#fiphoboserver-UnsupportedRequestHandler-onBody) | | -| [onEOM](#fiphoboserver-UnsupportedRequestHandler-onEOM) | | -| [onUpgrade](#fiphoboserver-UnsupportedRequestHandler-onUpgrade) | | -| [requestComplete](#fiphoboserver-UnsupportedRequestHandler-requestComplete) | | -| [onError](#fiphoboserver-UnsupportedRequestHandler-onError) | | -| [onEgressPaused](#fiphoboserver-UnsupportedRequestHandler-onEgressPaused) | | -| [onEgressResumed](#fiphoboserver-UnsupportedRequestHandler-onEgressResumed) | | - - - -## Public Functions -### public void fiphoboserver::UnsupportedRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override - - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< proxygen::HTTPMessage > | headers | | - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override - - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | body | | - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onEOM () noexcept override - - - - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override - - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::UpgradeProtocol | proto | | - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::requestComplete () noexcept override - - - - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onError (proxygen::ProxygenError err) noexcept override - - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | proxygen::ProxygenError | err | | - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onEgressPaused () noexcept override - - - - - - - - -### public void fiphoboserver::UnsupportedRequestHandler::onEgressResumed () noexcept override - - - - - - - diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index b05abb2..51753d2 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -22,10 +22,8 @@ -## Classes -## Namespaces +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md [fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index f577ee7..2e08261 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -22,11 +22,11 @@ constructor #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | const char * | caller | the function that threw this exception | -| | const char * | functionName | the phobos function that returned an error | -| | int | returnValue | the return value of the phobos function | +| Type | Name | Description | +| ---- | ---- | ---- | +| const char * | caller | the function that threw this exception | +| const char * | functionName | the phobos function that returned an error | +| int | returnValue | the return value of the phobos function | @@ -39,4 +39,11 @@ constructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/PhobosException.md) + [fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 6af6d5b..809e5e4 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -13,12 +13,6 @@ default constructor ## Inheritance: Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. -## Private Attributes -| Name | Description | -| ---- | ---- | -| [descriptor](#fiphoboserver-storage-Phobos_file-descriptor) | | - - ## Public Functions | Name | Description | | ---- | ---- | @@ -32,16 +26,11 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. | [prepare_get](#fiphoboserver-storage-Phobos_file-prepare_get) | Starts a get operation to the database. | -## Private Functions -| Name | Description | -| ---- | ---- | -| [set_bucket_name](#fiphoboserver-storage-Phobos_file-set_bucket_name) | | -| [close_file](#fiphoboserver-storage-Phobos_file-close_file) | | - +## Public Functions +### public fiphoboserver::storage::Phobos_file::Phobos_file () -## Private Attributes -### private fiphoboserver::storage::Phobos_file::descriptor +default constructor @@ -49,27 +38,27 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. - -## Public Functions -### public fiphoboserver::storage::Phobos_file::Phobos_file () - -default constructor - +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +### public fiphoboserver::storage::Phobos_file::~Phobos_file () +default destructor -### public fiphoboserver::storage::Phobos_file::~Phobos_file () -default destructor +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) ### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override @@ -78,10 +67,12 @@ set the metadata an object that is added to the database should get #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | + + @@ -92,8 +83,12 @@ set the metadata an object that is added to the database should get +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) + ### public void fiphoboserver::storage::Phobos_file::get_meta_data () const override get the metadata associated to the current object as a vector of key:value pairs @@ -104,6 +99,13 @@ get the metadata associated to the current object as a vector of key:value pairs +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) + ### public ssize_t fiphoboserver::storage::Phobos_file::db_put (size_t size) override Puts data to the databse. @@ -111,9 +113,9 @@ Puts data to the databse. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | size_t | size | the number of bytes to add to the file in the database | +| Type | Name | Description | +| ---- | ---- | ---- | +| size_t | size | the number of bytes to add to the file in the database | #### Returns: | Type | Description | @@ -131,6 +133,12 @@ Puts data to the databse. +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) + ### public ssize_t fiphoboserver::storage::Phobos_file::db_get () override Gets data from the databse. @@ -154,40 +162,28 @@ Gets data from the databse. -### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override - -Starts a put operation to the database. - - - -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data that will be added to the database shouldbe read from | -| | std::string | object_id | the internal storage id the data should get | - - - - +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override +Starts a put operation to the database. +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| std::string | object_id | the internal storage id the data should get | -### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override -Starts a get operation to the database. -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data from the database should be written to | -| | std::string | object_id | the internal storage id from the data that should be retrieved | @@ -195,33 +191,39 @@ Starts a get operation to the database. +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override +Starts a get operation to the database. -## Private Functions -### private void fiphoboserver::storage::Phobos_file::set_bucket_name (std::string bucket_name) +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | file_name | the filename the data from the database should be written to | +| std::string | object_id | the internal storage id from the data that should be retrieved | -#### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | bucket_name | | -### private void fiphoboserver::storage::Phobos_file::close_file () +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index f3c680f..9159843 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -47,10 +47,10 @@ set the metadata an object that is added to the database should get #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -63,6 +63,12 @@ set the metadata an object that is added to the database should get +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ### public void fiphoboserver::storage::Storage::get_meta_data () const =0 get the metadata associated to the current object as a vector of key:value pairs @@ -73,6 +79,13 @@ get the metadata associated to the current object as a vector of key:value pairs +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ## PUT functions ### public void fiphoboserver::storage::Storage::prepare_put (std::string file_name, std::string object_id)=0 @@ -81,10 +94,10 @@ Starts a put operation to the database. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data that will be added to the database shouldbe read from | -| | std::string | object_id | the internal storage id the data should get | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| std::string | object_id | the internal storage id the data should get | @@ -97,6 +110,12 @@ Starts a put operation to the database. +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ### public ssize_t fiphoboserver::storage::Storage::db_put (size_t size)=0 Puts data to the databse. @@ -104,9 +123,9 @@ Puts data to the databse. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | size_t | size | the number of bytes to add to the file in the database | +| Type | Name | Description | +| ---- | ---- | ---- | +| size_t | size | the number of bytes to add to the file in the database | #### Returns: | Type | Description | @@ -124,6 +143,12 @@ Puts data to the databse. +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ## GET functions ### public void fiphoboserver::storage::Storage::prepare_get (std::string file_name, std::string object_id)=0 @@ -132,10 +157,10 @@ Starts a get operation to the database. #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | file_name | the filename the data from the database should be written to | -| | std::string | object_id | the internal storage id from the data that should be retrieved | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | file_name | the filename the data from the database should be written to | +| std::string | object_id | the internal storage id from the data that should be retrieved | @@ -148,6 +173,12 @@ Starts a get operation to the database. +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ### public ssize_t fiphoboserver::storage::Storage::db_get ()=0 Gets data from the databse. @@ -171,6 +202,12 @@ Gets data from the databse. +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ## Public Functions ### public fiphoboserver::storage::Storage::Storage () @@ -182,6 +219,13 @@ default constructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + ### public fiphoboserver::storage::Storage::~Storage () default destructor @@ -192,4 +236,11 @@ default destructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) + [fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index f2563e5..245aea3 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -13,7 +13,6 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: -## Classes -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md [fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index c0c71cd..f75c796 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -20,13 +20,6 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | [get_fifo_name](#fiphoboserver-stream-Fifo-get_fifo_name) | get the name of the fifo | -## Private Attributes -| Name | Description | -| ---- | ---- | -| [fifo_name](#fiphoboserver-stream-Fifo-fifo_name) | | -| [fifo_descriptor](#fiphoboserver-stream-Fifo-fifo_descriptor) | | - - ## Public Functions | Name | Description | | ---- | ---- | @@ -42,12 +35,6 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation | -## Private Functions -| Name | Description | -| ---- | ---- | -| [create_fifo](#fiphoboserver-stream-Fifo-create_fifo) | | - - ## Implementation specific details ### public int fiphoboserver::stream::Fifo::get_fifo_descriptor () const @@ -76,6 +63,14 @@ get the descriptor of the fifo +#### Qualifiers: +* const +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) + ### public std::string fiphoboserver::stream::Fifo::get_fifo_name () const get the name of the fifo @@ -102,24 +97,13 @@ get the name of the fifo -## Private Attributes -### private fiphoboserver::stream::Fifo::fifo_name - - - - - - - - -### private fiphoboserver::stream::Fifo::fifo_descriptor - - - - - +#### Qualifiers: +* const +* inline +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) ## Public Functions ### public fiphoboserver::stream::Fifo::Fifo (std::unique_ptr< storage::Storage > input_storage) @@ -129,13 +113,19 @@ default constructor #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | | + +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) ### public fiphoboserver::stream::Fifo::~Fifo () @@ -147,6 +137,12 @@ default destructor +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) + ### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override setting the metadata that the created object should get @@ -154,10 +150,10 @@ setting the metadata that the created object should get #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -174,6 +170,13 @@ setting the metadata that the created object should get +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) + ### public void fiphoboserver::stream::Fifo::get_meta_data () const override get the metadata from a stored object @@ -184,6 +187,13 @@ get the metadata from a stored object +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) + ### public void fiphoboserver::stream::Fifo::start_put (ssize_t size, std::string object_id) override start a put operation @@ -191,10 +201,13 @@ start a put operation #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | ssize_t | size | amount of bytes that shall be added to the internal storage | -| | std::string | object_id | the identifier the object should get | +| Type | Name | Description | +| ---- | ---- | ---- | +| ssize_t | size | amount of bytes that shall be added to the internal storage | +| std::string | object_id | the identifier the object should get | + + + @@ -204,8 +217,11 @@ start a put operation +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) ### public void fiphoboserver::stream::Fifo::put (std::unique_ptr< folly::IOBuf > buffer) const override @@ -214,9 +230,12 @@ add a chunk of data to the object #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | + + + @@ -226,8 +245,12 @@ add a chunk of data to the object +#### Qualifiers: +* const +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) ### public void fiphoboserver::stream::Fifo::finish_put () override @@ -239,6 +262,12 @@ end a put operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) + ### public void fiphoboserver::stream::Fifo::start_get (std::string object_id) override start a gett operation @@ -246,9 +275,10 @@ start a gett operation #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | object_id | the identifier of the object to retrieve | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | the identifier of the object to retrieve | + @@ -260,6 +290,11 @@ start a gett operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) ### public ssize_t fiphoboserver::stream::Fifo::get (void *buf, size_t count) const override @@ -268,10 +303,10 @@ gets a chunk of data of the object #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | void * | buf | a buffer for the data to be filled in this function | -| | size_t | count | the size of the buffer / chunk | +| Type | Name | Description | +| ---- | ---- | ---- | +| void * | buf | a buffer for the data to be filled in this function | +| size_t | count | the size of the buffer / chunk | #### Returns: | Type | Description | @@ -291,25 +326,28 @@ gets a chunk of data of the object -### public void fiphoboserver::stream::Fifo::finish_get () override - -end a get operation - +#### Qualifiers: +* const +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +### public void fiphoboserver::stream::Fifo::finish_get () override +end a get operation -## Private Functions -### private void fiphoboserver::stream::Fifo::create_fifo () +#### Qualifiers: +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 967e0f5..37db827 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -44,10 +44,10 @@ start a put operation #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | ssize_t | size | amount of bytes that shall be added to the internal storage | -| | std::string | object_id | the identifier the object should get | +| Type | Name | Description | +| ---- | ---- | ---- | +| ssize_t | size | amount of bytes that shall be added to the internal storage | +| std::string | object_id | the identifier the object should get | @@ -60,6 +60,12 @@ start a put operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::put (std::unique_ptr< folly::IOBuf > buffer) const =0 add a chunk of data to the object @@ -67,9 +73,11 @@ add a chunk of data to the object #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | + + @@ -80,8 +88,13 @@ add a chunk of data to the object +#### Qualifiers: +* const +* virtual +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::finish_put ()=0 end a put operation @@ -92,6 +105,12 @@ end a put operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::start_get (std::string object_id)=0 start a gett operation @@ -99,9 +118,9 @@ start a gett operation #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::string | object_id | the identifier of the object to retrieve | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | the identifier of the object to retrieve | @@ -114,6 +133,12 @@ start a gett operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public ssize_t fiphoboserver::stream::Stream::get (void *buf, size_t count) const =0 gets a chunk of data of the object @@ -121,10 +146,10 @@ gets a chunk of data of the object #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | void * | buf | a buffer for the data to be filled in this function | -| | size_t | count | the size of the buffer / chunk | +| Type | Name | Description | +| ---- | ---- | ---- | +| void * | buf | a buffer for the data to be filled in this function | +| size_t | count | the size of the buffer / chunk | #### Returns: | Type | Description | @@ -144,6 +169,13 @@ gets a chunk of data of the object +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::finish_get ()=0 end a get operation @@ -154,6 +186,12 @@ end a get operation +#### Qualifiers: +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ## Protected Attributes ### protected std::unique_ptr< storage::Storage > @@ -184,6 +222,8 @@ a future object that can be used to get the results from the asynchronous backen +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ## Public Functions ### public fiphoboserver::stream::Stream::Stream () @@ -195,6 +235,13 @@ default constructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public fiphoboserver::stream::Stream::Stream (std::unique_ptr< storage::Storage > input_storage) constructor with storage implementation @@ -202,9 +249,9 @@ constructor with storage implementation #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | pointer to a implementation of the [storage::Storage][fiphoboserver-storage-Storage] class that will be used to store the data | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< [storage::Storage][fiphoboserver-storage-Storage] > | input_storage | pointer to a implementation of the [storage::Storage][fiphoboserver-storage-Storage] class that will be used to store the data | @@ -221,6 +268,13 @@ constructor with storage implementation +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public fiphoboserver::stream::Stream::~Stream () default destructor @@ -231,6 +285,13 @@ default destructor +#### Qualifiers: +* inline +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const =0 setting the metadata that the created object should get @@ -238,10 +299,10 @@ setting the metadata that the created object should get #### Parameters: -| Direction | Type | Name | Description | -| ---- | ---- | ---- | ---- | -| | std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | -| | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | +| Type | Name | Description | +| ---- | ---- | ---- | +| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -258,6 +319,13 @@ setting the metadata that the created object should get +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + ### public void fiphoboserver::stream::Stream::get_meta_data () const =0 get the metadata from a stored object @@ -268,6 +336,13 @@ get the metadata from a stored object -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +#### Qualifiers: +* const +* virtual + + +[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) + [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 9978a91..e2efc9e 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -12,7 +12,6 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: -## Classes -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md -- GitLab From 5796ef672fbded2b7e741905706930d60090ef22 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 21 Feb 2020 11:29:42 +0000 Subject: [PATCH 16/62] Add todo list to Markdown output --- .../fiphoboserver/GetRequestHandler.md | 3 ++- .../fiphoboserver/PutRequestHandler.md | 3 ++- doc/Markdown/fiphoboserver/index.md | 4 ++-- .../fiphoboserver/storage/Phobos_file.md | 3 ++- doc/Markdown/fiphoboserver/storage/index.md | 3 ++- doc/Markdown/fiphoboserver/stream/Fifo.md | 7 ++++--- doc/Markdown/fiphoboserver/stream/Stream.md | 5 +++-- doc/Markdown/fiphoboserver/stream/index.md | 3 ++- doc/Markdown/todo.md | 19 +++++++++++++++++++ 9 files changed, 38 insertions(+), 12 deletions(-) diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index c5a1d18..e7ec91d 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -135,7 +135,7 @@ function ... -> **Todo:** find out what this does? +> **[Todo][todo]:** find out what this does? @@ -249,3 +249,4 @@ Constructor for stream class initialization. [Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[todo]:doc/Markdown/todo.md#todo diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index fe5f57f..ddc1d81 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -133,7 +133,7 @@ function ... -> **Todo:** find out what this does? +> **[Todo][todo]:** find out what this does? @@ -203,3 +203,4 @@ Constructor for stream class initialization. [Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[todo]:doc/Markdown/todo.md#todo diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 51753d2..b6b6270 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -22,8 +22,8 @@ +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md [fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md [fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 809e5e4..f42a865 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -3,7 +3,7 @@ default constructor -> **Todo:** maybe not only copy from interface but add specific phobos information? +> **[Todo][todo]:** maybe not only copy from interface but add specific phobos information? @@ -226,4 +226,5 @@ Starts a get operation to the database. [Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[todo]:doc/Markdown/todo.md#todo [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index 245aea3..7cd98d0 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -14,5 +14,6 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: [fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[todo]:doc/Markdown/todo.md#todo [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index f75c796..af04e70 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -3,7 +3,7 @@ default constructor -> **Todo:** maybe not only copy from interface but add specific fifo information? +> **[Todo][todo]:** maybe not only copy from interface but add specific fifo information? @@ -56,7 +56,7 @@ get the descriptor of the fifo -> **Todo:** Do we use this? should this really be public? +> **[Todo][todo]:** Do we use this? should this really be public? @@ -90,7 +90,7 @@ get the name of the fifo -> **Todo:** Do we use this? should this really be public? +> **[Todo][todo]:** Do we use this? should this really be public? @@ -350,4 +350,5 @@ end a get operation [Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) [fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[todo]:doc/Markdown/todo.md#todo [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 37db827..857633f 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -215,7 +215,7 @@ a future object that can be used to get the results from the asynchronous backen -> **Todo:** should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? +> **[Todo][todo]:** should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? @@ -343,6 +343,7 @@ get the metadata from a stored object [Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[todo]:doc/Markdown/todo.md#todo [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index e2efc9e..9bd539d 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -12,6 +12,7 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md [fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md +[todo]:doc/Markdown/todo.md#todo [fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 0c44059..ae7188d 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -2,9 +2,28 @@ +* Member [fiphoboserver::GetRequestHandler::requestComplete][fiphoboserver-GetRequestHandler-requestComplete] () noexcept override + * find out what this does? +* Class [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phobos_file] + * maybe not only copy from interface but add specific phobos information? +* Class [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo] + * maybe not only copy from interface but add specific fifo information? +* Member [fiphoboserver::stream::Fifo::get_fifo_descriptor][fiphoboserver-stream-Fifo-get_fifo_descriptor] () const + * Do we use this? should this really be public? +* Member [fiphoboserver::stream::Fifo::get_fifo_name][fiphoboserver-stream-Fifo-get_fifo_name] () const + * Do we use this? should this really be public? +* Member [fiphoboserver::stream::Stream::db_result][fiphoboserver-stream-Stream-db_result] + * should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? + +[fiphoboserver-stream-Fifo-get_fifo_descriptor]:doc/Markdown/fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor +[fiphoboserver-stream-Fifo-get_fifo_name]:doc/Markdown/fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name +[fiphoboserver-GetRequestHandler-requestComplete]:doc/Markdown/fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete +[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md +[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md +[fiphoboserver-stream-Stream-db_result]:doc/Markdown/fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result -- GitLab From 7b5195a25a4603c23ba767b9a6392bb986abbbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Fri, 21 Feb 2020 12:30:52 +0000 Subject: [PATCH 17/62] Tests default to off --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80d70d7..8c00126 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ find_package(proxygen REQUIRED) option( FIPHOBOSERVER_BUILD_TESTS "build FiPhoboServer tests. Depends on Catch2." - ON + OFF ) add_subdirectory(src) -- GitLab From a5f3283e7446266acaa457e32d2c16f487ac1f28 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 21 Feb 2020 16:21:25 +0000 Subject: [PATCH 18/62] Fix links in Markdown output --- .../fiphoboserver/FIPhoboServerException.md | 12 +++---- .../fiphoboserver/GetRequestHandler.md | 22 ++++++------- doc/Markdown/fiphoboserver/IOException.md | 4 +-- .../fiphoboserver/PutRequestHandler.md | 18 +++++------ doc/Markdown/fiphoboserver/S3_header.md | 12 +++---- doc/Markdown/fiphoboserver/index.md | 26 +++++++-------- .../fiphoboserver/storage/PhobosException.md | 4 +-- .../fiphoboserver/storage/Phobos_file.md | 20 ++++++------ doc/Markdown/fiphoboserver/storage/Storage.md | 18 +++++------ doc/Markdown/fiphoboserver/storage/index.md | 14 ++++---- doc/Markdown/fiphoboserver/stream/Fifo.md | 30 ++++++++--------- doc/Markdown/fiphoboserver/stream/Stream.md | 32 +++++++++---------- doc/Markdown/fiphoboserver/stream/index.md | 12 +++---- doc/Markdown/todo.md | 12 +++---- 14 files changed, 118 insertions(+), 118 deletions(-) diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index 31e9f2d..c135ac1 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -33,7 +33,7 @@ the internal message -[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) +[Go to Top](./FIPhoboServerException.md) ## Public Functions ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException () @@ -51,7 +51,7 @@ default constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) +[Go to Top](./FIPhoboServerException.md) ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException (const char *message) @@ -80,7 +80,7 @@ constructor for a message with information on the exception * virtual -[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) +[Go to Top](./FIPhoboServerException.md) ### public const char * fiphoboserver::FIPhoboServerException::what () const noexcept override @@ -111,7 +111,7 @@ get information on this exception * virtual -[Go to Top](doc/Markdown/fiphoboserver/FIPhoboServerException.md) +[Go to Top](./FIPhoboServerException.md) -[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md -[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md +[fiphoboserver-IOException]:./IOException.md +[fiphoboserver-storage-PhobosException]:./storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index e7ec91d..c630167 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -49,7 +49,7 @@ first function to be called when a new request comes in * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -78,7 +78,7 @@ This is not used in this case because GET requests don't usually have a body * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override @@ -94,7 +94,7 @@ function called when the incoming message is finished * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override @@ -121,7 +121,7 @@ Not supported in our case! * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::requestComplete () noexcept override @@ -146,7 +146,7 @@ function ... * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onError (proxygen::ProxygenError err) noexcept override @@ -173,7 +173,7 @@ Not supported in our case! * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onEgressPaused () noexcept override @@ -195,7 +195,7 @@ Contents are copies from a proxygen example * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public void fiphoboserver::GetRequestHandler::onEgressResumed () noexcept override @@ -217,7 +217,7 @@ Contents are copies from a proxygen example * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) ### public fiphoboserver::GetRequestHandler::GetRequestHandler (std::unique_ptr< stream::Stream > input_stream) @@ -246,7 +246,7 @@ Constructor for stream class initialization. * virtual -[Go to Top](doc/Markdown/fiphoboserver/GetRequestHandler.md) +[Go to Top](./GetRequestHandler.md) -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[todo]:doc/Markdown/todo.md#todo +[todo]:./../todo.md#todo +[fiphoboserver-stream-Stream]:./stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index 4fb2f44..a168bd4 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -44,6 +44,6 @@ constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/IOException.md) +[Go to Top](./IOException.md) -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index ddc1d81..7f5acce 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -47,7 +47,7 @@ first function to be called when a new request comes in * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -76,7 +76,7 @@ This is not used in this case because GET requests don't usually have a body * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::onEOM () noexcept override @@ -92,7 +92,7 @@ function called when the incoming message is finished * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override @@ -119,7 +119,7 @@ Not supported in our case! * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::requestComplete () noexcept override @@ -144,7 +144,7 @@ function ... * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public void fiphoboserver::PutRequestHandler::onError (proxygen::ProxygenError err) noexcept override @@ -171,7 +171,7 @@ Not supported in our case! * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) ### public fiphoboserver::PutRequestHandler::PutRequestHandler (std::unique_ptr< stream::Stream > input_stream) @@ -200,7 +200,7 @@ Constructor for stream class initialization. * virtual -[Go to Top](doc/Markdown/fiphoboserver/PutRequestHandler.md) +[Go to Top](./PutRequestHandler.md) -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[todo]:doc/Markdown/todo.md#todo +[todo]:./../todo.md#todo +[fiphoboserver-stream-Stream]:./stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index 84e1da1..c654a6c 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -44,7 +44,7 @@ sets the internal header instance to the given header object * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) ### public std::string fiphoboserver::S3_header::getBucket () @@ -74,7 +74,7 @@ extracts the name of the S3 bucket requested in the HTTP message * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) ### public std::string fiphoboserver::S3_header::getKey () @@ -104,7 +104,7 @@ extracts the name of the key id requested in the HTTP message * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) ### public bool fiphoboserver::S3_header::isCreateBucketRequest () @@ -135,7 +135,7 @@ This is convenient since we don't really support buckets themselves and can easi * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) ### public size_t fiphoboserver::S3_header::getBodyLength () @@ -166,7 +166,7 @@ This reads out the header "Content-Length" * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) ### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::getMetaData () @@ -197,5 +197,5 @@ In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE * virtual -[Go to Top](doc/Markdown/fiphoboserver/S3_header.md) +[Go to Top](./S3_header.md) diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index b6b6270..72964a5 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -6,24 +6,24 @@ ## Classes | Name | Description | | ---- | ---- | -| [FIPhoboServerException](doc/Markdown/fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | -| [GetRequestHandler](doc/Markdown/fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests | -| [IOException](doc/Markdown/fiphoboserver/IOException.md) | exceptions specifically for errors in I/O | -| [PutRequestHandler](doc/Markdown/fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests | -| [S3_header](doc/Markdown/fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | -| [UnsupportedRequestHandler](doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | +| [FIPhoboServerException](./../../fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | +| [GetRequestHandler](./../../fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests | +| [IOException](./../../fiphoboserver/IOException.md) | exceptions specifically for errors in I/O | +| [PutRequestHandler](./../../fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests | +| [S3_header](./../../fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | +| [UnsupportedRequestHandler](./../../fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | ## Namespaces | Name | Description | | ---- | ---- | -| [storage](doc/Markdown/fiphoboserver/storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | -| [stream](doc/Markdown/fiphoboserver/stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | +| [storage](./../fiphoboserver/storage) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | +| [stream](./../fiphoboserver/stream) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[fiphoboserver-storage-PhobosException]:doc/Markdown/fiphoboserver/storage/PhobosException.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md -[fiphoboserver-IOException]:doc/Markdown/fiphoboserver/IOException.md -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Stream]:./../../fiphoboserver/stream/Stream.md +[fiphoboserver-storage-PhobosException]:./../../fiphoboserver/storage/PhobosException.md +[fiphoboserver-FIPhoboServerException]:./../../fiphoboserver/FIPhoboServerException.md +[fiphoboserver-IOException]:./../../fiphoboserver/IOException.md +[fiphoboserver-storage-Storage]:./../../fiphoboserver/storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index 2e08261..cdd7b60 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -44,6 +44,6 @@ constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/PhobosException.md) +[Go to Top](./PhobosException.md) -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index f42a865..a8cd2a3 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -42,7 +42,7 @@ default constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public fiphoboserver::storage::Phobos_file::~Phobos_file () @@ -58,7 +58,7 @@ default destructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override @@ -87,7 +87,7 @@ set the metadata an object that is added to the database should get * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public void fiphoboserver::storage::Phobos_file::get_meta_data () const override @@ -104,7 +104,7 @@ get the metadata associated to the current object as a vector of key:value pairs * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public ssize_t fiphoboserver::storage::Phobos_file::db_put (size_t size) override @@ -137,7 +137,7 @@ Puts data to the databse. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public ssize_t fiphoboserver::storage::Phobos_file::db_get () override @@ -166,7 +166,7 @@ Gets data from the databse. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override @@ -195,7 +195,7 @@ Starts a put operation to the database. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) ### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override @@ -224,7 +224,7 @@ Starts a get operation to the database. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Phobos_file.md) +[Go to Top](./Phobos_file.md) -[todo]:doc/Markdown/todo.md#todo -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index 9159843..cc1356a 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -67,7 +67,7 @@ set the metadata an object that is added to the database should get * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ### public void fiphoboserver::storage::Storage::get_meta_data () const =0 @@ -84,7 +84,7 @@ get the metadata associated to the current object as a vector of key:value pairs * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ## PUT functions ### public void fiphoboserver::storage::Storage::prepare_put (std::string file_name, std::string object_id)=0 @@ -114,7 +114,7 @@ Starts a put operation to the database. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ### public ssize_t fiphoboserver::storage::Storage::db_put (size_t size)=0 @@ -147,7 +147,7 @@ Puts data to the databse. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ## GET functions ### public void fiphoboserver::storage::Storage::prepare_get (std::string file_name, std::string object_id)=0 @@ -177,7 +177,7 @@ Starts a get operation to the database. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ### public ssize_t fiphoboserver::storage::Storage::db_get ()=0 @@ -206,7 +206,7 @@ Gets data from the databse. * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ## Public Functions ### public fiphoboserver::storage::Storage::Storage () @@ -224,7 +224,7 @@ default constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) ### public fiphoboserver::storage::Storage::~Storage () @@ -241,6 +241,6 @@ default destructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/storage/Storage.md) +[Go to Top](./Storage.md) -[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md +[fiphoboserver-storage-Phobos_file]:./Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index 7cd98d0..31d0447 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -7,13 +7,13 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: ## Classes | Name | Description | | ---- | ---- | -| [Phobos_file](doc/Markdown/fiphoboserver/storage/Phobos_file.md) | default constructor | -| [PhobosException](doc/Markdown/fiphoboserver/storage/PhobosException.md) | exceptions specifically for the phobos backend library | -| [Storage](doc/Markdown/fiphoboserver/storage/Storage.md) | virtual storage class to be implemented by backend storage | +| [Phobos_file](./Phobos_file.md) | default constructor | +| [PhobosException](./PhobosException.md) | exceptions specifically for the phobos backend library | +| [Storage](./Storage.md) | virtual storage class to be implemented by backend storage | -[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md -[todo]:doc/Markdown/todo.md#todo -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md -[fiphoboserver-FIPhoboServerException]:doc/Markdown/fiphoboserver/FIPhoboServerException.md +[fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md +[todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./Storage.md +[fiphoboserver-storage-Phobos_file]:./Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index af04e70..c68208d 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -69,7 +69,7 @@ get the descriptor of the fifo * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public std::string fiphoboserver::stream::Fifo::get_fifo_name () const @@ -103,7 +103,7 @@ get the name of the fifo * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ## Public Functions ### public fiphoboserver::stream::Fifo::Fifo (std::unique_ptr< storage::Storage > input_storage) @@ -125,7 +125,7 @@ default constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public fiphoboserver::stream::Fifo::~Fifo () @@ -141,7 +141,7 @@ default destructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override @@ -175,7 +175,7 @@ setting the metadata that the created object should get * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::get_meta_data () const override @@ -192,7 +192,7 @@ get the metadata from a stored object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::start_put (ssize_t size, std::string object_id) override @@ -221,7 +221,7 @@ start a put operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::put (std::unique_ptr< folly::IOBuf > buffer) const override @@ -250,7 +250,7 @@ add a chunk of data to the object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::finish_put () override @@ -266,7 +266,7 @@ end a put operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::start_get (std::string object_id) override @@ -294,7 +294,7 @@ start a gett operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public ssize_t fiphoboserver::stream::Fifo::get (void *buf, size_t count) const override @@ -331,7 +331,7 @@ gets a chunk of data of the object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) ### public void fiphoboserver::stream::Fifo::finish_get () override @@ -347,8 +347,8 @@ end a get operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Fifo.md) +[Go to Top](./Fifo.md) -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[todo]:doc/Markdown/todo.md#todo -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[todo]:./../../todo.md#todo +[fiphoboserver-stream-Stream]:./Stream.md +[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 857633f..4206c96 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -64,7 +64,7 @@ start a put operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::put (std::unique_ptr< folly::IOBuf > buffer) const =0 @@ -93,7 +93,7 @@ add a chunk of data to the object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::finish_put ()=0 @@ -109,7 +109,7 @@ end a put operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::start_get (std::string object_id)=0 @@ -137,7 +137,7 @@ start a gett operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public ssize_t fiphoboserver::stream::Stream::get (void *buf, size_t count) const =0 @@ -174,7 +174,7 @@ gets a chunk of data of the object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::finish_get ()=0 @@ -190,7 +190,7 @@ end a get operation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ## Protected Attributes ### protected std::unique_ptr< storage::Storage > @@ -222,7 +222,7 @@ a future object that can be used to get the results from the asynchronous backen -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ## Public Functions ### public fiphoboserver::stream::Stream::Stream () @@ -240,7 +240,7 @@ default constructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public fiphoboserver::stream::Stream::Stream (std::unique_ptr< storage::Storage > input_storage) @@ -273,7 +273,7 @@ constructor with storage implementation * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public fiphoboserver::stream::Stream::~Stream () @@ -290,7 +290,7 @@ default destructor * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const =0 @@ -324,7 +324,7 @@ setting the metadata that the created object should get * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) ### public void fiphoboserver::stream::Stream::get_meta_data () const =0 @@ -341,9 +341,9 @@ get the metadata from a stored object * virtual -[Go to Top](doc/Markdown/fiphoboserver/stream/Stream.md) +[Go to Top](./Stream.md) -[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[todo]:doc/Markdown/todo.md#todo -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Fifo]:./Fifo.md +[fiphoboserver-stream-Stream]:./Stream.md +[todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 9bd539d..59ebc76 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -7,12 +7,12 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: ## Classes | Name | Description | | ---- | ---- | -| [Fifo](doc/Markdown/fiphoboserver/stream/Fifo.md) | default constructor | -| [Stream](doc/Markdown/fiphoboserver/stream/Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class | +| [Fifo](./Fifo.md) | default constructor | +| [Stream](./Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class | -[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md -[fiphoboserver-stream-Stream]:doc/Markdown/fiphoboserver/stream/Stream.md -[todo]:doc/Markdown/todo.md#todo -[fiphoboserver-storage-Storage]:doc/Markdown/fiphoboserver/storage/Storage.md +[fiphoboserver-stream-Fifo]:./Fifo.md +[fiphoboserver-stream-Stream]:./Stream.md +[fiphoboserver-storage-Storage]:./../storage/Storage.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index ae7188d..b99a3ad 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -21,9 +21,9 @@ -[fiphoboserver-stream-Fifo-get_fifo_descriptor]:doc/Markdown/fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor -[fiphoboserver-stream-Fifo-get_fifo_name]:doc/Markdown/fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name -[fiphoboserver-GetRequestHandler-requestComplete]:doc/Markdown/fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-stream-Fifo]:doc/Markdown/fiphoboserver/stream/Fifo.md -[fiphoboserver-storage-Phobos_file]:doc/Markdown/fiphoboserver/storage/Phobos_file.md -[fiphoboserver-stream-Stream-db_result]:doc/Markdown/fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result +[fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete +[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor +[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name +[fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md +[fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Stream-db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result -- GitLab From 01bb1409673ee660e14d9795b588381234eb0fb4 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Mon, 24 Feb 2020 11:07:57 +0000 Subject: [PATCH 19/62] More fixing of links (fiphoboserver index.md) --- .../fiphoboserver/FIPhoboServerException.md | 2 +- .../fiphoboserver/GetRequestHandler.md | 2 +- .../fiphoboserver/PutRequestHandler.md | 2 +- doc/Markdown/fiphoboserver/index.md | 26 +++++++++---------- .../fiphoboserver/storage/Phobos_file.md | 2 +- doc/Markdown/fiphoboserver/storage/index.md | 4 +-- doc/Markdown/fiphoboserver/stream/Fifo.md | 4 +-- doc/Markdown/fiphoboserver/stream/Stream.md | 4 +-- doc/Markdown/fiphoboserver/stream/index.md | 4 +-- doc/Markdown/todo.md | 6 ++--- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index c135ac1..7d9d2db 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -113,5 +113,5 @@ get information on this exception [Go to Top](./FIPhoboServerException.md) -[fiphoboserver-IOException]:./IOException.md [fiphoboserver-storage-PhobosException]:./storage/PhobosException.md +[fiphoboserver-IOException]:./IOException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index c630167..9197316 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -248,5 +248,5 @@ Constructor for stream class initialization. [Go to Top](./GetRequestHandler.md) -[todo]:./../todo.md#todo [fiphoboserver-stream-Stream]:./stream/Stream.md +[todo]:./../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 7f5acce..5518778 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -202,5 +202,5 @@ Constructor for stream class initialization. [Go to Top](./PutRequestHandler.md) -[todo]:./../todo.md#todo [fiphoboserver-stream-Stream]:./stream/Stream.md +[todo]:./../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 72964a5..520d6ce 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -6,24 +6,24 @@ ## Classes | Name | Description | | ---- | ---- | -| [FIPhoboServerException](./../../fiphoboserver/FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | -| [GetRequestHandler](./../../fiphoboserver/GetRequestHandler.md) | proxygen class implementation for handling GET requests | -| [IOException](./../../fiphoboserver/IOException.md) | exceptions specifically for errors in I/O | -| [PutRequestHandler](./../../fiphoboserver/PutRequestHandler.md) | proxygen class implementation for handling PUT requests | -| [S3_header](./../../fiphoboserver/S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | -| [UnsupportedRequestHandler](./../../fiphoboserver/UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | +| [FIPhoboServerException](./FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | +| [GetRequestHandler](./GetRequestHandler.md) | proxygen class implementation for handling GET requests | +| [IOException](./IOException.md) | exceptions specifically for errors in I/O | +| [PutRequestHandler](./PutRequestHandler.md) | proxygen class implementation for handling PUT requests | +| [S3_header](./S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | +| [UnsupportedRequestHandler](./UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | ## Namespaces | Name | Description | | ---- | ---- | -| [storage](./../fiphoboserver/storage) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | -| [stream](./../fiphoboserver/stream) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | +| [storage](./storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | +| [stream](./stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | -[fiphoboserver-stream-Stream]:./../../fiphoboserver/stream/Stream.md -[fiphoboserver-storage-PhobosException]:./../../fiphoboserver/storage/PhobosException.md -[fiphoboserver-FIPhoboServerException]:./../../fiphoboserver/FIPhoboServerException.md -[fiphoboserver-IOException]:./../../fiphoboserver/IOException.md -[fiphoboserver-storage-Storage]:./../../fiphoboserver/storage/Storage.md +[fiphoboserver-storage-Storage]:./storage/Storage.md +[fiphoboserver-storage-PhobosException]:./storage/PhobosException.md +[fiphoboserver-stream-Stream]:./stream/Stream.md +[fiphoboserver-IOException]:./IOException.md +[fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index a8cd2a3..8136424 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -226,5 +226,5 @@ Starts a get operation to the database. [Go to Top](./Phobos_file.md) -[todo]:./../../todo.md#todo [fiphoboserver-storage-Storage]:./Storage.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index 31d0447..e0fb3e3 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -14,6 +14,6 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: [fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md -[todo]:./../../todo.md#todo -[fiphoboserver-storage-Storage]:./Storage.md [fiphoboserver-storage-Phobos_file]:./Phobos_file.md +[fiphoboserver-storage-Storage]:./Storage.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index c68208d..16846d8 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -349,6 +349,6 @@ end a get operation [Go to Top](./Fifo.md) -[todo]:./../../todo.md#todo -[fiphoboserver-stream-Stream]:./Stream.md [fiphoboserver-storage-Storage]:./../storage/Storage.md +[fiphoboserver-stream-Stream]:./Stream.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 4206c96..9196bb9 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -343,7 +343,7 @@ get the metadata from a stored object [Go to Top](./Stream.md) -[fiphoboserver-stream-Fifo]:./Fifo.md +[fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Stream]:./Stream.md +[fiphoboserver-stream-Fifo]:./Fifo.md [todo]:./../../todo.md#todo -[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 59ebc76..871406f 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -12,7 +12,7 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: -[fiphoboserver-stream-Fifo]:./Fifo.md -[fiphoboserver-stream-Stream]:./Stream.md [fiphoboserver-storage-Storage]:./../storage/Storage.md +[fiphoboserver-stream-Stream]:./Stream.md +[fiphoboserver-stream-Fifo]:./Fifo.md [todo]:./../../todo.md#todo diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index b99a3ad..826c6ea 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -21,9 +21,9 @@ -[fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor -[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name [fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md +[fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete [fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name +[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor [fiphoboserver-stream-Stream-db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result -- GitLab From 04d24af20c2945d63b79e1744be28b6c8a4c1446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Mon, 24 Feb 2020 11:58:28 +0000 Subject: [PATCH 20/62] Moved main. Added Phobos_file test --- src/main.cc | 100 ++++++++++++++++++++++++++++++++++++++++++++ test/phobos_file.cc | 60 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 src/main.cc create mode 100644 test/phobos_file.cc diff --git a/src/main.cc b/src/main.cc new file mode 100644 index 0000000..c9e1930 --- /dev/null +++ b/src/main.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "server/get_request_handler.h" +#include "server/put_request_handler.h" +#include "server/unsupported_request_handler.h" + +#include "stream/fifo.h" +#include "storage/phobos_file.h" + +using folly::SocketAddress; + +using Protocol = proxygen::HTTPServer::Protocol; + +DEFINE_int32(http_port, 11000, "Port to listen on with HTTP protocol"); +DEFINE_int32(h2_port, 11002, "Port to listen on with HTTP/2 protocol"); +DEFINE_string(ip, "localhost", "IP/Hostname to bind to"); +DEFINE_int32( + threads, + 0, + "Number of threads to listen on. Numbers <= 0 " + "will use the number of cores on this machine."); + +namespace { + +class HandlerFactory : public proxygen::RequestHandlerFactory { + public: + void onServerStart(folly::EventBase* /*evb*/) noexcept override {} + + void onServerStop() noexcept override {} + + proxygen::RequestHandler* onRequest( + proxygen::RequestHandler*, + proxygen::HTTPMessage* headers) noexcept override + { + if (headers->getMethod() == proxygen::HTTPMethod::GET) { + return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); + } + else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { + return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); + } + else { + return new fiphoboserver::UnsupportedRequestHandler; + } + } +}; +} // namespace + +int main(int argc, char* argv[]) +{ + folly::init(&argc, &argv, true); + + std::vector IPs = { + {SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP}, + {SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2}, + }; + + if (FLAGS_threads <= 0) { + FLAGS_threads = sysconf(_SC_NPROCESSORS_ONLN); + CHECK_GT(FLAGS_threads, 0); + } + + proxygen::HTTPServerOptions options; + options.threads = static_cast(FLAGS_threads); + options.idleTimeout = std::chrono::milliseconds(60000); + options.shutdownOn = {SIGINT, SIGTERM}; + options.enableContentCompression = false; + options.handlerFactories = + proxygen::RequestHandlerChain().addThen().build(); + options.h2cEnabled = true; + + auto diskIOThreadPool = std::make_shared( + FLAGS_threads, + std::make_shared("FIFOIOThread")); + folly::setCPUExecutor(diskIOThreadPool); + + proxygen::HTTPServer server(std::move(options)); + server.bind(IPs); + + /* Start HTTPServer mainloop in a separate thread */ + std::thread t([&]() { server.start(); }); + + t.join(); + return 0; +} diff --git a/test/phobos_file.cc b/test/phobos_file.cc new file mode 100644 index 0000000..a7167a6 --- /dev/null +++ b/test/phobos_file.cc @@ -0,0 +1,60 @@ +#define CATCH_CONFIG_MAIN + +#include +#include "../src/storage/phobos_file.h" +#include +#include + +namespace fiphoboserver { + +TEST_CASE("file ownership", "[basic]") +{ + SECTION("move constructor") + { + storage::Phobos_file pho; + + /* generate a file descriptor from a file */ + pho.prepare_put("data/dummy", "dummy"); + int fd = pho.get_fd(); + + storage::Phobos_file pho2(std::move(pho)); + REQUIRE(fd == pho2.get_fd()); + //REQUIRE(fd == pho.get_fd()); + } + SECTION("assignment move constructor") + { + storage::Phobos_file pho; + + /* generate a file descriptor from a file */ + pho.prepare_put("data/dummy", "dummy"); + int fd = pho.get_fd(); + + storage::Phobos_file pho2 = std::move(pho); + REQUIRE(fd == pho2.get_fd()); + //REQUIRE(fd == pho.get_fd()); + } + //SECTION("copy constructor") + //{ + // storage::Phobos_file pho; + + // /* generate a file descriptor from a file */ + // pho.prepare_put("/tmp/dummy", "dummy"); + // int fd = pho.get_fd(); + + // storage::Phobos_filepho2(pho); + // REQUIRE(fd == pho2.get_fd()); + //} + //SECTION("assignment copy constructor") + //{ + // storage::Phobos_file pho; + + // /* generate a file descriptor from a file */ + // pho.prepare_put("/tmp/dummy", "dummy"); + // int fd = pho.get_fd(); + + // storage::Phobos_file pho2 = pho; + // REQUIRE(fd == pho2.get_fd()); + //} +} + +} // namespace fiphoboserver -- GitLab From 3269d433bf6cd46479b5210ad763ab007fd81218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Tue, 25 Feb 2020 13:10:03 +0000 Subject: [PATCH 21/62] Support for getmd feature Change log: * Send bad message upon entering unsupported request handler * add c++ wrapper for pho_attrs_foreach * Phobos_file::get_meta_data returns vector of key-value pairs * Phobos_file::get_meta_data functions like: * phobos_getmd_cpp fills the descriptor with attributes for the requested object * phobos_attrs_foreach_cpp loops over the attrs and returns them to the stream * Fifo::get_meta_data simply passes on the result of the storage call * Fifo::get_meta_data returns vector of key-value pairs * add Getmd request handler * add getmd_request_handler.h to headers * request handler has a meta_data member variable * fill meta_data variable with Phobos attributes * use repeated calls to ResponseBuilder.header to fill header with Phobos attributes * send entire message at once * add support for testing getmd --- src/fiphoboserver_exception.h | 12 +-- src/main.cc | 18 +++- src/server/CMakeLists.txt | 10 +- src/server/get_request_handler.cc | 23 +++-- src/server/get_request_handler.h | 8 +- src/server/getmd_request_handler.cc | 94 +++++++++++++++++++ src/server/getmd_request_handler.h | 50 ++++++++++ src/server/put_request_handler.cc | 8 +- src/server/put_request_handler.h | 8 +- src/server/unsupported_request_handler.cc | 2 +- .../phobos_cpp_wrapper/phobos_cpp_wrapper.c | 6 ++ .../phobos_cpp_wrapper/phobos_cpp_wrapper.h | 16 +++- src/storage/phobos_exception.h | 10 +- src/storage/phobos_file.cc | 76 ++++++++++----- src/storage/phobos_file.h | 60 ++++++------ src/storage/storage.h | 29 +++--- src/stream/fifo.cc | 38 ++++---- src/stream/fifo.h | 63 +++++++------ src/stream/stream.h | 69 +++++++------- test/s3_call_script.py | 14 +-- 20 files changed, 420 insertions(+), 194 deletions(-) create mode 100644 src/server/getmd_request_handler.cc create mode 100644 src/server/getmd_request_handler.h diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index f01f47b..813a5f3 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -7,27 +7,27 @@ namespace fiphoboserver { class FIPhoboServerException : public std::exception { -public: - FIPhoboServerException() : message("") { } + public: + FIPhoboServerException() : message("") {} - FIPhoboServerException(const char* message) : message(message) { } + FIPhoboServerException(const char* message) : message(message) {} virtual const char* what() const noexcept override { return message.c_str(); } -protected: + protected: std::string message; }; class IOException : public FIPhoboServerException { -public: + public: IOException(const char* caller, const char* functionName, int returnValue) { std::stringstream ss; ss << "[" << caller << "]: IO Exception occurred: " << functionName - << ": " << std::strerror(returnValue) << " (" << returnValue << ")"; + << ": " << std::strerror(returnValue) << " (" << returnValue << ")"; message = ss.str(); } }; diff --git a/src/main.cc b/src/main.cc index c9e1930..5a33a22 100644 --- a/src/main.cc +++ b/src/main.cc @@ -7,7 +7,7 @@ */ #include -#include +#include #include #include #include @@ -17,11 +17,12 @@ #include #include "server/get_request_handler.h" +#include "server/getmd_request_handler.h" #include "server/put_request_handler.h" #include "server/unsupported_request_handler.h" -#include "stream/fifo.h" #include "storage/phobos_file.h" +#include "stream/fifo.h" using folly::SocketAddress; @@ -49,10 +50,19 @@ class HandlerFactory : public proxygen::RequestHandlerFactory { proxygen::HTTPMessage* headers) noexcept override { if (headers->getMethod() == proxygen::HTTPMethod::GET) { - return new fiphoboserver::GetRequestHandler(std::make_unique(std::make_unique())); + return new fiphoboserver::GetRequestHandler( + std::make_unique( + std::make_unique())); } else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { - return new fiphoboserver::PutRequestHandler(std::make_unique(std::make_unique())); + return new fiphoboserver::PutRequestHandler( + std::make_unique( + std::make_unique())); + } + else if (headers->getMethod() == proxygen::HTTPMethod::HEAD) { + return new fiphoboserver::GetmdRequestHandler( + std::make_unique( + std::make_unique())); } else { return new fiphoboserver::UnsupportedRequestHandler; diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 14ad367..4be3d55 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -6,18 +6,10 @@ add_library( server get_request_handler.cc put_request_handler.cc + getmd_request_handler.cc unsupported_request_handler.cc ) -if(CUSTOM_OUTPUT_DIRECTORY) - set_target_properties( server - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - RUNTIME_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - ) -endif(CUSTOM_OUTPUT_DIRECTORY) - target_compile_features(server PUBLIC cxx_std_14) target_link_libraries(server PUBLIC proxygen::proxygen) diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index c8a9f03..e78dd2c 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -36,11 +36,13 @@ void GetRequestHandler::onRequest( try { /* Send meta data to backend through the stream */ - stream->set_storage_meta_data(s3_header.getMetaData(), s3_header.getBucket()); - /* Tell stream to coordinate with backend to prepare for GET operation */ + stream->set_storage_meta_data( + s3_header.getMetaData(), s3_header.getBucket()); + /* Tell stream to coordinate with backend to prepare for GET operation + */ stream->start_get(s3_header.getKey()); - file_closed_ = false; // TODO: Better way of communicating this + file_closed_ = false; // TODO: Better way of communicating this } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -75,7 +77,7 @@ void GetRequestHandler::readFile(folly::EventBase* evb) while (!file_closed_ && !paused_) { /* read 4k-ish chunks and foward each one to the client */ auto data = buf.preallocate(4000, 4000); - /* + /* * rc is set to error code or bytes read * This informs the server how to proceed * Note: zero will only be returned if the @@ -83,9 +85,10 @@ void GetRequestHandler::readFile(folly::EventBase* evb) * for writing */ try { - auto rc = stream->get(data.first, data.second); + auto rc = stream->get(data.first, data.second); if (rc < 0) { - // should not happen as an exception should have been thrown before + // should not happen as an exception should have been thrown + // before throw FIPhoboServerException("stream->get returned an error"); } else if (rc == 0) { @@ -107,8 +110,7 @@ void GetRequestHandler::readFile(folly::EventBase* evb) }); } } - catch(const FIPhoboServerException& ex) - { + catch (const FIPhoboServerException& ex) { std::cerr << "Read error=" << ex.what() << '\n'; evb->runInEventBaseThread([this] { LOG(ERROR) << "Error reading file"; @@ -155,10 +157,7 @@ void GetRequestHandler::onBody(std::unique_ptr /*body*/) noexcept // ignore, only support GET } -void GetRequestHandler::onEOM() noexcept -{ - -} +void GetRequestHandler::onEOM() noexcept {} void GetRequestHandler::onUpgrade( proxygen::UpgradeProtocol /*protocol*/) noexcept diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 7f67791..7cd7efe 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -12,8 +12,8 @@ #include #include -#include "S3_header.h" #include "../stream/stream.h" +#include "S3_header.h" namespace proxygen { class ResponseHandler; @@ -40,8 +40,10 @@ class GetRequestHandler : public proxygen::RequestHandler { void onEgressResumed() noexcept override; - GetRequestHandler(std::unique_ptr input_stream): - stream(std::move(input_stream)) {} + GetRequestHandler(std::unique_ptr input_stream) : + stream(std::move(input_stream)) + { + } private: void readFile(folly::EventBase* evb); diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc new file mode 100644 index 0000000..b4616fa --- /dev/null +++ b/src/server/getmd_request_handler.cc @@ -0,0 +1,94 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "getmd_request_handler.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "../fiphoboserver_exception.h" + +namespace fiphoboserver { + +void GetmdRequestHandler::onRequest( + std::unique_ptr headers) noexcept +{ + if (headers->getMethod() != proxygen::HTTPMethod::HEAD) { + proxygen::ResponseBuilder(downstream_) + .status(400, "Bad method") + .body("Only HEAD is supported") + .sendWithEOM(); + return; + } + + /* Creating Bucket queries */ + s3_header.setHeaders(std::move(headers)); + try { + + /* Get meta data to backend through the stream */ + meta_data = std::move(stream->get_meta_data(s3_header.getKey())); + } + catch (const std::system_error& ex) { + proxygen::ResponseBuilder(downstream_) + .status(404, "Not Found") + .body(folly::to( + "Could not find ", headers->getPathAsStringPiece(), + " ex=", folly::exceptionStr(ex))) + .sendWithEOM(); + return; + } + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught exception: " << ex.what() << '\n'; + proxygen::ResponseBuilder(downstream_) + .status(409, "Conflict") + .body(ex.what()) + .sendWithEOM(); + return; + } + + // proxygen::ResponseBuilder(downstream_).status(200, "Ok").send(); + proxygen::ResponseBuilder response(downstream_); + response.status(200, "OK"); + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + response.header("x-amz-meta-" + pair.first, pair.second); + }); + response.sendWithEOM(); +} + +void GetmdRequestHandler::onBody( + std::unique_ptr /*body*/) noexcept +{ + // ignore, only support HEAD +} + +void GetmdRequestHandler::onEOM() noexcept {} + +void GetmdRequestHandler::onUpgrade( + proxygen::UpgradeProtocol /*protocol*/) noexcept +{ + // handler doesn't support upgrades +} + +void GetmdRequestHandler::requestComplete() noexcept +{ + delete this; +} + +void GetmdRequestHandler::onError(proxygen::ProxygenError /*err*/) noexcept +{ + delete this; +} + +} // namespace fiphoboserver diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h new file mode 100644 index 0000000..5d44b77 --- /dev/null +++ b/src/server/getmd_request_handler.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +//#include +#include +#include + +#include "../stream/stream.h" +#include "S3_header.h" + +namespace proxygen { +class ResponseHandler; +} + +namespace fiphoboserver { + +class GetmdRequestHandler : public proxygen::RequestHandler { + public: + void onRequest( + std::unique_ptr headers) noexcept override; + + void onBody(std::unique_ptr body) noexcept override; + + void onEOM() noexcept override; + + void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + + void requestComplete() noexcept override; + + void onError(proxygen::ProxygenError err) noexcept override; + + GetmdRequestHandler(std::unique_ptr input_stream) : + stream(std::move(input_stream)) + { + } + + private: + std::unique_ptr stream; + S3_header s3_header; + std::vector> meta_data; +}; + +} // namespace fiphoboserver diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 2cb95e0..5052fa6 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -28,7 +28,7 @@ void PutRequestHandler::onRequest( /* Creating Bucket queries */ s3_header.setHeaders(std::move(headers)); if (s3_header.isCreateBucketRequest()) { - /* + /* * Ignore, since we don't really have buckets. * Or do we want a list of the actual buckets to give an error, when a * requested bucket doesn't exist? @@ -38,8 +38,10 @@ void PutRequestHandler::onRequest( } try { /* Send meta data to backend through the stream */ - stream->set_storage_meta_data(s3_header.getMetaData(), s3_header.getBucket()); - /* Tell stream to coordinate with backend to prepare for PUT operation */ + stream->set_storage_meta_data( + s3_header.getMetaData(), s3_header.getBucket()); + /* Tell stream to coordinate with backend to prepare for PUT operation + */ stream->start_put(s3_header.getBodyLength(), s3_header.getKey()); } catch (const std::system_error& ex) { diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 7f0166c..555ac9f 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -11,8 +11,8 @@ #include #include -#include "S3_header.h" #include "../stream/stream.h" +#include "S3_header.h" namespace proxygen { class ResponseHandler; @@ -35,8 +35,10 @@ class PutRequestHandler : public proxygen::RequestHandler { void onError(proxygen::ProxygenError err) noexcept override; - PutRequestHandler(std::unique_ptr input_stream): - stream(std::move(input_stream)) {} + PutRequestHandler(std::unique_ptr input_stream) : + stream(std::move(input_stream)) + { + } private: S3_header s3_header; diff --git a/src/server/unsupported_request_handler.cc b/src/server/unsupported_request_handler.cc index 279a2bd..3a5067e 100644 --- a/src/server/unsupported_request_handler.cc +++ b/src/server/unsupported_request_handler.cc @@ -25,7 +25,7 @@ void UnsupportedRequestHandler::onRequest( ss << "The method " << method << " is not supported!"; proxygen::ResponseBuilder(downstream_) - .status(200, "Ok") // TODO: This is NOT "Ok" + .status(400, "Bad method") .body(ss.str()) .sendWithEOM(); return; diff --git a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c index 476949b..9cf23d7 100644 --- a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c +++ b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.c @@ -1,3 +1,4 @@ +#include "pho_attrs.h" #include "phobos_store.h" @@ -14,6 +15,11 @@ int phobos_get_cpp( return phobos_get(xfers, n, cb, udata); } +int phobos_attrs_foreach_cpp( + const struct pho_attrs* md, pho_attrs_iter_t cb, void* udata) +{ + return pho_attrs_foreach(md, cb, udata); +} int phobos_getmd_cpp( struct pho_xfer_desc* xfers, size_t n, pho_completion_cb_t cb, void* udata) diff --git a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h index cb2daf4..b69d4bb 100644 --- a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h +++ b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h @@ -5,7 +5,6 @@ #include "pho_attrs.h" - /** * Transfer (GET / PUT / MPUT) flags. * Exact semantic depends on the operation it is applied on. @@ -112,6 +111,21 @@ int phobos_get_cpp( int phobos_getmd_cpp( struct pho_xfer_desc* xfers, size_t n, pho_completion_cb_t cb, void* udata); +/** + * Loops over pho_attr key/value pairs and calls + * the cb function argument + * desc contains: + * - md: Phobos meta data struct + * - cb: callback function + * - udata: implementation details + * + * Individual completion notifications are issued via xd_callback. + * This function returns the first encountered error of 0 if all + * sub-operations have succeeded. + */ +int phobos_attrs_foreach_cpp( + const struct pho_attrs* md, pho_attrs_iter_t cb, void* udata); + /** query metadata of the object store */ /* TODO int phobos_query(criteria, &obj_list); */ /* TODO int phobos_del(); */ diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index 129941d..109b0a5 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include #include "../fiphoboserver_exception.h" @@ -10,12 +10,14 @@ namespace fiphoboserver { namespace storage { class PhobosException : public FIPhoboServerException { -public: - PhobosException(const char* caller, const char* functionName, int returnValue) + public: + PhobosException( + const char* caller, const char* functionName, int returnValue) { std::stringstream ss; ss << "[" << caller << "]: Phobos returned an error: " << functionName - << ": " << std::strerror(-returnValue) << " (" << -returnValue << ")"; + << ": " << std::strerror(-returnValue) << " (" << -returnValue + << ")"; message = ss.str(); } }; diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index 6db9222..96653e8 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -1,16 +1,16 @@ #include "phobos_file.h" +#include #include #include +#include #include #include -#include -#include #include -#include #include #include #include +#include #include "../fiphoboserver_exception.h" #include "phobos_exception.h" @@ -29,12 +29,12 @@ Phobos_file::Phobos_file() Phobos_file::~Phobos_file() { // TODO: See issue #23 - // pho_attrs_free(&descriptor.xd_attrs); - + // pho_attrs_free(&descriptor.xd_attrs); + pho_xfer_desc_destroy_cpp(&descriptor); } -/* +/* * Open the file for reading * fill the descriptor with relevant data */ @@ -55,19 +55,18 @@ void Phobos_file::prepare_put(std::string file_name, std::string object_id) /* Tell Phobos to start reading from the file */ ssize_t Phobos_file::db_put(size_t size) { - descriptor.xd_size = size; - ssize_t rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); + descriptor.xd_size = size; + ssize_t rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); close_file(); - if(rc) - { + if (rc) { throw PhobosException("Phobos_file::db_put", "phobos_put", rc); } return rc; } -/* - * Open the file for writing +/* + * Open the file for writing * fill the descriptor with relevant data */ void Phobos_file::prepare_get(std::string file_name, std::string object_id) @@ -81,21 +80,20 @@ void Phobos_file::prepare_get(std::string file_name, std::string object_id) strcpy(unconsted_object_id, object_id.c_str()); descriptor.xd_objid = unconsted_object_id; - descriptor.xd_op = PHO_XFER_OP_GET; + descriptor.xd_op = PHO_XFER_OP_GET; } /* Tell Phobos to write the object to file */ ssize_t Phobos_file::db_get() { ssize_t rc = phobos_get_cpp(&descriptor, 1, NULL, NULL); - /* + /* * Need to close file here so the stream knows * there will be no more data to read */ close_file(); - if(rc) - { + if (rc) { throw PhobosException("Phobos_file::db_get", "phobos_get", rc); } @@ -106,13 +104,15 @@ void Phobos_file::set_bucket_name(std::string bucket_name) { int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucket_name.c_str()); if (rc) { - throw PhobosException("Phobos_file::set_bucket_name", "pho_attr_set", rc); - } + throw PhobosException( + "Phobos_file::set_bucket_name", "pho_attr_set", rc); + } } /* Pass s3 meta data to Phobos */ void Phobos_file::set_meta_data( - std::vector> meta_data, std::string bucket_name) + std::vector> meta_data, + std::string bucket_name) { set_bucket_name(bucket_name); std::for_each( @@ -121,14 +121,46 @@ void Phobos_file::set_meta_data( int rc = pho_attr_set( &descriptor.xd_attrs, pair.first.c_str(), pair.second.c_str()); if (rc) { - throw PhobosException("Phobos_file::set_meta_data", "pho_attr_set", rc); + throw PhobosException( + "Phobos_file::set_meta_data", "pho_attr_set", rc); } }); } +int add_to_meta_data(const char* key, const char* value, void* udata) +{ + std::vector>* meta_data = + static_cast>*>(udata); + meta_data->push_back(std::make_pair(key, value)); + return 0; +} + // TODO -void Phobos_file::get_meta_data() const +std::vector> Phobos_file::get_meta_data( + std::string object_id) { + std::vector> meta_data; + + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + descriptor.xd_objid = unconsted_object_id; + descriptor.xd_op = PHO_XFER_OP_GETMD; + + int rc = phobos_getmd_cpp(&descriptor, 1, NULL, NULL); + if (rc) { + throw PhobosException( + "Phobos_file::get_meta_data", "phobos_getmd_cpp", rc); + } + + rc = phobos_attrs_foreach_cpp( + &descriptor.xd_attrs, &add_to_meta_data, &meta_data); + if (rc) { + throw PhobosException( + "Phobos_file::get_meta_data", "phobos_attrs_foreach_cpp", rc); + } + + return meta_data; } int Phobos_file::get_fd() const @@ -144,5 +176,5 @@ void Phobos_file::close_file() } } -} // namespace storage +} // namespace storage } // namespace fiphoboserver diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 2cf5fed..d796ca9 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -8,31 +8,35 @@ namespace fiphoboserver { namespace storage { class Phobos_file : public Storage { - public: - Phobos_file(); - ~Phobos_file(); - - Phobos_file(Phobos_file&&) = default; - Phobos_file& operator= (Phobos_file&&) = default; - Phobos_file(const Phobos_file&) = delete; - Phobos_file& operator= (const Phobos_file&) = delete; - - void set_meta_data(std::vector> meta_data, std::string bucket_name) override; - void get_meta_data() const override; - - ssize_t db_put(size_t size) override; - ssize_t db_get() override; - - void prepare_put(std::string file_name, std::string object_id) override; - void prepare_get(std::string file_name, std::string object_id) override; - - int get_fd() const; - private: - /* Implementation specific variables */ - struct pho_xfer_desc descriptor = {0}; - void set_bucket_name(std::string bucket_name); - void close_file(); -}; - -} // namespace storage -} // namespace fiphoboserver + public: + Phobos_file(); + ~Phobos_file(); + + Phobos_file(Phobos_file&&) = default; + Phobos_file& operator=(Phobos_file&&) = default; + Phobos_file(const Phobos_file&) = delete; + Phobos_file& operator=(const Phobos_file&) = delete; + + void set_meta_data( + std::vector> meta_data, + std::string bucket_name) override; + std::vector> get_meta_data( + std::string object_id) override; + + ssize_t db_put(size_t size) override; + ssize_t db_get() override; + + void prepare_put(std::string file_name, std::string object_id) override; + void prepare_get(std::string file_name, std::string object_id) override; + + int get_fd() const; + + private: + /* Implementation specific variables */ + struct pho_xfer_desc descriptor = {0}; + void set_bucket_name(std::string bucket_name); + void close_file(); +}; + +} // namespace storage +} // namespace fiphoboserver diff --git a/src/storage/storage.h b/src/storage/storage.h index e5ee624..30fbd6c 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -1,26 +1,29 @@ #pragma once #include -#include #include +#include namespace fiphoboserver { namespace storage { class Storage { - public: - Storage() {}; - virtual ~Storage() {}; + public: + Storage(){}; + virtual ~Storage(){}; - virtual void set_meta_data(std::vector> meta_data, std::string bucket_name) = 0; - virtual void get_meta_data() const = 0; + virtual void set_meta_data( + std::vector> meta_data, + std::string bucket_name) = 0; + virtual std::vector> get_meta_data( + std::string) = 0; - virtual void prepare_put(std::string, std::string) = 0; - virtual ssize_t db_put(size_t size) = 0; + virtual void prepare_put(std::string, std::string) = 0; + virtual ssize_t db_put(size_t size) = 0; - virtual void prepare_get(std::string, std::string) = 0; - virtual ssize_t db_get() = 0; -}; + virtual void prepare_get(std::string, std::string) = 0; + virtual ssize_t db_get() = 0; +}; -} // namespace storage -} // namespace fiphoboserver +} // namespace storage +} // namespace fiphoboserver diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index cd42c92..27b57aa 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -4,9 +4,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -15,8 +15,9 @@ namespace fiphoboserver { namespace stream { /* We need to pass the storage implementation to the base constructor */ -Fifo::Fifo(std::unique_ptr input_storage) : Stream(std::move(input_storage)) -{ +Fifo::Fifo(std::unique_ptr input_storage) : + Stream(std::move(input_storage)) +{ create_fifo(); } @@ -30,24 +31,28 @@ Fifo::~Fifo() } } /* Simply pass along to the backend */ -void Fifo::set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const -{ +void Fifo::set_storage_meta_data( + std::vector> meta_data, + std::string bucket_name) const +{ storage->set_meta_data(meta_data, bucket_name); } -// TODO -void Fifo::get_meta_data() const +// TODO +std::vector> Fifo::get_meta_data( + std::string object_id) const { + return std::move(storage->get_meta_data(object_id)); } -/* +/* * Generate a randomly named FIFO * Opening here would block so we don't */ void Fifo::create_fifo() { fifo_name = std::tmpnam(nullptr); - int rc = mkfifo(fifo_name.c_str(), 0777); + int rc = mkfifo(fifo_name.c_str(), 0777); if (rc < 0) { if (errno != EEXIST) { throw IOException("Fifo::create_fifo", "mkfifo", errno); @@ -56,7 +61,7 @@ void Fifo::create_fifo() } } -/* +/* * Tell the backend that it must be ready for a PUT operation * Opening the FIFO here will not block as the backend will * open on the other end @@ -79,7 +84,7 @@ void Fifo::start_put(ssize_t size, std::string object_id) } } -/* +/* * Repeatedly called to write buffer to FIFO until * the server's message body is depleted */ @@ -98,7 +103,7 @@ void Fifo::finish_put() db_result.get(); } -/* +/* * Tell the backend that it must be ready for a GET operation * Opening the FIFO here will not block as the backend will * open on the other end @@ -121,10 +126,10 @@ void Fifo::start_get(std::string object_id) } } -/* +/* * Repeatedly called to read from FIFO into buffer * until the FIFO is empty and closed on the - * writing end + * writing end * Returns the number of bytes read */ ssize_t Fifo::get(void* buf, size_t count) const @@ -142,6 +147,5 @@ void Fifo::finish_get() db_result.get(); } -} // namespace stream -} // namespace fiphoboserver - +} // namespace stream +} // namespace fiphoboserver diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 4523e65..fe1c776 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -4,35 +4,38 @@ namespace fiphoboserver { namespace stream { class Fifo : public Stream { - public: - Fifo(std::unique_ptr input_storage); - ~Fifo(); - - Fifo(Fifo&&) = default; - Fifo& operator= (Fifo&&) = default; - Fifo(const Fifo&) = delete; - Fifo& operator= (const Fifo&) = delete; - - void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const override; - void get_meta_data() const override; - - void start_put(ssize_t size, std::string object_id) override; - void put(std::unique_ptr buffer) const override; - void finish_put() override; - - void start_get(std::string object_id) override; - ssize_t get(void *buf, size_t count) const override; - void finish_get() override; - - /* Implementation specific details */ - int get_fifo_descriptor() const { return fifo_descriptor; } - std::string get_fifo_name() const { return fifo_name; } - - private: - void create_fifo(); - std::string fifo_name; - int fifo_descriptor = -1; + public: + Fifo(std::unique_ptr input_storage); + ~Fifo(); + + Fifo(Fifo&&) = default; + Fifo& operator=(Fifo&&) = default; + Fifo(const Fifo&) = delete; + Fifo& operator=(const Fifo&) = delete; + + void set_storage_meta_data( + std::vector> meta_data, + std::string bucket_name) const override; + std::vector> get_meta_data( + std::string object_id) const override; + + void start_put(ssize_t size, std::string object_id) override; + void put(std::unique_ptr buffer) const override; + void finish_put() override; + + void start_get(std::string object_id) override; + ssize_t get(void* buf, size_t count) const override; + void finish_get() override; + + /* Implementation specific details */ + int get_fifo_descriptor() const { return fifo_descriptor; } + std::string get_fifo_name() const { return fifo_name; } + + private: + void create_fifo(); + std::string fifo_name; + int fifo_descriptor = -1; }; -} // namespace stream -} // namespace fiphoboserver +} // namespace stream +} // namespace fiphoboserver diff --git a/src/stream/stream.h b/src/stream/stream.h index 9632e93..276f99d 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -2,45 +2,50 @@ #include "../storage/storage.h" +#include +#include #include #include #include -#include -#include -#include namespace fiphoboserver { namespace stream { class Stream { - public: - Stream() {}; - /* - * Derived class must be initialised with a - * storage implementation - */ - Stream(std::unique_ptr input_storage): - storage(std::move(input_storage)) {} - - virtual ~Stream() {}; - - /* Responsible for getting meta data to the storage implementation */ - virtual void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const = 0; - virtual void get_meta_data() const = 0; - - virtual void start_put(ssize_t size, std::string object_id) = 0; - virtual void put(std::unique_ptr buffer) const = 0; - virtual void finish_put() = 0; - - virtual void start_get(std::string object_id) = 0; - virtual ssize_t get(void *buf, size_t count) const = 0; - virtual void finish_get() = 0; - protected: - /* Abstract class that requires implementation */ - std::unique_ptr storage; - /* This may be used to wait on the asynchronous backend */ - std::future db_result; + public: + Stream(){}; + /* + * Derived class must be initialised with a + * storage implementation + */ + Stream(std::unique_ptr input_storage) : + storage(std::move(input_storage)) + { + } + + virtual ~Stream(){}; + + /* Responsible for getting meta data to the storage implementation */ + virtual void set_storage_meta_data( + std::vector> meta_data, + std::string bucket_name) const = 0; + virtual std::vector> get_meta_data( + std::string) const = 0; + + virtual void start_put(ssize_t size, std::string object_id) = 0; + virtual void put(std::unique_ptr buffer) const = 0; + virtual void finish_put() = 0; + + virtual void start_get(std::string object_id) = 0; + virtual ssize_t get(void* buf, size_t count) const = 0; + virtual void finish_get() = 0; + + protected: + /* Abstract class that requires implementation */ + std::unique_ptr storage; + /* This may be used to wait on the asynchronous backend */ + std::future db_result; }; -} // namespace stream -} // namespace fiphoboserver +} // namespace stream +} // namespace fiphoboserver diff --git a/test/s3_call_script.py b/test/s3_call_script.py index c6c4159..c1201c2 100644 --- a/test/s3_call_script.py +++ b/test/s3_call_script.py @@ -6,6 +6,7 @@ import argparse parser = argparse.ArgumentParser() parser.add_argument('-put', action = 'store_true') parser.add_argument('-get', action = 'store_true') +parser.add_argument('-getmd', action = 'store_true') parser.add_argument('-key', dest = 'key_string', default = 'MyKey') parser.add_argument('-putfile', dest = 'put_file_string', default = 'EmperorWu.txt') parser.add_argument('-getfile', dest = 'get_file_string', default = 'get_file.txt') @@ -68,12 +69,13 @@ if(args.get): # get_file.write(response["Body"].read()) # This is another command (HEAD request) that will result in a getmd call to phobos! -# response = s3_client.head_object( -# Bucket=first_bucket_name, -# Key=first_file_name -# ) -# print("head_object: ") -# print(response) +if(args.getmd): + response = s3_client.head_object( + Bucket=first_bucket_name, + Key=args.key_string + ) + print("Meta data:") + print(response["Metadata"]) # response = s3_client.put_object( # Bucket=first_bucket_name, -- GitLab From 7db353d5bffb9052c6ad6696229423ceb56f2d79 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 28 Feb 2020 15:23:00 +0000 Subject: [PATCH 22/62] Apdate documentation to new Markdown and add OpenAPI Yaml file --- .../fiphoboserver/FIPhoboServerException.md | 10 +- .../fiphoboserver/GetRequestHandler.md | 18 +- doc/Markdown/fiphoboserver/IOException.md | 2 +- .../fiphoboserver/PutRequestHandler.md | 14 +- doc/Markdown/fiphoboserver/S3_header.md | 12 +- doc/Markdown/fiphoboserver/index.md | 4 +- .../fiphoboserver/storage/PhobosException.md | 2 +- .../fiphoboserver/storage/Phobos_file.md | 18 +- doc/Markdown/fiphoboserver/storage/Storage.md | 16 +- doc/Markdown/fiphoboserver/storage/index.md | 4 +- doc/Markdown/fiphoboserver/stream/Fifo.md | 26 +-- doc/Markdown/fiphoboserver/stream/Stream.md | 30 +-- doc/Markdown/fiphoboserver/stream/index.md | 4 +- doc/Markdown/todo.md | 6 +- doc/openapi.yaml | 193 ++++++++++++++++++ 15 files changed, 277 insertions(+), 82 deletions(-) create mode 100644 doc/openapi.yaml diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index 7d9d2db..a413408 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -33,7 +33,7 @@ the internal message -[Go to Top](./FIPhoboServerException.md) +[Go to Top](#fiphoboserver-FIPhoboServerException) ## Public Functions ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException () @@ -51,7 +51,7 @@ default constructor * virtual -[Go to Top](./FIPhoboServerException.md) +[Go to Top](#fiphoboserver-FIPhoboServerException) ### public fiphoboserver::FIPhoboServerException::FIPhoboServerException (const char *message) @@ -80,7 +80,7 @@ constructor for a message with information on the exception * virtual -[Go to Top](./FIPhoboServerException.md) +[Go to Top](#fiphoboserver-FIPhoboServerException) ### public const char * fiphoboserver::FIPhoboServerException::what () const noexcept override @@ -111,7 +111,7 @@ get information on this exception * virtual -[Go to Top](./FIPhoboServerException.md) +[Go to Top](#fiphoboserver-FIPhoboServerException) -[fiphoboserver-storage-PhobosException]:./storage/PhobosException.md [fiphoboserver-IOException]:./IOException.md +[fiphoboserver-storage-PhobosException]:./storage/PhobosException.md diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index 9197316..d916132 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -49,7 +49,7 @@ first function to be called when a new request comes in * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -78,7 +78,7 @@ This is not used in this case because GET requests don't usually have a body * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override @@ -94,7 +94,7 @@ function called when the incoming message is finished * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override @@ -121,7 +121,7 @@ Not supported in our case! * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::requestComplete () noexcept override @@ -146,7 +146,7 @@ function ... * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onError (proxygen::ProxygenError err) noexcept override @@ -173,7 +173,7 @@ Not supported in our case! * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onEgressPaused () noexcept override @@ -195,7 +195,7 @@ Contents are copies from a proxygen example * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onEgressResumed () noexcept override @@ -217,7 +217,7 @@ Contents are copies from a proxygen example * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) ### public fiphoboserver::GetRequestHandler::GetRequestHandler (std::unique_ptr< stream::Stream > input_stream) @@ -246,7 +246,7 @@ Constructor for stream class initialization. * virtual -[Go to Top](./GetRequestHandler.md) +[Go to Top](#fiphoboserver-GetRequestHandler) [fiphoboserver-stream-Stream]:./stream/Stream.md [todo]:./../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index a168bd4..1a6f57b 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -44,6 +44,6 @@ constructor * virtual -[Go to Top](./IOException.md) +[Go to Top](#fiphoboserver-IOException) [fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 5518778..4f85e4a 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -47,7 +47,7 @@ first function to be called when a new request comes in * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -76,7 +76,7 @@ This is not used in this case because GET requests don't usually have a body * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onEOM () noexcept override @@ -92,7 +92,7 @@ function called when the incoming message is finished * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override @@ -119,7 +119,7 @@ Not supported in our case! * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::requestComplete () noexcept override @@ -144,7 +144,7 @@ function ... * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onError (proxygen::ProxygenError err) noexcept override @@ -171,7 +171,7 @@ Not supported in our case! * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) ### public fiphoboserver::PutRequestHandler::PutRequestHandler (std::unique_ptr< stream::Stream > input_stream) @@ -200,7 +200,7 @@ Constructor for stream class initialization. * virtual -[Go to Top](./PutRequestHandler.md) +[Go to Top](#fiphoboserver-PutRequestHandler) [fiphoboserver-stream-Stream]:./stream/Stream.md [todo]:./../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index c654a6c..1fe1a29 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -44,7 +44,7 @@ sets the internal header instance to the given header object * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) ### public std::string fiphoboserver::S3_header::getBucket () @@ -74,7 +74,7 @@ extracts the name of the S3 bucket requested in the HTTP message * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) ### public std::string fiphoboserver::S3_header::getKey () @@ -104,7 +104,7 @@ extracts the name of the key id requested in the HTTP message * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) ### public bool fiphoboserver::S3_header::isCreateBucketRequest () @@ -135,7 +135,7 @@ This is convenient since we don't really support buckets themselves and can easi * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) ### public size_t fiphoboserver::S3_header::getBodyLength () @@ -166,7 +166,7 @@ This reads out the header "Content-Length" * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) ### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::getMetaData () @@ -197,5 +197,5 @@ In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE * virtual -[Go to Top](./S3_header.md) +[Go to Top](#fiphoboserver-S3_header) diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 520d6ce..d22d9b6 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -22,8 +22,8 @@ +[fiphoboserver-IOException]:./IOException.md [fiphoboserver-storage-Storage]:./storage/Storage.md [fiphoboserver-storage-PhobosException]:./storage/PhobosException.md -[fiphoboserver-stream-Stream]:./stream/Stream.md -[fiphoboserver-IOException]:./IOException.md [fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md +[fiphoboserver-stream-Stream]:./stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index cdd7b60..5ad7b6b 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -44,6 +44,6 @@ constructor * virtual -[Go to Top](./PhobosException.md) +[Go to Top](#fiphoboserver-storage-PhobosException) [fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 8136424..65767e4 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -42,7 +42,7 @@ default constructor * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public fiphoboserver::storage::Phobos_file::~Phobos_file () @@ -58,7 +58,7 @@ default destructor * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override @@ -87,7 +87,7 @@ set the metadata an object that is added to the database should get * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public void fiphoboserver::storage::Phobos_file::get_meta_data () const override @@ -104,7 +104,7 @@ get the metadata associated to the current object as a vector of key:value pairs * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public ssize_t fiphoboserver::storage::Phobos_file::db_put (size_t size) override @@ -137,7 +137,7 @@ Puts data to the databse. * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public ssize_t fiphoboserver::storage::Phobos_file::db_get () override @@ -166,7 +166,7 @@ Gets data from the databse. * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override @@ -195,7 +195,7 @@ Starts a put operation to the database. * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) ### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override @@ -224,7 +224,7 @@ Starts a get operation to the database. * virtual -[Go to Top](./Phobos_file.md) +[Go to Top](#fiphoboserver-storage-Phobos_file) -[fiphoboserver-storage-Storage]:./Storage.md [todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./Storage.md diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index cc1356a..01f3e80 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -67,7 +67,7 @@ set the metadata an object that is added to the database should get * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ### public void fiphoboserver::storage::Storage::get_meta_data () const =0 @@ -84,7 +84,7 @@ get the metadata associated to the current object as a vector of key:value pairs * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ## PUT functions ### public void fiphoboserver::storage::Storage::prepare_put (std::string file_name, std::string object_id)=0 @@ -114,7 +114,7 @@ Starts a put operation to the database. * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ### public ssize_t fiphoboserver::storage::Storage::db_put (size_t size)=0 @@ -147,7 +147,7 @@ Puts data to the databse. * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ## GET functions ### public void fiphoboserver::storage::Storage::prepare_get (std::string file_name, std::string object_id)=0 @@ -177,7 +177,7 @@ Starts a get operation to the database. * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ### public ssize_t fiphoboserver::storage::Storage::db_get ()=0 @@ -206,7 +206,7 @@ Gets data from the databse. * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ## Public Functions ### public fiphoboserver::storage::Storage::Storage () @@ -224,7 +224,7 @@ default constructor * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) ### public fiphoboserver::storage::Storage::~Storage () @@ -241,6 +241,6 @@ default destructor * virtual -[Go to Top](./Storage.md) +[Go to Top](#fiphoboserver-storage-Storage) [fiphoboserver-storage-Phobos_file]:./Phobos_file.md diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index e0fb3e3..8c918ef 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -13,7 +13,7 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: -[fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md [fiphoboserver-storage-Phobos_file]:./Phobos_file.md -[fiphoboserver-storage-Storage]:./Storage.md +[fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md [todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 16846d8..a38d97c 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -69,7 +69,7 @@ get the descriptor of the fifo * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public std::string fiphoboserver::stream::Fifo::get_fifo_name () const @@ -103,7 +103,7 @@ get the name of the fifo * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ## Public Functions ### public fiphoboserver::stream::Fifo::Fifo (std::unique_ptr< storage::Storage > input_storage) @@ -125,7 +125,7 @@ default constructor * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public fiphoboserver::stream::Fifo::~Fifo () @@ -141,7 +141,7 @@ default destructor * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override @@ -175,7 +175,7 @@ setting the metadata that the created object should get * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::get_meta_data () const override @@ -192,7 +192,7 @@ get the metadata from a stored object * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::start_put (ssize_t size, std::string object_id) override @@ -221,7 +221,7 @@ start a put operation * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::put (std::unique_ptr< folly::IOBuf > buffer) const override @@ -250,7 +250,7 @@ add a chunk of data to the object * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::finish_put () override @@ -266,7 +266,7 @@ end a put operation * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::start_get (std::string object_id) override @@ -294,7 +294,7 @@ start a gett operation * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public ssize_t fiphoboserver::stream::Fifo::get (void *buf, size_t count) const override @@ -331,7 +331,7 @@ gets a chunk of data of the object * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::finish_get () override @@ -347,8 +347,8 @@ end a get operation * virtual -[Go to Top](./Fifo.md) +[Go to Top](#fiphoboserver-stream-Fifo) -[fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 9196bb9..29b6e94 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -64,7 +64,7 @@ start a put operation * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::put (std::unique_ptr< folly::IOBuf > buffer) const =0 @@ -93,7 +93,7 @@ add a chunk of data to the object * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::finish_put ()=0 @@ -109,7 +109,7 @@ end a put operation * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::start_get (std::string object_id)=0 @@ -137,7 +137,7 @@ start a gett operation * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public ssize_t fiphoboserver::stream::Stream::get (void *buf, size_t count) const =0 @@ -174,7 +174,7 @@ gets a chunk of data of the object * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::finish_get ()=0 @@ -190,7 +190,7 @@ end a get operation * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ## Protected Attributes ### protected std::unique_ptr< storage::Storage > @@ -203,6 +203,8 @@ an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will +[Go to Top](#fiphoboserver-stream-Stream) + ### protected fiphoboserver::stream::Stream::db_result a future object that can be used to get the results from the asynchronous backend @@ -222,7 +224,7 @@ a future object that can be used to get the results from the asynchronous backen -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ## Public Functions ### public fiphoboserver::stream::Stream::Stream () @@ -240,7 +242,7 @@ default constructor * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public fiphoboserver::stream::Stream::Stream (std::unique_ptr< storage::Storage > input_storage) @@ -273,7 +275,7 @@ constructor with storage implementation * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public fiphoboserver::stream::Stream::~Stream () @@ -290,7 +292,7 @@ default destructor * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const =0 @@ -324,7 +326,7 @@ setting the metadata that the created object should get * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::get_meta_data () const =0 @@ -341,9 +343,9 @@ get the metadata from a stored object * virtual -[Go to Top](./Stream.md) +[Go to Top](#fiphoboserver-stream-Stream) -[fiphoboserver-storage-Storage]:./../storage/Storage.md -[fiphoboserver-stream-Stream]:./Stream.md [fiphoboserver-stream-Fifo]:./Fifo.md +[fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 871406f..d82da19 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -12,7 +12,7 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: -[fiphoboserver-storage-Storage]:./../storage/Storage.md -[fiphoboserver-stream-Stream]:./Stream.md [fiphoboserver-stream-Fifo]:./Fifo.md +[fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo +[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 826c6ea..573c7ca 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -21,9 +21,9 @@ -[fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md [fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md -[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name [fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor +[fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md +[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name +[fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md [fiphoboserver-stream-Stream-db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result diff --git a/doc/openapi.yaml b/doc/openapi.yaml new file mode 100644 index 0000000..b4dcb61 --- /dev/null +++ b/doc/openapi.yaml @@ -0,0 +1,193 @@ +openapi: 3.0.1 +info: + title: FiPhoboServer + description: >- + Server to accept S3 requests to store and retrieve data and use the phobos + tape library as storage. + version: 1.0.1 +tags: + - name: bucket + description: Operations on the buckets + - name: object + description: Operations on the objects inside buckets +servers: + - url: 'http://localhost:11000/' + description: Local host server for testing purposes +paths: + '/{bucket}': + parameters: + - name: bucket + in: path + description: Name of the bucket + required: true + schema: + type: string + minimum: 1 + get: + tags: + - bucket + summary: Lists all objects in the bucket + operationId: getBucket + responses: + '200': + description: Success + content: + application/xml: + schema: + $ref: '#/components/schemas/ListObjectsV2Output' + '403': + description: Access denied + '404': + description: No such bucket + '500': + description: Internal Server Error + put: + tags: + - bucket + summary: Creates a new bucket + operationId: putBucket + requestBody: + description: Information on the bucket that should be created + content: + application/xml: + schema: + $ref: '#/components/schemas/CreateBucketInfo' + required: true + responses: + '200': + description: Success + '403': + description: Access denied + '500': + description: Internal Server Error + '/{bucket}/{key}': + parameters: + - name: bucket + in: path + description: Name of the bucket + required: true + schema: + type: string + minimum: 1 + - name: key + in: path + description: Identifier of the object to store + required: true + schema: + type: string + minimum: 1 + get: + tags: + - object + summary: Retrieves an object from the storage + operationId: getObject + description: >- + The object `key` will be searched in bucket `bucket` and the contents + returned in chunks to the user + responses: + '200': + description: Success + content: + application/text: + schema: + type: string + '403': + description: Access denied + '404': + description: No such key + '500': + description: Internal Server Error + put: + tags: + - object + summary: Add an object to the storage + operationId: putObject + description: >- + The contents of the body will be added as a new object with id `key` to + the bucket `bucket` + parameters: + - in: header + name: Content-Length + schema: + type: integer + required: true + requestBody: + description: File contents to add + content: + application/text: + schema: + type: string + required: true + responses: + '100': + description: Continue + '200': + description: Success + '403': + description: Access denied + '500': + description: Internal Server Error + delete: + tags: + - object + summary: Delete an object + operationId: deleteObject + description: Deletes an object from the internal storage + responses: + '204': + description: No Content + '403': + description: Access denied + '500': + description: Internal Server Error + '501': + description: Not Implemented +components: + schemas: + CreateBucketInfo: + type: object + properties: + xmlns: + type: string + default: 'http://s3.amazonaws.com/doc/2006-03-01/' + xml: + attribute: true + LocationContraint: + type: string + xml: + name: CreateBucketConfiguration + ListObjectsV2Output: + type: object + properties: + IsTruncated: + type: boolean + Contents: + type: array + items: + properties: + ETag: + type: string + Key: + type: string + LastModified: + type: string + Owner: + properties: + DisplayName: + type: string + ID: + type: string + Size: + type: integer + Name: + type: string + Delimiter: + type: string + MaxKeys: + type: string + EncodingType: + type: string + KeyCount: + type: string + xml: + name: ListObjectsV2Output \ No newline at end of file -- GitLab From b3fc20ca330cc138ea021d2ba61d80b04f7b1e60 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 28 Feb 2020 15:28:43 +0000 Subject: [PATCH 23/62] Test openapi with aws --- doc/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/openapi.yaml b/doc/openapi.yaml index b4dcb61..e34c17f 100644 --- a/doc/openapi.yaml +++ b/doc/openapi.yaml @@ -11,7 +11,7 @@ tags: - name: object description: Operations on the objects inside buckets servers: - - url: 'http://localhost:11000/' + - url: 'https://s3-eu-west-1.amazonaws.com' description: Local host server for testing purposes paths: '/{bucket}': -- GitLab From 1e665162166fd18b835e399e034b3f5bdfce6356 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 28 Feb 2020 15:32:09 +0000 Subject: [PATCH 24/62] Revert AWS test --- doc/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/openapi.yaml b/doc/openapi.yaml index e34c17f..b4dcb61 100644 --- a/doc/openapi.yaml +++ b/doc/openapi.yaml @@ -11,7 +11,7 @@ tags: - name: object description: Operations on the objects inside buckets servers: - - url: 'https://s3-eu-west-1.amazonaws.com' + - url: 'http://localhost:11000/' description: Local host server for testing purposes paths: '/{bucket}': -- GitLab From 780c3b6bcec98db3b697e3507685303a1ebbb73f Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 5 Mar 2020 12:18:40 +0000 Subject: [PATCH 25/62] Add optional Doxygen XML parser command to cmake --- doc/CMakeLists.txt | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index c6e517f..61ac82f 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -15,18 +15,36 @@ set(INCLUDE_DOXYFILE "@INCLUDE = ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile") # TODO: The main dependency is currently doc/Doxyfile, but there should be # a depencency on *every* file being documented! add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/html/index.html + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in - DEPENDS ${EXSEISDAT_DOCUMENTED_EXAMPLES} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Building the documentation..." ) +if(CUSTOM_DOCU_PATH) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/output.json + COMMAND python3 + ARGS "${CUSTOM_DOCU_PATH}/xml_to_json_parser/xml_to_json_parser.py" "${CMAKE_CURRENT_BINARY_DIR}/xml" output.json + MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Parsing the documentation XML..." + ) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/Markdown/index.md + COMMAND python3 + ARGS "${CUSTOM_DOCU_PATH}/json_to_Markdown_converter/json_to_Markdown_converter.py" output.json "${CMAKE_CURRENT_SOURCE_DIR}/Markdown" + MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/output.json + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Creating the Markdown documentation..." + ) +endif(CUSTOM_DOCU_PATH) + # Target to wire up building the documentation to the ALL target. add_custom_target( doc ALL DEPENDS - ${CMAKE_CURRENT_BINARY_DIR}/html/index.html + ${CMAKE_CURRENT_SOURCE_DIR}/Markdown/index.md ) -- GitLab From 855c8e8157df1c9ef01b0bfc1a072c150d813bf8 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 5 Mar 2020 12:22:12 +0000 Subject: [PATCH 26/62] Change output.json to docu.json --- doc/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 61ac82f..fb98d0b 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -24,9 +24,9 @@ add_custom_command( if(CUSTOM_DOCU_PATH) add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/output.json + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/docu.json COMMAND python3 - ARGS "${CUSTOM_DOCU_PATH}/xml_to_json_parser/xml_to_json_parser.py" "${CMAKE_CURRENT_BINARY_DIR}/xml" output.json + ARGS "${CUSTOM_DOCU_PATH}/xml_to_json_parser/xml_to_json_parser.py" "${CMAKE_CURRENT_BINARY_DIR}/xml" docu.json MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Parsing the documentation XML..." @@ -34,8 +34,8 @@ if(CUSTOM_DOCU_PATH) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/Markdown/index.md COMMAND python3 - ARGS "${CUSTOM_DOCU_PATH}/json_to_Markdown_converter/json_to_Markdown_converter.py" output.json "${CMAKE_CURRENT_SOURCE_DIR}/Markdown" - MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/output.json + ARGS "${CUSTOM_DOCU_PATH}/json_to_Markdown_converter/json_to_Markdown_converter.py" docu.json "${CMAKE_CURRENT_SOURCE_DIR}/Markdown" + MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/docu.json WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Creating the Markdown documentation..." ) -- GitLab From 166a7194d003d4f16f7dd3793568302999f26b8a Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 5 Mar 2020 15:14:37 +0000 Subject: [PATCH 27/62] Update Markdown (Links now sorted) and gitignore --- .gitignore | 1 + doc/Markdown/fiphoboserver/index.md | 4 ++-- doc/Markdown/fiphoboserver/storage/Phobos_file.md | 2 +- doc/Markdown/fiphoboserver/storage/index.md | 4 ++-- doc/Markdown/fiphoboserver/stream/Fifo.md | 2 +- doc/Markdown/fiphoboserver/stream/Stream.md | 2 +- doc/Markdown/fiphoboserver/stream/index.md | 2 +- doc/Markdown/todo.md | 4 ++-- 8 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index d171c66..0a2f2bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.swp todo.txt +.DS_Store diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index d22d9b6..e90faf3 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -22,8 +22,8 @@ +[fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md [fiphoboserver-IOException]:./IOException.md -[fiphoboserver-storage-Storage]:./storage/Storage.md [fiphoboserver-storage-PhobosException]:./storage/PhobosException.md -[fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md +[fiphoboserver-storage-Storage]:./storage/Storage.md [fiphoboserver-stream-Stream]:./stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 65767e4..b309a6e 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -226,5 +226,5 @@ Starts a get operation to the database. [Go to Top](#fiphoboserver-storage-Phobos_file) -[todo]:./../../todo.md#todo [fiphoboserver-storage-Storage]:./Storage.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index 8c918ef..e0fb3e3 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -13,7 +13,7 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: -[fiphoboserver-storage-Phobos_file]:./Phobos_file.md [fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md -[todo]:./../../todo.md#todo +[fiphoboserver-storage-Phobos_file]:./Phobos_file.md [fiphoboserver-storage-Storage]:./Storage.md +[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index a38d97c..a186930 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -349,6 +349,6 @@ end a get operation [Go to Top](#fiphoboserver-stream-Fifo) +[fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo -[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 29b6e94..1e915b1 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -345,7 +345,7 @@ get the metadata from a stored object [Go to Top](#fiphoboserver-stream-Stream) +[fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Fifo]:./Fifo.md [fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo -[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index d82da19..301d295 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -12,7 +12,7 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: +[fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Fifo]:./Fifo.md [fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo -[fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 573c7ca..2d58c8a 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -22,8 +22,8 @@ [fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor [fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md -[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name [fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md +[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor +[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name [fiphoboserver-stream-Stream-db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result -- GitLab From e35b9eb71514a317dacf6be2c1b745de0be8e87f Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Thu, 5 Mar 2020 16:29:27 +0000 Subject: [PATCH 28/62] Document the new members --- .../fiphoboserver/PutRequestHandler.md | 7 +- doc/Markdown/fiphoboserver/index.md | 1 + .../fiphoboserver/storage/Phobos_file.md | 101 ++++++++++++++++- doc/Markdown/fiphoboserver/storage/Storage.md | 22 +++- doc/Markdown/fiphoboserver/stream/Fifo.md | 104 +++++++++++++++++- doc/Markdown/fiphoboserver/stream/Stream.md | 25 ++++- src/server/getmd_request_handler.h | 25 +++++ src/server/put_request_handler.h | 4 +- src/storage/phobos_file.h | 4 + src/storage/storage.h | 4 +- src/stream/fifo.h | 4 + src/stream/stream.h | 14 ++- 12 files changed, 295 insertions(+), 20 deletions(-) diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index 4f85e4a..f9d6922 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -11,7 +11,7 @@ Inherits from RequestHandler. | Name | Description | | ---- | ---- | | [onRequest](#fiphoboserver-PutRequestHandler-onRequest) | first function to be called when a new request comes in | -| [onBody](#fiphoboserver-PutRequestHandler-onBody) | function called on every body chunk belonging to this message | +| [onBody](#fiphoboserver-PutRequestHandler-onBody) | ccalled with each chunk of the body coming in for the request | | [onEOM](#fiphoboserver-PutRequestHandler-onEOM) | function called when the incoming message is finished | | [onUpgrade](#fiphoboserver-PutRequestHandler-onUpgrade) | function called on upgrade | | [requestComplete](#fiphoboserver-PutRequestHandler-requestComplete) | function ... | @@ -51,14 +51,14 @@ first function to be called when a new request comes in ### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override -function called on every body chunk belonging to this message +ccalled with each chunk of the body coming in for the request #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::unique_ptr< folly::IOBuf > | body | buffer for the body | +| std::unique_ptr< folly::IOBuf > | body | chunk of data to be added to the file | @@ -66,7 +66,6 @@ function called on every body chunk belonging to this message -This is not used in this case because GET requests don't usually have a body diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index e90faf3..5096e6b 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -7,6 +7,7 @@ | Name | Description | | ---- | ---- | | [FIPhoboServerException](./FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | +| [GetmdRequestHandler](./GetmdRequestHandler.md) | proxygen class implementation for handling HEAD requests | | [GetRequestHandler](./GetRequestHandler.md) | proxygen class implementation for handling GET requests | | [IOException](./IOException.md) | exceptions specifically for errors in I/O | | [PutRequestHandler](./PutRequestHandler.md) | proxygen class implementation for handling PUT requests | diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index b309a6e..aa49695 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -18,6 +18,10 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. | ---- | ---- | | [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file) | default constructor | | [~Phobos_file](#fiphoboserver-storage-Phobos_file-~Phobos_file) | default destructor | +| [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file-1) | default move contructor | +| [operator=](#fiphoboserver-storage-Phobos_file-operator=) | default assignment operator for rvalue references | +| [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file-2) | copy constructor deleted | +| [operator=](#fiphoboserver-storage-Phobos_file-operator=-1) | assignment operator deleted | | [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata an object that is added to the database should get | | [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | | [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. | @@ -54,6 +58,80 @@ default destructor +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +### public fiphoboserver::storage::Phobos_file::Phobos_file (Phobos_file &&)=default + +default move contructor + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +### public [Phobos_file][fiphoboserver-storage-Phobos_file] & fiphoboserver::storage::Phobos_file::operator= (Phobos_file &&)=default + +default assignment operator for rvalue references + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| [Phobos_file][fiphoboserver-storage-Phobos_file] & | | + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +### public fiphoboserver::storage::Phobos_file::Phobos_file (const Phobos_file &)=delete + +copy constructor deleted + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +### public [Phobos_file][fiphoboserver-storage-Phobos_file] & fiphoboserver::storage::Phobos_file::operator= (const Phobos_file &)=delete + +assignment operator deleted + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| [Phobos_file][fiphoboserver-storage-Phobos_file] & | | + + + + + #### Qualifiers: * virtual @@ -89,18 +167,36 @@ set the metadata an object that is added to the database should get [Go to Top](#fiphoboserver-storage-Phobos_file) -### public void fiphoboserver::storage::Phobos_file::get_meta_data () const override +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::storage::Phobos_file::get_meta_data (std::string object_id) override get the metadata associated to the current object as a vector of key:value pairs +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | The object id of the requested object | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | + + + + + + + + + + #### Qualifiers: -* const * virtual @@ -226,5 +322,6 @@ Starts a get operation to the database. [Go to Top](#fiphoboserver-storage-Phobos_file) +[fiphoboserver-storage-Phobos_file]:./Phobos_file.md [fiphoboserver-storage-Storage]:./Storage.md [todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index 01f3e80..4531cf0 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -69,18 +69,36 @@ set the metadata an object that is added to the database should get [Go to Top](#fiphoboserver-storage-Storage) -### public void fiphoboserver::storage::Storage::get_meta_data () const =0 +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::storage::Storage::get_meta_data (std::string object_id)=0 get the metadata associated to the current object as a vector of key:value pairs +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | The object id of the requested object | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | + + + + + + + + + + #### Qualifiers: -* const * virtual diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index a186930..ef2e9ba 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -25,8 +25,12 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | ---- | ---- | | [Fifo](#fiphoboserver-stream-Fifo-Fifo) | default constructor | | [~Fifo](#fiphoboserver-stream-Fifo-~Fifo) | default destructor | +| [Fifo](#fiphoboserver-stream-Fifo-Fifo-1) | default move contructor | +| [operator=](#fiphoboserver-stream-Fifo-operator=) | default assignment operator for rvalue references | +| [Fifo](#fiphoboserver-stream-Fifo-Fifo-2) | copy constructor deleted | +| [operator=](#fiphoboserver-stream-Fifo-operator=-1) | assignment operator deleted | | [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get | -| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata from a stored object | +| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | | [start_put](#fiphoboserver-stream-Fifo-start_put) | start a put operation | | [put](#fiphoboserver-stream-Fifo-put) | add a chunk of data to the object | | [finish_put](#fiphoboserver-stream-Fifo-finish_put) | end a put operation | @@ -137,6 +141,80 @@ default destructor +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-stream-Fifo) + +### public fiphoboserver::stream::Fifo::Fifo (Fifo &&)=default + +default move contructor + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-stream-Fifo) + +### public [Fifo][fiphoboserver-stream-Fifo] & fiphoboserver::stream::Fifo::operator= (Fifo &&)=default + +default assignment operator for rvalue references + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| [Fifo][fiphoboserver-stream-Fifo] & | | + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-stream-Fifo) + +### public fiphoboserver::stream::Fifo::Fifo (const Fifo &)=delete + +copy constructor deleted + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-stream-Fifo) + +### public [Fifo][fiphoboserver-stream-Fifo] & fiphoboserver::stream::Fifo::operator= (const Fifo &)=delete + +assignment operator deleted + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| [Fifo][fiphoboserver-stream-Fifo] & | | + + + + + #### Qualifiers: * virtual @@ -177,9 +255,28 @@ setting the metadata that the created object should get [Go to Top](#fiphoboserver-stream-Fifo) -### public void fiphoboserver::stream::Fifo::get_meta_data () const override +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::stream::Fifo::get_meta_data (std::string object_id) const override + +get the metadata associated to the current object as a vector of key:value pairs + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | The object id of the requested object | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | + + + + + + -get the metadata from a stored object @@ -350,5 +447,6 @@ end a get operation [Go to Top](#fiphoboserver-stream-Fifo) [fiphoboserver-storage-Storage]:./../storage/Storage.md +[fiphoboserver-stream-Fifo]:./Fifo.md [fiphoboserver-stream-Stream]:./Stream.md [todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 1e915b1..af0da81 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -32,7 +32,7 @@ Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. | [Stream](#fiphoboserver-stream-Stream-Stream-1) | constructor with storage implementation | | [~Stream](#fiphoboserver-stream-Stream-~Stream) | default destructor | | [set_storage_meta_data](#fiphoboserver-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get | -| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata from a stored object | +| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | @@ -328,9 +328,28 @@ setting the metadata that the created object should get [Go to Top](#fiphoboserver-stream-Stream) -### public void fiphoboserver::stream::Stream::get_meta_data () const =0 +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::stream::Stream::get_meta_data (std::string object_id) const =0 + +get the metadata associated to the current object as a vector of key:value pairs + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | object_id | The object id of the requested object | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | + + + + + + -get the metadata from a stored object diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index 5d44b77..949f09d 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -21,21 +21,46 @@ class ResponseHandler; namespace fiphoboserver { +/// +/// @brief proxygen class implementation for handling HEAD requests +/// class GetmdRequestHandler : public proxygen::RequestHandler { public: + + /// + /// @copydoc GetRequestHandler::onRequest + /// void onRequest( std::unique_ptr headers) noexcept override; + /// + /// @copydoc GetRequestHandler::onBody + /// void onBody(std::unique_ptr body) noexcept override; + /// + /// @copydoc GetRequestHandler::onEOM + /// void onEOM() noexcept override; + /// + /// @copydoc GetRequestHandler::onUpgrade + /// void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + /// + /// @copydoc GetRequestHandler::requestComplete + /// void requestComplete() noexcept override; + /// + /// @copydoc GetRequestHandler::onError + /// void onError(proxygen::ProxygenError err) noexcept override; + /// + /// @copydoc GetRequestHandler::GetRequestHandler + /// GetmdRequestHandler(std::unique_ptr input_stream) : stream(std::move(input_stream)) { diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 6b8ee32..c4a752c 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -29,7 +29,9 @@ class PutRequestHandler : public proxygen::RequestHandler { std::unique_ptr headers) noexcept override; /// - /// @copydoc GetRequestHandler::onBody + /// @brief ccalled with each chunk of the body coming in for the request + /// + /// @param body chunk of data to be added to the file /// void onBody(std::unique_ptr body) noexcept override; diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 58be6ad..16f75ef 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -19,9 +19,13 @@ class Phobos_file : public Storage { /// @copydoc Storage::~Storage ~Phobos_file(); + /// @brief default move contructor Phobos_file(Phobos_file&&) = default; + /// @brief default assignment operator for rvalue references Phobos_file& operator=(Phobos_file&&) = default; + /// @brief copy constructor deleted Phobos_file(const Phobos_file&) = delete; + /// @brief assignment operator deleted Phobos_file& operator=(const Phobos_file&) = delete; /// @copydoc Storage::set_meta_data diff --git a/src/storage/storage.h b/src/storage/storage.h index c161e03..0f29e20 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -50,12 +50,12 @@ class Storage { /// @brief get the metadata associated to the current object as a /// vector of key:value pairs /// - /// @param string The object id of the requested object + /// @param object_id The object id of the requested object /// /// @returns a vector of the key:value metadata pairs /// virtual std::vector> get_meta_data( - std::string) = 0; + std::string object_id) = 0; /// @} /// diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 0df5d56..2f77a57 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -16,9 +16,13 @@ class Fifo : public Stream { /// @copydoc Stream::~Stream ~Fifo(); + /// @brief default move contructor Fifo(Fifo&&) = default; + /// @brief default assignment operator for rvalue references Fifo& operator=(Fifo&&) = default; + /// @brief copy constructor deleted Fifo(const Fifo&) = delete; + /// @brief assignment operator deleted Fifo& operator=(const Fifo&) = delete; /// @copydoc Stream::set_storage_meta_data diff --git a/src/stream/stream.h b/src/stream/stream.h index 9973386..92d47b1 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -55,11 +55,19 @@ class Stream { /// @warning this function is responsible for getting metadata to /// the storage implementation /// - virtual void set_storage_meta_data(std::vector> meta_data, std::string bucket_name) const = 0; + virtual void set_storage_meta_data( + std::vector> meta_data, + std::string bucket_name) const = 0; /// - /// @brief get the metadata from a stored object + /// @brief get the metadata associated to the current object as a + /// vector of key:value pairs /// - virtual void get_meta_data() const = 0; + /// @param object_id The object id of the requested object + /// + /// @returns a vector of the key:value metadata pairs + /// + virtual std::vector> get_meta_data( + std::string object_id) const = 0; /// @name PUT functions /// -- GitLab From daa9c80e517e8bfa80993bbd2e176b8f66f26b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Wed, 11 Mar 2020 14:12:06 +0000 Subject: [PATCH 29/62] Implement Clang Tidy Change log: * use std::move to return from get_meta_data * Fifo::put: replace Folly object argument with void pointer and size_t as arguments and return ssize_t * Fifo::put considers rc < count to be an exception * run Clang Tidy * rename i_ps to IPs in main.cc --- .clang-tidy | 4 +- README.md | 2 +- src/CMakeLists.txt | 2 + src/fiphoboserver_exception.h | 17 +++-- src/main.cc | 4 +- src/server/CMakeLists.txt | 2 - src/server/S3_header.h | 70 +++++++++-------- src/server/get_request_handler.cc | 66 ++++++++-------- src/server/get_request_handler.h | 20 ++--- src/server/getmd_request_handler.cc | 9 +-- src/server/getmd_request_handler.h | 8 +- src/server/put_request_handler.cc | 21 ++--- src/server/put_request_handler.h | 6 +- src/storage/CMakeLists.txt | 2 - .../phobos_cpp_wrapper/phobos_cpp_wrapper.h | 4 +- src/storage/phobos_exception.h | 8 +- src/storage/phobos_file.cc | 76 +++++++++---------- src/storage/phobos_file.h | 10 ++- src/stream/CMakeLists.txt | 2 - src/stream/fifo.cc | 56 +++++++------- src/stream/fifo.h | 20 +++-- src/stream/stream.h | 18 ++--- test/CMakeLists.txt | 44 ----------- test/phobos_file.cc | 60 --------------- tools/run_format.sh | 2 +- tools/run_lint.sh | 2 +- 26 files changed, 218 insertions(+), 317 deletions(-) delete mode 100644 test/CMakeLists.txt delete mode 100644 test/phobos_file.cc diff --git a/.clang-tidy b/.clang-tidy index 6aaac06..298375f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -16,7 +16,7 @@ WarningsAsErrors: '*' HeaderFilterRegex: '' AnalyzeTemporaryDtors: false FormatStyle: none -User: ExSeisDat +User: FiPhoboServer CheckOptions: - key: google-readability-braces-around-statements.ShortStatementLines value: '1' @@ -74,6 +74,6 @@ CheckOptions: - key: readability-identifier-naming.MacroDefinitionCase value: UPPER_CASE - key: readability-identifier-naming.MacroDefinitionPrefix - value: EXSEISDAT_ + value: FIPHOBOSERVER_ ... diff --git a/README.md b/README.md index 1595e79..3d7b885 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ export PHOBOS_LAYOUT_RAID1_repl_count={1,2,3,...} Build with ``` -cmake -DDEBUG=ON -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/ +cmake -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/ make ``` diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4ffe6b2..6426566 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,6 +8,8 @@ add_subdirectory(server) add_library(libfiphoboserver main.cc) +target_compile_features(libfiphoboserver PUBLIC cxx_std_14) + target_link_libraries(libfiphoboserver PUBLIC stream) target_link_libraries(libfiphoboserver PUBLIC storage) target_link_libraries(libfiphoboserver PUBLIC server) diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index 813a5f3..f9c5e94 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -8,27 +8,28 @@ namespace fiphoboserver { class FIPhoboServerException : public std::exception { public: - FIPhoboServerException() : message("") {} + FIPhoboServerException() : m_message("") {} - FIPhoboServerException(const char* message) : message(message) {} + FIPhoboServerException(const char* message) : m_message(message) {} virtual const char* what() const noexcept override { - return message.c_str(); + return m_message.c_str(); } protected: - std::string message; + std::string m_message; }; class IOException : public FIPhoboServerException { public: - IOException(const char* caller, const char* functionName, int returnValue) + IOException(const char* caller, const char* function_name, int return_value) { std::stringstream ss; - ss << "[" << caller << "]: IO Exception occurred: " << functionName - << ": " << std::strerror(returnValue) << " (" << returnValue << ")"; - message = ss.str(); + ss << "[" << caller << "]: IO Exception occurred: " << function_name + << ": " << std::strerror(return_value) << " (" << return_value + << ")"; + m_message = ss.str(); } }; diff --git a/src/main.cc b/src/main.cc index 5a33a22..274e2a2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -94,10 +94,10 @@ int main(int argc, char* argv[]) proxygen::RequestHandlerChain().addThen().build(); options.h2cEnabled = true; - auto diskIOThreadPool = std::make_shared( + auto disk_io_thread_pool = std::make_shared( FLAGS_threads, std::make_shared("FIFOIOThread")); - folly::setCPUExecutor(diskIOThreadPool); + folly::setCPUExecutor(disk_io_thread_pool); proxygen::HTTPServer server(std::move(options)); server.bind(IPs); diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 4be3d55..a1244ae 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -10,7 +10,5 @@ add_library( unsupported_request_handler.cc ) -target_compile_features(server PUBLIC cxx_std_14) - target_link_libraries(server PUBLIC proxygen::proxygen) target_link_libraries(server PUBLIC proxygen::proxygenhttpserver) diff --git a/src/server/S3_header.h b/src/server/S3_header.h index 520408d..c9edf7f 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -5,73 +5,77 @@ namespace fiphoboserver { class S3_header { public: - void setHeaders(std::unique_ptr newHeaders) + void set_headers(std::unique_ptr new_m_headers) { - headers = std::move(newHeaders); + m_headers = std::move(new_m_headers); } - std::string getBucket() + std::string get_bucket() { - if (!headers) { + if (!m_headers) { return ""; } - std::string path = headers->getPath(); - path = path.substr(1, path.size()); - std::string bucketName = path.substr(0, path.find('/')); - return bucketName; + std::string path = m_headers->getPath(); + path = path.substr(1, path.size()); + std::string bucket_name = path.substr(0, path.find('/')); + return bucket_name; } - std::string getKey() + std::string get_key() { - if (!headers) { + if (!m_headers) { return ""; } - std::string path = headers->getPath(); - path = path.substr(1, path.size()); - std::string fileKey = path.substr(path.find('/') + 1, path.size() - 1); + std::string path = m_headers->getPath(); + path = path.substr(1, path.size()); + std::string file_key = path.substr(path.find('/') + 1, path.size() - 1); - if (fileKey.find('/') != std::string::npos - || fileKey.find('\\') != std::string::npos) { + if (file_key.find('/') != std::string::npos + || file_key.find('\\') != std::string::npos) { std::cout << "Bad Key! " << std::endl; return ""; } - return fileKey; + return file_key; } - bool isCreateBucketRequest() + bool is_create_bucket_request() { - if (!headers) { + if (!m_headers) { return false; } - if (getKey() == "") return true; + if (get_key().empty()) { + return true; + } return false; } - size_t getBodyLength() + size_t get_body_length() { - std::string contentLength = - headers->getHeaders().getSingleOrEmpty("Content-Length"); - return static_cast(std::stoi(contentLength)); + std::string content_length = + m_headers->getHeaders().getSingleOrEmpty("Content-Length"); + return static_cast(std::stoi(content_length)); } - std::vector> getMetaData() + std::vector> get_meta_data() { std::vector> metadata; - std::string metaSearchString = "x-amz-meta-"; + std::string meta_search_string = "x-amz-meta-"; - headers->getHeaders().forEach([&](const std::string& header, - const std::string& val) { - if (header.find(metaSearchString) != std::string::npos) { - std::string metaName = - header.substr(metaSearchString.size(), header.size() - 1); - std::pair currentPair(metaName, val); - metadata.push_back(currentPair); + m_headers->getHeaders().forEach([&](const std::string& header, + const std::string& val) { + if (header.find(meta_search_string) != std::string::npos) { + std::string meta_name = + header.substr(meta_search_string.size(), header.size() - 1); + std::pair current_pair( + meta_name, val); + metadata.push_back(current_pair); } }); return metadata; } private: - std::unique_ptr headers; + std::unique_ptr m_headers; }; + } // namespace fiphoboserver diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index e78dd2c..0e47752 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -32,17 +32,17 @@ void GetRequestHandler::onRequest( } /* Creating Bucket queries */ - s3_header.setHeaders(std::move(headers)); + m_s3_header.set_headers(std::move(headers)); try { /* Send meta data to backend through the stream */ - stream->set_storage_meta_data( - s3_header.getMetaData(), s3_header.getBucket()); + m_stream->set_storage_meta_data( + m_s3_header.get_meta_data(), m_s3_header.get_bucket()); /* Tell stream to coordinate with backend to prepare for GET operation */ - stream->start_get(s3_header.getKey()); + m_stream->start_get(m_s3_header.get_key()); - file_closed_ = false; // TODO: Better way of communicating this + m_file_closed = false; // TODO: Better way of communicating this } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -65,16 +65,16 @@ void GetRequestHandler::onRequest( proxygen::ResponseBuilder(downstream_).status(200, "Ok").send(); /* Initiating a read from the stream and creating a body to send */ - readFileScheduled_ = true; + m_read_file_scheduled = true; folly::getCPUExecutor()->add(std::bind( - &GetRequestHandler::readFile, this, + &GetRequestHandler::read_file, this, folly::EventBaseManager::get()->getEventBase())); } -void GetRequestHandler::readFile(folly::EventBase* evb) +void GetRequestHandler::read_file(folly::EventBase* evb) { folly::IOBufQueue buf; - while (!file_closed_ && !paused_) { + while (!m_file_closed && !m_paused) { /* read 4k-ish chunks and foward each one to the client */ auto data = buf.preallocate(4000, 4000); /* @@ -85,16 +85,16 @@ void GetRequestHandler::readFile(folly::EventBase* evb) * for writing */ try { - auto rc = stream->get(data.first, data.second); + auto rc = m_stream->get(data.first, data.second); if (rc < 0) { // should not happen as an exception should have been thrown // before - throw FIPhoboServerException("stream->get returned an error"); + throw FIPhoboServerException("m_stream->get returned an error"); } else if (rc == 0) { // done - file_closed_ = true; - stream->finish_get(); + m_file_closed = true; + m_stream->finish_get(); VLOG(4) << "Read EOF"; evb->runInEventBaseThread([this] { proxygen::ResponseBuilder(downstream_).sendWithEOM(); @@ -119,11 +119,11 @@ void GetRequestHandler::readFile(folly::EventBase* evb) break; } } - /* Notify the request thread that we terminated the readFile loop */ + /* Notify the request thread that we terminated the read_file loop */ evb->runInEventBaseThread([this] { - readFileScheduled_ = false; - if (!checkForCompletion() && !paused_) { - VLOG(4) << "Resuming deferred readFile"; + m_read_file_scheduled = false; + if (!check_for_completion() && !m_paused) { + VLOG(4) << "Resuming deferred read_file"; onEgressResumed(); } }); @@ -131,24 +131,24 @@ void GetRequestHandler::readFile(folly::EventBase* evb) void GetRequestHandler::onEgressPaused() noexcept { - /* This will terminate readFile soon */ + /* This will terminate read_file soon */ VLOG(4) << "GetRequestHandler pause"; - paused_ = true; + m_paused = true; } void GetRequestHandler::onEgressResumed() noexcept { VLOG(4) << "GetRequestHandler resumed"; - paused_ = false; - /* If readFileScheduled_, it will reschedule itself */ - if (!readFileScheduled_) { - readFileScheduled_ = true; + m_paused = false; + /* If m_read_file_scheduled, it will reschedule itself */ + if (!m_read_file_scheduled) { + m_read_file_scheduled = true; folly::getCPUExecutor()->add(std::bind( - &GetRequestHandler::readFile, this, + &GetRequestHandler::read_file, this, folly::EventBaseManager::get()->getEventBase())); } else { - VLOG(4) << "Deferred scheduling readFile"; + VLOG(4) << "Deferred scheduling read_file"; } } @@ -167,21 +167,21 @@ void GetRequestHandler::onUpgrade( void GetRequestHandler::requestComplete() noexcept { - finished_ = true; - paused_ = true; - checkForCompletion(); + m_finished = true; + m_paused = true; + check_for_completion(); } void GetRequestHandler::onError(proxygen::ProxygenError /*err*/) noexcept { - finished_ = true; - paused_ = true; - checkForCompletion(); + m_finished = true; + m_paused = true; + check_for_completion(); } -bool GetRequestHandler::checkForCompletion() +bool GetRequestHandler::check_for_completion() { - if (finished_ && !readFileScheduled_) { + if (m_finished && !m_read_file_scheduled) { VLOG(4) << "deleting GetRequestHandler"; delete this; return true; diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 7cd7efe..f4bd00d 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -41,20 +41,20 @@ class GetRequestHandler : public proxygen::RequestHandler { void onEgressResumed() noexcept override; GetRequestHandler(std::unique_ptr input_stream) : - stream(std::move(input_stream)) + m_stream(std::move(input_stream)) { } private: - void readFile(folly::EventBase* evb); - bool checkForCompletion(); - - std::unique_ptr stream; - S3_header s3_header; - bool readFileScheduled_{false}; - std::atomic paused_{false}; - bool finished_{false}; - bool file_closed_{false}; + void read_file(folly::EventBase* evb); + bool check_for_completion(); + + std::unique_ptr m_stream; + S3_header m_s3_header; + bool m_read_file_scheduled{false}; + std::atomic m_paused{false}; + bool m_finished{false}; + bool m_file_closed{false}; }; } // namespace fiphoboserver diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index b4616fa..c17c408 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -32,11 +32,11 @@ void GetmdRequestHandler::onRequest( } /* Creating Bucket queries */ - s3_header.setHeaders(std::move(headers)); + m_s3_header.set_headers(std::move(headers)); try { - /* Get meta data to backend through the stream */ - meta_data = std::move(stream->get_meta_data(s3_header.getKey())); + /* Get meta data from backend through the stream */ + m_meta_data = m_stream->get_meta_data(m_s3_header.get_key()); } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -56,11 +56,10 @@ void GetmdRequestHandler::onRequest( return; } - // proxygen::ResponseBuilder(downstream_).status(200, "Ok").send(); proxygen::ResponseBuilder response(downstream_); response.status(200, "OK"); std::for_each( - meta_data.begin(), meta_data.end(), + m_meta_data.begin(), m_meta_data.end(), [&](std::pair pair) { response.header("x-amz-meta-" + pair.first, pair.second); }); diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index 5d44b77..d2a49c4 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -37,14 +37,14 @@ class GetmdRequestHandler : public proxygen::RequestHandler { void onError(proxygen::ProxygenError err) noexcept override; GetmdRequestHandler(std::unique_ptr input_stream) : - stream(std::move(input_stream)) + m_stream(std::move(input_stream)) { } private: - std::unique_ptr stream; - S3_header s3_header; - std::vector> meta_data; + std::unique_ptr m_stream; + S3_header m_s3_header; + std::vector> m_meta_data; }; } // namespace fiphoboserver diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 5052fa6..0c57ee3 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -20,14 +20,14 @@ namespace fiphoboserver { void PutRequestHandler::onRequest( std::unique_ptr headers) noexcept { - if (!downstream_) { + if (downstream_ == nullptr) { // can't push return; } /* Creating Bucket queries */ - s3_header.setHeaders(std::move(headers)); - if (s3_header.isCreateBucketRequest()) { + m_s3_header.set_headers(std::move(headers)); + if (m_s3_header.is_create_bucket_request()) { /* * Ignore, since we don't really have buckets. * Or do we want a list of the actual buckets to give an error, when a @@ -38,11 +38,14 @@ void PutRequestHandler::onRequest( } try { /* Send meta data to backend through the stream */ - stream->set_storage_meta_data( - s3_header.getMetaData(), s3_header.getBucket()); - /* Tell stream to coordinate with backend to prepare for PUT operation + m_stream->set_storage_meta_data( + m_s3_header.get_meta_data(), m_s3_header.get_bucket()); + + /* + * Tell stream to coordinate with backend to prepare for PUT operation */ - stream->start_put(s3_header.getBodyLength(), s3_header.getKey()); + m_stream->start_put( + m_s3_header.get_body_length(), m_s3_header.get_key()); } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -67,7 +70,7 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept { try { /* Hand message body over to stream for PUT operation */ - stream->put(std::move(body)); + m_stream->put(std::move(body->data()), std::move(body->length())); } catch (const FIPhoboServerException& ex) { std::cerr << "Caught an exception in put: " << ex.what() << '\n'; @@ -85,7 +88,7 @@ void PutRequestHandler::onEOM() noexcept { try { /* Tell stream it's time to clean up */ - stream->finish_put(); + m_stream->finish_put(); } catch (const FIPhoboServerException& ex) { std::cerr << "Caught an exception in finish put: " << ex.what() << '\n'; diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 555ac9f..3368bdf 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -36,13 +36,13 @@ class PutRequestHandler : public proxygen::RequestHandler { void onError(proxygen::ProxygenError err) noexcept override; PutRequestHandler(std::unique_ptr input_stream) : - stream(std::move(input_stream)) + m_stream(std::move(input_stream)) { } private: - S3_header s3_header; - std::unique_ptr stream; + S3_header m_s3_header; + std::unique_ptr m_stream; }; } // namespace fiphoboserver diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 2e9e38e..9027e05 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -9,8 +9,6 @@ target_include_directories(storage PUBLIC ${PHOBOS_INCLUDE_DIRECTORY}) target_include_directories(storage PUBLIC /usr/include/glib-2.0) target_include_directories(storage PUBLIC /usr/lib64/glib-2.0/include) -target_compile_features(storage PUBLIC cxx_std_14) - target_link_libraries(storage PUBLIC phobos_store) target_link_libraries(storage PUBLIC phobos_cpp_wrapper) target_link_libraries(storage PUBLIC proxygen::proxygen) diff --git a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h index b69d4bb..945bef5 100644 --- a/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h +++ b/src/storage/phobos_cpp_wrapper/phobos_cpp_wrapper.h @@ -1,7 +1,7 @@ // Header to be included by cpp projects that intend to use the phobos API -#ifndef _PHO_CPP_LIB_H -#define _PHO_CPP_LIB_H +#ifndef FIPHOBOSERVER_PHO_CPP_LIB_H +#define FIPHOBOSERVER_PHO_CPP_LIB_H #include "pho_attrs.h" diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index 109b0a5..3569f40 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -12,13 +12,13 @@ namespace storage { class PhobosException : public FIPhoboServerException { public: PhobosException( - const char* caller, const char* functionName, int returnValue) + const char* caller, const char* function_name, int return_value) { std::stringstream ss; - ss << "[" << caller << "]: Phobos returned an error: " << functionName - << ": " << std::strerror(-returnValue) << " (" << -returnValue + ss << "[" << caller << "]: Phobos returned an error: " << function_name + << ": " << std::strerror(-return_value) << " (" << -return_value << ")"; - message = ss.str(); + m_message = ss.str(); } }; diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index 96653e8..a89de5f 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -18,48 +18,48 @@ namespace fiphoboserver { namespace storage { -/* Just need to allocate Phobos' descriptor struct */ +/* Just need to allocate Phobos' m_descriptor struct */ Phobos_file::Phobos_file() { - memset(&descriptor, 0, sizeof(descriptor)); - descriptor.xd_attrs = {0}; + memset(&m_descriptor, 0, sizeof(m_descriptor)); + m_descriptor.xd_attrs = {0}; } -/* Destroy Phobos' descriptor struct */ +/* Destroy Phobos' m_descriptor struct */ Phobos_file::~Phobos_file() { // TODO: See issue #23 - // pho_attrs_free(&descriptor.xd_attrs); + // pho_attrs_free(&m_descriptor.xd_attrs); - pho_xfer_desc_destroy_cpp(&descriptor); + pho_xfer_desc_destroy_cpp(&m_descriptor); } /* * Open the file for reading - * fill the descriptor with relevant data + * fill the m_descriptor with relevant data */ void Phobos_file::prepare_put(std::string file_name, std::string object_id) { - descriptor.xd_fd = open(file_name.c_str(), O_RDONLY); - if (descriptor.xd_fd < 0) { + m_descriptor.xd_fd = open(file_name.c_str(), O_RDONLY); + if (m_descriptor.xd_fd < 0) { throw IOException("Phobos_file::prepare_put", file_name.c_str(), errno); return; } char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); - descriptor.xd_objid = unconsted_object_id; - descriptor.xd_op = PHO_XFER_OP_PUT; + m_descriptor.xd_objid = unconsted_object_id; + m_descriptor.xd_op = PHO_XFER_OP_PUT; } /* Tell Phobos to start reading from the file */ ssize_t Phobos_file::db_put(size_t size) { - descriptor.xd_size = size; - ssize_t rc = phobos_put_cpp(&descriptor, 1, NULL, NULL); + m_descriptor.xd_size = size; + ssize_t rc = phobos_put_cpp(&m_descriptor, 1, NULL, NULL); close_file(); - if (rc) { + if (rc != 0) { throw PhobosException("Phobos_file::db_put", "phobos_put", rc); } return rc; @@ -67,33 +67,33 @@ ssize_t Phobos_file::db_put(size_t size) /* * Open the file for writing - * fill the descriptor with relevant data + * fill the m_descriptor with relevant data */ void Phobos_file::prepare_get(std::string file_name, std::string object_id) { - descriptor.xd_fd = open(file_name.c_str(), O_WRONLY); - if (descriptor.xd_fd < 0) { + m_descriptor.xd_fd = open(file_name.c_str(), O_WRONLY); + if (m_descriptor.xd_fd < 0) { throw IOException("Phobos_file::prepare_get", file_name.c_str(), errno); return; } char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); - descriptor.xd_objid = unconsted_object_id; - descriptor.xd_op = PHO_XFER_OP_GET; + m_descriptor.xd_objid = unconsted_object_id; + m_descriptor.xd_op = PHO_XFER_OP_GET; } /* Tell Phobos to write the object to file */ ssize_t Phobos_file::db_get() { - ssize_t rc = phobos_get_cpp(&descriptor, 1, NULL, NULL); + ssize_t rc = phobos_get_cpp(&m_descriptor, 1, NULL, NULL); /* * Need to close file here so the stream knows * there will be no more data to read */ close_file(); - if (rc) { + if (rc != 0) { throw PhobosException("Phobos_file::db_get", "phobos_get", rc); } @@ -102,8 +102,9 @@ ssize_t Phobos_file::db_get() void Phobos_file::set_bucket_name(std::string bucket_name) { - int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucket_name.c_str()); - if (rc) { + int rc = + pho_attr_set(&m_descriptor.xd_attrs, "bucket", bucket_name.c_str()); + if (rc != 0) { throw PhobosException( "Phobos_file::set_bucket_name", "pho_attr_set", rc); } @@ -119,8 +120,9 @@ void Phobos_file::set_meta_data( meta_data.begin(), meta_data.end(), [&](std::pair pair) { int rc = pho_attr_set( - &descriptor.xd_attrs, pair.first.c_str(), pair.second.c_str()); - if (rc) { + &m_descriptor.xd_attrs, pair.first.c_str(), + pair.second.c_str()); + if (rc != 0) { throw PhobosException( "Phobos_file::set_meta_data", "pho_attr_set", rc); } @@ -135,7 +137,6 @@ int add_to_meta_data(const char* key, const char* value, void* udata) return 0; } -// TODO std::vector> Phobos_file::get_meta_data( std::string object_id) { @@ -144,35 +145,30 @@ std::vector> Phobos_file::get_meta_data( char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); - descriptor.xd_objid = unconsted_object_id; - descriptor.xd_op = PHO_XFER_OP_GETMD; + m_descriptor.xd_objid = unconsted_object_id; + m_descriptor.xd_op = PHO_XFER_OP_GETMD; - int rc = phobos_getmd_cpp(&descriptor, 1, NULL, NULL); - if (rc) { + int rc = phobos_getmd_cpp(&m_descriptor, 1, NULL, NULL); + if (rc != 0) { throw PhobosException( "Phobos_file::get_meta_data", "phobos_getmd_cpp", rc); } rc = phobos_attrs_foreach_cpp( - &descriptor.xd_attrs, &add_to_meta_data, &meta_data); - if (rc) { + &m_descriptor.xd_attrs, &add_to_meta_data, &meta_data); + if (rc != 0) { throw PhobosException( "Phobos_file::get_meta_data", "phobos_attrs_foreach_cpp", rc); } - return meta_data; -} - -int Phobos_file::get_fd() const -{ - return descriptor.xd_fd; + return std::move(meta_data); } /* Close the file */ void Phobos_file::close_file() { - if (descriptor.xd_fd > 0) { - close(descriptor.xd_fd); + if (m_descriptor.xd_fd > 0) { + close(m_descriptor.xd_fd); } } diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index d796ca9..b17fb7e 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -29,13 +29,15 @@ class Phobos_file : public Storage { void prepare_put(std::string file_name, std::string object_id) override; void prepare_get(std::string file_name, std::string object_id) override; - int get_fd() const; + int get_fd() const { return m_descriptor.xd_fd; } + std::string get_object_id() const { return m_descriptor.xd_objid; } - private: /* Implementation specific variables */ - struct pho_xfer_desc descriptor = {0}; - void set_bucket_name(std::string bucket_name); void close_file(); + + private: + struct pho_xfer_desc m_descriptor = {0}; + void set_bucket_name(std::string bucket_name); }; } // namespace storage diff --git a/src/stream/CMakeLists.txt b/src/stream/CMakeLists.txt index a297ff2..e04cc21 100644 --- a/src/stream/CMakeLists.txt +++ b/src/stream/CMakeLists.txt @@ -12,6 +12,4 @@ if(CUSTOM_OUTPUT_DIRECTORY) ) endif(CUSTOM_OUTPUT_DIRECTORY) -target_compile_features(stream PUBLIC cxx_std_14) - target_link_libraries(stream PUBLIC proxygen::proxygen) diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index 27b57aa..d55599a 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -23,11 +23,11 @@ Fifo::Fifo(std::unique_ptr input_storage) : Fifo::~Fifo() { - if (access(fifo_name.c_str(), F_OK) != -1) { - remove(fifo_name.c_str()); + if (access(m_fifo_name.c_str(), F_OK) != -1) { + remove(m_fifo_name.c_str()); } - if (fifo_descriptor > 0) { - close(fifo_descriptor); + if (m_fifo_descriptor > 0) { + close(m_fifo_descriptor); } } /* Simply pass along to the backend */ @@ -35,14 +35,13 @@ void Fifo::set_storage_meta_data( std::vector> meta_data, std::string bucket_name) const { - storage->set_meta_data(meta_data, bucket_name); + m_storage->set_meta_data(meta_data, bucket_name); } -// TODO std::vector> Fifo::get_meta_data( std::string object_id) const { - return std::move(storage->get_meta_data(object_id)); + return std::move(m_storage->get_meta_data(object_id)); } /* @@ -51,8 +50,8 @@ std::vector> Fifo::get_meta_data( */ void Fifo::create_fifo() { - fifo_name = std::tmpnam(nullptr); - int rc = mkfifo(fifo_name.c_str(), 0777); + m_fifo_name = std::tmpnam(nullptr); + int rc = mkfifo(m_fifo_name.c_str(), 0777); if (rc < 0) { if (errno != EEXIST) { throw IOException("Fifo::create_fifo", "mkfifo", errno); @@ -69,16 +68,16 @@ void Fifo::create_fifo() void Fifo::start_put(ssize_t size, std::string object_id) { /* Start backend storage solution asynchronously */ - db_result = std::async([this, object_id, size]() { - storage->prepare_put(fifo_name, object_id); - return storage->db_put(size); + m_db_result = std::async([this, object_id, size]() { + m_storage->prepare_put(m_fifo_name, object_id); + return m_storage->db_put(size); }); /* Open fifo on the writing end */ - fifo_descriptor = open(fifo_name.c_str(), O_WRONLY); - if (fifo_descriptor < 0) { + m_fifo_descriptor = open(m_fifo_name.c_str(), O_WRONLY); + if (m_fifo_descriptor < 0) { std::stringstream ss; - ss << "open (" << fifo_name.c_str() << ")"; + ss << "open (" << m_fifo_name.c_str() << ")"; throw IOException("Fifo::start_put", ss.str().c_str(), errno); return; } @@ -88,19 +87,20 @@ void Fifo::start_put(ssize_t size, std::string object_id) * Repeatedly called to write buffer to FIFO until * the server's message body is depleted */ -void Fifo::put(std::unique_ptr buffer) const +ssize_t Fifo::put(const void* data, const size_t count) const { - ssize_t rc = write(fifo_descriptor, buffer->data(), buffer->length()); - if (rc < 0) { + ssize_t rc = write(m_fifo_descriptor, data, count); + if (rc < count) { throw IOException("Fifo::put", "write", rc); } + return rc; } // TODO see git issue #26 void Fifo::finish_put() { // will throw the inner exception if one occurred - db_result.get(); + m_db_result.get(); } /* @@ -111,16 +111,16 @@ void Fifo::finish_put() void Fifo::start_get(std::string object_id) { /* Start backend storage solution asynchronously */ - db_result = std::async([this, object_id]() { - storage->prepare_get(fifo_name, object_id); - return storage->db_get(); + m_db_result = std::async([this, object_id]() { + m_storage->prepare_get(m_fifo_name, object_id); + return m_storage->db_get(); }); /* Open fifo on the reading end */ - fifo_descriptor = open(fifo_name.c_str(), O_RDONLY); - if (fifo_descriptor < 0) { + m_fifo_descriptor = open(m_fifo_name.c_str(), O_RDONLY); + if (m_fifo_descriptor < 0) { std::stringstream ss; - ss << "open (" << fifo_name.c_str() << ")"; + ss << "open (" << m_fifo_name.c_str() << ")"; throw IOException("Fifo::start_get", ss.str().c_str(), errno); return; } @@ -132,9 +132,9 @@ void Fifo::start_get(std::string object_id) * writing end * Returns the number of bytes read */ -ssize_t Fifo::get(void* buf, size_t count) const +ssize_t Fifo::get(void* buf, const size_t count) const { - ssize_t rc = read(fifo_descriptor, buf, count); + ssize_t rc = read(m_fifo_descriptor, buf, count); if (rc < 0) { throw IOException("Fifo::get", "read", rc); } @@ -144,7 +144,7 @@ ssize_t Fifo::get(void* buf, size_t count) const // TODO see git issue #26 void Fifo::finish_get() { - db_result.get(); + m_db_result.get(); } } // namespace stream diff --git a/src/stream/fifo.h b/src/stream/fifo.h index fe1c776..850254a 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -8,9 +8,13 @@ class Fifo : public Stream { Fifo(std::unique_ptr input_storage); ~Fifo(); - Fifo(Fifo&&) = default; - Fifo& operator=(Fifo&&) = default; - Fifo(const Fifo&) = delete; + Fifo(Fifo&& input) : Stream(std::move(input.m_storage)) {} + Fifo& operator=(Fifo&& input) + { + Stream::m_storage = std::move(input.m_storage); + return *this; + } + Fifo(const Fifo&) = delete; Fifo& operator=(const Fifo&) = delete; void set_storage_meta_data( @@ -20,7 +24,7 @@ class Fifo : public Stream { std::string object_id) const override; void start_put(ssize_t size, std::string object_id) override; - void put(std::unique_ptr buffer) const override; + ssize_t put(const void* data, size_t count) const override; void finish_put() override; void start_get(std::string object_id) override; @@ -28,13 +32,13 @@ class Fifo : public Stream { void finish_get() override; /* Implementation specific details */ - int get_fifo_descriptor() const { return fifo_descriptor; } - std::string get_fifo_name() const { return fifo_name; } + int get_fifo_descriptor() const { return m_fifo_descriptor; } + std::string get_fifo_name() const { return m_fifo_name; } private: void create_fifo(); - std::string fifo_name; - int fifo_descriptor = -1; + std::string m_fifo_name; + int m_fifo_descriptor = -1; }; } // namespace stream diff --git a/src/stream/stream.h b/src/stream/stream.h index 276f99d..925106a 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -19,7 +19,7 @@ class Stream { * storage implementation */ Stream(std::unique_ptr input_storage) : - storage(std::move(input_storage)) + m_storage(std::move(input_storage)) { } @@ -32,19 +32,19 @@ class Stream { virtual std::vector> get_meta_data( std::string) const = 0; - virtual void start_put(ssize_t size, std::string object_id) = 0; - virtual void put(std::unique_ptr buffer) const = 0; - virtual void finish_put() = 0; + virtual void start_put(ssize_t size, std::string object_id) = 0; + virtual ssize_t put(const void*, size_t) const = 0; + virtual void finish_put() = 0; - virtual void start_get(std::string object_id) = 0; - virtual ssize_t get(void* buf, size_t count) const = 0; - virtual void finish_get() = 0; + virtual void start_get(std::string object_id) = 0; + virtual ssize_t get(void*, size_t) const = 0; + virtual void finish_get() = 0; protected: /* Abstract class that requires implementation */ - std::unique_ptr storage; + std::unique_ptr m_storage; /* This may be used to wait on the asynchronous backend */ - std::future db_result; + std::future m_db_result; }; } // namespace stream diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt deleted file mode 100644 index 2d4aefc..0000000 --- a/test/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -include_directories(/usr/include/glib-2.0) -include_directories(/usr/lib64/glib-2.0/include) - -# -# Define the options -# - -set( - FIPHOBOSERVER_TEST_DATA_DIR "data" - CACHE PATH - "The directory to read/write test data to/from." -) - - -# -# Find Catch2 -# - -find_package(Catch2 REQUIRED) - -# -# Setup the test executables -# - -add_executable(phobos_file phobos_file.cc) - -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${FIPHOBOSERVER_TEST_DATA_DIR}) - -target_link_libraries(phobos_file PUBLIC libfiphoboserver) -target_link_libraries(phobos_file PUBLIC Catch2::Catch2) - -# -# Create directory for data I/O -# -add_custom_target(make_data_directory) -add_custom_target( - TARGET make_data_directory PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${FIPHOBOSERVER_TEST_DATA_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} -) - -include(CTest) -include(Catch) -catch_discover_tests(phobos_file) diff --git a/test/phobos_file.cc b/test/phobos_file.cc deleted file mode 100644 index a7167a6..0000000 --- a/test/phobos_file.cc +++ /dev/null @@ -1,60 +0,0 @@ -#define CATCH_CONFIG_MAIN - -#include -#include "../src/storage/phobos_file.h" -#include -#include - -namespace fiphoboserver { - -TEST_CASE("file ownership", "[basic]") -{ - SECTION("move constructor") - { - storage::Phobos_file pho; - - /* generate a file descriptor from a file */ - pho.prepare_put("data/dummy", "dummy"); - int fd = pho.get_fd(); - - storage::Phobos_file pho2(std::move(pho)); - REQUIRE(fd == pho2.get_fd()); - //REQUIRE(fd == pho.get_fd()); - } - SECTION("assignment move constructor") - { - storage::Phobos_file pho; - - /* generate a file descriptor from a file */ - pho.prepare_put("data/dummy", "dummy"); - int fd = pho.get_fd(); - - storage::Phobos_file pho2 = std::move(pho); - REQUIRE(fd == pho2.get_fd()); - //REQUIRE(fd == pho.get_fd()); - } - //SECTION("copy constructor") - //{ - // storage::Phobos_file pho; - - // /* generate a file descriptor from a file */ - // pho.prepare_put("/tmp/dummy", "dummy"); - // int fd = pho.get_fd(); - - // storage::Phobos_filepho2(pho); - // REQUIRE(fd == pho2.get_fd()); - //} - //SECTION("assignment copy constructor") - //{ - // storage::Phobos_file pho; - - // /* generate a file descriptor from a file */ - // pho.prepare_put("/tmp/dummy", "dummy"); - // int fd = pho.get_fd(); - - // storage::Phobos_file pho2 = pho; - // REQUIRE(fd == pho2.get_fd()); - //} -} - -} // namespace fiphoboserver diff --git a/tools/run_format.sh b/tools/run_format.sh index 6b74e4d..cd01c94 100755 --- a/tools/run_format.sh +++ b/tools/run_format.sh @@ -13,7 +13,7 @@ FORMAT_EXECUTABLE="${script_dir}/format.sh" cd ${source_dir} # All the directories containing source files -source_dirs="src" +source_dirs="src test" # All the extensions of source files source_exts="cc c hh h" diff --git a/tools/run_lint.sh b/tools/run_lint.sh index 465a80f..de6aec0 100755 --- a/tools/run_lint.sh +++ b/tools/run_lint.sh @@ -31,7 +31,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source_dir="$( cd "${script_dir}/.." && pwd)" # All the directories containing source files -source_dirs="src" +source_dirs="src test" # Run lint.sh on every source file in phoboserver. # -- GitLab From a5fa57a263949837e0b325d93280c6b1e4b0dd8f Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 13 Mar 2020 13:41:34 +0000 Subject: [PATCH 30/62] Fix errors and rebuild documentation --- doc/Doxyfile.in | 2 +- .../fiphoboserver/FIPhoboServerException.md | 4 +- .../fiphoboserver/GetRequestHandler.md | 6 +- .../fiphoboserver/GetmdRequestHandler.md | 208 ++++++++++++++++++ doc/Markdown/fiphoboserver/IOException.md | 6 +- .../fiphoboserver/PutRequestHandler.md | 6 +- doc/Markdown/fiphoboserver/S3_header.md | 26 +-- .../fiphoboserver/storage/PhobosException.md | 6 +- .../fiphoboserver/storage/Phobos_file.md | 71 ++++++ doc/Markdown/fiphoboserver/stream/Fifo.md | 36 ++- doc/Markdown/fiphoboserver/stream/Stream.md | 20 +- doc/Markdown/todo.md | 4 +- src/server/S3_header.h | 4 +- src/server/get_request_handler.h | 4 + src/server/getmd_request_handler.h | 4 - src/storage/phobos_exception.h | 4 +- src/storage/phobos_file.h | 4 +- src/stream/stream.h | 4 +- 18 files changed, 365 insertions(+), 54 deletions(-) create mode 100644 doc/Markdown/fiphoboserver/GetmdRequestHandler.md diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 044c0d7..6842916 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -29,7 +29,7 @@ WARN_AS_ERROR = YES # Input directories INPUT = @CMAKE_SOURCE_DIR@/src -EXCLUDE = @CMAKE_SOURCE_DIR@/src/server/server.main.cc +EXCLUDE = @CMAKE_SOURCE_DIR@/src/main.cc EXCLUDE_PATTERNS = */phobos_cpp_wrapper/* RECURSIVE = YES diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index a413408..51d0c53 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -10,7 +10,7 @@ Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphob ## Protected Attributes | Name | Description | | ---- | ---- | -| [message](#fiphoboserver-FIPhoboServerException-message) | the internal message | +| [m_message](#fiphoboserver-FIPhoboServerException-m_message) | the internal message | ## Public Functions @@ -23,7 +23,7 @@ Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphob ## Protected Attributes -### protected fiphoboserver::FIPhoboServerException::message +### protected fiphoboserver::FIPhoboServerException::m_message the internal message diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index d916132..97c0edc 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -105,7 +105,8 @@ function called on upgrade #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| proxygen::UpgradeProtocol | proto | | +| proxygen::UpgradeProtocol | proto | the new protocol | + @@ -157,7 +158,8 @@ function called when an error occurred #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| proxygen::ProxygenError | err | | +| proxygen::ProxygenError | err | the error | + diff --git a/doc/Markdown/fiphoboserver/GetmdRequestHandler.md b/doc/Markdown/fiphoboserver/GetmdRequestHandler.md new file mode 100644 index 0000000..b724ed2 --- /dev/null +++ b/doc/Markdown/fiphoboserver/GetmdRequestHandler.md @@ -0,0 +1,208 @@ +# public fiphoboserver::GetmdRequestHandler + +proxygen class implementation for handling HEAD requests + + + +## Inheritance: +Inherits from RequestHandler. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onRequest](#fiphoboserver-GetmdRequestHandler-onRequest) | first function to be called when a new request comes in | +| [onBody](#fiphoboserver-GetmdRequestHandler-onBody) | function called on every body chunk belonging to this message | +| [onEOM](#fiphoboserver-GetmdRequestHandler-onEOM) | function called when the incoming message is finished | +| [onUpgrade](#fiphoboserver-GetmdRequestHandler-onUpgrade) | function called on upgrade | +| [requestComplete](#fiphoboserver-GetmdRequestHandler-requestComplete) | function ... | +| [onError](#fiphoboserver-GetmdRequestHandler-onError) | function called when an error occurred | +| [GetmdRequestHandler](#fiphoboserver-GetmdRequestHandler-GetmdRequestHandler) | Constructor for stream class initialization. | + + + +## Public Functions +### public void fiphoboserver::GetmdRequestHandler::onRequest (std::unique_ptr< proxygen::HTTPMessage > headers) noexcept override + +first function to be called when a new request comes in + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< proxygen::HTTPMessage > | headers | headers of the HTTP message this handler was created for | + + + + + + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public void fiphoboserver::GetmdRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override + +function called on every body chunk belonging to this message + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< folly::IOBuf > | body | buffer for the body | + + + + + + + +This is not used in this case because GET requests don't usually have a body + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public void fiphoboserver::GetmdRequestHandler::onEOM () noexcept override + +function called when the incoming message is finished + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public void fiphoboserver::GetmdRequestHandler::onUpgrade (proxygen::UpgradeProtocol proto) noexcept override + +function called on upgrade + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::UpgradeProtocol | proto | the new protocol | + + + + + + + + +Not supported in our case! + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public void fiphoboserver::GetmdRequestHandler::requestComplete () noexcept override + +function ... + + + + + + + + + +> **[Todo][todo]:** find out what this does? + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public void fiphoboserver::GetmdRequestHandler::onError (proxygen::ProxygenError err) noexcept override + +function called when an error occurred + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::ProxygenError | err | the error | + + + + + + + + +Not supported in our case! + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +### public fiphoboserver::GetmdRequestHandler::GetmdRequestHandler (std::unique_ptr< stream::Stream > input_stream) + +Constructor for stream class initialization. + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< [stream::Stream][fiphoboserver-stream-Stream] > | input_stream | [stream::Stream][fiphoboserver-stream-Stream] class instance to initialize the server | + + + + + + + + + + + + +#### Qualifiers: +* inline +* virtual + + +[Go to Top](#fiphoboserver-GetmdRequestHandler) + +[fiphoboserver-stream-Stream]:./stream/Stream.md +[todo]:./../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index 1a6f57b..82c057e 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -15,7 +15,7 @@ Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServe ## Public Functions -### public fiphoboserver::IOException::IOException (const char *caller, const char *functionName, int returnValue) +### public fiphoboserver::IOException::IOException (const char *caller, const char *function_name, int return_value) constructor @@ -25,8 +25,8 @@ constructor | Type | Name | Description | | ---- | ---- | ---- | | const char * | caller | the function that threw this exception | -| const char * | functionName | the io function that returned an error | -| int | returnValue | the return value of the io function | +| const char * | function_name | the io function that returned an error | +| int | return_value | the return value of the io function | diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index f9d6922..b02ad59 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -102,7 +102,8 @@ function called on upgrade #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| proxygen::UpgradeProtocol | proto | | +| proxygen::UpgradeProtocol | proto | the new protocol | + @@ -154,7 +155,8 @@ function called when an error occurred #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| proxygen::ProxygenError | err | | +| proxygen::ProxygenError | err | the error | + diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index 1fe1a29..8aff8cf 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -7,17 +7,17 @@ class to extract the S3 specific information from proxygens HTTPMessage headers ## Public Functions | Name | Description | | ---- | ---- | -| [setHeaders](#fiphoboserver-S3_header-setHeaders) | sets the internal header instance to the given header object | -| [getBucket](#fiphoboserver-S3_header-getBucket) | extracts the name of the S3 bucket requested in the HTTP message | -| [getKey](#fiphoboserver-S3_header-getKey) | extracts the name of the key id requested in the HTTP message | -| [isCreateBucketRequest](#fiphoboserver-S3_header-isCreateBucketRequest) | checks if the message we got is one for just creating a bucket | -| [getBodyLength](#fiphoboserver-S3_header-getBodyLength) | gets the total length of all body chunks that is expected | -| [getMetaData](#fiphoboserver-S3_header-getMetaData) | gets the S3 metadata stored in the HTTP headers | +| [set_headers](#fiphoboserver-S3_header-set_headers) | sets the internal header instance to the given header object | +| [get_bucket](#fiphoboserver-S3_header-get_bucket) | extracts the name of the S3 bucket requested in the HTTP message | +| [get_key](#fiphoboserver-S3_header-get_key) | extracts the name of the key id requested in the HTTP message | +| [is_create_bucket_request](#fiphoboserver-S3_header-is_create_bucket_request) | checks if the message we got is one for just creating a bucket | +| [get_body_length](#fiphoboserver-S3_header-get_body_length) | gets the total length of all body chunks that is expected | +| [get_meta_data](#fiphoboserver-S3_header-get_meta_data) | gets the S3 metadata stored in the HTTP headers | ## Public Functions -### public void fiphoboserver::S3_header::setHeaders (std::unique_ptr< proxygen::HTTPMessage > newHeaders) +### public void fiphoboserver::S3_header::set_headers (std::unique_ptr< proxygen::HTTPMessage > new_m_headers) sets the internal header instance to the given header object @@ -26,7 +26,7 @@ sets the internal header instance to the given header object #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::unique_ptr< proxygen::HTTPMessage > | newHeaders | headers that should be read out in this instance | +| std::unique_ptr< proxygen::HTTPMessage > | new_m_headers | headers that should be read out in this instance | @@ -46,7 +46,7 @@ sets the internal header instance to the given header object [Go to Top](#fiphoboserver-S3_header) -### public std::string fiphoboserver::S3_header::getBucket () +### public std::string fiphoboserver::S3_header::get_bucket () extracts the name of the S3 bucket requested in the HTTP message @@ -76,7 +76,7 @@ extracts the name of the S3 bucket requested in the HTTP message [Go to Top](#fiphoboserver-S3_header) -### public std::string fiphoboserver::S3_header::getKey () +### public std::string fiphoboserver::S3_header::get_key () extracts the name of the key id requested in the HTTP message @@ -106,7 +106,7 @@ extracts the name of the key id requested in the HTTP message [Go to Top](#fiphoboserver-S3_header) -### public bool fiphoboserver::S3_header::isCreateBucketRequest () +### public bool fiphoboserver::S3_header::is_create_bucket_request () checks if the message we got is one for just creating a bucket @@ -137,7 +137,7 @@ This is convenient since we don't really support buckets themselves and can easi [Go to Top](#fiphoboserver-S3_header) -### public size_t fiphoboserver::S3_header::getBodyLength () +### public size_t fiphoboserver::S3_header::get_body_length () gets the total length of all body chunks that is expected @@ -168,7 +168,7 @@ This reads out the header "Content-Length" [Go to Top](#fiphoboserver-S3_header) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::getMetaData () +### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::get_meta_data () gets the S3 metadata stored in the HTTP headers diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index 5ad7b6b..c8f6eef 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -15,7 +15,7 @@ Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServe ## Public Functions -### public fiphoboserver::storage::PhobosException::PhobosException (const char *caller, const char *functionName, int returnValue) +### public fiphoboserver::storage::PhobosException::PhobosException (const char *caller, const char *function_name, int return_value) constructor @@ -25,8 +25,8 @@ constructor | Type | Name | Description | | ---- | ---- | ---- | | const char * | caller | the function that threw this exception | -| const char * | functionName | the phobos function that returned an error | -| int | returnValue | the return value of the phobos function | +| const char * | function_name | the phobos function that returned an error | +| int | return_value | the return value of the phobos function | diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index aa49695..86427c4 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -13,6 +13,12 @@ default constructor ## Inheritance: Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_descriptor](#fiphoboserver-storage-Phobos_file-m_descriptor) | struct to contain information for phobos | + + ## Public Functions | Name | Description | | ---- | ---- | @@ -30,6 +36,26 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. | [prepare_get](#fiphoboserver-storage-Phobos_file-prepare_get) | Starts a get operation to the database. | +## Private Functions +| Name | Description | +| ---- | ---- | +| [set_bucket_name](#fiphoboserver-storage-Phobos_file-set_bucket_name) | set the name for the bucket an object blongs to | +| [close_file](#fiphoboserver-storage-Phobos_file-close_file) | close the file opened for phobos | + + + +## Private Attributes +### private fiphoboserver::storage::Phobos_file::m_descriptor + +struct to contain information for phobos + + + + + + + +[Go to Top](#fiphoboserver-storage-Phobos_file) ## Public Functions ### public fiphoboserver::storage::Phobos_file::Phobos_file () @@ -316,6 +342,51 @@ Starts a get operation to the database. +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +## Private Functions +### private void fiphoboserver::storage::Phobos_file::set_bucket_name (std::string bucket_name) + +set the name for the bucket an object blongs to + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | bucket_name | name of the bucket | + + + + + + + + + + + + +#### Qualifiers: +* virtual + + +[Go to Top](#fiphoboserver-storage-Phobos_file) + +### private void fiphoboserver::storage::Phobos_file::close_file () + +close the file opened for phobos + + + + + + + #### Qualifiers: * virtual diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index ef2e9ba..75db4c1 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -25,8 +25,8 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | ---- | ---- | | [Fifo](#fiphoboserver-stream-Fifo-Fifo) | default constructor | | [~Fifo](#fiphoboserver-stream-Fifo-~Fifo) | default destructor | -| [Fifo](#fiphoboserver-stream-Fifo-Fifo-1) | default move contructor | -| [operator=](#fiphoboserver-stream-Fifo-operator=) | default assignment operator for rvalue references | +| [Fifo](#fiphoboserver-stream-Fifo-Fifo-1) | move contructor | +| [operator=](#fiphoboserver-stream-Fifo-operator=) | assignment operator for rvalue references | | [Fifo](#fiphoboserver-stream-Fifo-Fifo-2) | copy constructor deleted | | [operator=](#fiphoboserver-stream-Fifo-operator=-1) | assignment operator deleted | | [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get | @@ -147,28 +147,39 @@ default destructor [Go to Top](#fiphoboserver-stream-Fifo) -### public fiphoboserver::stream::Fifo::Fifo (Fifo &&)=default +### public fiphoboserver::stream::Fifo::Fifo (Fifo &&input) -default move contructor +move contructor +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| [Fifo][fiphoboserver-stream-Fifo] && | input | | + #### Qualifiers: +* inline * virtual [Go to Top](#fiphoboserver-stream-Fifo) -### public [Fifo][fiphoboserver-stream-Fifo] & fiphoboserver::stream::Fifo::operator= (Fifo &&)=default +### public [Fifo][fiphoboserver-stream-Fifo] & fiphoboserver::stream::Fifo::operator= (Fifo &&input) -default assignment operator for rvalue references +assignment operator for rvalue references +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| [Fifo][fiphoboserver-stream-Fifo] && | input | | + #### Returns: | Type | Description | | ---- | ---- | @@ -179,6 +190,7 @@ default assignment operator for rvalue references #### Qualifiers: +* inline * virtual @@ -320,7 +332,7 @@ start a put operation [Go to Top](#fiphoboserver-stream-Fifo) -### public void fiphoboserver::stream::Fifo::put (std::unique_ptr< folly::IOBuf > buffer) const override +### public ssize_t fiphoboserver::stream::Fifo::put (const void *data, size_t count) const override add a chunk of data to the object @@ -329,7 +341,15 @@ add a chunk of data to the object #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | +| const void * | data | the chunked data to add | +| size_t | count | the size of the buffer / chunk | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the amount of bytes written to internal storage | + + diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index af0da81..1bfd99d 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -21,8 +21,8 @@ Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. ## Protected Attributes | Name | Description | | ---- | ---- | -| [Storage >](#fiphoboserver-stream-Stream-storage) | an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects | -| [db_result](#fiphoboserver-stream-Stream-db_result) | a future object that can be used to get the results from the asynchronous backend | +| [m_storage](#fiphoboserver-stream-Stream-m_storage) | an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects | +| [m_db_result](#fiphoboserver-stream-Stream-m_db_result) | a future object that can be used to get the results from the asynchronous backend | ## Public Functions @@ -66,7 +66,7 @@ start a put operation [Go to Top](#fiphoboserver-stream-Stream) -### public void fiphoboserver::stream::Stream::put (std::unique_ptr< folly::IOBuf > buffer) const =0 +### public ssize_t fiphoboserver::stream::Stream::put (const void *data, size_t count) const =0 add a chunk of data to the object @@ -75,7 +75,15 @@ add a chunk of data to the object #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::unique_ptr< folly::IOBuf > | buffer | the chunked data to add | +| const void * | data | the chunked data to add | +| size_t | count | the size of the buffer / chunk | + +#### Returns: +| Type | Description | +| ---- | ---- | +| ssize_t | the amount of bytes written to internal storage | + + @@ -193,7 +201,7 @@ end a get operation [Go to Top](#fiphoboserver-stream-Stream) ## Protected Attributes -### protected std::unique_ptr< storage::Storage > +### protected fiphoboserver::stream::Stream::m_storage an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will be used as a database for the objects @@ -205,7 +213,7 @@ an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will [Go to Top](#fiphoboserver-stream-Stream) -### protected fiphoboserver::stream::Stream::db_result +### protected fiphoboserver::stream::Stream::m_db_result a future object that can be used to get the results from the asynchronous backend diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 2d58c8a..3766e41 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -12,7 +12,7 @@ * Do we use this? should this really be public? * Member [fiphoboserver::stream::Fifo::get_fifo_name][fiphoboserver-stream-Fifo-get_fifo_name] () const * Do we use this? should this really be public? -* Member [fiphoboserver::stream::Stream::db_result][fiphoboserver-stream-Stream-db_result] +* Member [fiphoboserver::stream::Stream::m_db_result][fiphoboserver-stream-Stream-m_db_result] * should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? @@ -26,4 +26,4 @@ [fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md [fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor [fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name -[fiphoboserver-stream-Stream-db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-db_result +[fiphoboserver-stream-Stream-m_db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-m_db_result diff --git a/src/server/S3_header.h b/src/server/S3_header.h index 21d754f..c7055ed 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -14,9 +14,9 @@ class S3_header { /// /// @brief sets the internal header instance to the given header object /// - /// @param newHeaders headers that should be read out in this instance + /// @param new_m_headers headers that should be read out in this instance /// - void setHeaders(std::unique_ptr new_m_headers) + void set_headers(std::unique_ptr new_m_headers) { m_headers = std::move(new_m_headers); } diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index fcda65c..c4889da 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -49,6 +49,8 @@ class GetRequestHandler : public proxygen::RequestHandler { /// /// @brief function called on upgrade /// + /// @param proto the new protocol + /// /// Not supported in our case! /// void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; @@ -63,6 +65,8 @@ class GetRequestHandler : public proxygen::RequestHandler { /// /// @brief function called when an error occurred /// + /// @param err the error + /// /// Not supported in our case! /// void onError(proxygen::ProxygenError err) noexcept override; diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index 686d32f..df49c64 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -15,10 +15,6 @@ #include "../stream/stream.h" #include "S3_header.h" -namespace proxygen { -class ResponseHandler; -} - namespace fiphoboserver { /// diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index dd73ced..bf14108 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -18,8 +18,8 @@ public: /// @brief constructor /// /// @param caller the function that threw this exception - /// @param functionName the phobos function that returned an error - /// @param returnValue the return value of the phobos function + /// @param function_name the phobos function that returned an error + /// @param return_value the return value of the phobos function /// PhobosException( const char* caller, const char* function_name, int return_value) diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 9507962..db9f17d 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -50,8 +50,8 @@ class Phobos_file : public Storage { private: /* Implementation specific variables */ - /// @brief struct to cintain information for phobos - struct pho_xfer_desc descriptor = {0}; + /// @brief struct to contain information for phobos + struct pho_xfer_desc m_descriptor = {0}; /// /// @brief set the name for the bucket an object blongs to diff --git a/src/stream/stream.h b/src/stream/stream.h index 8c6fe42..9ac7954 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -86,12 +86,12 @@ class Stream { /// /// @brief add a chunk of data to the object /// - /// @param buffer the chunked data to add + /// @param data the chunked data to add /// @param count the size of the buffer / chunk /// /// @returns the amount of bytes written to internal storage /// - virtual ssize_t put(const void* buffer, size_t count) const = 0; + virtual ssize_t put(const void* data, size_t count) const = 0; /// /// @brief end a put operation /// -- GitLab From 7440baffc1b889d59988f9a09e3061ac7c9c685a Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Mon, 16 Mar 2020 16:10:52 +0000 Subject: [PATCH 31/62] Add more specific information to Phobos and FIFO and add text from comments in the source code --- src/storage/phobos_file.h | 7 ++++--- src/storage/storage.h | 4 ++-- src/stream/fifo.h | 31 ++++++++++++++++++++++++++++--- src/stream/stream.h | 3 +++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index db9f17d..c9b73e1 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -7,10 +7,11 @@ extern "C" { namespace fiphoboserver { namespace storage { -/// @copydoc Storage +/// +/// @brief Phobos specific mplementation of the @ref Storage class /// -/// @todo maybe not only copy from interface but add specific phobos -/// information? +/// This class uses the phobos library as a backend to store and retrieve files +/// in chunks. /// class Phobos_file : public Storage { public: diff --git a/src/storage/storage.h b/src/storage/storage.h index 0f29e20..3a9ba96 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -35,8 +35,8 @@ class Storage { /// @{ /// - /// @brief set the metadata an object that is added to the database - /// should get + /// @brief set the metadata that an object that is added to the + /// database should get /// /// @param meta_data a vector of all key:value pairs that should be /// added to the data diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 070658e..307d77c 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -4,10 +4,12 @@ namespace fiphoboserver { namespace stream { /// -/// @copydoc Stream +/// @brief FIFO specific implementation of the @ref Stream class /// -/// @todo maybe not only copy from interface but add specific fifo -/// information? +/// This class uses a fifo to stream data from the server to a backend storage +/// class. Therefore, all data gets written to the fifo and the implementation +/// of the internal @ref storage::Storage class can read and process it from the +/// other end of the fifo. /// class Fifo : public Stream { public: @@ -29,7 +31,11 @@ class Fifo : public Stream { /// @brief assignment operator deleted Fifo& operator=(const Fifo&) = delete; + /// /// @copydoc Stream::set_storage_meta_data + /// + /// Just passes the data aling to the backend + /// void set_storage_meta_data( std::vector> meta_data, std::string bucket_name) const override; @@ -38,6 +44,11 @@ class Fifo : public Stream { std::string object_id) const override; /// @copydoc Stream::start_put + /// + /// Telling the backend that it must be ready for a PUT operation + /// Opening the FIFO here will not block as the backend will + /// open on the other end + /// void start_put(ssize_t size, std::string object_id) override; /// @copydoc Stream::put ssize_t put(const void* data, size_t count) const override; @@ -45,8 +56,18 @@ class Fifo : public Stream { void finish_put() override; /// @copydoc Stream::start_get + /// + /// Telling the backend that it must be ready for a PUT operation + /// Opening the FIFO here will not block as the backend will + /// open on the other end + /// void start_get(std::string object_id) override; /// @copydoc Stream::get + /// + /// Repeatedly called to read from FIFO into buffer + /// until the FIFO is empty and closed on the + /// writing end + /// ssize_t get(void *buf, size_t count) const override; /// @copydoc Stream::finish_get void finish_get() override; @@ -76,6 +97,10 @@ class Fifo : public Stream { /// @} private: + /// + /// @brief create a randomly named fifo + /// + /// The fifo is just created here, because opening it would block it void create_fifo(); std::string m_fifo_name; int m_fifo_descriptor = -1; diff --git a/src/stream/stream.h b/src/stream/stream.h index 9ac7954..cd679c0 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -86,6 +86,9 @@ class Stream { /// /// @brief add a chunk of data to the object /// + /// This function can be called repeatedly until the server's message + /// body is depleted + /// /// @param data the chunked data to add /// @param count the size of the buffer / chunk /// -- GitLab From 0f921d05c433200c62844c72edff174997917e9b Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Mon, 16 Mar 2020 16:16:20 +0000 Subject: [PATCH 32/62] Rebuild documentation --- .../fiphoboserver/FIPhoboServerException.md | 8 +- .../fiphoboserver/GetRequestHandler.md | 40 ++------ .../fiphoboserver/GetmdRequestHandler.md | 32 ++----- doc/Markdown/fiphoboserver/IOException.md | 3 +- .../fiphoboserver/PutRequestHandler.md | 31 ++----- doc/Markdown/fiphoboserver/S3_header.md | 21 ++--- .../UnsupportedRequestHandler.md | 1 + .../fiphoboserver/storage/PhobosException.md | 3 +- .../fiphoboserver/storage/Phobos_file.md | 52 +++-------- doc/Markdown/fiphoboserver/storage/Storage.md | 18 ++-- doc/Markdown/fiphoboserver/storage/index.md | 4 +- doc/Markdown/fiphoboserver/stream/Fifo.md | 91 ++++++++++++------- doc/Markdown/fiphoboserver/stream/Stream.md | 37 ++++---- doc/Markdown/fiphoboserver/stream/index.md | 7 +- doc/Markdown/todo.md | 20 ++-- src/stream/fifo.h | 2 +- 16 files changed, 158 insertions(+), 212 deletions(-) diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index 51d0c53..65d03cb 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -4,6 +4,7 @@ exception class for all user defined exceptions in FiPhboServer + ## Inheritance: Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphoboserver::storage::PhobosException][fiphoboserver-storage-PhobosException]. @@ -33,6 +34,7 @@ the internal message + [Go to Top](#fiphoboserver-FIPhoboServerException) ## Public Functions @@ -46,9 +48,9 @@ default constructor + #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-FIPhoboServerException) @@ -59,6 +61,7 @@ constructor for a message with information on the exception + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -77,7 +80,6 @@ constructor for a message with information on the exception #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-FIPhoboServerException) @@ -88,6 +90,7 @@ get information on this exception + #### Returns: | Type | Description | | ---- | ---- | @@ -104,7 +107,6 @@ get information on this exception - #### Qualifiers: * const * inline diff --git a/doc/Markdown/fiphoboserver/GetRequestHandler.md b/doc/Markdown/fiphoboserver/GetRequestHandler.md index 97c0edc..098b25f 100644 --- a/doc/Markdown/fiphoboserver/GetRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetRequestHandler.md @@ -4,6 +4,7 @@ proxygen class implementation for handling GET requests + ## Inheritance: Inherits from RequestHandler. @@ -29,6 +30,7 @@ first function to be called when a new request comes in + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -45,10 +47,6 @@ first function to be called when a new request comes in -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -57,6 +55,7 @@ function called on every body chunk belonging to this message + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -73,11 +72,6 @@ This is not used in this case because GET requests don't usually have a body - -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-GetRequestHandler) ### public void fiphoboserver::GetRequestHandler::onEOM () noexcept override @@ -90,9 +84,6 @@ function called when the incoming message is finished -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetRequestHandler) @@ -102,6 +93,7 @@ function called on upgrade + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -118,9 +110,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetRequestHandler) @@ -136,15 +125,10 @@ function ... -> **[Todo][todo]:** find out what this does? - - - +> **[Todo][todo]:** find out what this does? -#### Qualifiers: -* virtual [Go to Top](#fiphoboserver-GetRequestHandler) @@ -155,6 +139,7 @@ function called when an error occurred + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -171,9 +156,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetRequestHandler) @@ -189,12 +171,10 @@ called when the queue is full. -Contents are copies from a proxygen example +Contents are copies from a proxygen example -#### Qualifiers: -* virtual [Go to Top](#fiphoboserver-GetRequestHandler) @@ -211,12 +191,10 @@ called when the queue is not longer full. -Contents are copies from a proxygen example +Contents are copies from a proxygen example -#### Qualifiers: -* virtual [Go to Top](#fiphoboserver-GetRequestHandler) @@ -227,6 +205,7 @@ Constructor for stream class initialization. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -245,7 +224,6 @@ Constructor for stream class initialization. #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-GetRequestHandler) diff --git a/doc/Markdown/fiphoboserver/GetmdRequestHandler.md b/doc/Markdown/fiphoboserver/GetmdRequestHandler.md index b724ed2..205a20a 100644 --- a/doc/Markdown/fiphoboserver/GetmdRequestHandler.md +++ b/doc/Markdown/fiphoboserver/GetmdRequestHandler.md @@ -4,6 +4,7 @@ proxygen class implementation for handling HEAD requests + ## Inheritance: Inherits from RequestHandler. @@ -27,6 +28,7 @@ first function to be called when a new request comes in + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -43,10 +45,6 @@ first function to be called when a new request comes in -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-GetmdRequestHandler) ### public void fiphoboserver::GetmdRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -55,6 +53,7 @@ function called on every body chunk belonging to this message + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -71,11 +70,6 @@ This is not used in this case because GET requests don't usually have a body - -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-GetmdRequestHandler) ### public void fiphoboserver::GetmdRequestHandler::onEOM () noexcept override @@ -88,9 +82,6 @@ function called when the incoming message is finished -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetmdRequestHandler) @@ -100,6 +91,7 @@ function called on upgrade + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -116,9 +108,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetmdRequestHandler) @@ -134,15 +123,10 @@ function ... -> **[Todo][todo]:** find out what this does? - - - +> **[Todo][todo]:** find out what this does? -#### Qualifiers: -* virtual [Go to Top](#fiphoboserver-GetmdRequestHandler) @@ -153,6 +137,7 @@ function called when an error occurred + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -169,9 +154,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-GetmdRequestHandler) @@ -181,6 +163,7 @@ Constructor for stream class initialization. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -199,7 +182,6 @@ Constructor for stream class initialization. #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-GetmdRequestHandler) diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index 82c057e..a164c7a 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -4,6 +4,7 @@ exceptions specifically for errors in I/O + ## Inheritance: Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. @@ -21,6 +22,7 @@ constructor + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -41,7 +43,6 @@ constructor #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-IOException) diff --git a/doc/Markdown/fiphoboserver/PutRequestHandler.md b/doc/Markdown/fiphoboserver/PutRequestHandler.md index b02ad59..82bcdb6 100644 --- a/doc/Markdown/fiphoboserver/PutRequestHandler.md +++ b/doc/Markdown/fiphoboserver/PutRequestHandler.md @@ -4,6 +4,7 @@ proxygen class implementation for handling PUT requests + ## Inheritance: Inherits from RequestHandler. @@ -27,6 +28,7 @@ first function to be called when a new request comes in + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -43,10 +45,6 @@ first function to be called when a new request comes in -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onBody (std::unique_ptr< folly::IOBuf > body) noexcept override @@ -55,6 +53,7 @@ ccalled with each chunk of the body coming in for the request + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -71,10 +70,6 @@ ccalled with each chunk of the body coming in for the request -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-PutRequestHandler) ### public void fiphoboserver::PutRequestHandler::onEOM () noexcept override @@ -87,9 +82,6 @@ function called when the incoming message is finished -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-PutRequestHandler) @@ -99,6 +91,7 @@ function called on upgrade + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -115,9 +108,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-PutRequestHandler) @@ -133,15 +123,10 @@ function ... -> **[Todo][todo]:** find out what this does? - - - +> **[Todo][todo]:** find out what this does? -#### Qualifiers: -* virtual [Go to Top](#fiphoboserver-PutRequestHandler) @@ -152,6 +137,7 @@ function called when an error occurred + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -168,9 +154,6 @@ Not supported in our case! -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-PutRequestHandler) @@ -180,6 +163,7 @@ Constructor for stream class initialization. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -198,7 +182,6 @@ Constructor for stream class initialization. #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-PutRequestHandler) diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index 8aff8cf..296fc0e 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -4,6 +4,7 @@ class to extract the S3 specific information from proxygens HTTPMessage headers + ## Public Functions | Name | Description | | ---- | ---- | @@ -23,6 +24,7 @@ sets the internal header instance to the given header object + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -41,7 +43,6 @@ sets the internal header instance to the given header object #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) @@ -52,6 +53,7 @@ extracts the name of the S3 bucket requested in the HTTP message + #### Returns: | Type | Description | | ---- | ---- | @@ -68,10 +70,8 @@ extracts the name of the S3 bucket requested in the HTTP message - #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) @@ -82,6 +82,7 @@ extracts the name of the key id requested in the HTTP message + #### Returns: | Type | Description | | ---- | ---- | @@ -98,10 +99,8 @@ extracts the name of the key id requested in the HTTP message - #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) @@ -112,6 +111,7 @@ checks if the message we got is one for just creating a bucket + #### Returns: | Type | Description | | ---- | ---- | @@ -128,11 +128,8 @@ This is convenient since we don't really support buckets themselves and can easi - - #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) @@ -143,6 +140,7 @@ gets the total length of all body chunks that is expected + #### Returns: | Type | Description | | ---- | ---- | @@ -159,11 +157,8 @@ This reads out the header "Content-Length" - - #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) @@ -174,6 +169,7 @@ gets the S3 metadata stored in the HTTP headers + #### Returns: | Type | Description | | ---- | ---- | @@ -190,11 +186,8 @@ In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE - - #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-S3_header) diff --git a/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md index c54d3ab..04aee3a 100644 --- a/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md +++ b/doc/Markdown/fiphoboserver/UnsupportedRequestHandler.md @@ -4,6 +4,7 @@ proxygen class implementation for handling all requests that we don't support + ## Inheritance: Inherits from RequestHandler. diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index c8f6eef..9763107 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -4,6 +4,7 @@ exceptions specifically for the phobos backend library + ## Inheritance: Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. @@ -21,6 +22,7 @@ constructor + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -41,7 +43,6 @@ constructor #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-storage-PhobosException) diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 86427c4..be61797 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -1,11 +1,10 @@ # public fiphoboserver::storage::Phobos_file -default constructor - +Phobos specific mplementation of the [Storage][fiphoboserver-storage-Storage] class. -> **[Todo][todo]:** maybe not only copy from interface but add specific phobos information? +This class uses the phobos library as a backend to store and retrieve files in chunks. @@ -28,7 +27,7 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. | [operator=](#fiphoboserver-storage-Phobos_file-operator=) | default assignment operator for rvalue references | | [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file-2) | copy constructor deleted | | [operator=](#fiphoboserver-storage-Phobos_file-operator=-1) | assignment operator deleted | -| [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata an object that is added to the database should get | +| [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata that an object that is added to the database should get | | [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | | [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. | | [db_get](#fiphoboserver-storage-Phobos_file-db_get) | Gets data from the databse. | @@ -45,7 +44,7 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. ## Private Attributes -### private fiphoboserver::storage::Phobos_file::m_descriptor +### private fiphoboserver::storage::Phobos_file::m_descriptor = {0} struct to contain information for phobos @@ -55,6 +54,7 @@ struct to contain information for phobos + [Go to Top](#fiphoboserver-storage-Phobos_file) ## Public Functions @@ -68,9 +68,6 @@ default constructor -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-storage-Phobos_file) @@ -84,9 +81,6 @@ default destructor -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-storage-Phobos_file) @@ -100,9 +94,6 @@ default move contructor -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-storage-Phobos_file) @@ -112,6 +103,7 @@ default assignment operator for rvalue references + #### Returns: | Type | Description | | ---- | ---- | @@ -121,10 +113,6 @@ default assignment operator for rvalue references -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-storage-Phobos_file) ### public fiphoboserver::storage::Phobos_file::Phobos_file (const Phobos_file &)=delete @@ -137,9 +125,6 @@ copy constructor deleted -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-storage-Phobos_file) @@ -149,6 +134,7 @@ assignment operator deleted + #### Returns: | Type | Description | | ---- | ---- | @@ -158,15 +144,12 @@ assignment operator deleted -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-storage-Phobos_file) ### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override -set the metadata an object that is added to the database should get +set the metadata that an object that is added to the database should get + @@ -199,6 +182,7 @@ get the metadata associated to the current object as a vector of key:value pairs + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -220,8 +204,6 @@ get the metadata associated to the current object as a vector of key:value pairs - - #### Qualifiers: * virtual @@ -234,6 +216,7 @@ Puts data to the databse. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -267,6 +250,7 @@ Gets data from the databse. + #### Returns: | Type | Description | | ---- | ---- | @@ -283,7 +267,6 @@ Gets data from the databse. - #### Qualifiers: * virtual @@ -296,6 +279,7 @@ Starts a put operation to the database. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -325,6 +309,7 @@ Starts a get operation to the database. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -355,6 +340,7 @@ set the name for the bucket an object blongs to + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -371,10 +357,6 @@ set the name for the bucket an object blongs to -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-storage-Phobos_file) ### private void fiphoboserver::storage::Phobos_file::close_file () @@ -387,12 +369,8 @@ close the file opened for phobos -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-storage-Phobos_file) [fiphoboserver-storage-Phobos_file]:./Phobos_file.md [fiphoboserver-storage-Storage]:./Storage.md -[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index 4531cf0..10bb683 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -3,17 +3,19 @@ virtual storage class to be implemented by backend storage + This class is a prototype for every class that handles to internal database that should be accessed by the FiPhoboServer. This class represents exactly one object file in the internal database + ## Inheritance: Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phobos_file]. ## Metadata functions | Name | Description | | ---- | ---- | -| [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata an object that is added to the database should get | +| [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata that an object that is added to the database should get | | [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | @@ -42,7 +44,8 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob ## Metadata functions ### public void fiphoboserver::storage::Storage::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name)=0 -set the metadata an object that is added to the database should get +set the metadata that an object that is added to the database should get + @@ -75,6 +78,7 @@ get the metadata associated to the current object as a vector of key:value pairs + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -96,8 +100,6 @@ get the metadata associated to the current object as a vector of key:value pairs - - #### Qualifiers: * virtual @@ -111,6 +113,7 @@ Starts a put operation to the database. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -140,6 +143,7 @@ Puts data to the databse. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -174,6 +178,7 @@ Starts a get operation to the database. + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -203,6 +208,7 @@ Gets data from the databse. + #### Returns: | Type | Description | | ---- | ---- | @@ -219,7 +225,6 @@ Gets data from the databse. - #### Qualifiers: * virtual @@ -237,9 +242,9 @@ default constructor + #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-storage-Storage) @@ -254,6 +259,7 @@ default destructor + #### Qualifiers: * inline * virtual diff --git a/doc/Markdown/fiphoboserver/storage/index.md b/doc/Markdown/fiphoboserver/storage/index.md index e0fb3e3..e21c298 100644 --- a/doc/Markdown/fiphoboserver/storage/index.md +++ b/doc/Markdown/fiphoboserver/storage/index.md @@ -4,10 +4,11 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: + ## Classes | Name | Description | | ---- | ---- | -| [Phobos_file](./Phobos_file.md) | default constructor | +| [Phobos_file](./Phobos_file.md) | Phobos specific mplementation of the [Storage][fiphoboserver-storage-Storage] class. | | [PhobosException](./PhobosException.md) | exceptions specifically for the phobos backend library | | [Storage](./Storage.md) | virtual storage class to be implemented by backend storage | @@ -16,4 +17,3 @@ namespace for storage classes that belong to / inherit from from [fiphoboserver: [fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md [fiphoboserver-storage-Phobos_file]:./Phobos_file.md [fiphoboserver-storage-Storage]:./Storage.md -[todo]:./../../todo.md#todo diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 75db4c1..6227457 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -1,11 +1,10 @@ # public fiphoboserver::stream::Fifo -default constructor - +FIFO specific implementation of the [Stream][fiphoboserver-stream-Stream] class. -> **[Todo][todo]:** maybe not only copy from interface but add specific fifo information? +This class uses a fifo to stream data from the server to a backend storage class. Therefore, all data gets written to the fifo and the implementation of the internal [storage::Storage][fiphoboserver-storage-Storage] class can read and process it from the other end of the fifo. @@ -39,6 +38,12 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation | +## Private Functions +| Name | Description | +| ---- | ---- | +| [create_fifo](#fiphoboserver-stream-Fifo-create_fifo) | create a randomly named fifo | + + ## Implementation specific details ### public int fiphoboserver::stream::Fifo::get_fifo_descriptor () const @@ -47,6 +52,7 @@ get the descriptor of the fifo + #### Returns: | Type | Description | | ---- | ---- | @@ -60,9 +66,8 @@ get the descriptor of the fifo -> **[Todo][todo]:** Do we use this? should this really be public? - +> **[Todo][todo]:** Do we use this? should this really be public? @@ -70,7 +75,6 @@ get the descriptor of the fifo #### Qualifiers: * const * inline -* virtual [Go to Top](#fiphoboserver-stream-Fifo) @@ -81,6 +85,7 @@ get the name of the fifo + #### Returns: | Type | Description | | ---- | ---- | @@ -94,9 +99,8 @@ get the name of the fifo -> **[Todo][todo]:** Do we use this? should this really be public? - +> **[Todo][todo]:** Do we use this? should this really be public? @@ -104,7 +108,6 @@ get the name of the fifo #### Qualifiers: * const * inline -* virtual [Go to Top](#fiphoboserver-stream-Fifo) @@ -116,6 +119,7 @@ default constructor + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -125,10 +129,6 @@ default constructor -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-stream-Fifo) ### public fiphoboserver::stream::Fifo::~Fifo () @@ -141,9 +141,6 @@ default destructor -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-stream-Fifo) @@ -153,6 +150,7 @@ move contructor + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -164,7 +162,6 @@ move contructor #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-stream-Fifo) @@ -175,6 +172,7 @@ assignment operator for rvalue references + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -191,7 +189,6 @@ assignment operator for rvalue references #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-stream-Fifo) @@ -206,9 +203,6 @@ copy constructor deleted -#### Qualifiers: -* virtual - [Go to Top](#fiphoboserver-stream-Fifo) @@ -218,6 +212,7 @@ assignment operator deleted + #### Returns: | Type | Description | | ---- | ---- | @@ -227,10 +222,6 @@ assignment operator deleted -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override @@ -239,6 +230,7 @@ setting the metadata that the created object should get + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -252,11 +244,14 @@ setting the metadata that the created object should get -> **warning:** this function is responsible for getting metadata to the storage implementation + +> **warning:** this function is responsible for getting metadata to the storage implementation +Just passes the data aling to the backend + @@ -273,6 +268,7 @@ get the metadata associated to the current object as a vector of key:value pairs + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -294,8 +290,6 @@ get the metadata associated to the current object as a vector of key:value pairs - - #### Qualifiers: * const * virtual @@ -309,11 +303,12 @@ start a put operation + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | | ssize_t | size | amount of bytes that shall be added to the internal storage | -| std::string | object_id | the identifier the object should get | +| std::string | object_id | the identifier the object should get | @@ -322,6 +317,7 @@ start a put operation +Telling the backend that it must be ready for a PUT operation Opening the FIFO here will not block as the backend will open on the other end @@ -338,6 +334,7 @@ add a chunk of data to the object + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -355,9 +352,7 @@ add a chunk of data to the object - - - +This function can be called repeatedly until the server's message body is depleted @@ -379,6 +374,7 @@ end a put operation + #### Qualifiers: * virtual @@ -391,10 +387,11 @@ start a gett operation + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | object_id | the identifier of the object to retrieve | +| std::string | object_id | the identifier of the object to retrieve | @@ -403,6 +400,7 @@ start a gett operation +Telling the backend that it must be ready for a PUT operation Opening the FIFO here will not block as the backend will open on the other end @@ -419,6 +417,7 @@ gets a chunk of data of the object + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -428,7 +427,7 @@ gets a chunk of data of the object #### Returns: | Type | Description | | ---- | ---- | -| ssize_t | the number of bytes the buffer has been filled with | +| ssize_t | the number of bytes the buffer has been filled with | @@ -440,6 +439,8 @@ gets a chunk of data of the object +Repeatedly called to read from FIFO into buffer until the FIFO is empty and closed on the writing end + @@ -460,10 +461,32 @@ end a get operation + #### Qualifiers: * virtual +[Go to Top](#fiphoboserver-stream-Fifo) + +## Private Functions +### private void fiphoboserver::stream::Fifo::create_fifo () + +create a randomly named fifo + + + + + + + + + + +The fifo is just created here, because opening it would block it + + + + [Go to Top](#fiphoboserver-stream-Fifo) [fiphoboserver-storage-Storage]:./../storage/Storage.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index 1bfd99d..d37f819 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -1,6 +1,8 @@ # public fiphoboserver::stream::Stream -virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class +virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend +[storage::Storage][fiphoboserver-storage-Storage] class + @@ -43,6 +45,7 @@ start a put operation + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -72,6 +75,7 @@ add a chunk of data to the object + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -89,9 +93,7 @@ add a chunk of data to the object - - - +This function can be called repeatedly until the server's message body is depleted @@ -113,6 +115,7 @@ end a put operation + #### Qualifiers: * virtual @@ -125,6 +128,7 @@ start a gett operation + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -153,6 +157,7 @@ gets a chunk of data of the object + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -175,8 +180,6 @@ gets a chunk of data of the object - - #### Qualifiers: * const * virtual @@ -194,6 +197,7 @@ end a get operation + #### Qualifiers: * virtual @@ -211,6 +215,7 @@ an implementation of [storage::Storage][fiphoboserver-storage-Storage] that will + [Go to Top](#fiphoboserver-stream-Stream) ### protected fiphoboserver::stream::Stream::m_db_result @@ -225,9 +230,8 @@ a future object that can be used to get the results from the asynchronous backen -> **[Todo][todo]:** should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? - +> **[Todo][todo]:** should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? @@ -245,9 +249,9 @@ default constructor + #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-stream-Stream) @@ -258,6 +262,7 @@ constructor with storage implementation + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -270,17 +275,14 @@ constructor with storage implementation -> **warning:** every class that inherits this interface must have an internal [storage::Storage][fiphoboserver-storage-Storage] implementation and initialize this here. - - +> **warning:** every class that inherits this interface must have an internal [storage::Storage][fiphoboserver-storage-Storage] implementation and initialize this here. #### Qualifiers: * inline -* virtual [Go to Top](#fiphoboserver-stream-Stream) @@ -295,6 +297,7 @@ default destructor + #### Qualifiers: * inline * virtual @@ -308,6 +311,7 @@ setting the metadata that the created object should get + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -321,10 +325,8 @@ setting the metadata that the created object should get -> **warning:** this function is responsible for getting metadata to the storage implementation - - +> **warning:** this function is responsible for getting metadata to the storage implementation @@ -342,6 +344,7 @@ get the metadata associated to the current object as a vector of key:value pairs + #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | @@ -363,8 +366,6 @@ get the metadata associated to the current object as a vector of key:value pairs - - #### Qualifiers: * const * virtual diff --git a/doc/Markdown/fiphoboserver/stream/index.md b/doc/Markdown/fiphoboserver/stream/index.md index 301d295..fd4bfce 100644 --- a/doc/Markdown/fiphoboserver/stream/index.md +++ b/doc/Markdown/fiphoboserver/stream/index.md @@ -4,15 +4,16 @@ namespace for stream classes that belong to / inherit from from [fiphoboserver:: + ## Classes | Name | Description | | ---- | ---- | -| [Fifo](./Fifo.md) | default constructor | -| [Stream](./Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend [storage::Storage][fiphoboserver-storage-Storage] class | +| [Fifo](./Fifo.md) | FIFO specific implementation of the [Stream][fiphoboserver-stream-Stream] class. | +| [Stream](./Stream.md) | virtual [Stream][fiphoboserver-stream-Stream] class to be implemented for streaming chunks of data between the server and a backend +[storage::Storage][fiphoboserver-storage-Storage] class | [fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Fifo]:./Fifo.md [fiphoboserver-stream-Stream]:./Stream.md -[todo]:./../../todo.md#todo diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 3766e41..832b609 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -2,28 +2,24 @@ -* Member [fiphoboserver::GetRequestHandler::requestComplete][fiphoboserver-GetRequestHandler-requestComplete] () noexcept override +* Member +[fiphoboserver::GetRequestHandler::requestComplete][fiphoboserver-GetRequestHandler-requestComplete] () noexcept override * find out what this does? -* Class [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phobos_file] - * maybe not only copy from interface but add specific phobos information? -* Class [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo] - * maybe not only copy from interface but add specific fifo information? -* Member [fiphoboserver::stream::Fifo::get_fifo_descriptor][fiphoboserver-stream-Fifo-get_fifo_descriptor] () const +* Member +[fiphoboserver::stream::Fifo::get_fifo_descriptor][fiphoboserver-stream-Fifo-get_fifo_descriptor] () const * Do we use this? should this really be public? -* Member [fiphoboserver::stream::Fifo::get_fifo_name][fiphoboserver-stream-Fifo-get_fifo_name] () const +* Member +[fiphoboserver::stream::Fifo::get_fifo_name][fiphoboserver-stream-Fifo-get_fifo_name] () const * Do we use this? should this really be public? -* Member [fiphoboserver::stream::Stream::m_db_result][fiphoboserver-stream-Stream-m_db_result] +* Member +[fiphoboserver::stream::Stream::m_db_result][fiphoboserver-stream-Stream-m_db_result] * should this really be a protected member in the interface? Isn't it the decision of the implementation to make this asynchronous? - - [fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-storage-Phobos_file]:./fiphoboserver/storage/Phobos_file.md -[fiphoboserver-stream-Fifo]:./fiphoboserver/stream/Fifo.md [fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor [fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name [fiphoboserver-stream-Stream-m_db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-m_db_result diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 307d77c..b09e977 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -34,7 +34,7 @@ class Fifo : public Stream { /// /// @copydoc Stream::set_storage_meta_data /// - /// Just passes the data aling to the backend + /// Just passes the data along to the backend /// void set_storage_meta_data( std::vector> meta_data, -- GitLab From 25c2bf4c201d8e92e59cb439a77b84d53c1906e1 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 24 Mar 2020 10:31:47 +0000 Subject: [PATCH 33/62] Minor Code Improvements --- doc/Markdown/fiphoboserver/S3_header.md | 32 +++- .../fiphoboserver/storage/Phobos_file.md | 8 +- doc/Markdown/fiphoboserver/storage/Storage.md | 8 +- doc/Markdown/fiphoboserver/stream/Fifo.md | 71 ++++----- doc/Markdown/todo.md | 8 +- src/CMakeLists.txt | 14 +- src/fiphoboserver_exception.h | 2 +- src/main.cc | 3 - src/server/CMakeLists.txt | 5 +- src/server/S3_header.h | 39 ++++- src/server/get_request_handler.cc | 6 +- src/server/get_request_handler.h | 7 +- src/server/getmd_request_handler.cc | 8 +- src/server/getmd_request_handler.h | 6 +- src/server/put_request_handler.cc | 3 - src/server/put_request_handler.h | 6 +- src/server/unsupported_request_handler.cc | 4 +- src/storage/CMakeLists.txt | 4 - src/storage/phobos_cpp_wrapper/CMakeLists.txt | 10 +- src/storage/phobos_file.cc | 34 +--- src/storage/phobos_file.h | 7 +- src/storage/storage.h | 20 +-- src/stream/CMakeLists.txt | 11 +- src/stream/fifo.cc | 71 +++------ src/stream/fifo.h | 147 +++++++++--------- src/stream/stream.h | 1 - 26 files changed, 248 insertions(+), 287 deletions(-) diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index 296fc0e..d76e484 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -10,6 +10,7 @@ class to extract the S3 specific information from proxygens HTTPMessage headers | ---- | ---- | | [set_headers](#fiphoboserver-S3_header-set_headers) | sets the internal header instance to the given header object | | [get_bucket](#fiphoboserver-S3_header-get_bucket) | extracts the name of the S3 bucket requested in the HTTP message | +| [get_key_without_bucket](#fiphoboserver-S3_header-get_key_without_bucket) | extracts the name of the key id requested in the HTTP message | | [get_key](#fiphoboserver-S3_header-get_key) | extracts the name of the key id requested in the HTTP message | | [is_create_bucket_request](#fiphoboserver-S3_header-is_create_bucket_request) | checks if the message we got is one for just creating a bucket | | [get_body_length](#fiphoboserver-S3_header-get_body_length) | gets the total length of all body chunks that is expected | @@ -76,7 +77,7 @@ extracts the name of the S3 bucket requested in the HTTP message [Go to Top](#fiphoboserver-S3_header) -### public std::string fiphoboserver::S3_header::get_key () +### public std::string fiphoboserver::S3_header::get_key_without_bucket () extracts the name of the key id requested in the HTTP message @@ -94,7 +95,36 @@ extracts the name of the key id requested in the HTTP message +This will only return the key itself without the leading bucket name! + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-S3_header) + +### public std::string fiphoboserver::S3_header::get_key () + +extracts the name of the key id requested in the HTTP message + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the key | + + + + + + +This is the whole path argument without the leading '/' but including the bucket name! diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index be61797..a08ec06 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -273,7 +273,7 @@ Gets data from the databse. [Go to Top](#fiphoboserver-storage-Phobos_file) -### public void fiphoboserver::storage::Phobos_file::prepare_put (std::string file_name, std::string object_id) override +### public void fiphoboserver::storage::Phobos_file::prepare_put (int file_descriptor, std::string object_id) override Starts a put operation to the database. @@ -283,7 +283,7 @@ Starts a put operation to the database. #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| int | file_descriptor | the descriptor of the open file, which contents should be added to the database | | std::string | object_id | the internal storage id the data should get | @@ -303,7 +303,7 @@ Starts a put operation to the database. [Go to Top](#fiphoboserver-storage-Phobos_file) -### public void fiphoboserver::storage::Phobos_file::prepare_get (std::string file_name, std::string object_id) override +### public void fiphoboserver::storage::Phobos_file::prepare_get (int file_descriptor, std::string object_id) override Starts a get operation to the database. @@ -313,7 +313,7 @@ Starts a get operation to the database. #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | file_name | the filename the data from the database should be written to | +| int | file_descriptor | the descriptor of the open file, to which the contents of the database should be written | | std::string | object_id | the internal storage id from the data that should be retrieved | diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index 10bb683..384495f 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -107,7 +107,7 @@ get the metadata associated to the current object as a vector of key:value pairs [Go to Top](#fiphoboserver-storage-Storage) ## PUT functions -### public void fiphoboserver::storage::Storage::prepare_put (std::string file_name, std::string object_id)=0 +### public void fiphoboserver::storage::Storage::prepare_put (int file_descriptor, std::string object_id)=0 Starts a put operation to the database. @@ -117,7 +117,7 @@ Starts a put operation to the database. #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | file_name | the filename the data that will be added to the database shouldbe read from | +| int | file_descriptor | the descriptor of the open file, which contents should be added to the database | | std::string | object_id | the internal storage id the data should get | @@ -172,7 +172,7 @@ Puts data to the databse. [Go to Top](#fiphoboserver-storage-Storage) ## GET functions -### public void fiphoboserver::storage::Storage::prepare_get (std::string file_name, std::string object_id)=0 +### public void fiphoboserver::storage::Storage::prepare_get (int file_descriptor, std::string object_id)=0 Starts a get operation to the database. @@ -182,7 +182,7 @@ Starts a get operation to the database. #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | file_name | the filename the data from the database should be written to | +| int | file_descriptor | the descriptor of the open file, to which the contents of the database should be written | | std::string | object_id | the internal storage id from the data that should be retrieved | diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 6227457..6309703 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -15,8 +15,13 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. ## Implementation specific details | Name | Description | | ---- | ---- | -| [get_fifo_descriptor](#fiphoboserver-stream-Fifo-get_fifo_descriptor) | get the descriptor of the fifo | -| [get_fifo_name](#fiphoboserver-stream-Fifo-get_fifo_name) | get the name of the fifo | +| [get_fifo_descriptors](#fiphoboserver-stream-Fifo-get_fifo_descriptors) | get the descriptor of the fifo | + + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_fifo_descriptor[2]](#fiphoboserver-stream-Fifo-m_fifo_descriptor) | the file descriptors for the FIFO. | ## Public Functions @@ -41,12 +46,12 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. ## Private Functions | Name | Description | | ---- | ---- | -| [create_fifo](#fiphoboserver-stream-Fifo-create_fifo) | create a randomly named fifo | +| [create_fifo](#fiphoboserver-stream-Fifo-create_fifo) | create a fifo by using the pipe command on the m_fifo_descriptor | ## Implementation specific details -### public int fiphoboserver::stream::Fifo::get_fifo_descriptor () const +### public const int * fiphoboserver::stream::Fifo::get_fifo_descriptors () const get the descriptor of the fifo @@ -56,7 +61,7 @@ get the descriptor of the fifo #### Returns: | Type | Description | | ---- | ---- | -| int | the descriptor of the fifo | +| const int * | the descriptor of the fifo | @@ -79,17 +84,10 @@ get the descriptor of the fifo [Go to Top](#fiphoboserver-stream-Fifo) -### public std::string fiphoboserver::stream::Fifo::get_fifo_name () const - -get the name of the fifo +## Private Attributes +### private fiphoboserver::stream::Fifo::m_fifo_descriptor[2] [2] = {-1, -1} - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| std::string | the name of the fifo | +the file descriptors for the FIFO. @@ -100,14 +98,12 @@ get the name of the fifo -> **[Todo][todo]:** Do we use this? should this really be public? +- Index 0 for read end +- Index 1 for write end -#### Qualifiers: -* const -* inline [Go to Top](#fiphoboserver-stream-Fifo) @@ -154,7 +150,14 @@ move contructor #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| [Fifo][fiphoboserver-stream-Fifo] && | input | | +| [Fifo][fiphoboserver-stream-Fifo] && | input | the other object to be moved | + + + + + + + @@ -176,12 +179,19 @@ assignment operator for rvalue references #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| [Fifo][fiphoboserver-stream-Fifo] && | input | | +| [Fifo][fiphoboserver-stream-Fifo] && | input | the other object to assign | #### Returns: | Type | Description | | ---- | ---- | -| [Fifo][fiphoboserver-stream-Fifo] & | | +| [Fifo][fiphoboserver-stream-Fifo] & | reference to newly created object | + + + + + + + @@ -250,7 +260,7 @@ setting the metadata that the created object should get -Just passes the data aling to the backend +Just passes the data along to the backend @@ -308,7 +318,7 @@ start a put operation | Type | Name | Description | | ---- | ---- | ---- | | ssize_t | size | amount of bytes that shall be added to the internal storage | -| std::string | object_id | the identifier the object should get | +| std::string | object_id | the identifier the object should get | @@ -317,7 +327,6 @@ start a put operation -Telling the backend that it must be ready for a PUT operation Opening the FIFO here will not block as the backend will open on the other end @@ -391,7 +400,7 @@ start a gett operation #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::string | object_id | the identifier of the object to retrieve | +| std::string | object_id | the identifier of the object to retrieve | @@ -400,7 +409,6 @@ start a gett operation -Telling the backend that it must be ready for a PUT operation Opening the FIFO here will not block as the backend will open on the other end @@ -471,18 +479,11 @@ end a get operation ## Private Functions ### private void fiphoboserver::stream::Fifo::create_fifo () -create a randomly named fifo - - - - - - +create a fifo by using the pipe command on the m_fifo_descriptor -The fifo is just created here, because opening it would block it diff --git a/doc/Markdown/todo.md b/doc/Markdown/todo.md index 832b609..4cc34ac 100644 --- a/doc/Markdown/todo.md +++ b/doc/Markdown/todo.md @@ -6,10 +6,7 @@ [fiphoboserver::GetRequestHandler::requestComplete][fiphoboserver-GetRequestHandler-requestComplete] () noexcept override * find out what this does? * Member -[fiphoboserver::stream::Fifo::get_fifo_descriptor][fiphoboserver-stream-Fifo-get_fifo_descriptor] () const - * Do we use this? should this really be public? -* Member -[fiphoboserver::stream::Fifo::get_fifo_name][fiphoboserver-stream-Fifo-get_fifo_name] () const +[fiphoboserver::stream::Fifo::get_fifo_descriptors][fiphoboserver-stream-Fifo-get_fifo_descriptors] () const * Do we use this? should this really be public? * Member [fiphoboserver::stream::Stream::m_db_result][fiphoboserver-stream-Stream-m_db_result] @@ -20,6 +17,5 @@ [fiphoboserver-GetRequestHandler-requestComplete]:./fiphoboserver/GetRequestHandler.md#fiphoboserver-GetRequestHandler-requestComplete -[fiphoboserver-stream-Fifo-get_fifo_descriptor]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptor -[fiphoboserver-stream-Fifo-get_fifo_name]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_name +[fiphoboserver-stream-Fifo-get_fifo_descriptors]:./fiphoboserver/stream/Fifo.md#fiphoboserver-stream-Fifo-get_fifo_descriptors [fiphoboserver-stream-Stream-m_db_result]:./fiphoboserver/stream/Stream.md#fiphoboserver-stream-Stream-m_db_result diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6426566..0daa8c0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,13 +6,7 @@ add_subdirectory(storage) add_subdirectory(stream) add_subdirectory(server) -add_library(libfiphoboserver main.cc) - -target_compile_features(libfiphoboserver PUBLIC cxx_std_14) - -target_link_libraries(libfiphoboserver PUBLIC stream) -target_link_libraries(libfiphoboserver PUBLIC storage) -target_link_libraries(libfiphoboserver PUBLIC server) +set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) add_executable( fiphoboserver @@ -20,4 +14,8 @@ add_executable( main.cc ) -target_link_libraries(fiphoboserver PUBLIC libfiphoboserver) +target_compile_features(fiphoboserver PUBLIC cxx_std_14) + +target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen) +target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver) +target_link_libraries(fiphoboserver PUBLIC server) diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index bafdcf1..8f50074 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -1,8 +1,8 @@ #pragma once #include -#include #include +#include namespace fiphoboserver { diff --git a/src/main.cc b/src/main.cc index 274e2a2..25662da 100644 --- a/src/main.cc +++ b/src/main.cc @@ -6,13 +6,10 @@ * LICENSE file in the root directory of this source tree. */ -#include #include #include #include -#include #include -#include #include #include diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index a1244ae..9360316 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -1,6 +1,3 @@ -include_directories(${PHOBOS_INCLUDE_DIRECTORY}) -include_directories(/usr/include/glib-2.0) -include_directories(/usr/lib64/glib-2.0/include) add_library( server @@ -11,4 +8,4 @@ add_library( ) target_link_libraries(server PUBLIC proxygen::proxygen) -target_link_libraries(server PUBLIC proxygen::proxygenhttpserver) +target_link_libraries(server PUBLIC stream) diff --git a/src/server/S3_header.h b/src/server/S3_header.h index c7055ed..b377cd6 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -2,6 +2,13 @@ #include +#include +#include +#include +#include +#include +#include + namespace fiphoboserver { /// @@ -40,9 +47,11 @@ class S3_header { /// /// @brief extracts the name of the key id requested in the HTTP message /// + /// This will only return the key itself without the leading bucket name! + /// /// @returns name of the key /// - std::string get_key() + std::string get_key_without_bucket() { if (!m_headers) { return ""; @@ -51,12 +60,30 @@ class S3_header { path = path.substr(1, path.size()); std::string file_key = path.substr(path.find('/') + 1, path.size() - 1); - if (file_key.find('/') != std::string::npos - || file_key.find('\\') != std::string::npos) { - std::cout << "Bad Key! " << std::endl; + return file_key; + } + + /// + /// @brief extracts the name of the key id requested in the HTTP message + /// + /// This is the whole path argument without the leading '/' but including + /// the bucket name! + /// + /// @returns name of the key + /// + std::string get_key() + { + if (!m_headers) { return ""; } - return file_key; + + std::string path = m_headers->getPath(); + if(path.rfind('/', 0) == 0) + { + path = path.substr(1, path.size()); + } + + return path; } /// @@ -73,7 +100,7 @@ class S3_header { if (!m_headers) { return false; } - if (get_key().empty()) { + if (get_key_without_bucket().empty()) { return true; } return false; diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index 0e47752..bbfc440 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -7,16 +7,12 @@ #include "get_request_handler.h" -#include -#include +#include "../fiphoboserver_exception.h" -#include #include #include -#include #include -#include "../fiphoboserver_exception.h" namespace fiphoboserver { diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index c4889da..9913ffe 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -8,13 +8,12 @@ #pragma once -//#include -#include -#include - #include "../stream/stream.h" #include "S3_header.h" +#include +#include + namespace fiphoboserver { /// diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index c17c408..8bf3e05 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -7,16 +7,10 @@ #include "getmd_request_handler.h" -#include -#include +#include "../fiphoboserver_exception.h" -#include -#include -#include -#include #include -#include "../fiphoboserver_exception.h" namespace fiphoboserver { diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index df49c64..187a49d 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -8,12 +8,12 @@ #pragma once -//#include +#include "../stream/stream.h" +#include "S3_header.h" + #include #include -#include "../stream/stream.h" -#include "S3_header.h" namespace fiphoboserver { diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 0c57ee3..171ff26 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -8,9 +8,6 @@ #include "put_request_handler.h" -#include -#include -#include #include #include "../fiphoboserver_exception.h" diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index b5786d3..c4722e6 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -8,12 +8,12 @@ #pragma once -#include -#include - #include "../stream/stream.h" #include "S3_header.h" +#include +#include + namespace fiphoboserver { /// diff --git a/src/server/unsupported_request_handler.cc b/src/server/unsupported_request_handler.cc index 3a5067e..9996891 100644 --- a/src/server/unsupported_request_handler.cc +++ b/src/server/unsupported_request_handler.cc @@ -8,11 +8,9 @@ #include "unsupported_request_handler.h" -#include #include -#include +#include -#include #include namespace fiphoboserver { diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 9027e05..9e13e7d 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -5,10 +5,6 @@ add_library( phobos_file.cc ) -target_include_directories(storage PUBLIC ${PHOBOS_INCLUDE_DIRECTORY}) target_include_directories(storage PUBLIC /usr/include/glib-2.0) -target_include_directories(storage PUBLIC /usr/lib64/glib-2.0/include) -target_link_libraries(storage PUBLIC phobos_store) target_link_libraries(storage PUBLIC phobos_cpp_wrapper) -target_link_libraries(storage PUBLIC proxygen::proxygen) diff --git a/src/storage/phobos_cpp_wrapper/CMakeLists.txt b/src/storage/phobos_cpp_wrapper/CMakeLists.txt index a948b9f..9bdbcf8 100644 --- a/src/storage/phobos_cpp_wrapper/CMakeLists.txt +++ b/src/storage/phobos_cpp_wrapper/CMakeLists.txt @@ -1,17 +1,9 @@ -include_directories(${PHOBOS_INCLUDE_DIRECTORY}) -include_directories(/usr/include/glib-2.0) -include_directories(/usr/lib64/glib-2.0/include) add_library( phobos_cpp_wrapper phobos_cpp_wrapper.c ) -set_target_properties( phobos_cpp_wrapper - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver/phobos_cpp_wrapper" - LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver/phobos_cpp_wrapper" - RUNTIME_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver/phobos_cpp_wrapper" -) +target_include_directories(phobos_cpp_wrapper PUBLIC /usr/include/glib-2.0) target_link_libraries(phobos_cpp_wrapper PUBLIC phobos_store) diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index a89de5f..bd11124 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -1,19 +1,10 @@ #include "phobos_file.h" +#include "phobos_exception.h" + #include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include - -#include "../fiphoboserver_exception.h" -#include "phobos_exception.h" +#include namespace fiphoboserver { namespace storage { @@ -35,16 +26,12 @@ Phobos_file::~Phobos_file() } /* - * Open the file for reading * fill the m_descriptor with relevant data */ -void Phobos_file::prepare_put(std::string file_name, std::string object_id) +void Phobos_file::prepare_put(int file_descriptor, std::string object_id) { - m_descriptor.xd_fd = open(file_name.c_str(), O_RDONLY); - if (m_descriptor.xd_fd < 0) { - throw IOException("Phobos_file::prepare_put", file_name.c_str(), errno); - return; - } + m_descriptor.xd_fd = file_descriptor; + char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); @@ -66,16 +53,11 @@ ssize_t Phobos_file::db_put(size_t size) } /* - * Open the file for writing * fill the m_descriptor with relevant data */ -void Phobos_file::prepare_get(std::string file_name, std::string object_id) +void Phobos_file::prepare_get(int file_descriptor, std::string object_id) { - m_descriptor.xd_fd = open(file_name.c_str(), O_WRONLY); - if (m_descriptor.xd_fd < 0) { - throw IOException("Phobos_file::prepare_get", file_name.c_str(), errno); - return; - } + m_descriptor.xd_fd = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index c9b73e1..94deafe 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -4,6 +4,9 @@ extern "C" { #include "phobos_cpp_wrapper/phobos_cpp_wrapper.h" } +#include +#include + namespace fiphoboserver { namespace storage { @@ -43,9 +46,9 @@ class Phobos_file : public Storage { ssize_t db_get() override; /// @copydoc Storage::prepare_put - void prepare_put(std::string file_name, std::string object_id) override; + void prepare_put(int file_descriptor, std::string object_id) override; /// @copydoc Storage::prepare_get - void prepare_get(std::string file_name, std::string object_id) override; + void prepare_get(int file_descriptor, std::string object_id) override; int get_fd() const { return m_descriptor.xd_fd; } private: diff --git a/src/storage/storage.h b/src/storage/storage.h index 3a9ba96..21cc071 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -66,11 +65,11 @@ class Storage { /// /// @brief Starts a put operation to the database /// - /// @param file_name the filename the data that will be added to the - /// database shouldbe read from - /// @param object_id the internal storage id the data should get + /// @param file_descriptor the descriptor of the open file, which + /// contents should be added to the database + /// @param object_id the internal storage id the data should get /// - virtual void prepare_put(std::string file_name, std::string object_id) = 0; + virtual void prepare_put(int file_descriptor, std::string object_id) = 0; /// /// @brief Puts data to the databse /// @@ -87,12 +86,13 @@ class Storage { /// /// @brief Starts a get operation to the database /// - /// @param file_name the filename the data from the database should - /// be written to - /// @param object_id the internal storage id from the data that - /// should be retrieved + /// @param file_descriptor the descriptor of the open file, to which + /// the contents of the database should be + /// written + /// @param object_id the internal storage id from the data that + /// should be retrieved /// - virtual void prepare_get(std::string file_name, std::string object_id) = 0; + virtual void prepare_get(int file_descriptor, std::string object_id) = 0; /// /// @brief Gets data from the databse /// diff --git a/src/stream/CMakeLists.txt b/src/stream/CMakeLists.txt index e04cc21..17a5265 100644 --- a/src/stream/CMakeLists.txt +++ b/src/stream/CMakeLists.txt @@ -3,13 +3,4 @@ add_library( fifo.cc ) -if(CUSTOM_OUTPUT_DIRECTORY) - set_target_properties(stream - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - RUNTIME_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver" - ) -endif(CUSTOM_OUTPUT_DIRECTORY) - -target_link_libraries(stream PUBLIC proxygen::proxygen) +target_link_libraries(stream PUBLIC storage) diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index d55599a..28a5da0 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -2,14 +2,8 @@ #include "../fiphoboserver_exception.h" -#include -#include -#include -#include -#include -#include -#include #include +#include namespace fiphoboserver { namespace stream { @@ -23,11 +17,11 @@ Fifo::Fifo(std::unique_ptr input_storage) : Fifo::~Fifo() { - if (access(m_fifo_name.c_str(), F_OK) != -1) { - remove(m_fifo_name.c_str()); + if (m_fifo_descriptor[0] > 0) { + close(m_fifo_descriptor[0]); } - if (m_fifo_descriptor > 0) { - close(m_fifo_descriptor); + if (m_fifo_descriptor[1] > 0) { + close(m_fifo_descriptor[1]); } } /* Simply pass along to the backend */ @@ -44,43 +38,17 @@ std::vector> Fifo::get_meta_data( return std::move(m_storage->get_meta_data(object_id)); } -/* - * Generate a randomly named FIFO - * Opening here would block so we don't - */ -void Fifo::create_fifo() -{ - m_fifo_name = std::tmpnam(nullptr); - int rc = mkfifo(m_fifo_name.c_str(), 0777); - if (rc < 0) { - if (errno != EEXIST) { - throw IOException("Fifo::create_fifo", "mkfifo", errno); - return; - } - } -} /* * Tell the backend that it must be ready for a PUT operation - * Opening the FIFO here will not block as the backend will - * open on the other end */ void Fifo::start_put(ssize_t size, std::string object_id) { /* Start backend storage solution asynchronously */ m_db_result = std::async([this, object_id, size]() { - m_storage->prepare_put(m_fifo_name, object_id); + m_storage->prepare_put(m_fifo_descriptor[0], object_id); return m_storage->db_put(size); }); - - /* Open fifo on the writing end */ - m_fifo_descriptor = open(m_fifo_name.c_str(), O_WRONLY); - if (m_fifo_descriptor < 0) { - std::stringstream ss; - ss << "open (" << m_fifo_name.c_str() << ")"; - throw IOException("Fifo::start_put", ss.str().c_str(), errno); - return; - } } /* @@ -89,7 +57,7 @@ void Fifo::start_put(ssize_t size, std::string object_id) */ ssize_t Fifo::put(const void* data, const size_t count) const { - ssize_t rc = write(m_fifo_descriptor, data, count); + ssize_t rc = write(m_fifo_descriptor[1], data, count); if (rc < count) { throw IOException("Fifo::put", "write", rc); } @@ -105,25 +73,14 @@ void Fifo::finish_put() /* * Tell the backend that it must be ready for a GET operation - * Opening the FIFO here will not block as the backend will - * open on the other end */ void Fifo::start_get(std::string object_id) { /* Start backend storage solution asynchronously */ m_db_result = std::async([this, object_id]() { - m_storage->prepare_get(m_fifo_name, object_id); + m_storage->prepare_get(m_fifo_descriptor[1], object_id); return m_storage->db_get(); }); - - /* Open fifo on the reading end */ - m_fifo_descriptor = open(m_fifo_name.c_str(), O_RDONLY); - if (m_fifo_descriptor < 0) { - std::stringstream ss; - ss << "open (" << m_fifo_name.c_str() << ")"; - throw IOException("Fifo::start_get", ss.str().c_str(), errno); - return; - } } /* @@ -134,7 +91,7 @@ void Fifo::start_get(std::string object_id) */ ssize_t Fifo::get(void* buf, const size_t count) const { - ssize_t rc = read(m_fifo_descriptor, buf, count); + ssize_t rc = read(m_fifo_descriptor[0], buf, count); if (rc < 0) { throw IOException("Fifo::get", "read", rc); } @@ -147,5 +104,15 @@ void Fifo::finish_get() m_db_result.get(); } + +void Fifo::create_fifo() +{ + int rc = pipe(m_fifo_descriptor); + if ( rc == -1) { + throw IOException("Fifo::create_fifo", "mkfifo", errno); + return; + } +} + } // namespace stream } // namespace fiphoboserver diff --git a/src/stream/fifo.h b/src/stream/fifo.h index b09e977..594fa40 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -1,5 +1,10 @@ #include "stream.h" +#include +#include +#include +#include + namespace fiphoboserver { namespace stream { @@ -12,66 +17,66 @@ namespace stream { /// other end of the fifo. /// class Fifo : public Stream { - public: - /// @copydoc Stream::Stream - Fifo(std::unique_ptr input_storage); - /// @copydoc Stream::~Stream - ~Fifo(); +public: + /// @copydoc Stream::Stream + Fifo(std::unique_ptr input_storage); + /// @copydoc Stream::~Stream + ~Fifo(); - /// @brief move contructor - Fifo(Fifo&& input) : Stream(std::move(input.m_storage)) {} - /// @brief assignment operator for rvalue references - Fifo& operator=(Fifo&& input) - { - Stream::m_storage = std::move(input.m_storage); - return *this; - } - /// @brief copy constructor deleted - Fifo(const Fifo&) = delete; - /// @brief assignment operator deleted - Fifo& operator=(const Fifo&) = delete; + /// + /// @brief move contructor + /// + /// @param input the other object to be moved + /// + Fifo(Fifo&& input) : Stream(std::move(input.m_storage)) {} - /// - /// @copydoc Stream::set_storage_meta_data - /// - /// Just passes the data along to the backend - /// - void set_storage_meta_data( - std::vector> meta_data, - std::string bucket_name) const override; - /// @copydoc Stream::get_meta_data - std::vector> get_meta_data( - std::string object_id) const override; + /// + /// @brief assignment operator for rvalue references + /// + /// @param input the other object to assign + /// + /// @return reference to newly created object + /// + Fifo& operator=(Fifo&& input) + { + Stream::m_storage = std::move(input.m_storage); + return *this; + } + /// @brief copy constructor deleted + Fifo(const Fifo&) = delete; + /// @brief assignment operator deleted + Fifo& operator=(const Fifo&) = delete; - /// @copydoc Stream::start_put - /// - /// Telling the backend that it must be ready for a PUT operation - /// Opening the FIFO here will not block as the backend will - /// open on the other end - /// - void start_put(ssize_t size, std::string object_id) override; - /// @copydoc Stream::put - ssize_t put(const void* data, size_t count) const override; - /// @copydoc Stream::finish_put - void finish_put() override; + /// + /// @copydoc Stream::set_storage_meta_data + /// + /// Just passes the data along to the backend + /// + void set_storage_meta_data( + std::vector> meta_data, + std::string bucket_name) const override; + /// @copydoc Stream::get_meta_data + std::vector> get_meta_data( + std::string object_id) const override; - /// @copydoc Stream::start_get - /// - /// Telling the backend that it must be ready for a PUT operation - /// Opening the FIFO here will not block as the backend will - /// open on the other end - /// - void start_get(std::string object_id) override; - /// @copydoc Stream::get - /// - /// Repeatedly called to read from FIFO into buffer - /// until the FIFO is empty and closed on the - /// writing end - /// - ssize_t get(void *buf, size_t count) const override; - /// @copydoc Stream::finish_get - void finish_get() override; + /// @copydoc Stream::start_put + void start_put(ssize_t size, std::string object_id) override; + /// @copydoc Stream::put + ssize_t put(const void* data, size_t count) const override; + /// @copydoc Stream::finish_put + void finish_put() override; + /// @copydoc Stream::start_get + void start_get(std::string object_id) override; + /// @copydoc Stream::get + /// + /// Repeatedly called to read from FIFO into buffer + /// until the FIFO is empty and closed on the + /// writing end + /// + ssize_t get(void* buf, size_t count) const override; + /// @copydoc Stream::finish_get + void finish_get() override; /// @name Implementation specific details /// @@ -84,26 +89,22 @@ class Fifo : public Stream { /// /// @todo Do we use this? should this really be public? /// - int get_fifo_descriptor() const { return m_fifo_descriptor; } - /// - /// @brief get the name of the fifo - /// - /// @returns the name of the fifo - /// - /// @todo Do we use this? should this really be public? - /// - std::string get_fifo_name() const { return m_fifo_name; } - + const int* get_fifo_descriptors() const { return m_fifo_descriptor; } + /// @} - private: - /// - /// @brief create a randomly named fifo - /// - /// The fifo is just created here, because opening it would block it - void create_fifo(); - std::string m_fifo_name; - int m_fifo_descriptor = -1; + private: + /// + /// @brief create a fifo by using the pipe command on the m_fifo_descriptor + /// + void create_fifo(); + + /// + /// @brief the file descriptors for the FIFO. + /// * Index 0 for read end + /// * Index 1 for write end + /// + int m_fifo_descriptor[2] = {-1, -1}; }; } // namespace stream diff --git a/src/stream/stream.h b/src/stream/stream.h index cd679c0..c1d7794 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -2,7 +2,6 @@ #include "../storage/storage.h" -#include #include #include #include -- GitLab From 1c7a83c11e1949c8853d154ecfdbda4ef3d13348 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 24 Mar 2020 14:18:00 +0000 Subject: [PATCH 34/62] Put a variable around the documentation build --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec2eafa..887d14c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,9 @@ option( ) add_subdirectory(src) -add_subdirectory(doc) +if(CUSTOM_DOCU_PATH) + add_subdirectory(doc) +endif(CUSTOM_DOCU_PATH) # Build tests if(FIPHOBOSERVER_BUILD_TESTS) -- GitLab From 3adfd80afeaf840f7c4ffccc3c6e68bfde5a30e1 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 24 Mar 2020 14:21:13 +0000 Subject: [PATCH 35/62] Revert "Put a variable around the documentation build" This reverts commit 1c7a83c11e1949c8853d154ecfdbda4ef3d13348 --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 887d14c..ec2eafa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,9 +17,7 @@ option( ) add_subdirectory(src) -if(CUSTOM_DOCU_PATH) - add_subdirectory(doc) -endif(CUSTOM_DOCU_PATH) +add_subdirectory(doc) # Build tests if(FIPHOBOSERVER_BUILD_TESTS) -- GitLab From ec41e6adef35dec0187b552408beee2569ea6f71 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 24 Mar 2020 14:41:22 +0000 Subject: [PATCH 36/62] Fix a documentation build issue: Now builds correctly without documentation enabled (Cmake variable CUSTOM_DOCU_PATH) --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec2eafa..1ce01ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,10 @@ option( ) add_subdirectory(src) -add_subdirectory(doc) + +if(CUSTOM_DOCU_PATH) + add_subdirectory(doc) +endif(CUSTOM_DOCU_PATH) # Build tests if(FIPHOBOSERVER_BUILD_TESTS) -- GitLab From 755f0d1a384fe931d7a5d8c8dbacdeebca4e4182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Mon, 30 Mar 2020 15:23:39 +0100 Subject: [PATCH 37/62] Squashed commit of the following: * utility functions for tests (utils.h, utils.cc) * copy test text files to test binary directory during build * tests for Phobos_file constructors * tests for Phobos_file PUT and GET * Phobos_file PUT and GET tests for empty file * Phobos_file GETMD test * in-memory (disk) storage solution for testing stream objects * Fifo file descriptors must be copied explicitly as they are now an array * Fifo tests for PUT and GET * repeat Fifo tests for empty file * Fifo tests for GETMD * tests for Fifo object construction * need to determine file size before db_get * Phobos_file PUT test for bad file descriptor exception * Phobos_file PUT test for used object_id exception * Phobos_file GETMD test for bad object_id * Phobos_file GET test for bad object_id * meta data is only written upon PUT operation in disk storage --- src/stream/fifo.h | 4 +- test/CMakeLists.txt | 31 ++++++ test/empty.txt | 0 test/fifo.cc | 146 ++++++++++++++++++++++++++ test/phobos_file.cc | 242 +++++++++++++++++++++++++++++++++++++++++++ test/storage/disk.cc | 159 ++++++++++++++++++++++++++++ test/storage/disk.h | 42 ++++++++ test/tests-main.cc | 2 + test/utils.cc | 37 +++++++ test/utils.h | 10 ++ 10 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 test/CMakeLists.txt create mode 100644 test/empty.txt create mode 100644 test/fifo.cc create mode 100644 test/phobos_file.cc create mode 100644 test/storage/disk.cc create mode 100644 test/storage/disk.h create mode 100644 test/tests-main.cc create mode 100644 test/utils.cc create mode 100644 test/utils.h diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 594fa40..71c3a94 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -28,7 +28,7 @@ public: /// /// @param input the other object to be moved /// - Fifo(Fifo&& input) : Stream(std::move(input.m_storage)) {} + Fifo(Fifo&& input) : Stream(std::move(input.m_storage)), m_fifo_descriptor{input.m_fifo_descriptor[0], input.m_fifo_descriptor[1]} {} /// /// @brief assignment operator for rvalue references @@ -40,6 +40,8 @@ public: Fifo& operator=(Fifo&& input) { Stream::m_storage = std::move(input.m_storage); + m_fifo_descriptor[0] = input.m_fifo_descriptor[0]; + m_fifo_descriptor[1] = input.m_fifo_descriptor[1]; return *this; } /// @brief copy constructor deleted diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..f91d83a --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,31 @@ +find_package(Catch2 REQUIRED) + +add_executable( + tests + + tests-main.cc + phobos_file.cc + fifo.cc + + storage/disk.cc + utils.cc +) + +target_compile_features(tests PUBLIC cxx_std_14) + +target_link_libraries(tests PUBLIC Catch2::Catch2) +target_link_libraries(tests PUBLIC server) + +add_custom_command( + TARGET tests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/EmperorWu.txt + ${CMAKE_CURRENT_BINARY_DIR}/EmperorWu.txt + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/empty.txt + ${CMAKE_CURRENT_BINARY_DIR}/empty.txt +) + +include(CTest) +include(Catch) +catch_discover_tests(tests) diff --git a/test/empty.txt b/test/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/fifo.cc b/test/fifo.cc new file mode 100644 index 0000000..0fc4386 --- /dev/null +++ b/test/fifo.cc @@ -0,0 +1,146 @@ +#include + +#include +#include + +#include "storage/disk.h" +#include "../src/stream/fifo.h" + +#include "utils.h" + +namespace fiphoboserver { +namespace stream { + +SCENARIO("stream file ownership", "[basic]") +{ + GIVEN("a Fifo object with a Disk backend") + { + /* generates a file descriptor from a fifo */ + Fifo fifo(std::make_unique()); + int fd[2]; + fd[0] = fifo.get_fifo_descriptors()[0]; + fd[1] = fifo.get_fifo_descriptors()[1]; + + WHEN("a Fifo object is move constructed") + { + Fifo fifo2(std::move(fifo)); + THEN("file ownership is transfered") + { + REQUIRE(fd[0] == fifo2.get_fifo_descriptors()[0]); + REQUIRE(fd[1] == fifo2.get_fifo_descriptors()[1]); + } + } + WHEN("a Fifo object is move assigned") + { + Fifo fifo2 = std::move(fifo); + THEN("file ownership is transfered") + { + REQUIRE(fd[0] == fifo2.get_fifo_descriptors()[0]); + REQUIRE(fd[1] == fifo2.get_fifo_descriptors()[1]); + } + } + } +} + +std::string bucket_name = "bucket_" + random_string(10); +std::string key = "id_" + random_string(10); +std::string dummy_key = "id_" + random_string(10); +std::string buffer; +std::vector> meta_data; + +SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") +{ + Fifo fifo(std::make_unique()); + + GIVEN("a bucket name, object ID, meta data, and a file") + { + WHEN("PUT operation is executed") + { + buffer = file_to_buffer("./EmperorWu.txt"); + for (int i = 0; i < 5; i++) { + meta_data.push_back( + std::make_pair(key + std::to_string(i), std::to_string(i))); + } + fifo.set_storage_meta_data(meta_data, bucket_name); + + fifo.start_put(buffer.size(), key); + ssize_t rc = fifo.put(buffer.data(), buffer.size()); + fifo.finish_put(); + THEN("the correct amount of data is written to the Disk backend") + { + REQUIRE(rc == buffer.size()); + } + } + WHEN("meta data is retreived") + { + std::vector> meta_data2 = + fifo.get_meta_data(key); + THEN("the retreived meta data matches the input meta data") + { + std::string bucket = "bucket"; + REQUIRE( + meta_data2.end() + != std::find( + meta_data2.begin(), meta_data2.end(), + std::make_pair(bucket, bucket_name))); + for (int i = 0; i < 5; i++) { + REQUIRE( + meta_data2.end() + != std::find( + meta_data2.begin(), meta_data2.end(), + meta_data[i])); + } + } + } + WHEN("GET operation is executed") + { + std::string buffer2; + buffer2.resize(buffer.size()); + fifo.start_get(key); + ssize_t rc = fifo.get(&buffer2.front(), buffer.size()); + fifo.finish_get(); + + THEN("the correct amount of data is read and the retreived data matches the input data") + { + REQUIRE(rc == buffer.size()); + REQUIRE(buffer == buffer2); + } + } + } + GIVEN("a bucket name, object ID, meta data, and an empty file") + { + WHEN("PUT operation is executed") + { + buffer = file_to_buffer("./empty.txt"); + for (int i = 0; i < 5; i++) { + meta_data.push_back( + std::make_pair(key + std::to_string(i), std::to_string(i))); + } + + fifo.set_storage_meta_data(meta_data, bucket_name); + fifo.start_put(buffer.size(), dummy_key); + ssize_t rc = fifo.put(buffer.data(), buffer.size()); + fifo.finish_put(); + THEN("the correct amount of data is written to the Disk backend") + { + REQUIRE(rc == buffer.size()); + } + } + WHEN("GET operation is executed") + { + std::string buffer2; + fifo.start_get(dummy_key); + ssize_t rc = fifo.get(&buffer2.front(), buffer.size()); + fifo.finish_get(); + + THEN("the correct amount of data is read and the retreived data matches the input data") + { + REQUIRE(rc == buffer2.size()); + REQUIRE(buffer == buffer2); + } + } + } +} + +} // namespace stream +} // namespace fiphoboserver diff --git a/test/phobos_file.cc b/test/phobos_file.cc new file mode 100644 index 0000000..24aa4c8 --- /dev/null +++ b/test/phobos_file.cc @@ -0,0 +1,242 @@ +#include + +#include +#include +#include + +#include "../src/storage/phobos_file.h" + +#include "utils.h" + +/* temporary includes */ +#include + +namespace fiphoboserver { +namespace storage { + +SCENARIO("Phobos_file construction", "[basic]") +{ + GIVEN("a Phobos_file object with a file descriptor") + { + std::string filename = "tmpfile" + random_string(5); + std::string object_id = "id" + random_string(5); + std::ofstream file(filename); + /* file is created when stream is closed */ + file.close(); + + /* generate a file descriptor from a file */ + int fd = open(filename.c_str(), O_WRONLY); + REQUIRE(0 < fd); + + Phobos_file store; + REQUIRE(0 >= store.get_fd()); + + /* have store take ownership of the file descriptor */ + store.prepare_put(fd, "dummy"); + WHEN("file descriptor is retrieved from Phobos_file object") + { + THEN("the value is that of the original file descriptor") + { + REQUIRE(fd == store.get_fd()); + } + remove(filename.c_str()); + } + WHEN("a move constructor is executed") + { + Phobos_file store2(std::move(store)); + + THEN("the file descriptor changes ownership") + { + REQUIRE(fd == store2.get_fd()); + } + remove(filename.c_str()); + } + WHEN("an assigment constructor is executed") + { + Phobos_file store2 = std::move(store); + + THEN("the file descriptor changes ownership") + { + REQUIRE(fd == store2.get_fd()); + } + remove(filename.c_str()); + } + } +} + +std::string bucket_name = "bucket_" + random_string(10); +std::string object_id = "id_" + random_string(10); +std::string dummy_key = "id_" + random_string(10); +std::vector> meta_data; + +SCENARIO("Phobos_file storage operations", "[I/O]") +{ + Phobos_file store; + REQUIRE(0 >= store.get_fd()); + + GIVEN("a bucket name, an object ID, meta data, and a file") + { + std::string buffer = file_to_buffer("./EmperorWu.txt"); + + WHEN("a PUT operation is prepared and executed") + { + /* generate a file descriptor from the file */ + int fd = open("./EmperorWu.txt", O_RDONLY); + REQUIRE(0 < fd); + + for (int i = 0; i < 5; i++) { + meta_data.push_back( + std::make_pair(object_id + std::to_string(i), std::to_string(i))); + } + store.set_meta_data(meta_data, bucket_name); + + store.prepare_put(fd, object_id); + REQUIRE(0 < store.get_fd()); + + ssize_t rc = store.db_put(buffer.size()); + THEN("The data is writen to the Phobos backend") + { + REQUIRE(0 == rc); + } + } + WHEN("meta data is retreived") + { + std::vector> meta_data2 = + store.get_meta_data(object_id); + THEN("the retreived meta data matches the input meta data") + { + std::string bucket = "bucket"; + REQUIRE( + meta_data2.end() + != std::find( + meta_data2.begin(), meta_data2.end(), + std::make_pair(bucket, bucket_name))); + for (int i = 0; i < 5; i++) { + REQUIRE( + meta_data2.end() + != std::find( + meta_data2.begin(), meta_data2.end(), + meta_data[i])); + } + } + } + std::string filename = "tmpfile" + random_string(5); + + WHEN("a GET operation is prepared and executed") + { + int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + + store.prepare_get(fd, object_id); + REQUIRE(fd == store.get_fd()); + + ssize_t rc = store.db_get(); + THEN("the data is read and the retreived data matches the input data") + { + REQUIRE(rc == 0); + std::string buffer2 = file_to_buffer(filename); + REQUIRE(buffer == buffer2); + } + remove(filename.c_str()); + } + } + GIVEN("a bucket name, an object ID, and an empty file") + { + std::string buffer = file_to_buffer("./empty.txt"); + + WHEN("a PUT operation is prepared and executed") + { + /* generate a file descriptor from the file */ + int fd = open("./empty.txt", O_RDONLY); + REQUIRE(0 < fd); + + store.prepare_put(fd, dummy_key); + REQUIRE(0 < store.get_fd()); + + ssize_t rc = store.db_put(buffer.size()); + THEN("The data is writen to the Phobos backend") + { + REQUIRE(0 == rc); + } + } + + WHEN("a GET operation is prepared and executed") + { + std::string filename = "tmpfile" + random_string(5); + int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + + store.prepare_get(fd, dummy_key); + REQUIRE(fd == store.get_fd()); + + ssize_t rc = store.db_get(); + THEN("the data is read and the retreived data matches the input data") + { + REQUIRE(rc == 0); + std::string buffer2 = file_to_buffer(filename); + REQUIRE(buffer == buffer2); + } + remove(filename.c_str()); + } + } + GIVEN("a used object ID") + { + std::string buffer = file_to_buffer("./EmperorWu.txt"); + + WHEN("a PUT operation is prepared and executed") + { + /* generate a file descriptor from the file */ + int fd = open("./EmperorWu.txt", O_RDONLY); + REQUIRE(0 < fd); + + store.prepare_put(fd, object_id); + REQUIRE(0 < store.get_fd()); + + THEN("an exception is thrown") + { + REQUIRE_THROWS(store.db_put(buffer.size())); + } + } + } + GIVEN("a bad file descriptor") + { + std::string buffer = file_to_buffer("./EmperorWu.txt"); + + WHEN("a PUT operation is prepared and executed") + { + int fd = -1; + + store.prepare_put(fd, "bad_fd"); + + THEN("an exception is thrown") + { + REQUIRE_THROWS(store.db_put(buffer.size())); + } + } + } + GIVEN("a bad object_id") + { + WHEN("a GET operation is prepared and executed") + { + std::string filename = "tmpfile" + random_string(5); + int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + + store.prepare_get(fd, "bad_object_id"); + REQUIRE(fd == store.get_fd()); + + THEN("an exception is thrown") + { + REQUIRE_THROWS(store.db_get()); + } + remove(filename.c_str()); + } + WHEN("meta data is retreived") + { + THEN("the retreived meta data matches the input meta data") + { + REQUIRE_THROWS(store.get_meta_data("bad_object_id")); + } + } + } +} + +} // namespace storage +} // namespace fiphoboserver diff --git a/test/storage/disk.cc b/test/storage/disk.cc new file mode 100644 index 0000000..3cc4b5d --- /dev/null +++ b/test/storage/disk.cc @@ -0,0 +1,159 @@ +#include "disk.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fiphoboserver { +namespace storage { + +/* + * fill the object related variables with relevant data + */ +void Disk::prepare_put(int file_descriptor, std::string object_id) +{ + m_file_descriptor = file_descriptor; + + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + m_objid = unconsted_object_id; +} + +/* Tell Disk to start reading from the file */ +ssize_t Disk::db_put(size_t size) +{ + m_size = size; + char buffer[size + 1]; + + /* create file for the Disk backend */ + std::string data_file = m_objid + ".data"; + int fd = open(data_file.c_str(), O_CREAT | O_WRONLY, 0644); + if (fd < 0) { + /* object ID already in use */ + return -1; + } + ssize_t rc = read(m_file_descriptor, buffer, size); + if (rc < 0) { + return rc; + } + rc = write(fd, buffer, size); + close(fd); + close_file(); + + /* write to meta data file */ + std::ofstream file(m_objid + ".meta"); + + std::for_each( + m_meta_data.begin(), m_meta_data.end(), + [&](std::pair pair) { + file << pair.first << " " << pair.second << std::endl; + }); + file.close(); + + return rc; +} + +/* + * fill the object related variables with relevant data + */ +void Disk::prepare_get(int file_descriptor, std::string object_id) +{ + m_file_descriptor = file_descriptor; + + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + m_objid = unconsted_object_id; +} + +/* Tell Phobos to write the object to file */ +ssize_t Disk::db_get() +{ + ssize_t rc; + + std::string data_file = m_objid + ".data"; + + int fd = open(data_file.c_str(), O_RDONLY); + + struct stat file_stat; + fstat(fd, &file_stat); + char *buffer = (char *)malloc(file_stat.st_size + 1); + + rc = read(fd, buffer, file_stat.st_size); + if (rc < file_stat.st_size) { + return rc; + } + close(fd); + + rc = write(m_file_descriptor, buffer, rc); + free(buffer); + /* + * Need to close file here so the stream knows + * there will be no more data to read + */ + close_file(); + + return rc; +} + +/* Pass s3 meta data to Phobos */ +void Disk::set_meta_data( + std::vector> meta_data, + std::string bucket_name) +{ + m_meta_data.push_back(std::pair("bucket", bucket_name)); + + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + m_meta_data.push_back(pair); + }); + /* + std::ofstream file(m_objid + ".meta"); + file << "bucket " << bucket_name << std::endl; + + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + file << pair.first << " " << pair.second << std::endl; + }); + file.close(); + */ +} + +std::vector> Disk::get_meta_data( + std::string object_id) +{ + std::vector> meta_data; + std::pair pair; + + char* unconsted_object_id = new char[object_id.length() + 1]; + strcpy(unconsted_object_id, object_id.c_str()); + + m_objid = unconsted_object_id; + + std::ifstream file(m_objid + ".meta"); + + while (file >> pair.first >> pair.second) { + meta_data.push_back(pair); + } + file.close(); + + return std::move(meta_data); +} + +/* Close the file */ +void Disk::close_file() +{ + if (m_file_descriptor > 0) { + close(m_file_descriptor); + } +} + +} // namespace storage +} // namespace fiphoboserver diff --git a/test/storage/disk.h b/test/storage/disk.h new file mode 100644 index 0000000..3c30a15 --- /dev/null +++ b/test/storage/disk.h @@ -0,0 +1,42 @@ +#include "../../src/storage/storage.h" + +namespace fiphoboserver { +namespace storage { + +class Disk: public Storage { + public: + Disk() = default; + + Disk(Disk&&) = default; + Disk& operator=(Disk&&) = default; + Disk(const Disk&) = delete; + Disk& operator=(const Disk&) = delete; + + void set_meta_data( + std::vector> meta_data, + std::string bucket_name) override; + std::vector> get_meta_data( + std::string object_id) override; + + ssize_t db_put(size_t size) override; + ssize_t db_get() override; + + void prepare_put(int file_descriptor, std::string object_id) override; + void prepare_get(int file_descriptor, std::string object_id) override; + + int get_fd() const { return m_file_descriptor; } + + private: + /* Implementation specific variables */ + + int m_file_descriptor = -1; + std::string m_objid; + std::vector> m_meta_data; + size_t m_size; + + void set_bucket_name(std::string bucket_name); + void close_file(); +}; + +} // namespace storage +} // namespace fiphoboserver diff --git a/test/tests-main.cc b/test/tests-main.cc new file mode 100644 index 0000000..62bf747 --- /dev/null +++ b/test/tests-main.cc @@ -0,0 +1,2 @@ +#define CATCH_CONFIG_MAIN +#include "catch2/catch.hpp" diff --git a/test/utils.cc b/test/utils.cc new file mode 100644 index 0000000..d0ef4c2 --- /dev/null +++ b/test/utils.cc @@ -0,0 +1,37 @@ +#include +#include +#include +#include + +#include "utils.h" + +namespace fiphoboserver { + +std::string file_to_buffer(const std::string& path) +{ + std::ifstream file(path); + std::string buffer( + (std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + + return buffer; +} + +std::string random_string(size_t length) +{ + auto randchar = []() -> char { + const char set[] = "0123456789" + "qwertyuiopasdfghjklzxcvbnm" + "QWERTYUIOPASDFGHJKLZXCVBNM"; + std::random_device dev; + std::mt19937 rng(dev()); + std::uniform_int_distribution dist( + 0, sizeof(set) - 2); + return set[dist(rng)]; + }; + std::string str(length, 0); + std::generate_n(str.begin(), length, randchar); + return str; +} + +} // namespace fiphoboserver diff --git a/test/utils.h b/test/utils.h new file mode 100644 index 0000000..2644e1f --- /dev/null +++ b/test/utils.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace fiphoboserver { + + std::string file_to_buffer(const std::string& path); + std::string random_string(size_t length); + +} // namespace fiphoboserver -- GitLab From dedbfa775d5a6f68578227b7466e3126407c0e24 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Tue, 31 Mar 2020 11:03:52 +0100 Subject: [PATCH 38/62] Add a creation timestamp to the metadata and return only original Metadata in getmd (not the server related stuff) --- doc/Markdown/fiphoboserver/HandlerFactory.md | 107 ++++++++++++++++++ doc/Markdown/fiphoboserver/S3_header.md | 4 +- doc/Markdown/fiphoboserver/index.md | 2 + .../fiphoboserver/storage/Phobos_file.md | 12 +- doc/Markdown/fiphoboserver/storage/Storage.md | 12 +- doc/Markdown/fiphoboserver/stream/Fifo.md | 46 ++++---- doc/Markdown/fiphoboserver/stream/Stream.md | 36 ++---- src/CMakeLists.txt | 1 + src/main.cc | 45 +------- src/server/CMakeLists.txt | 1 + src/server/S3_header.h | 12 +- src/server/get_request_handler.cc | 2 +- src/server/getmd_request_handler.cc | 3 +- src/server/getmd_request_handler.h | 4 +- src/server/handler_factory.h | 72 ++++++++++++ src/server/put_request_handler.cc | 12 +- src/storage/CMakeLists.txt | 1 + src/storage/phobos_file.cc | 17 ++- src/storage/phobos_file.h | 6 +- src/storage/storage.h | 12 +- src/stream/CMakeLists.txt | 1 + src/stream/fifo.cc | 15 +-- src/stream/fifo.h | 15 +-- src/stream/stream.h | 20 ++-- 24 files changed, 288 insertions(+), 170 deletions(-) create mode 100644 doc/Markdown/fiphoboserver/HandlerFactory.md create mode 100644 src/server/handler_factory.h diff --git a/doc/Markdown/fiphoboserver/HandlerFactory.md b/doc/Markdown/fiphoboserver/HandlerFactory.md new file mode 100644 index 0000000..95f4e92 --- /dev/null +++ b/doc/Markdown/fiphoboserver/HandlerFactory.md @@ -0,0 +1,107 @@ +# public fiphoboserver::HandlerFactory + +factory to create the handlers for incoming HTTP requests + + + +Depending on the method (GET, PUT, HEAD) this class creates an object of the corresponding request handler. + +All other requests result in an object of [UnsupportedRequestHandler][fiphoboserver-UnsupportedRequestHandler], that will then send an 400 Bad request response. + + + + +## Inheritance: +Inherits from RequestHandlerFactory. + +## Public Functions +| Name | Description | +| ---- | ---- | +| [onServerStart](#fiphoboserver-HandlerFactory-onServerStart) | function that gets called when the server starts | +| [onServerStop](#fiphoboserver-HandlerFactory-onServerStop) | function that gets called when the server stops | +| [onRequest](#fiphoboserver-HandlerFactory-onRequest) | this function gets called whenever a request reaches the server | + + + +## Public Functions +### public void fiphoboserver::HandlerFactory::onServerStart (folly::EventBase *) noexcept override + +function that gets called when the server starts + + + + + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-HandlerFactory) + +### public void fiphoboserver::HandlerFactory::onServerStop () noexcept override + +function that gets called when the server stops + + + + + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-HandlerFactory) + +### public proxygen::RequestHandler * fiphoboserver::HandlerFactory::onRequest (proxygen::RequestHandler *, proxygen::HTTPMessage *headers) noexcept override + +this function gets called whenever a request reaches the server + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| proxygen::HTTPMessage * | headers | The headers of the HTTP message that reached the server | + +#### Returns: +| Type | Description | +| ---- | ---- | +| proxygen::RequestHandler * | A handler for the incoming request | + + + + + + + + +This function creates the follosing request handlers: + +- GET: [GetRequestHandler][fiphoboserver-GetRequestHandler] + +- PUT: [PutRequestHandler][fiphoboserver-PutRequestHandler] + +- HEAD: [GetmdRequestHandler][fiphoboserver-GetmdRequestHandler] + +- or [UnsupportedRequestHandler][fiphoboserver-UnsupportedRequestHandler] in all other cases + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-HandlerFactory) + +[fiphoboserver-GetRequestHandler]:./GetRequestHandler.md +[fiphoboserver-GetmdRequestHandler]:./GetmdRequestHandler.md +[fiphoboserver-PutRequestHandler]:./PutRequestHandler.md +[fiphoboserver-UnsupportedRequestHandler]:./UnsupportedRequestHandler.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md index d76e484..45b3a89 100644 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ b/doc/Markdown/fiphoboserver/S3_header.md @@ -193,7 +193,7 @@ This reads out the header "Content-Length" [Go to Top](#fiphoboserver-S3_header) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::S3_header::get_meta_data () +### public std::map< std::string, std::string > fiphoboserver::S3_header::get_meta_data () gets the S3 metadata stored in the HTTP headers @@ -203,7 +203,7 @@ gets the S3 metadata stored in the HTTP headers #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector consisting of all key value pairs found in the header | +| std::map< std::string, std::string > | a map consisting of all key value pairs found in the header | diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 5096e6b..7f52e6e 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -9,6 +9,7 @@ | [FIPhoboServerException](./FIPhoboServerException.md) | exception class for all user defined exceptions in FiPhboServer | | [GetmdRequestHandler](./GetmdRequestHandler.md) | proxygen class implementation for handling HEAD requests | | [GetRequestHandler](./GetRequestHandler.md) | proxygen class implementation for handling GET requests | +| [HandlerFactory](./HandlerFactory.md) | factory to create the handlers for incoming HTTP requests | | [IOException](./IOException.md) | exceptions specifically for errors in I/O | | [PutRequestHandler](./PutRequestHandler.md) | proxygen class implementation for handling PUT requests | | [S3_header](./S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | @@ -25,6 +26,7 @@ [fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md [fiphoboserver-IOException]:./IOException.md +[fiphoboserver-UnsupportedRequestHandler]:./UnsupportedRequestHandler.md [fiphoboserver-storage-PhobosException]:./storage/PhobosException.md [fiphoboserver-storage-Storage]:./storage/Storage.md [fiphoboserver-stream-Stream]:./stream/Stream.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index a08ec06..3bd1ab7 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -28,7 +28,7 @@ Inherits from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage]. | [Phobos_file](#fiphoboserver-storage-Phobos_file-Phobos_file-2) | copy constructor deleted | | [operator=](#fiphoboserver-storage-Phobos_file-operator=-1) | assignment operator deleted | | [set_meta_data](#fiphoboserver-storage-Phobos_file-set_meta_data) | set the metadata that an object that is added to the database should get | -| [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | +| [get_meta_data](#fiphoboserver-storage-Phobos_file-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs | | [db_put](#fiphoboserver-storage-Phobos_file-db_put) | Puts data to the databse. | | [db_get](#fiphoboserver-storage-Phobos_file-db_get) | Gets data from the databse. | | [prepare_put](#fiphoboserver-storage-Phobos_file-prepare_put) | Starts a put operation to the database. | @@ -146,7 +146,7 @@ assignment operator deleted [Go to Top](#fiphoboserver-storage-Phobos_file) -### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) override +### public void fiphoboserver::storage::Phobos_file::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) override set the metadata that an object that is added to the database should get @@ -156,7 +156,7 @@ set the metadata that an object that is added to the database should get #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data | | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -176,9 +176,9 @@ set the metadata that an object that is added to the database should get [Go to Top](#fiphoboserver-storage-Phobos_file) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::storage::Phobos_file::get_meta_data (std::string object_id) override +### public std::map< std::string, std::string > fiphoboserver::storage::Phobos_file::get_meta_data (std::string object_id) override -get the metadata associated to the current object as a vector of key:value pairs +get the metadata associated to the current object as a map of key:value pairs @@ -191,7 +191,7 @@ get the metadata associated to the current object as a vector of key:value pairs #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | +| std::map< std::string, std::string > | a map of the key:value metadata pairs | diff --git a/doc/Markdown/fiphoboserver/storage/Storage.md b/doc/Markdown/fiphoboserver/storage/Storage.md index 384495f..fb75a4a 100644 --- a/doc/Markdown/fiphoboserver/storage/Storage.md +++ b/doc/Markdown/fiphoboserver/storage/Storage.md @@ -16,7 +16,7 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob | Name | Description | | ---- | ---- | | [set_meta_data](#fiphoboserver-storage-Storage-set_meta_data) | set the metadata that an object that is added to the database should get | -| [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | +| [get_meta_data](#fiphoboserver-storage-Storage-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs | ## PUT functions @@ -42,7 +42,7 @@ Is inherited by [fiphoboserver::storage::Phobos_file][fiphoboserver-storage-Phob ## Metadata functions -### public void fiphoboserver::storage::Storage::set_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name)=0 +### public void fiphoboserver::storage::Storage::set_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name)=0 set the metadata that an object that is added to the database should get @@ -52,7 +52,7 @@ set the metadata that an object that is added to the database should get #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data | | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -72,9 +72,9 @@ set the metadata that an object that is added to the database should get [Go to Top](#fiphoboserver-storage-Storage) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::storage::Storage::get_meta_data (std::string object_id)=0 +### public std::map< std::string, std::string > fiphoboserver::storage::Storage::get_meta_data (std::string object_id)=0 -get the metadata associated to the current object as a vector of key:value pairs +get the metadata associated to the current object as a map of key:value pairs @@ -87,7 +87,7 @@ get the metadata associated to the current object as a vector of key:value pairs #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | +| std::map< std::string, std::string > | a map of the key:value metadata pairs | diff --git a/doc/Markdown/fiphoboserver/stream/Fifo.md b/doc/Markdown/fiphoboserver/stream/Fifo.md index 6309703..9544d42 100644 --- a/doc/Markdown/fiphoboserver/stream/Fifo.md +++ b/doc/Markdown/fiphoboserver/stream/Fifo.md @@ -34,13 +34,12 @@ Inherits from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream]. | [Fifo](#fiphoboserver-stream-Fifo-Fifo-2) | copy constructor deleted | | [operator=](#fiphoboserver-stream-Fifo-operator=-1) | assignment operator deleted | | [set_storage_meta_data](#fiphoboserver-stream-Fifo-set_storage_meta_data) | setting the metadata that the created object should get | -| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | +| [get_meta_data](#fiphoboserver-stream-Fifo-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs | | [start_put](#fiphoboserver-stream-Fifo-start_put) | start a put operation | | [put](#fiphoboserver-stream-Fifo-put) | add a chunk of data to the object | -| [finish_put](#fiphoboserver-stream-Fifo-finish_put) | end a put operation | | [start_get](#fiphoboserver-stream-Fifo-start_get) | start a gett operation | | [get](#fiphoboserver-stream-Fifo-get) | gets a chunk of data of the object | -| [finish_get](#fiphoboserver-stream-Fifo-finish_get) | end a get operation | +| [finish_io](#fiphoboserver-stream-Fifo-finish_io) | end an I/O operation | ## Private Functions @@ -234,7 +233,7 @@ assignment operator deleted [Go to Top](#fiphoboserver-stream-Fifo) -### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const override +### public void fiphoboserver::stream::Fifo::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const override setting the metadata that the created object should get @@ -244,7 +243,7 @@ setting the metadata that the created object should get #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data | | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -272,9 +271,9 @@ Just passes the data along to the backend [Go to Top](#fiphoboserver-stream-Fifo) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::stream::Fifo::get_meta_data (std::string object_id) const override +### public std::map< std::string, std::string > fiphoboserver::stream::Fifo::get_meta_data (std::string object_id) const override -get the metadata associated to the current object as a vector of key:value pairs +get the metadata associated to the current object as a map of key:value pairs @@ -287,7 +286,7 @@ get the metadata associated to the current object as a vector of key:value pairs #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | +| std::map< std::string, std::string > | a map of the key:value metadata pairs | @@ -371,23 +370,6 @@ This function can be called repeatedly until the server's message body is deplet * virtual -[Go to Top](#fiphoboserver-stream-Fifo) - -### public void fiphoboserver::stream::Fifo::finish_put () override - -end a put operation - - - - - - - - -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-stream-Fifo) ### public void fiphoboserver::stream::Fifo::start_get (std::string object_id) override @@ -459,13 +441,21 @@ Repeatedly called to read from FIFO into buffer until the FIFO is empty and clos [Go to Top](#fiphoboserver-stream-Fifo) -### public void fiphoboserver::stream::Fifo::finish_get () override +### public void fiphoboserver::stream::Fifo::finish_io () override + +end an I/O operation + + + + + -end a get operation +This will throw a [storage::PhobosException][fiphoboserver-storage-PhobosException] in case an error occurred in +[storage::Phobos_file][fiphoboserver-storage-Phobos_file] during I/O. @@ -490,6 +480,8 @@ create a fifo by using the pipe command on the m_fifo_descriptor [Go to Top](#fiphoboserver-stream-Fifo) +[fiphoboserver-storage-PhobosException]:./../storage/PhobosException.md +[fiphoboserver-storage-Phobos_file]:./../storage/Phobos_file.md [fiphoboserver-storage-Storage]:./../storage/Storage.md [fiphoboserver-stream-Fifo]:./Fifo.md [fiphoboserver-stream-Stream]:./Stream.md diff --git a/doc/Markdown/fiphoboserver/stream/Stream.md b/doc/Markdown/fiphoboserver/stream/Stream.md index d37f819..e60a898 100644 --- a/doc/Markdown/fiphoboserver/stream/Stream.md +++ b/doc/Markdown/fiphoboserver/stream/Stream.md @@ -14,10 +14,9 @@ Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. | ---- | ---- | | [start_put](#fiphoboserver-stream-Stream-start_put) | start a put operation | | [put](#fiphoboserver-stream-Stream-put) | add a chunk of data to the object | -| [finish_put](#fiphoboserver-stream-Stream-finish_put) | end a put operation | | [start_get](#fiphoboserver-stream-Stream-start_get) | start a gett operation | | [get](#fiphoboserver-stream-Stream-get) | gets a chunk of data of the object | -| [finish_get](#fiphoboserver-stream-Stream-finish_get) | end a get operation | +| [finish_io](#fiphoboserver-stream-Stream-finish_io) | end an I/O operation | ## Protected Attributes @@ -34,7 +33,7 @@ Is inherited by [fiphoboserver::stream::Fifo][fiphoboserver-stream-Fifo]. | [Stream](#fiphoboserver-stream-Stream-Stream-1) | constructor with storage implementation | | [~Stream](#fiphoboserver-stream-Stream-~Stream) | default destructor | | [set_storage_meta_data](#fiphoboserver-stream-Stream-set_storage_meta_data) | setting the metadata that the created object should get | -| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata associated to the current object as a vector of key:value pairs | +| [get_meta_data](#fiphoboserver-stream-Stream-get_meta_data) | get the metadata associated to the current object as a map of key:value pairs | @@ -103,23 +102,6 @@ This function can be called repeatedly until the server's message body is deplet * virtual -[Go to Top](#fiphoboserver-stream-Stream) - -### public void fiphoboserver::stream::Stream::finish_put ()=0 - -end a put operation - - - - - - - - -#### Qualifiers: -* virtual - - [Go to Top](#fiphoboserver-stream-Stream) ### public void fiphoboserver::stream::Stream::start_get (std::string object_id)=0 @@ -187,9 +169,9 @@ gets a chunk of data of the object [Go to Top](#fiphoboserver-stream-Stream) -### public void fiphoboserver::stream::Stream::finish_get ()=0 +### public void fiphoboserver::stream::Stream::finish_io ()=0 -end a get operation +end an I/O operation @@ -305,7 +287,7 @@ default destructor [Go to Top](#fiphoboserver-stream-Stream) -### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::vector< std::pair< std::string, std::string >> meta_data, std::string bucket_name) const =0 +### public void fiphoboserver::stream::Stream::set_storage_meta_data (std::map< std::string, std::string > meta_data, std::string bucket_name) const =0 setting the metadata that the created object should get @@ -315,7 +297,7 @@ setting the metadata that the created object should get #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| std::vector< std::pair< std::string, std::string >> | meta_data | a vector of all key:value pairs that should be added to the data | +| std::map< std::string, std::string > | meta_data | a map of all key:value pairs that should be added to the data | | std::string | bucket_name | the name of the S3 bucket - currently also handled as normal metadata | @@ -338,9 +320,9 @@ setting the metadata that the created object should get [Go to Top](#fiphoboserver-stream-Stream) -### public std::vector< std::pair< std::string, std::string > > fiphoboserver::stream::Stream::get_meta_data (std::string object_id) const =0 +### public std::map< std::string, std::string > fiphoboserver::stream::Stream::get_meta_data (std::string object_id) const =0 -get the metadata associated to the current object as a vector of key:value pairs +get the metadata associated to the current object as a map of key:value pairs @@ -353,7 +335,7 @@ get the metadata associated to the current object as a vector of key:value pairs #### Returns: | Type | Description | | ---- | ---- | -| std::vector< std::pair< std::string, std::string > > | a vector of the key:value metadata pairs | +| std::map< std::string, std::string > | a map of the key:value metadata pairs | diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0daa8c0..438978c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,7 @@ add_executable( ) target_compile_features(fiphoboserver PUBLIC cxx_std_14) +#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen) target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver) diff --git a/src/main.cc b/src/main.cc index 25662da..b70d36a 100644 --- a/src/main.cc +++ b/src/main.cc @@ -13,13 +13,7 @@ #include #include -#include "server/get_request_handler.h" -#include "server/getmd_request_handler.h" -#include "server/put_request_handler.h" -#include "server/unsupported_request_handler.h" - -#include "storage/phobos_file.h" -#include "stream/fifo.h" +#include "server/handler_factory.h" using folly::SocketAddress; @@ -34,39 +28,6 @@ DEFINE_int32( "Number of threads to listen on. Numbers <= 0 " "will use the number of cores on this machine."); -namespace { - -class HandlerFactory : public proxygen::RequestHandlerFactory { - public: - void onServerStart(folly::EventBase* /*evb*/) noexcept override {} - - void onServerStop() noexcept override {} - - proxygen::RequestHandler* onRequest( - proxygen::RequestHandler*, - proxygen::HTTPMessage* headers) noexcept override - { - if (headers->getMethod() == proxygen::HTTPMethod::GET) { - return new fiphoboserver::GetRequestHandler( - std::make_unique( - std::make_unique())); - } - else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { - return new fiphoboserver::PutRequestHandler( - std::make_unique( - std::make_unique())); - } - else if (headers->getMethod() == proxygen::HTTPMethod::HEAD) { - return new fiphoboserver::GetmdRequestHandler( - std::make_unique( - std::make_unique())); - } - else { - return new fiphoboserver::UnsupportedRequestHandler; - } - } -}; -} // namespace int main(int argc, char* argv[]) { @@ -88,7 +49,8 @@ int main(int argc, char* argv[]) options.shutdownOn = {SIGINT, SIGTERM}; options.enableContentCompression = false; options.handlerFactories = - proxygen::RequestHandlerChain().addThen().build(); + proxygen::RequestHandlerChain().addThen() + .build(); options.h2cEnabled = true; auto disk_io_thread_pool = std::make_shared( @@ -100,6 +62,7 @@ int main(int argc, char* argv[]) server.bind(IPs); /* Start HTTPServer mainloop in a separate thread */ + std::cout << "Starting server" << std::endl; std::thread t([&]() { server.start(); }); t.join(); diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 9360316..c08cbf6 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -7,5 +7,6 @@ add_library( unsupported_request_handler.cc ) +#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") target_link_libraries(server PUBLIC proxygen::proxygen) target_link_libraries(server PUBLIC stream) diff --git a/src/server/S3_header.h b/src/server/S3_header.h index b377cd6..dfc408e 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include namespace fiphoboserver { @@ -127,11 +127,11 @@ class S3_header { /// x-amx-meta-KEY: VALUE /// for for example a KEY:VALUE pair. /// - /// @returns a vector consisting of all key value pairs found in the header + /// @returns a map consisting of all key value pairs found in the header /// - std::vector> get_meta_data() + std::map get_meta_data() { - std::vector> metadata; + std::map metadata; std::string meta_search_string = "x-amz-meta-"; m_headers->getHeaders().forEach([&](const std::string& header, @@ -139,9 +139,7 @@ class S3_header { if (header.find(meta_search_string) != std::string::npos) { std::string meta_name = header.substr(meta_search_string.size(), header.size() - 1); - std::pair current_pair( - meta_name, val); - metadata.push_back(current_pair); + metadata[meta_name] = val; } }); return metadata; diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index bbfc440..510741f 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -90,7 +90,7 @@ void GetRequestHandler::read_file(folly::EventBase* evb) else if (rc == 0) { // done m_file_closed = true; - m_stream->finish_get(); + m_stream->finish_io(); VLOG(4) << "Read EOF"; evb->runInEventBaseThread([this] { proxygen::ResponseBuilder(downstream_).sendWithEOM(); diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index 8bf3e05..f6ad0f9 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -55,7 +55,8 @@ void GetmdRequestHandler::onRequest( std::for_each( m_meta_data.begin(), m_meta_data.end(), [&](std::pair pair) { - response.header("x-amz-meta-" + pair.first, pair.second); + if(pair.first != "creation_time" && pair.first != "bucket") + response.header("x-amz-meta-" + pair.first, pair.second); }); response.sendWithEOM(); } diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index 187a49d..1ff4cb0 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -14,6 +14,8 @@ #include #include +#include + namespace fiphoboserver { @@ -65,7 +67,7 @@ class GetmdRequestHandler : public proxygen::RequestHandler { private: std::unique_ptr m_stream; S3_header m_s3_header; - std::vector> m_meta_data; + std::map m_meta_data; }; } // namespace fiphoboserver diff --git a/src/server/handler_factory.h b/src/server/handler_factory.h new file mode 100644 index 0000000..fb8cd31 --- /dev/null +++ b/src/server/handler_factory.h @@ -0,0 +1,72 @@ +#pragma once + +#include "get_request_handler.h" +#include "getmd_request_handler.h" +#include "put_request_handler.h" +#include "unsupported_request_handler.h" + +#include "../storage/phobos_file.h" +#include "../stream/fifo.h" + +#include + + +namespace fiphoboserver { + + +/// +/// @brief factory to create the handlers for incoming HTTP requests +/// +/// Depending on the method (GET, PUT, HEAD) this class creates an object of the +/// corresponding request handler. +/// +/// All other requests result in an object of @ref UnsupportedRequestHandler, +/// that will then send an 400 Bad request response. +/// +class HandlerFactory : public proxygen::RequestHandlerFactory { + public: + + /// @brief function that gets called when the server starts + void onServerStart(folly::EventBase* /*evb*/) noexcept override {} + + /// @brief function that gets called when the server stops + void onServerStop() noexcept override {} + + /// + /// @brief this function gets called whenever a request reaches the server + /// + /// @param headers The headers of the HTTP message that reached the server + /// + /// This function creates the follosing request handlers: + /// - GET: @ref GetRequestHandler + /// - PUT: @ref PutRequestHandler + /// - HEAD: @ref GetmdRequestHandler + /// - or @ref UnsupportedRequestHandler in all other cases + /// + /// @returns A handler for the incoming request + /// + proxygen::RequestHandler* onRequest( + proxygen::RequestHandler*, + proxygen::HTTPMessage* headers) noexcept override + { + if (headers->getMethod() == proxygen::HTTPMethod::GET) { + return new GetRequestHandler( + std::make_unique( + std::make_unique())); + } + else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { + return new PutRequestHandler( + std::make_unique( + std::make_unique())); + } + else if (headers->getMethod() == proxygen::HTTPMethod::HEAD) { + return new GetmdRequestHandler( + std::make_unique( + std::make_unique())); + } + else { + return new UnsupportedRequestHandler(); + } + } +}; +} // namespace fiphoboserver diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 171ff26..6a679aa 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -10,6 +10,8 @@ #include +#include + #include "../fiphoboserver_exception.h" namespace fiphoboserver { @@ -34,9 +36,13 @@ void PutRequestHandler::onRequest( return; } try { - /* Send meta data to backend through the stream */ + /* + * Add timestamp to meta data and send it to backend through the stream + */ + auto metadata = m_s3_header.get_meta_data(); + metadata["creation_time"] = std::to_string(std::time(nullptr)); m_stream->set_storage_meta_data( - m_s3_header.get_meta_data(), m_s3_header.get_bucket()); + metadata, m_s3_header.get_bucket()); /* * Tell stream to coordinate with backend to prepare for PUT operation @@ -85,7 +91,7 @@ void PutRequestHandler::onEOM() noexcept { try { /* Tell stream it's time to clean up */ - m_stream->finish_put(); + m_stream->finish_io(); } catch (const FIPhoboServerException& ex) { std::cerr << "Caught an exception in finish put: " << ex.what() << '\n'; diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 9e13e7d..e493939 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -5,6 +5,7 @@ add_library( phobos_file.cc ) +#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") target_include_directories(storage PUBLIC /usr/include/glib-2.0) target_link_libraries(storage PUBLIC phobos_cpp_wrapper) diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index bd11124..d120477 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -19,9 +19,8 @@ Phobos_file::Phobos_file() /* Destroy Phobos' m_descriptor struct */ Phobos_file::~Phobos_file() { - // TODO: See issue #23 - // pho_attrs_free(&m_descriptor.xd_attrs); - + if(m_descriptor.xd_objid) + delete[] m_descriptor.xd_objid; pho_xfer_desc_destroy_cpp(&m_descriptor); } @@ -94,7 +93,7 @@ void Phobos_file::set_bucket_name(std::string bucket_name) /* Pass s3 meta data to Phobos */ void Phobos_file::set_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) { set_bucket_name(bucket_name); @@ -113,16 +112,16 @@ void Phobos_file::set_meta_data( int add_to_meta_data(const char* key, const char* value, void* udata) { - std::vector>* meta_data = - static_cast>*>(udata); - meta_data->push_back(std::make_pair(key, value)); + std::map* meta_data = + static_cast*>(udata); + (*meta_data)[key] = value; return 0; } -std::vector> Phobos_file::get_meta_data( +std::map Phobos_file::get_meta_data( std::string object_id) { - std::vector> meta_data; + std::map meta_data; char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 94deafe..655ab9e 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -5,7 +5,7 @@ extern "C" { } #include -#include +#include namespace fiphoboserver { namespace storage { @@ -34,10 +34,10 @@ class Phobos_file : public Storage { /// @copydoc Storage::set_meta_data void set_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) override; /// @copydoc Storage::get_meta_data - std::vector> get_meta_data( + std::map get_meta_data( std::string object_id) override; /// @copydoc Storage::db_put diff --git a/src/storage/storage.h b/src/storage/storage.h index 21cc071..1d61da0 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace fiphoboserver { /// @@ -37,23 +37,23 @@ class Storage { /// @brief set the metadata that an object that is added to the /// database should get /// - /// @param meta_data a vector of all key:value pairs that should be + /// @param meta_data a map of all key:value pairs that should be /// added to the data /// @param bucket_name the name of the S3 bucket - currently also /// handled as normal metadata /// virtual void set_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) = 0; /// /// @brief get the metadata associated to the current object as a - /// vector of key:value pairs + /// map of key:value pairs /// /// @param object_id The object id of the requested object /// - /// @returns a vector of the key:value metadata pairs + /// @returns a map of the key:value metadata pairs /// - virtual std::vector> get_meta_data( + virtual std::map get_meta_data( std::string object_id) = 0; /// @} diff --git a/src/stream/CMakeLists.txt b/src/stream/CMakeLists.txt index 17a5265..026b1d2 100644 --- a/src/stream/CMakeLists.txt +++ b/src/stream/CMakeLists.txt @@ -3,4 +3,5 @@ add_library( fifo.cc ) +#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") target_link_libraries(stream PUBLIC storage) diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index 28a5da0..bb08f5f 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -26,13 +26,13 @@ Fifo::~Fifo() } /* Simply pass along to the backend */ void Fifo::set_storage_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) const { m_storage->set_meta_data(meta_data, bucket_name); } -std::vector> Fifo::get_meta_data( +std::map Fifo::get_meta_data( std::string object_id) const { return std::move(m_storage->get_meta_data(object_id)); @@ -64,13 +64,6 @@ ssize_t Fifo::put(const void* data, const size_t count) const return rc; } -// TODO see git issue #26 -void Fifo::finish_put() -{ - // will throw the inner exception if one occurred - m_db_result.get(); -} - /* * Tell the backend that it must be ready for a GET operation */ @@ -98,9 +91,9 @@ ssize_t Fifo::get(void* buf, const size_t count) const return rc; } -// TODO see git issue #26 -void Fifo::finish_get() +void Fifo::finish_io() { + // will throw the inner exception if one occurred m_db_result.get(); } diff --git a/src/stream/fifo.h b/src/stream/fifo.h index 594fa40..0ca60f0 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace fiphoboserver { namespace stream { @@ -53,18 +53,16 @@ public: /// Just passes the data along to the backend /// void set_storage_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) const override; /// @copydoc Stream::get_meta_data - std::vector> get_meta_data( + std::map get_meta_data( std::string object_id) const override; /// @copydoc Stream::start_put void start_put(ssize_t size, std::string object_id) override; /// @copydoc Stream::put ssize_t put(const void* data, size_t count) const override; - /// @copydoc Stream::finish_put - void finish_put() override; /// @copydoc Stream::start_get void start_get(std::string object_id) override; @@ -75,8 +73,11 @@ public: /// writing end /// ssize_t get(void* buf, size_t count) const override; - /// @copydoc Stream::finish_get - void finish_get() override; + /// @copydoc Stream::finish_io + /// + /// This will throw a @ref storage::PhobosException in case an error + /// occurred in @ref storage::Phobos_file during I/O. + void finish_io() override; /// @name Implementation specific details /// diff --git a/src/stream/stream.h b/src/stream/stream.h index c1d7794..34df868 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace fiphoboserver { /// @@ -48,7 +48,7 @@ class Stream { /// /// @brief setting the metadata that the created object should get /// - /// @param meta_data a vector of all key:value pairs that should be + /// @param meta_data a map of all key:value pairs that should be /// added to the data /// @param bucket_name the name of the S3 bucket - currently also /// handled as normal metadata @@ -57,17 +57,17 @@ class Stream { /// the storage implementation /// virtual void set_storage_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) const = 0; /// /// @brief get the metadata associated to the current object as a - /// vector of key:value pairs + /// map of key:value pairs /// /// @param object_id The object id of the requested object /// - /// @returns a vector of the key:value metadata pairs + /// @returns a map of the key:value metadata pairs /// - virtual std::vector> get_meta_data( + virtual std::map get_meta_data( std::string object_id) const = 0; /// @name PUT functions @@ -94,10 +94,6 @@ class Stream { /// @returns the amount of bytes written to internal storage /// virtual ssize_t put(const void* data, size_t count) const = 0; - /// - /// @brief end a put operation - /// - virtual void finish_put() = 0; /// @} /// @@ -121,9 +117,9 @@ class Stream { /// virtual ssize_t get(void *buf, size_t count) const = 0; /// - /// @brief end a get operation + /// @brief end an I/O operation /// - virtual void finish_get() = 0; + virtual void finish_io() = 0; /// @} -- GitLab From 1392c1f97d2b6fa551c55144f260ea52bcb34747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Thu, 2 Apr 2020 16:43:00 +0100 Subject: [PATCH 39/62] * src/storage/phobos_file.cc: need to set xd_objid to NULL to avoid double free * src/storage/phobos_file.hexplicit copy constructors to avoid double free of xd_objid * tests: reflect meta data change to std::map from std::vector * run Clang Format * resolve Clang Tidy warnings * add Gitlab-CI * use fiphoboserver docker image * update DockerHub image with Doxygen requirements * CI: Clang Format job * CI: Clang Tidy job * CI: Doxygen build job * CI: Unit Tests job --- .gitlab-ci.yml | 114 ++++++++++++++ src/fiphoboserver_exception.h | 13 +- src/main.cc | 15 +- src/server/S3_header.h | 18 +-- src/server/get_request_handler.h | 5 +- src/server/getmd_request_handler.cc | 3 +- src/server/getmd_request_handler.h | 1 - src/server/handler_factory.h | 20 +-- src/server/put_request_handler.cc | 9 +- src/server/put_request_handler.h | 3 +- src/storage/phobos_exception.h | 2 +- src/storage/phobos_file.cc | 15 +- src/storage/phobos_file.h | 106 +++++++------ src/storage/storage.h | 158 ++++++++++--------- src/stream/fifo.cc | 5 +- src/stream/fifo.h | 45 +++--- src/stream/stream.h | 226 ++++++++++++++-------------- test/fifo.cc | 53 +++---- test/phobos_file.cc | 73 ++++----- test/storage/disk.cc | 43 ++---- test/storage/disk.h | 10 +- test/tests-main.cc | 2 +- test/utils.cc | 2 +- test/utils.h | 4 +- 24 files changed, 519 insertions(+), 426 deletions(-) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..7f8e6c4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,114 @@ +image: ciaranorourke/fiphoboserver:debian + +.snippets: + - &start_phobos + /home/superfiphoboserver/tools/build_phobos.sh -r + - &configure + mkdir build + && pushd build + && ( + set -x; + cmake + -DCMAKE_VERBOSE_MAKEFILE=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + -DCMAKE_C_COMPILER=${CC} + -DCMAKE_CXX_COMPILER=${CXX} + -DFIPHOBOSERVER_BUILD_TESTS="${FIPHOBOSERVER_BUILD_TESTS}" + -DCMAKE_PREFIX_PATH="${DEPS_DIR}" + -DCUSTOM_DOCU_PATH="${FIPHOBOSERVER_DOC_DIR}" + .. + ) + && popd + - &build + make -C build + - &test + make -C build test ARGS=-V + +.cmake_variables: + default_cmake_variables: &default_cmake_variables + DEPS_DIR: /home/superfiphoboserver/build/install + FIPHOBOSERVER_DOC_DIR: "" + FIPHOBOSERVER_BUILD_TESTS: "ON" + +.default_job: &default_job + tags: + - docker + image: ciaranorourke/fiphoboserver:debian + before_script: + - apt-get update + - apt-get install -y + cmake + clang clang-tidy + script: + - *configure + - *build + - *test + +stages: + - static analysis + - build + - test + +Clang Format: + <<: *default_job + stage: static analysis + variables: + CC: clang + CXX: clang++ + before_script: + - apt-get update + - apt-get install -y + git + clang-format + script: + - ./tools/run_format.sh + && git status + && git diff-index --quiet HEAD + || { echo "This commit contains unformatted files! Run tools/run_format.sh on the project to format them correctly."; false; } + +Clang Tidy: + <<: *default_job + stage: static analysis + variables: + <<: *default_cmake_variables + CC: clang + CXX: clang++ + script: + - *configure + - *build + - make -C build/test + - ./tools/run_lint.sh build + +Doxygen: + <<: *default_job + stage: build + variables: + <<: *default_cmake_variables + FIPHOBOSERVER_DOC_DIR: /home/superfiphoboserver/external/doxygen-xml-parser/src + CC: clang + CXX: clang++ + before_script: + - apt-get update + - apt-get install -y + cmake + doxygen + graphviz + clang + script: + - *configure + - make -C build/doc + +Unit Tests: + <<: *default_job + stage: test + variables: + <<: *default_cmake_variables + FIPHOBOSERVER_BUILD_TESTS: "ON" + CC: gcc + CXX: g++ + script: + - *start_phobos + - *configure + - *build + - *test + diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index 8f50074..9fd65cb 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include namespace fiphoboserver { @@ -10,8 +10,7 @@ namespace fiphoboserver { /// @brief exception class for all user defined exceptions in FiPhboServer /// class FIPhoboServerException : public std::exception { -public: - + public: /// /// @brief default constructor /// @@ -20,7 +19,7 @@ public: /// /// @brief constructor for a message with information on the exception /// - /// @param message message explaining what went wrong such that this + /// @param message message explaining what went wrong such that this /// exception was thrown. /// FIPhoboServerException(const char* message) : m_message(message) {} @@ -35,8 +34,8 @@ public: return m_message.c_str(); } -protected: - /// + protected: + /// /// @brief the internal message /// std::string m_message; @@ -46,7 +45,7 @@ protected: /// @brief exceptions specifically for errors in I/O /// class IOException : public FIPhoboServerException { -public: + public: /// /// @brief constructor /// diff --git a/src/main.cc b/src/main.cc index b70d36a..f372216 100644 --- a/src/main.cc +++ b/src/main.cc @@ -33,10 +33,11 @@ int main(int argc, char* argv[]) { folly::init(&argc, &argv, true); - std::vector IPs = { - {SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP}, - {SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2}, - }; + std::vector IPs // NOLINT + = { + {SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP}, + {SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2}, + }; if (FLAGS_threads <= 0) { FLAGS_threads = sysconf(_SC_NPROCESSORS_ONLN); @@ -48,9 +49,9 @@ int main(int argc, char* argv[]) options.idleTimeout = std::chrono::milliseconds(60000); options.shutdownOn = {SIGINT, SIGTERM}; options.enableContentCompression = false; - options.handlerFactories = - proxygen::RequestHandlerChain().addThen() - .build(); + options.handlerFactories = proxygen::RequestHandlerChain() + .addThen() + .build(); options.h2cEnabled = true; auto disk_io_thread_pool = std::make_shared( diff --git a/src/server/S3_header.h b/src/server/S3_header.h index dfc408e..e1dfe79 100644 --- a/src/server/S3_header.h +++ b/src/server/S3_header.h @@ -4,20 +4,19 @@ #include #include +#include #include #include #include -#include namespace fiphoboserver { /// -/// @brief class to extract the S3 specific information from proxygens +/// @brief class to extract the S3 specific information from proxygens /// HTTPMessage headers /// class S3_header { public: - /// /// @brief sets the internal header instance to the given header object /// @@ -65,8 +64,8 @@ class S3_header { /// /// @brief extracts the name of the key id requested in the HTTP message - /// - /// This is the whole path argument without the leading '/' but including + /// + /// This is the whole path argument without the leading '/' but including /// the bucket name! /// /// @returns name of the key @@ -78,8 +77,7 @@ class S3_header { } std::string path = m_headers->getPath(); - if(path.rfind('/', 0) == 0) - { + if (path.rfind('/', 0) == 0) { path = path.substr(1, path.size()); } @@ -89,10 +87,10 @@ class S3_header { /// /// @brief checks if the message we got is one for just creating a bucket /// - /// This is convenient since we don't really support buckets themselves and + /// This is convenient since we don't really support buckets themselves and /// can easily stop working if this returns true /// - /// @returns true if these headers belong to a create bucket request, + /// @returns true if these headers belong to a create bucket request, /// false for all other requests /// bool is_create_bucket_request() @@ -123,7 +121,7 @@ class S3_header { /// /// @brief gets the S3 metadata stored in the HTTP headers /// - /// In S3 arbitrary metadata can be defined. This has the form + /// In S3 arbitrary metadata can be defined. This has the form /// x-amx-meta-KEY: VALUE /// for for example a KEY:VALUE pair. /// diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 9913ffe..9ef05f6 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -21,7 +21,6 @@ namespace fiphoboserver { /// class GetRequestHandler : public proxygen::RequestHandler { public: - /// /// @brief first function to be called when a new request comes in /// @@ -33,7 +32,7 @@ class GetRequestHandler : public proxygen::RequestHandler { /// /// @brief function called on every body chunk belonging to this message /// - /// This is not used in this case because GET requests don't usually have a + /// This is not used in this case because GET requests don't usually have a /// body /// /// @param body buffer for the body @@ -87,7 +86,7 @@ class GetRequestHandler : public proxygen::RequestHandler { /// /// @brief Constructor for stream class initialization /// - /// @param input_stream @ref stream::Stream class instance to initialize the + /// @param input_stream @ref stream::Stream class instance to initialize the /// server /// GetRequestHandler(std::unique_ptr input_stream) : diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index f6ad0f9..b95d5f6 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -55,8 +55,9 @@ void GetmdRequestHandler::onRequest( std::for_each( m_meta_data.begin(), m_meta_data.end(), [&](std::pair pair) { - if(pair.first != "creation_time" && pair.first != "bucket") + if (pair.first != "creation_time" && pair.first != "bucket") { response.header("x-amz-meta-" + pair.first, pair.second); + } }); response.sendWithEOM(); } diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index 1ff4cb0..f3a1d16 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -24,7 +24,6 @@ namespace fiphoboserver { /// class GetmdRequestHandler : public proxygen::RequestHandler { public: - /// /// @copydoc GetRequestHandler::onRequest /// diff --git a/src/server/handler_factory.h b/src/server/handler_factory.h index fb8cd31..0705af9 100644 --- a/src/server/handler_factory.h +++ b/src/server/handler_factory.h @@ -20,12 +20,11 @@ namespace fiphoboserver { /// Depending on the method (GET, PUT, HEAD) this class creates an object of the /// corresponding request handler. /// -/// All other requests result in an object of @ref UnsupportedRequestHandler, +/// All other requests result in an object of @ref UnsupportedRequestHandler, /// that will then send an 400 Bad request response. -/// +/// class HandlerFactory : public proxygen::RequestHandlerFactory { public: - /// @brief function that gets called when the server starts void onServerStart(folly::EventBase* /*evb*/) noexcept override {} @@ -50,19 +49,16 @@ class HandlerFactory : public proxygen::RequestHandlerFactory { proxygen::HTTPMessage* headers) noexcept override { if (headers->getMethod() == proxygen::HTTPMethod::GET) { - return new GetRequestHandler( - std::make_unique( - std::make_unique())); + return new GetRequestHandler(std::make_unique( + std::make_unique())); } else if (headers->getMethod() == proxygen::HTTPMethod::PUT) { - return new PutRequestHandler( - std::make_unique( - std::make_unique())); + return new PutRequestHandler(std::make_unique( + std::make_unique())); } else if (headers->getMethod() == proxygen::HTTPMethod::HEAD) { - return new GetmdRequestHandler( - std::make_unique( - std::make_unique())); + return new GetmdRequestHandler(std::make_unique( + std::make_unique())); } else { return new UnsupportedRequestHandler(); diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index 6a679aa..a58d674 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -36,13 +36,12 @@ void PutRequestHandler::onRequest( return; } try { - /* - * Add timestamp to meta data and send it to backend through the stream + /* + * Add timestamp to meta data and send it to backend through the stream */ - auto metadata = m_s3_header.get_meta_data(); + auto metadata = m_s3_header.get_meta_data(); metadata["creation_time"] = std::to_string(std::time(nullptr)); - m_stream->set_storage_meta_data( - metadata, m_s3_header.get_bucket()); + m_stream->set_storage_meta_data(metadata, m_s3_header.get_bucket()); /* * Tell stream to coordinate with backend to prepare for PUT operation diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index c4722e6..d05aeed 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -21,7 +21,6 @@ namespace fiphoboserver { /// class PutRequestHandler : public proxygen::RequestHandler { public: - /// /// @copydoc GetRequestHandler::onRequest /// @@ -59,7 +58,7 @@ class PutRequestHandler : public proxygen::RequestHandler { /// @copydoc GetRequestHandler::GetRequestHandler /// PutRequestHandler(std::unique_ptr input_stream) : - m_stream(std::move(input_stream)) + m_stream(std::move(input_stream)) { } diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index bf14108..0199074 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -13,7 +13,7 @@ namespace storage { /// @brief exceptions specifically for the phobos backend library /// class PhobosException : public FIPhoboServerException { -public: + public: /// /// @brief constructor /// diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index d120477..d023d40 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -19,8 +19,10 @@ Phobos_file::Phobos_file() /* Destroy Phobos' m_descriptor struct */ Phobos_file::~Phobos_file() { - if(m_descriptor.xd_objid) + if (m_descriptor.xd_objid != nullptr) { delete[] m_descriptor.xd_objid; + m_descriptor.xd_objid = NULL; + } pho_xfer_desc_destroy_cpp(&m_descriptor); } @@ -32,7 +34,7 @@ void Phobos_file::prepare_put(int file_descriptor, std::string object_id) m_descriptor.xd_fd = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_descriptor.xd_objid = unconsted_object_id; m_descriptor.xd_op = PHO_XFER_OP_PUT; @@ -56,9 +58,9 @@ ssize_t Phobos_file::db_put(size_t size) */ void Phobos_file::prepare_get(int file_descriptor, std::string object_id) { - m_descriptor.xd_fd = file_descriptor; + m_descriptor.xd_fd = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_descriptor.xd_objid = unconsted_object_id; m_descriptor.xd_op = PHO_XFER_OP_GET; @@ -93,8 +95,7 @@ void Phobos_file::set_bucket_name(std::string bucket_name) /* Pass s3 meta data to Phobos */ void Phobos_file::set_meta_data( - std::map meta_data, - std::string bucket_name) + std::map meta_data, std::string bucket_name) { set_bucket_name(bucket_name); std::for_each( @@ -124,7 +125,7 @@ std::map Phobos_file::get_meta_data( std::map meta_data; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_descriptor.xd_objid = unconsted_object_id; m_descriptor.xd_op = PHO_XFER_OP_GETMD; diff --git a/src/storage/phobos_file.h b/src/storage/phobos_file.h index 655ab9e..e66d28c 100644 --- a/src/storage/phobos_file.h +++ b/src/storage/phobos_file.h @@ -4,71 +4,81 @@ extern "C" { #include "phobos_cpp_wrapper/phobos_cpp_wrapper.h" } -#include #include +#include namespace fiphoboserver { namespace storage { -/// +/// /// @brief Phobos specific mplementation of the @ref Storage class /// /// This class uses the phobos library as a backend to store and retrieve files /// in chunks. /// class Phobos_file : public Storage { - public: - /// @copydoc Storage::Storage - Phobos_file(); - /// @copydoc Storage::~Storage - ~Phobos_file(); + public: + /// @copydoc Storage::Storage + Phobos_file(); + /// @copydoc Storage::~Storage + ~Phobos_file(); + + /// @brief default move contructor + Phobos_file(Phobos_file&& input) : + m_descriptor(std::move(input.m_descriptor)) + { + input.m_descriptor.xd_objid = NULL; + } + /// @brief default assignment operator for rvalue references + Phobos_file& operator=(Phobos_file&& input) + { + m_descriptor = std::move(input.m_descriptor); + input.m_descriptor.xd_objid = NULL; + return *this; + } + /// @brief copy constructor deleted + Phobos_file(const Phobos_file&) = delete; + /// @brief assignment operator deleted + Phobos_file& operator=(const Phobos_file&) = delete; - /// @brief default move contructor - Phobos_file(Phobos_file&&) = default; - /// @brief default assignment operator for rvalue references - Phobos_file& operator=(Phobos_file&&) = default; - /// @brief copy constructor deleted - Phobos_file(const Phobos_file&) = delete; - /// @brief assignment operator deleted - Phobos_file& operator=(const Phobos_file&) = delete; + /// @copydoc Storage::set_meta_data + void set_meta_data( + std::map meta_data, + std::string bucket_name) override; + /// @copydoc Storage::get_meta_data + std::map get_meta_data( + std::string object_id) override; - /// @copydoc Storage::set_meta_data - void set_meta_data( - std::map meta_data, - std::string bucket_name) override; - /// @copydoc Storage::get_meta_data - std::map get_meta_data( - std::string object_id) override; + /// @copydoc Storage::db_put + ssize_t db_put(size_t size) override; + /// @copydoc Storage::db_get + ssize_t db_get() override; - /// @copydoc Storage::db_put - ssize_t db_put(size_t size) override; - /// @copydoc Storage::db_get - ssize_t db_get() override; + /// @copydoc Storage::prepare_put + void prepare_put(int file_descriptor, std::string object_id) override; + /// @copydoc Storage::prepare_get + void prepare_get(int file_descriptor, std::string object_id) override; - /// @copydoc Storage::prepare_put - void prepare_put(int file_descriptor, std::string object_id) override; - /// @copydoc Storage::prepare_get - void prepare_get(int file_descriptor, std::string object_id) override; + int get_fd() const { return m_descriptor.xd_fd; } - int get_fd() const { return m_descriptor.xd_fd; } - private: - /* Implementation specific variables */ + private: + /* Implementation specific variables */ - /// @brief struct to contain information for phobos - struct pho_xfer_desc m_descriptor = {0}; + /// @brief struct to contain information for phobos + struct pho_xfer_desc m_descriptor = {0}; - /// - /// @brief set the name for the bucket an object blongs to - /// - /// @param bucket_name name of the bucket - /// - void set_bucket_name(std::string bucket_name); + /// + /// @brief set the name for the bucket an object blongs to + /// + /// @param bucket_name name of the bucket + /// + void set_bucket_name(std::string bucket_name); - /// - /// @brief close the file opened for phobos - /// - void close_file(); -}; + /// + /// @brief close the file opened for phobos + /// + void close_file(); +}; -} // namespace storage -} // namespace fiphoboserver +} // namespace storage +} // namespace fiphoboserver diff --git a/src/storage/storage.h b/src/storage/storage.h index 1d61da0..2722439 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -1,11 +1,11 @@ #pragma once -#include #include +#include namespace fiphoboserver { /// -/// @brief namespace for storage classes that belong to / inherit from from +/// @brief namespace for storage classes that belong to / inherit from from /// @ref fiphoboserver::storage::Storage /// namespace storage { @@ -14,95 +14,93 @@ namespace storage { /// @brief virtual storage class to be implemented by backend storage /// /// This class is a prototype for every class that handles to internal database -/// that should be accessed by the FiPhoboServer. +/// that should be accessed by the FiPhoboServer. /// This class represents exactly one object file in the internal database /// class Storage { - public: + public: + /// + /// @brief default constructor + /// + Storage(){}; + /// + /// @brief default destructor + /// + virtual ~Storage(){}; - /// - /// @brief default constructor - /// - Storage() {}; - /// - /// @brief default destructor - /// - virtual ~Storage() {}; + /// @name Metadata functions + /// + /// @{ -/// @name Metadata functions -/// -/// @{ - - /// - /// @brief set the metadata that an object that is added to the - /// database should get - /// - /// @param meta_data a map of all key:value pairs that should be - /// added to the data - /// @param bucket_name the name of the S3 bucket - currently also - /// handled as normal metadata - /// - virtual void set_meta_data( - std::map meta_data, - std::string bucket_name) = 0; - /// - /// @brief get the metadata associated to the current object as a - /// map of key:value pairs - /// - /// @param object_id The object id of the requested object - /// - /// @returns a map of the key:value metadata pairs - /// - virtual std::map get_meta_data( + /// + /// @brief set the metadata that an object that is added to the + /// database should get + /// + /// @param meta_data a map of all key:value pairs that should be + /// added to the data + /// @param bucket_name the name of the S3 bucket - currently also + /// handled as normal metadata + /// + virtual void set_meta_data( + std::map meta_data, + std::string bucket_name) = 0; + /// + /// @brief get the metadata associated to the current object as a + /// map of key:value pairs + /// + /// @param object_id The object id of the requested object + /// + /// @returns a map of the key:value metadata pairs + /// + virtual std::map get_meta_data( std::string object_id) = 0; -/// @} -/// -/// @name PUT functions -/// -/// @{ - - /// - /// @brief Starts a put operation to the database - /// - /// @param file_descriptor the descriptor of the open file, which - /// contents should be added to the database - /// @param object_id the internal storage id the data should get - /// - virtual void prepare_put(int file_descriptor, std::string object_id) = 0; - /// - /// @brief Puts data to the databse - /// - /// @param size the number of bytes to add to the file in the database - /// - virtual ssize_t db_put(size_t size) = 0; + /// @} + /// + /// @name PUT functions + /// + /// @{ -/// @} -/// -/// @name GET functions -/// -/// @{ + /// + /// @brief Starts a put operation to the database + /// + /// @param file_descriptor the descriptor of the open file, which + /// contents should be added to the database + /// @param object_id the internal storage id the data should get + /// + virtual void prepare_put(int file_descriptor, std::string object_id) = 0; + /// + /// @brief Puts data to the databse + /// + /// @param size the number of bytes to add to the file in the database + /// + virtual ssize_t db_put(size_t size) = 0; - /// - /// @brief Starts a get operation to the database - /// - /// @param file_descriptor the descriptor of the open file, to which - /// the contents of the database should be - /// written - /// @param object_id the internal storage id from the data that - /// should be retrieved - /// - virtual void prepare_get(int file_descriptor, std::string object_id) = 0; - /// - /// @brief Gets data from the databse - /// - /// @returns the number of bytes read from the database - /// - virtual ssize_t db_get() = 0; + /// @} + /// + /// @name GET functions + /// + /// @{ -/// @} + /// + /// @brief Starts a get operation to the database + /// + /// @param file_descriptor the descriptor of the open file, to which + /// the contents of the database should be + /// written + /// @param object_id the internal storage id from the data that + /// should be retrieved + /// + virtual void prepare_get(int file_descriptor, std::string object_id) = 0; + /// + /// @brief Gets data from the databse + /// + /// @returns the number of bytes read from the database + /// + virtual ssize_t db_get() = 0; -}; + /// @} +}; } // namespace storage } // namespace fiphoboserver diff --git a/src/stream/fifo.cc b/src/stream/fifo.cc index bb08f5f..91887f9 100644 --- a/src/stream/fifo.cc +++ b/src/stream/fifo.cc @@ -26,8 +26,7 @@ Fifo::~Fifo() } /* Simply pass along to the backend */ void Fifo::set_storage_meta_data( - std::map meta_data, - std::string bucket_name) const + std::map meta_data, std::string bucket_name) const { m_storage->set_meta_data(meta_data, bucket_name); } @@ -101,7 +100,7 @@ void Fifo::finish_io() void Fifo::create_fifo() { int rc = pipe(m_fifo_descriptor); - if ( rc == -1) { + if (rc == -1) { throw IOException("Fifo::create_fifo", "mkfifo", errno); return; } diff --git a/src/stream/fifo.h b/src/stream/fifo.h index c8ed9b5..7bc5abb 100644 --- a/src/stream/fifo.h +++ b/src/stream/fifo.h @@ -1,9 +1,9 @@ #include "stream.h" #include +#include #include #include -#include namespace fiphoboserver { namespace stream { @@ -17,7 +17,7 @@ namespace stream { /// other end of the fifo. /// class Fifo : public Stream { -public: + public: /// @copydoc Stream::Stream Fifo(std::unique_ptr input_storage); /// @copydoc Stream::~Stream @@ -28,18 +28,23 @@ public: /// /// @param input the other object to be moved /// - Fifo(Fifo&& input) : Stream(std::move(input.m_storage)), m_fifo_descriptor{input.m_fifo_descriptor[0], input.m_fifo_descriptor[1]} {} + Fifo(Fifo&& input) : + Stream(std::move(input.m_storage)), + m_fifo_descriptor{input.m_fifo_descriptor[0], + input.m_fifo_descriptor[1]} + { + } /// /// @brief assignment operator for rvalue references /// - /// @param input the other object to assign + /// @param input the other object to assign /// /// @return reference to newly created object /// Fifo& operator=(Fifo&& input) { - Stream::m_storage = std::move(input.m_storage); + Stream::m_storage = std::move(input.m_storage); m_fifo_descriptor[0] = input.m_fifo_descriptor[0]; m_fifo_descriptor[1] = input.m_fifo_descriptor[1]; return *this; @@ -77,24 +82,24 @@ public: ssize_t get(void* buf, size_t count) const override; /// @copydoc Stream::finish_io /// - /// This will throw a @ref storage::PhobosException in case an error + /// This will throw a @ref storage::PhobosException in case an error /// occurred in @ref storage::Phobos_file during I/O. void finish_io() override; -/// @name Implementation specific details -/// -/// @{ + /// @name Implementation specific details + /// + /// @{ - /// - /// @brief get the descriptor of the fifo - /// - /// @returns the descriptor of the fifo - /// - /// @todo Do we use this? should this really be public? - /// + /// + /// @brief get the descriptor of the fifo + /// + /// @returns the descriptor of the fifo + /// + /// @todo Do we use this? should this really be public? + /// const int* get_fifo_descriptors() const { return m_fifo_descriptor; } - -/// @} + + /// @} private: /// @@ -103,10 +108,10 @@ public: void create_fifo(); /// - /// @brief the file descriptors for the FIFO. + /// @brief the file descriptors for the FIFO. /// * Index 0 for read end /// * Index 1 for write end - /// + /// int m_fifo_descriptor[2] = {-1, -1}; }; diff --git a/src/stream/stream.h b/src/stream/stream.h index 34df868..77bbf64 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -3,13 +3,13 @@ #include "../storage/storage.h" #include +#include #include #include -#include namespace fiphoboserver { /// -/// @brief namespace for stream classes that belong to / inherit from from +/// @brief namespace for stream classes that belong to / inherit from from /// @ref fiphoboserver::stream::Stream /// namespace stream { @@ -19,125 +19,125 @@ namespace stream { /// between the server and a backend @ref storage::Storage class /// class Stream { - public: - /// - /// @brief default constructor - /// - Stream() {}; - /// - /// @brief constructor with storage implementation - /// - /// @param input_storage pointer to a implementation of the @ref - /// storage::Storage class that will be used to - /// store the data - /// - /// @warning every class that inherits this interface must have an - /// internal @ref storage::Storage implementation and - /// initialize this here. - /// - Stream(std::unique_ptr input_storage) : - m_storage(std::move(input_storage)) - { - } + public: + /// + /// @brief default constructor + /// + Stream(){}; + /// + /// @brief constructor with storage implementation + /// + /// @param input_storage pointer to a implementation of the @ref + /// storage::Storage class that will be used to + /// store the data + /// + /// @warning every class that inherits this interface must have an + /// internal @ref storage::Storage implementation and + /// initialize this here. + /// + Stream(std::unique_ptr input_storage) : + m_storage(std::move(input_storage)) + { + } - /// - /// @brief default destructor - /// - virtual ~Stream() {}; + /// + /// @brief default destructor + /// + virtual ~Stream(){}; - /// - /// @brief setting the metadata that the created object should get - /// - /// @param meta_data a map of all key:value pairs that should be - /// added to the data - /// @param bucket_name the name of the S3 bucket - currently also - /// handled as normal metadata - /// - /// @warning this function is responsible for getting metadata to - /// the storage implementation - /// - virtual void set_storage_meta_data( - std::map meta_data, - std::string bucket_name) const = 0; - /// - /// @brief get the metadata associated to the current object as a - /// map of key:value pairs - /// - /// @param object_id The object id of the requested object - /// - /// @returns a map of the key:value metadata pairs - /// - virtual std::map get_meta_data( - std::string object_id) const = 0; + /// + /// @brief setting the metadata that the created object should get + /// + /// @param meta_data a map of all key:value pairs that should be + /// added to the data + /// @param bucket_name the name of the S3 bucket - currently also + /// handled as normal metadata + /// + /// @warning this function is responsible for getting metadata to + /// the storage implementation + /// + virtual void set_storage_meta_data( + std::map meta_data, + std::string bucket_name) const = 0; + /// + /// @brief get the metadata associated to the current object as a + /// map of key:value pairs + /// + /// @param object_id The object id of the requested object + /// + /// @returns a map of the key:value metadata pairs + /// + virtual std::map get_meta_data( + std::string object_id) const = 0; -/// @name PUT functions -/// -/// @{ + /// @name PUT functions + /// + /// @{ - /// - /// @brief start a put operation - /// - /// @param size amount of bytes that shall be added to the - /// internal storage - /// @param object_id the identifier the object should get - /// - virtual void start_put(ssize_t size, std::string object_id) = 0; - /// - /// @brief add a chunk of data to the object - /// - /// This function can be called repeatedly until the server's message - /// body is depleted - /// - /// @param data the chunked data to add - /// @param count the size of the buffer / chunk - /// - /// @returns the amount of bytes written to internal storage - /// - virtual ssize_t put(const void* data, size_t count) const = 0; + /// + /// @brief start a put operation + /// + /// @param size amount of bytes that shall be added to the + /// internal storage + /// @param object_id the identifier the object should get + /// + virtual void start_put(ssize_t size, std::string object_id) = 0; + /// + /// @brief add a chunk of data to the object + /// + /// This function can be called repeatedly until the server's message + /// body is depleted + /// + /// @param data the chunked data to add + /// @param count the size of the buffer / chunk + /// + /// @returns the amount of bytes written to internal storage + /// + virtual ssize_t put(const void* data, size_t count) const = 0; -/// @} -/// -/// @name PUT functions -/// -/// @{ + /// @} + /// + /// @name PUT functions + /// + /// @{ - /// - /// @brief start a gett operation - /// - /// @param object_id the identifier of the object to retrieve - /// - virtual void start_get(std::string object_id) = 0; - /// - /// @brief gets a chunk of data of the object - /// - /// @param buf a buffer for the data to be filled in this function - /// @param count the size of the buffer / chunk - /// - /// @returns the number of bytes the buffer has been filled with - /// - virtual ssize_t get(void *buf, size_t count) const = 0; - /// - /// @brief end an I/O operation - /// - virtual void finish_io() = 0; + /// + /// @brief start a gett operation + /// + /// @param object_id the identifier of the object to retrieve + /// + virtual void start_get(std::string object_id) = 0; + /// + /// @brief gets a chunk of data of the object + /// + /// @param buf a buffer for the data to be filled in this function + /// @param count the size of the buffer / chunk + /// + /// @returns the number of bytes the buffer has been filled with + /// + virtual ssize_t get(void* buf, size_t count) const = 0; + /// + /// @brief end an I/O operation + /// + virtual void finish_io() = 0; -/// @} + /// @} - protected: - /// - /// @brief an implementation of @ref storage::Storage that will be - /// used as a database for the objects - /// - std::unique_ptr m_storage; - /// - /// @brief a future object that can be used to get the results from the - /// asynchronous backend - /// - /// @todo should this really be a protected member in the interface? - /// Isn't it the decision of the implementation to make this - /// asynchronous? - /// - std::future m_db_result; + protected: + /// + /// @brief an implementation of @ref storage::Storage that will be + /// used as a database for the objects + /// + std::unique_ptr m_storage; + /// + /// @brief a future object that can be used to get the results from the + /// asynchronous backend + /// + /// @todo should this really be a protected member in the interface? + /// Isn't it the decision of the implementation to make this + /// asynchronous? + /// + std::future m_db_result; }; } // namespace stream diff --git a/test/fifo.cc b/test/fifo.cc index 0fc4386..6348b4e 100644 --- a/test/fifo.cc +++ b/test/fifo.cc @@ -3,8 +3,8 @@ #include #include -#include "storage/disk.h" #include "../src/stream/fifo.h" +#include "storage/disk.h" #include "utils.h" @@ -18,8 +18,8 @@ SCENARIO("stream file ownership", "[basic]") /* generates a file descriptor from a fifo */ Fifo fifo(std::make_unique()); int fd[2]; - fd[0] = fifo.get_fifo_descriptors()[0]; - fd[1] = fifo.get_fifo_descriptors()[1]; + fd[0] = fifo.get_fifo_descriptors()[0]; + fd[1] = fifo.get_fifo_descriptors()[1]; WHEN("a Fifo object is move constructed") { @@ -46,7 +46,7 @@ std::string bucket_name = "bucket_" + random_string(10); std::string key = "id_" + random_string(10); std::string dummy_key = "id_" + random_string(10); std::string buffer; -std::vector> meta_data; +std::map meta_data; SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") { @@ -57,15 +57,15 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") WHEN("PUT operation is executed") { buffer = file_to_buffer("./EmperorWu.txt"); + meta_data.clear(); for (int i = 0; i < 5; i++) { - meta_data.push_back( - std::make_pair(key + std::to_string(i), std::to_string(i))); + meta_data[key + std::to_string(i)] = std::to_string(i); } fifo.set_storage_meta_data(meta_data, bucket_name); fifo.start_put(buffer.size(), key); ssize_t rc = fifo.put(buffer.data(), buffer.size()); - fifo.finish_put(); + fifo.finish_io(); THEN("the correct amount of data is written to the Disk backend") { REQUIRE(rc == buffer.size()); @@ -73,23 +73,16 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") } WHEN("meta data is retreived") { - std::vector> meta_data2 = + std::map meta_data2 = fifo.get_meta_data(key); THEN("the retreived meta data matches the input meta data") { - std::string bucket = "bucket"; - REQUIRE( - meta_data2.end() - != std::find( - meta_data2.begin(), meta_data2.end(), - std::make_pair(bucket, bucket_name))); - for (int i = 0; i < 5; i++) { - REQUIRE( - meta_data2.end() - != std::find( - meta_data2.begin(), meta_data2.end(), - meta_data[i])); - } + REQUIRE(meta_data2["bucket"] == bucket_name); + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + REQUIRE(meta_data2[pair.first] == pair.second); + }); } } WHEN("GET operation is executed") @@ -98,9 +91,10 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") buffer2.resize(buffer.size()); fifo.start_get(key); ssize_t rc = fifo.get(&buffer2.front(), buffer.size()); - fifo.finish_get(); + fifo.finish_io(); - THEN("the correct amount of data is read and the retreived data matches the input data") + THEN( + "the correct amount of data is read and the retreived data matches the input data") { REQUIRE(rc == buffer.size()); REQUIRE(buffer == buffer2); @@ -112,15 +106,15 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") WHEN("PUT operation is executed") { buffer = file_to_buffer("./empty.txt"); + meta_data.clear(); for (int i = 0; i < 5; i++) { - meta_data.push_back( - std::make_pair(key + std::to_string(i), std::to_string(i))); + meta_data[key + std::to_string(i)] = std::to_string(i); } fifo.set_storage_meta_data(meta_data, bucket_name); fifo.start_put(buffer.size(), dummy_key); ssize_t rc = fifo.put(buffer.data(), buffer.size()); - fifo.finish_put(); + fifo.finish_io(); THEN("the correct amount of data is written to the Disk backend") { REQUIRE(rc == buffer.size()); @@ -131,9 +125,10 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") std::string buffer2; fifo.start_get(dummy_key); ssize_t rc = fifo.get(&buffer2.front(), buffer.size()); - fifo.finish_get(); + fifo.finish_io(); - THEN("the correct amount of data is read and the retreived data matches the input data") + THEN( + "the correct amount of data is read and the retreived data matches the input data") { REQUIRE(rc == buffer2.size()); REQUIRE(buffer == buffer2); @@ -142,5 +137,5 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") } } -} // namespace stream +} // namespace stream } // namespace fiphoboserver diff --git a/test/phobos_file.cc b/test/phobos_file.cc index 24aa4c8..72160aa 100644 --- a/test/phobos_file.cc +++ b/test/phobos_file.cc @@ -1,8 +1,8 @@ #include -#include -#include #include +#include +#include #include "../src/storage/phobos_file.h" @@ -31,7 +31,7 @@ SCENARIO("Phobos_file construction", "[basic]") Phobos_file store; REQUIRE(0 >= store.get_fd()); - /* have store take ownership of the file descriptor */ + /* have store take ownership of the file descriptor */ store.prepare_put(fd, "dummy"); WHEN("file descriptor is retrieved from Phobos_file object") { @@ -44,7 +44,7 @@ SCENARIO("Phobos_file construction", "[basic]") WHEN("a move constructor is executed") { Phobos_file store2(std::move(store)); - + THEN("the file descriptor changes ownership") { REQUIRE(fd == store2.get_fd()); @@ -54,7 +54,7 @@ SCENARIO("Phobos_file construction", "[basic]") WHEN("an assigment constructor is executed") { Phobos_file store2 = std::move(store); - + THEN("the file descriptor changes ownership") { REQUIRE(fd == store2.get_fd()); @@ -64,10 +64,10 @@ SCENARIO("Phobos_file construction", "[basic]") } } -std::string bucket_name = "bucket_" + random_string(10); -std::string object_id = "id_" + random_string(10); -std::string dummy_key = "id_" + random_string(10); -std::vector> meta_data; +std::string bucket_name = "bucket_" + random_string(10); +std::string object_id = "id_" + random_string(10); +std::string dummy_key = "id_" + random_string(10); +std::map meta_data; SCENARIO("Phobos_file storage operations", "[I/O]") { @@ -85,8 +85,7 @@ SCENARIO("Phobos_file storage operations", "[I/O]") REQUIRE(0 < fd); for (int i = 0; i < 5; i++) { - meta_data.push_back( - std::make_pair(object_id + std::to_string(i), std::to_string(i))); + meta_data[object_id + std::to_string(i)] = std::to_string(i); } store.set_meta_data(meta_data, bucket_name); @@ -101,36 +100,32 @@ SCENARIO("Phobos_file storage operations", "[I/O]") } WHEN("meta data is retreived") { - std::vector> meta_data2 = + std::map meta_data2 = store.get_meta_data(object_id); THEN("the retreived meta data matches the input meta data") { - std::string bucket = "bucket"; - REQUIRE( - meta_data2.end() - != std::find( - meta_data2.begin(), meta_data2.end(), - std::make_pair(bucket, bucket_name))); - for (int i = 0; i < 5; i++) { - REQUIRE( - meta_data2.end() - != std::find( - meta_data2.begin(), meta_data2.end(), - meta_data[i])); - } + REQUIRE(meta_data2["bucket"] == bucket_name); + + std::for_each( + meta_data.begin(), meta_data.end(), + [&](std::pair pair) { + REQUIRE(meta_data2[pair.first] == pair.second); + }); } } - std::string filename = "tmpfile" + random_string(5); + std::string filename = "tmpfile" + random_string(5); WHEN("a GET operation is prepared and executed") { - int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644); + std::cout << "******** get file name = " << filename << std::endl; store.prepare_get(fd, object_id); REQUIRE(fd == store.get_fd()); ssize_t rc = store.db_get(); - THEN("the data is read and the retreived data matches the input data") + THEN( + "the data is read and the retreived data matches the input data") { REQUIRE(rc == 0); std::string buffer2 = file_to_buffer(filename); @@ -161,14 +156,15 @@ SCENARIO("Phobos_file storage operations", "[I/O]") WHEN("a GET operation is prepared and executed") { - std::string filename = "tmpfile" + random_string(5); - int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + std::string filename = "tmpfile" + random_string(5); + int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644); store.prepare_get(fd, dummy_key); REQUIRE(fd == store.get_fd()); ssize_t rc = store.db_get(); - THEN("the data is read and the retreived data matches the input data") + THEN( + "the data is read and the retreived data matches the input data") { REQUIRE(rc == 0); std::string buffer2 = file_to_buffer(filename); @@ -202,7 +198,7 @@ SCENARIO("Phobos_file storage operations", "[I/O]") WHEN("a PUT operation is prepared and executed") { - int fd = -1; + int fd = -1; store.prepare_put(fd, "bad_fd"); @@ -216,16 +212,13 @@ SCENARIO("Phobos_file storage operations", "[I/O]") { WHEN("a GET operation is prepared and executed") { - std::string filename = "tmpfile" + random_string(5); - int fd = open(filename.c_str(), O_CREAT | O_WRONLY); + std::string filename = "tmpfile" + random_string(5); + int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644); store.prepare_get(fd, "bad_object_id"); REQUIRE(fd == store.get_fd()); - THEN("an exception is thrown") - { - REQUIRE_THROWS(store.db_get()); - } + THEN("an exception is thrown") { REQUIRE_THROWS(store.db_get()); } remove(filename.c_str()); } WHEN("meta data is retreived") @@ -238,5 +231,5 @@ SCENARIO("Phobos_file storage operations", "[I/O]") } } -} // namespace storage -} // namespace fiphoboserver +} // namespace storage +} // namespace fiphoboserver diff --git a/test/storage/disk.cc b/test/storage/disk.cc index 3cc4b5d..99f1181 100644 --- a/test/storage/disk.cc +++ b/test/storage/disk.cc @@ -1,12 +1,12 @@ #include "disk.h" #include +#include #include +#include #include #include #include -#include -#include namespace fiphoboserver { namespace storage { @@ -19,7 +19,7 @@ void Disk::prepare_put(int file_descriptor, std::string object_id) m_file_descriptor = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_objid = unconsted_object_id; } @@ -29,10 +29,10 @@ ssize_t Disk::db_put(size_t size) { m_size = size; char buffer[size + 1]; - + /* create file for the Disk backend */ std::string data_file = m_objid + ".data"; - int fd = open(data_file.c_str(), O_CREAT | O_WRONLY, 0644); + int fd = open(data_file.c_str(), O_CREAT | O_WRONLY, 0644); if (fd < 0) { /* object ID already in use */ return -1; @@ -41,7 +41,7 @@ ssize_t Disk::db_put(size_t size) if (rc < 0) { return rc; } - rc = write(fd, buffer, size); + rc = write(fd, buffer, size); close(fd); close_file(); @@ -66,7 +66,7 @@ void Disk::prepare_get(int file_descriptor, std::string object_id) m_file_descriptor = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_objid = unconsted_object_id; } @@ -82,7 +82,7 @@ ssize_t Disk::db_get() struct stat file_stat; fstat(fd, &file_stat); - char *buffer = (char *)malloc(file_stat.st_size + 1); + char* buffer = (char*)malloc(file_stat.st_size + 1); rc = read(fd, buffer, file_stat.st_size); if (rc < file_stat.st_size) { @@ -103,44 +103,31 @@ ssize_t Disk::db_get() /* Pass s3 meta data to Phobos */ void Disk::set_meta_data( - std::vector> meta_data, - std::string bucket_name) + std::map meta_data, std::string bucket_name) { - m_meta_data.push_back(std::pair("bucket", bucket_name)); - - std::for_each( - meta_data.begin(), meta_data.end(), - [&](std::pair pair) { - m_meta_data.push_back(pair); - }); - /* - std::ofstream file(m_objid + ".meta"); - file << "bucket " << bucket_name << std::endl; + m_meta_data["bucket"] = bucket_name; std::for_each( meta_data.begin(), meta_data.end(), [&](std::pair pair) { - file << pair.first << " " << pair.second << std::endl; + m_meta_data[pair.first] = pair.second; }); - file.close(); - */ } -std::vector> Disk::get_meta_data( - std::string object_id) +std::map Disk::get_meta_data(std::string object_id) { - std::vector> meta_data; + std::map meta_data; std::pair pair; char* unconsted_object_id = new char[object_id.length() + 1]; - strcpy(unconsted_object_id, object_id.c_str()); + strcpy(unconsted_object_id, object_id.c_str()); // NOLINT m_objid = unconsted_object_id; std::ifstream file(m_objid + ".meta"); while (file >> pair.first >> pair.second) { - meta_data.push_back(pair); + meta_data[pair.first] = pair.second; } file.close(); diff --git a/test/storage/disk.h b/test/storage/disk.h index 3c30a15..c2a3f9b 100644 --- a/test/storage/disk.h +++ b/test/storage/disk.h @@ -3,19 +3,19 @@ namespace fiphoboserver { namespace storage { -class Disk: public Storage { +class Disk : public Storage { public: Disk() = default; - Disk(Disk&&) = default; + Disk(Disk&&) = default; Disk& operator=(Disk&&) = default; Disk(const Disk&) = delete; Disk& operator=(const Disk&) = delete; void set_meta_data( - std::vector> meta_data, + std::map meta_data, std::string bucket_name) override; - std::vector> get_meta_data( + std::map get_meta_data( std::string object_id) override; ssize_t db_put(size_t size) override; @@ -31,7 +31,7 @@ class Disk: public Storage { int m_file_descriptor = -1; std::string m_objid; - std::vector> m_meta_data; + std::map m_meta_data; size_t m_size; void set_bucket_name(std::string bucket_name); diff --git a/test/tests-main.cc b/test/tests-main.cc index 62bf747..270a69d 100644 --- a/test/tests-main.cc +++ b/test/tests-main.cc @@ -1,2 +1,2 @@ -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_MAIN // NOLINT #include "catch2/catch.hpp" diff --git a/test/utils.cc b/test/utils.cc index d0ef4c2..3cfc4fd 100644 --- a/test/utils.cc +++ b/test/utils.cc @@ -28,7 +28,7 @@ std::string random_string(size_t length) std::uniform_int_distribution dist( 0, sizeof(set) - 2); return set[dist(rng)]; - }; + }; std::string str(length, 0); std::generate_n(str.begin(), length, randchar); return str; diff --git a/test/utils.h b/test/utils.h index 2644e1f..c68e8af 100644 --- a/test/utils.h +++ b/test/utils.h @@ -4,7 +4,7 @@ namespace fiphoboserver { - std::string file_to_buffer(const std::string& path); - std::string random_string(size_t length); +std::string file_to_buffer(const std::string& path); +std::string random_string(size_t length); } // namespace fiphoboserver -- GitLab From 9569a15f62f5b831d8939243322cc6c30e33bf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Fri, 3 Apr 2020 13:24:16 +0100 Subject: [PATCH 40/62] Develop integration testing and run during CI * separate directories for unit tests and integration tests * text files directory for testing server functions * module for s3 utility functions * module for file comparison utility functions * module for generating random keys and meta data * handle server executable within tests * basic test for PUT and GET * PUT and GET test for zero-byte file * add integration testing to CI * correct Clang Tidy job * PUT command supports optional meta data * test for GETMD * test that GET with bad key fails * test that PUT with used key fails --- .gitlab-ci.yml | 24 ++++- CMakeLists.txt | 2 +- test/integration_tests/client.py | 100 ++++++++++++++++++ .../data}/EmperorWu.txt | 0 test/{ => integration_tests/data}/empty.txt | 0 test/integration_tests/utils/compare.py | 20 ++++ test/integration_tests/utils/random_string.py | 13 +++ test/integration_tests/utils/s3_client.py | 34 ++++++ test/s3_call_script.py | 92 ---------------- test/{ => unit_tests}/CMakeLists.txt | 2 + test/unit_tests/EmperorWu.txt | 78 ++++++++++++++ test/unit_tests/empty.txt | 0 test/{ => unit_tests}/fifo.cc | 2 +- test/{ => unit_tests}/phobos_file.cc | 2 +- test/{ => unit_tests}/smallTestFile.txt | 0 test/{ => unit_tests}/storage/disk.cc | 0 test/{ => unit_tests}/storage/disk.h | 2 +- test/{ => unit_tests}/tests-main.cc | 0 test/{ => unit_tests}/utils.cc | 0 test/{ => unit_tests}/utils.h | 0 20 files changed, 271 insertions(+), 100 deletions(-) create mode 100644 test/integration_tests/client.py rename test/{ => integration_tests/data}/EmperorWu.txt (100%) rename test/{ => integration_tests/data}/empty.txt (100%) create mode 100644 test/integration_tests/utils/compare.py create mode 100644 test/integration_tests/utils/random_string.py create mode 100644 test/integration_tests/utils/s3_client.py delete mode 100644 test/s3_call_script.py rename test/{ => unit_tests}/CMakeLists.txt (91%) create mode 100644 test/unit_tests/EmperorWu.txt create mode 100644 test/unit_tests/empty.txt rename test/{ => unit_tests}/fifo.cc (99%) rename test/{ => unit_tests}/phobos_file.cc (99%) rename test/{ => unit_tests}/smallTestFile.txt (100%) rename test/{ => unit_tests}/storage/disk.cc (100%) rename test/{ => unit_tests}/storage/disk.h (96%) rename test/{ => unit_tests}/tests-main.cc (100%) rename test/{ => unit_tests}/utils.cc (100%) rename test/{ => unit_tests}/utils.h (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f8e6c4..54ddbc1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,8 +21,11 @@ image: ciaranorourke/fiphoboserver:debian && popd - &build make -C build - - &test + - &unit_test make -C build test ARGS=-V + - &integration_test + cd test/integration_tests + && python3 -m unittest client .cmake_variables: default_cmake_variables: &default_cmake_variables @@ -42,7 +45,8 @@ image: ciaranorourke/fiphoboserver:debian script: - *configure - *build - - *test + - *unit_test + - *integration_test stages: - static analysis @@ -71,12 +75,12 @@ Clang Tidy: stage: static analysis variables: <<: *default_cmake_variables + FIPHOBOSERVER_BUILD_TESTS: "ON" CC: clang CXX: clang++ script: - *configure - *build - - make -C build/test - ./tools/run_lint.sh build Doxygen: @@ -110,5 +114,17 @@ Unit Tests: - *start_phobos - *configure - *build - - *test + - *unit_test +Integration Tests: + <<: *default_job + stage: test + variables: + <<: *default_cmake_variables + CC: gcc + CXX: g++ + script: + - *start_phobos + - *configure + - *build + - *integration_test diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ce01ec..8973e69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,6 @@ endif(CUSTOM_DOCU_PATH) # Build tests if(FIPHOBOSERVER_BUILD_TESTS) - add_subdirectory(test) + add_subdirectory(test/unit_tests) enable_testing() endif(FIPHOBOSERVER_BUILD_TESTS) diff --git a/test/integration_tests/client.py b/test/integration_tests/client.py new file mode 100644 index 0000000..f4b7790 --- /dev/null +++ b/test/integration_tests/client.py @@ -0,0 +1,100 @@ +import unittest + +import subprocess +import signal +import os + +import utils.s3_client as s3_client +import utils.compare as compare +import utils.random_string as random_string + +class Server_test_case(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0" + cls.client = s3_client.S3_client('http://localhost:11000/') + cls.server = subprocess.Popen('../../build/fiphoboserver', shell=True, preexec_fn=os.setsid) + cls.put_file_name = 'data/EmperorWu.txt' + cls.get_file_name = 'data/EmperorWu_get.txt' + cls.put_empty_file_name = 'data/empty.txt' + cls.get_empty_file_name = 'data/empty_get.txt' + cls.key = random_string.random_string('key_', 12) + + @classmethod + def tearDownClass(cls): + os.remove(cls.get_file_name) + cls.server.terminate() + + # kill the fiphoboserver process + os.killpg(os.getpgid(cls.server.pid), signal.SIGTERM) + + def test_file_put_and_get(self): + self.client.put( + self.put_file_name, + self.bucket_name, + self.key + ) + self.client.get( + self.get_file_name, + self.bucket_name, + self.key + ) + self.assertTrue( + compare.files_are_equal(self.put_file_name, self.get_file_name) + ) + def test_empty_file_put_and_get(self): + key = random_string.random_string('key_', 12) + + self.client.put( + self.put_empty_file_name, + self.bucket_name, + key + ) + self.client.get( + self.get_empty_file_name, + self.bucket_name, + key + ) + self.assertTrue( + compare.files_are_equal(self.put_empty_file_name, self.get_empty_file_name) + ) + + @unittest.expectedFailure + def test_put_with_used_key(self): + self.assertTrue( + self.client.put( + self.put_file_name, + self.bucket_name, + self.key + ) + ) + + @unittest.expectedFailure + def test_get_with_bad_key(self): + self.assertTrue( + self.client.get( + self.get_file_name, + self.bucket_name, + 'bad_key' + ) + ) + + def test_get_meta_data(self): + key = random_string.random_string('key_', 12) + put_meta_data = random_string.random_map(5, 8) + + self.client.put( + self.put_file_name, + self.bucket_name, + key, + meta_data=put_meta_data + ) + + get_meta_data = self.client.get_md( + self.bucket_name, + key + ) + + self.assertTrue( + put_meta_data == get_meta_data + ) diff --git a/test/EmperorWu.txt b/test/integration_tests/data/EmperorWu.txt similarity index 100% rename from test/EmperorWu.txt rename to test/integration_tests/data/EmperorWu.txt diff --git a/test/empty.txt b/test/integration_tests/data/empty.txt similarity index 100% rename from test/empty.txt rename to test/integration_tests/data/empty.txt diff --git a/test/integration_tests/utils/compare.py b/test/integration_tests/utils/compare.py new file mode 100644 index 0000000..985109a --- /dev/null +++ b/test/integration_tests/utils/compare.py @@ -0,0 +1,20 @@ +import os + +def files_are_equal(filename1, filename2): + if not os.path.isfile(filename1): + return False, "Error: File not found: " + filename1 + if not os.path.isfile(filename2): + return False, "Error: File not found: " + filename2 + + file1 = open(filename1) + data1 = file1.read() + file1.close() + + file2 = open(filename2) + data2 = file2.read() + file2.close() + + if data1 != data2: + return False, "Error: " + filename1 + " and " + filename2 + "do not contain the same data" + + return True, "" diff --git a/test/integration_tests/utils/random_string.py b/test/integration_tests/utils/random_string.py new file mode 100644 index 0000000..f4ba7c3 --- /dev/null +++ b/test/integration_tests/utils/random_string.py @@ -0,0 +1,13 @@ +import random +import string + +def random_string(prefix = '', length = 16): + return prefix + ''.join([random.choice(string.ascii_letters + string.digits) for i in range(length)]) + +def random_map(size, length): + dictionary = {random_string(): random_string()} + + for i in range(size - 1): + dictionary.update({random_string(): random_string()}) + + return dictionary diff --git a/test/integration_tests/utils/s3_client.py b/test/integration_tests/utils/s3_client.py new file mode 100644 index 0000000..af36c4b --- /dev/null +++ b/test/integration_tests/utils/s3_client.py @@ -0,0 +1,34 @@ +import boto3 +import uuid +import sys + +class S3_client: + def __init__(self, server_url): + self.session = boto3.session.Session() + self.client = self.session.client( + service_name='s3', + endpoint_url=server_url + ) + + def put(self, filename, bucket_name, key, meta_data = {}): + self.client.upload_file( + Filename = filename, + Bucket = bucket_name, + Key = key, + ExtraArgs={"Metadata": meta_data} + ) + + def get(self, filename, bucket_name, key): + self.client.download_file( + Filename = filename, + Bucket = bucket_name, + Key = key + ) + + def get_md(self, bucket_name, key): + response = self.client.head_object( + Bucket = bucket_name, + Key = key + ) + return response["Metadata"] + diff --git a/test/s3_call_script.py b/test/s3_call_script.py deleted file mode 100644 index c1201c2..0000000 --- a/test/s3_call_script.py +++ /dev/null @@ -1,92 +0,0 @@ -import boto3 -import uuid -import sys -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('-put', action = 'store_true') -parser.add_argument('-get', action = 'store_true') -parser.add_argument('-getmd', action = 'store_true') -parser.add_argument('-key', dest = 'key_string', default = 'MyKey') -parser.add_argument('-putfile', dest = 'put_file_string', default = 'EmperorWu.txt') -parser.add_argument('-getfile', dest = 'get_file_string', default = 'get_file.txt') - -args = parser.parse_args() - -def create_bucket_name(bucket_prefix): - # The generated bucket name must be between 3 and 63 chars long - return ''.join([bucket_prefix, str(uuid.uuid4())]) - -def create_bucket(bucket_prefix, s3_connection): - session = boto3.session.Session() - current_region = session.region_name - bucket_name = create_bucket_name(bucket_prefix) - bucket_response = s3_connection.create_bucket( - Bucket=bucket_name, - CreateBucketConfiguration={ - 'LocationConstraint': current_region}) - print(bucket_name, current_region) - return bucket_name, bucket_response - - -# s3_resource = boto3.resource('s3') -# s3_client = s3_resource.meta.client -#first_bucket_name, first_response = create_bucket( - #bucket_prefix="ichec.phobos.", - #s3_connection=s3_resource.meta.client) - - -session = boto3.session.Session() -s3_client = session.client( - service_name='s3', - endpoint_url='http://localhost:11000/' -) -# print("Creating bucket...") -# first_bucket_name, first_response = create_bucket( -# bucket_prefix="ichec.phobos.", -# s3_connection=s3_client) - -# print(first_response) - -first_bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0" - -if(args.put): - s3_client.upload_file( - Filename=args.put_file_string, Bucket=first_bucket_name, - Key=args.key_string) - -if(args.get): - s3_client.download_file( - Filename=args.get_file_string, Bucket=first_bucket_name, - Key=args.key_string) - -# if(args.get): -# response = s3_client.get_object( -# Bucket=first_bucket_name, -# Key=args.key_string -# ) -# get_file = open(args.get_file_string, "w") -# get_file.write(response["Body"].read()) - -# This is another command (HEAD request) that will result in a getmd call to phobos! -if(args.getmd): - response = s3_client.head_object( - Bucket=first_bucket_name, - Key=args.key_string - ) - print("Meta data:") - print(response["Metadata"]) - -# response = s3_client.put_object( -# Bucket=first_bucket_name, -# Key=first_file_name, -# Body="smallTestFile.txt", -# Metadata={ -# 'metaKey': 'metaVal', -# 'metaKey2': 'metaVal2' -# } -# ) -# print("put_object: ") -# print(response) - -#print(s3_client.list_buckets()) diff --git a/test/CMakeLists.txt b/test/unit_tests/CMakeLists.txt similarity index 91% rename from test/CMakeLists.txt rename to test/unit_tests/CMakeLists.txt index f91d83a..675f823 100644 --- a/test/CMakeLists.txt +++ b/test/unit_tests/CMakeLists.txt @@ -1,5 +1,7 @@ find_package(Catch2 REQUIRED) +set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test) + add_executable( tests diff --git a/test/unit_tests/EmperorWu.txt b/test/unit_tests/EmperorWu.txt new file mode 100644 index 0000000..bb18fac --- /dev/null +++ b/test/unit_tests/EmperorWu.txt @@ -0,0 +1,78 @@ +Emperor Wu of Northern Zhou ((北)周武帝) (543–578), personal name Yuwen Yong (宇文邕), Xianbei name Miluotu (禰羅突), was an emperor of the Xianbei dynasty Northern Zhou. As was the case of the reigns of his brothers Emperor Xiaomin and Emperor Ming, the early part of his reign was dominated by his cousin Yuwen Hu, but in 572 he ambushed Yuwen Hu and seized power personally. He thereafter ruled ably and built up the power of his military, destroying rival Northern Qi in 577 and annexing its territory. His death the next year, however, ended his ambitions of uniting China, and under the reign of his erratic son Emperor Xuan (Yuwen Yun), Northern Zhou itself soon deteriorated and was usurped by Yang Jian in 581. + +Background + +Yuwen Yong was born in 543, as the fourth son of the Western Wei paramount general Yuwen Tai. His mother was Yuwen Tai's concubine Lady Chinu. He was born at Yuwen Tai's then-headquarters at Tong Province (同州, roughly modern Weinan, Shaanxi). He was considered filially pious, respectful, and intelligent in his youth. In 554, Emperor Fei of Western Wei created him the Duke of Fucheng. + +Yuwen Tai died in 556, and in spring 557, Yuwen Yong's cousin Yuwen Hu, entrusted with the governing authority by Yuwen Tai, forced Emperor Gong of Western Wei to yield the throne to Yuwen Yong's older brother Yuwen Jue, ending Western Wei and establishing Northern Zhou. Yuwen Jue took the throne as Emperor Xiaomin, but used the alternative title of "Heavenly Prince" (Tian Wang). Yuwen Hu served as regent, and later that year, when Emperor Xiaomin tried to seize power from him, Yuwen Hu deposed Emperor Xiaomin and then killed him, replacing him with another older brother of Yuwen Yong's, Yuwen Yu, who took the throne as Emperor Ming. Emperor Ming created Yuwen Yong the greater title of Duke of Lu and often consulted Yuwen Yong on important matters. Although Yuwen Yong did not speak much, Emperor Ming made the observation, "He did not often speak, but whatever he spoke was always right." + +In 559, Yuwen Hu formally returned his authorities to Emperor Ming, and Emperor Ming began to formally rule on governmental matters, but Yuwen Hu retained the command of the military. In 560, Yuwen Hu, apprehensive of Emperor Ming's abilities, had the imperial chef Li An (李安) poison him with sugar cookies. Emperor Ming, realizing that he was near death, designated Yuwen Yong as his successor, and after he soon died, Yuwen Yong took the throne as Emperor Wu. However, the control of the government again fell into Yuwen Hu's hands. + +Early reign + +Emperor Wu was said to be largely a silent emperor early in his reign, giving Yuwen Hu free rein over the government, although he appeared to start cultivating a group of officials who would be loyal to him as the years went by. He formally bestowed Yuwen Hu with not only the military authorities, but also authority over all six ministries. + +With the Liang Dynasty general Wang Lin and the throne claimant that he supported, Xiao Zhuang, having been defeated by Chen Dynasty in spring 560 and having fled to Northern Qi, Northern Zhou (and its vassal Western Liang, with Emperor Xuan of Western Liang as its emperor) contended for control of Xiao Zhuang's former territory with Chen, precipitating a confrontation. Starting in winter 560, the Northern Zhou generals Heruo Dun (賀若敦) and Dugu Sheng (獨孤盛) began a drawn-out stalemate with the Chen general Hou Tian (侯瑱), initially being successful in thwarting Hou's attacks. Around the new year 561, however, Dugu was forced to withdraw, and Heruo was isolated. In spring 561, Hou agreed to let Heruo withdraw if Heruo would yield, and so Heruo withdrew; the modern Hunan region thus became Chen territory. (Yuwen Hu, believing Heruo to be at fault for losing the region, removed him from his posts.) + +Also in 561, Emperor Wu honored his mother Lady Chinu empress dowager. + +In spring 562, to foster a peaceful relationship with Chen, Northern Zhou returned the brother of Emperor Wen of Chen, Chen Xu, as well as Chen Xu's wife Liu Jingyan and son Chen Shubao, to Chen. In exchange, Chen gave the city of Lushan (魯山, in modern Wuhan, Hubei) to Northern Zhou. + +In summer 562, Emperor Wu, seeing that previously, nobles were not receiving any material benefits from their titles, began to have the nobles receive stipends based on the size of their fiefs. + +In spring 563, while on a visit to Yuan Province (原州, roughly modern Guyuan, Ningxia), Emperor Wu suddenly returned to the capital Chang'an without explanation. One of his attendants, Houmochen Chong (侯莫陳崇) the Duke of Liang, speculated to his associates that Yuwen Hu had died. When Houmochen's speculations became known, Emperor Wu publicly rebuked Houmochen, and the same night, Yuwen Hu sent troops to surround Houmochen's mansion, forcing him to commit suicide. Soon thereafter, he publicly bestowed Yuwen Hu the honor of having his name be subject to naming taboo, an honor that Yuwen Hu declined. + +Also in spring 563, Emperor Wu promulgated a new 25-volume criminal code drafted by the official Tuoba Di (拓拔迪), which divided the criminal punishment into 25 classes. + +In fall 563, Northern Zhou entered into an alliance treaty with Tujue against Northern Qi, part of which involved a promise that Emperor Wu would marry the daughter of Ashina Qijin, Tujue's Mugan Khan. In winter 563, the joint forces of Northern Zhou and Tujue launched a two-prong attack on Northern Qi, with the northern prong attacking Northern Qi's secondary capital Jinyang (晉陽, in modern Taiyuan, Shanxi) and the southern prong attacking Pingyang (平陽, in modern Linfen, Shanxi). The northern prong, commanded by the general Yang Zhong (楊忠), put Jinyang under siege, but was soon defeated by the Northern Qi general Duan Shao (段韶) and forced to withdraw. In response, the southern prong, commanded by Daxi Wu (達奚武), also withdrew. Still, the attack demonstrated the growing Northern Zhou strength—as previously, in the winter months, Northern Zhou forces would break the ice on the Yellow River to prevent possible Northern Qi attacks, but around this time and thereafter, Northern Qi forces broke the ice on the river to prevent possible Northern Zhou attacks. + +In fall 564, in order to placate Yuwen Hu, Emperor Wucheng of Northern Qi returned Yuwen Hu's mother Lady Yan and his (and Emperor Wu's) aunt (Yuwen Tai's sister), who had been trapped in Northern Qi territory several decades earlier, to Northern Zhou. In order to celebrate Lady Yan's return, Emperor Wu issued a general pardon, and prostrated himself before her as an ordinary nephew would. In turn, Yuwen Hu considered calling off planned joint attacks with Tujue against Northern Qi, but was fearful that Tujue would believe that Northern Zhou was abandoning the alliance, and therefore launched another joint attack with Tujue in winter 564. The assault, the main brunt of which was against Luoyang, however, was unsuccessful, and soon was abandoned. + +In spring 565, Emperor Wu sent his brother Yuwen Chun (宇文純) the Duke of Chen, Yuwen Gui (宇文貴) the Duke of Xu, Dou Yi (竇毅) the Duke of Shenwu, and Yang Jian (楊薦, different person than the more-known Yang Jian, referenced above and below) the Duke of Nanyang, to lead a ceremonial guard corps to Tujue to welcome back Ashina Qijin's daughter for marriage to him. However, when they arrived at Ashina Qijin's headquarters, he turned against the treaty and detained Yuwen Chun and his attendants. + +Middle reign + +In 566, the non-Chinese tribes of Xin Province (信州, modern eastern Chongqing) rebelled and captured Baidicheng, under the leadership of the chieftains Ran Lingxian (冉令賢) and Xiang Wuziwang (向五子王). The general Lu Teng (陸騰), however, was able to persuade some of Rang's subjects to turn against him, and he subsequently defeated Rang and Xiang, killing them and suppressing the revolts. + +In 567, in light of the death of Chen's Emperor Wen and succession by his son Emperor Fei of Chen in 566, the high level Chen officials engaged in infighting, and Emperor Wen's brother Chen Xu was victorious. The general Hua Jiao (華皎), the governor of Xiang Province (roughly modern Changsha, Hunan), felt uneasy, and therefore sought aid from Northern Zhou and Western Liang. Yuwen Hu, over the opposition by the official Cui You (崔猷), sent an army commanded by Emperor Wu's brother Yuwen Zhi (宇文直) the Duke of Wei to assist Hua and Western Liang, which was also aiding Hua. The Chen general Wu Mingche, however, quickly defeated the joint forces of Northern Zhou, Western Liang, and Hua, forcing Hua and Yuwen Zhi to both give up the war and flee to the Western Liang capital Jiangling. Chen was able to retain all of Hua's territory and further make minor territorial gains against both Northern Zhou and Western Liang as well. Yuwen Hu relieved Yuwen Zhi from his posts, and while Yuwen Zhi was eventually restored to them, Yuwen Zhi, who had previously had a cordial relationship with Yuwen Hu, bore a grudge against Yuwen Hu and secretly encouraged Emperor Wu to act against Yuwen Hu. + +In spring 568, a major storm at Tujue's headquarters inflicted substantial damage, and Ashina Qijin took it as a sign of divine displeasure at his rescission of the marriage agreement with Northern Zhou. He therefore returned Yuwen Chun, along with the daughter he promised Emperor Wu, back to Northern Zhou. Emperor Wu personally welcomed her and created her empress. + +Perhaps in light of the new adversarial relationship with Chen, when Northern Qi made peace overtures in fall 568, Northern Zhou accepted, and there was peace between the states for about a year, until fall 569, when Emperor Wu's brother Yuwen Xian the Prince of Qi led an army to siege Northern Qi's city of Yiyang (宜陽, in modern Luoyang, Henan) -- and for more than a year, the two states would engage in struggle for the control of Yiyang. Meanwhile, in fall 570, the Chen general Zhang Zhaoda (章昭達) put siege to Jiangling, nearly capturing it, but was eventually fought off by Northern Zhou and Western Liang's joint forces. + +In 569-570, Emperor Wu organized a debate between Buddhists and Daoists and commissioned two reports - the Xiaodao Lun and the Erjiao Lun - on the suitability of either religion for their adoption by the Chinese government.[1] He came out with a more favorable impression of Daoism, and would found the Tongdao Guan (通道观) for Daoist research, which would eventually compile the first Daoist encyclopedia, the Wushang Biyao (无上秘要).[1][2] + +In winter 570—as forewarned by the Northern Zhou general Wei Xiaokuan, who advised against the Yiyang campaign—the famed Northern Qi general Hulü Guang left Yiyang and instead advanced onto Northern Zhou territory north of the Fen River (汾水, flowing through modern Linfen), building forts and capturing substantial territory from Northern Zhou. While a counterattack by Yuwen Xian subsequently fought Hulü to a stalemate, damage had been done, and Northern Zhou was further forced to give up on the Yiyang campaign in fall 571 to concentrate against Hulü. + +Also in 571, Hua went to Chang'an, and on the way, he met Yuwen Zhi at Xiang Province (襄州, roughly modern Xiangfan, Hubei), suggesting to Yuwen Zhi that Western Liang was in such a desperate shape that if Northern Zhou wanted to see it preserved, Northern Zhou should lend some land to Western Liang. Yuwen Zhi agreed and made the proposal to Emperor Wu; in response, Emperor Wu gave three provinces—Ji (基州), Ping (平州), and Ruo (鄀州) (together making up about modern Jingmen and Yichang, Hubei) to Western Liang. + +By 572, Yuwen Hu had controlled the military for 16 years and the government for nearly as long. Emperor Wu had long wanted him out of the way, although he showed few outward signs of it. He conspired with Yuwen Zhi, distant relatives Yuwen Shenju (宇文神舉) and Yuwen Xiaobo (宇文孝伯), and Wang Gui (王軌) against Yuwen Hu. In spring 572, he made his move. After Emperor Wu and Yuwen Hu had a meeting, he invited Yuwen Hu into the palace to meet with Empress Dowager Chinu. On the way to her palace, he told Yuwen Hu that Empress Dowager Chinu was having problem with alcoholism and not listening to his advice to stop her drinking, so he wanted Yuwen Hu to advise her to change her ways as well. He further gave Yuwen Hu the text of the Jiu Gao (酒誥) -- an anti-alcoholism declaration written by King Cheng of Zhou—and suggested that he read the Jiu Gao to Empress Dowager Chinu. Once they reached her palace, Yuwen Hu, pursuant to Emperor Wu's request, started reading the Jiu Gao. Before he could finish it, Emperor Wu stepped behind him and used a jade tablet to strike the back of his head. Yuwen Hu fell to the ground, and Yuwen Zhi, who was hiding nearby, jumped out, and cut off Yuwen Hu's head, ending Yuwen Hu's hold on power. Yuwen Hu's sons, brothers, and key associates were all executed. + +Late reign +The epitaph of Emperor Wu's mausoleum. It reads: "The Xiao Mausoleum of Great Zhou Gaozu Emperor Wu" + +Having been instrumental in Yuwen Hu's death, Yuwen Zhi sought to take over Yuwen Hu's post, but Emperor Wu, who wanted to directly control the government, divided the authority between several officials, retaining most authorities in himself. He took the opportunity posthumously to honor his brother Emperor Xiaomin (Yuwen Hu had refused to do so previously) and create his son Yuwen Yun the Duke of Lu crown prince. He also began to oppose overt luxury and destroyed several palaces that he found overly luxurious as well as other items that he considered ornately decorated. + +Also in summer 572, Emperor Wu learned that Northern Qi's emperor Gao Wei, apprehensive of Hulü Guang, had executed Hulü. Being glad, Emperor Wu declared a general pardon. + +By 573, it had come to Emperor Wu's attention that Crown Prince Yun was not paying attention to matters of state but instead associated with immoral people. In response, Emperor Wu selected staff members for Crown Prince Yun who were known for their strict conduct. This made the crown prince unhappy. + +Around the new year 574, Emperor Wu gathered Confucian scholars, Taoist monks, and Buddhist monks, and had them debate about their philosophies. He ranked Confucianism the highest, then Taoism, and then Buddhism. Subsequently, in summer 574, he banned both Taoism and Buddhism, ordering their monks to return to secular life. He also banned the worship of minor deities whose cults were not registered with the government. (This became known as the second of the Three Disasters of Wu) + +In spring 574, Empress Dowager Chinu died. Emperor Wu mourned for more than a month, eating only a small amount of rice during this period. + +In fall 574, while Emperor Wu was at Yunyang (雲陽, in modern Xianyang, Shaanxi), Yuwen Zhi, who had long resented not receiving more authority, rebelled at Chang'an. The official Yuchi Yun (尉遲運), one of the officials in charge of the capital along with Crown Prince Yun, defeated Yuwen Zhi, forcing him to flee. Yuwen Zhi was soon captured and executed. + +Believing Northern Qi to have been substantially weakened not only by Hulü's death but also by the successful campaign that Chen waged against it in 573 (capturing the provinces between the Yangtze River and the Huai River), by 575, Emperor Wu was seriously considering a major campaign against Northern Qi. However, he kept the matter secret, consulting only Yuwen Xian, Wang Yi (王誼), and Yu Yi (于翼). Only until he was ready in fall 575 did he announce it generally. He aimed his attack at Luoyang, but he spent about 20 days sieging it and could not capture it, and became ill. He withdrew, with virtually no gain. + +In spring 576, pursuant to Emperor Wu's orders, Crown Prince Yun launched a campaign against Tuyuhun; a campaign that appeared to be moderately successful. Yet the campaign would bring another deterioration of the relationship between father and son, as Wang Gui, who officially served as the crown prince's lieutenant (along with Yuwen Xiaobo) but was in charge of the operation, reported a matter of immoral acts that the crown prince and his associates Zheng Yi (鄭譯) and Wang Duan (王端) engaged in. Emperor Wu himself caned the crown prince and his associates, expelling the associates from the crown prince's palace. (Crown Prince Yun, however, soon recalled his associates.) Emperor Wu was also exceedingly strict with Crown Prince Yun, disallowing him from resting or drinking. Whenever he had faults, Emperor Wu would batter him or whip him, and further warn him that he would be deposed. Emperor Wu further ordered the crown prince's staff to report all of his actions to the emperor. Fearful of his father, Crown Prince Yun learned to feign upright behavior, and the emperor thought that the crown prince had changed. + +In winter 576, Emperor Wu again attacked Northern Qi; this time, changing strategy and attacking Pingyang instead. He was able to capture Pingyang quickly, before Northern Qi troops could arrive. The Northern Qi emperor Gao Wei soon advanced toward Pingyang with a large army, and Emperor Wu, not wanting to engage Gao Wei's army directly, withdrew, leaving the general Liang Shiyan (梁士彥) in charge of defending Pingyang. Gao Wei put Pingyang under siege, and at one point nearly captured it. Emperor Wu, after reorganizing his forces, relaunched his army and headed for Pingyang, seeking to lift the siege. Around the new year 577, he arrived near Pingyang and Gao Wei chose to engage him—but, once the battle began, panicked when his favorite concubine Consort Feng Xiaolian falsely believed that the army had been defeated—and he abandoned the army, causing its collapse. Gao Wei fled to Jinyang, and Emperor Wu gave chase. No longer having the will to fight Emperor Wu, Gao Wei further fled back to the Northern Qi capital Yecheng, leaving his cousin Gao Yanzong in charge of Jinyang. Gao Yanzong launched a counterattack, catching Emperor Wu by surprise and nearly killing him. However, after the victory, Gao Yanzong's army went into a celebration, and he was unable to reorganize it, and Emperor Wu soon defeated and captured him, and headed for Yecheng. + +Gao Wei, after passing the throne to his young son Gao Heng to deflect ill omens, considered resisting, but instead decided to flee southeast across the Yellow River, planning to regroup and see if he could make a last stand—but if not, to flee to Chen. In spring 577, Emperor Wu entered Yecheng. With Gao Wei's official Gao Anagong feeding him intelligence on Gao Wei's location, he was able to capture Gao Wei. After Gao Wei was returned to Yecheng, he treated Gao Wei with respect and created Gao Wei the Duke of Wen. Gao Wei's uncle Gao Jie (高湝) and cousin Gao Xiaoheng (高孝珩), making one last stand at Xindu (信都, in modern Hengshui, Hebei), were also soon defeated and captured. Another of Gao Wei's cousins, Gao Shaoyi, after making a failed bid to resist, fled to Tujue and came under the protection of Ashin Qijin's successor Tuobo Khan. Other than Ying Province (營州, roughly modern Zhaoyang, Liaoning), held by the official Gao Baoning (高寶寧), a distant relative to Northern Qi's imperial Gao clan, all of Northern Qi's territory came under Northern Zhou rule. + +In summer 577, Emperor Wu returned to Chang'an with Gao Wei and other members of the Gao clan in tow. In winter 577, apprehensive of the Gao clan members, he falsely accused Gao Wei of conspiring with the former Northern Qi official Mu Tipo and killed Mu and ordered Gao Wei and the other members of the Gao clan to commit suicide. + +In light of Northern Qi's defeat, Chen, then ruled by Chen Xu (who had deposed Emperor Fei and took the throne himself as Emperor Xuan), launched an attack commanded by Wu Mingche on Pengcheng (modern Xuzhou, Jiangsu), an important city on the former Chen/Northern Qi border. Emperor Wu sent Wang Gui to relieve Pengcheng, and in spring 578, Wang defeated Wu, capturing him. + +By summer 578, Emperor Wu was engaging in military campaigns on two fronts: against Tujue in the north and against Chen in the south. However, he suddenly grew ill and, after stopping at Yunyang, ended the attack against Tujue. He entrusted the important matters to Yuwen Xiaobo, and he soon died at the age of 35. Crown Prince Yun succeeded him (as Emperor Xuan), and by 581 Northern Zhou had fallen, its throne having been seized by Emperor Xuan's father-in-law Yang Jian. diff --git a/test/unit_tests/empty.txt b/test/unit_tests/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/fifo.cc b/test/unit_tests/fifo.cc similarity index 99% rename from test/fifo.cc rename to test/unit_tests/fifo.cc index 6348b4e..3aeaaac 100644 --- a/test/fifo.cc +++ b/test/unit_tests/fifo.cc @@ -3,7 +3,7 @@ #include #include -#include "../src/stream/fifo.h" +#include "../../src/stream/fifo.h" #include "storage/disk.h" #include "utils.h" diff --git a/test/phobos_file.cc b/test/unit_tests/phobos_file.cc similarity index 99% rename from test/phobos_file.cc rename to test/unit_tests/phobos_file.cc index 72160aa..568d43c 100644 --- a/test/phobos_file.cc +++ b/test/unit_tests/phobos_file.cc @@ -4,7 +4,7 @@ #include #include -#include "../src/storage/phobos_file.h" +#include "../../src/storage/phobos_file.h" #include "utils.h" diff --git a/test/smallTestFile.txt b/test/unit_tests/smallTestFile.txt similarity index 100% rename from test/smallTestFile.txt rename to test/unit_tests/smallTestFile.txt diff --git a/test/storage/disk.cc b/test/unit_tests/storage/disk.cc similarity index 100% rename from test/storage/disk.cc rename to test/unit_tests/storage/disk.cc diff --git a/test/storage/disk.h b/test/unit_tests/storage/disk.h similarity index 96% rename from test/storage/disk.h rename to test/unit_tests/storage/disk.h index c2a3f9b..cd474a3 100644 --- a/test/storage/disk.h +++ b/test/unit_tests/storage/disk.h @@ -1,4 +1,4 @@ -#include "../../src/storage/storage.h" +#include "../../../src/storage/storage.h" namespace fiphoboserver { namespace storage { diff --git a/test/tests-main.cc b/test/unit_tests/tests-main.cc similarity index 100% rename from test/tests-main.cc rename to test/unit_tests/tests-main.cc diff --git a/test/utils.cc b/test/unit_tests/utils.cc similarity index 100% rename from test/utils.cc rename to test/unit_tests/utils.cc diff --git a/test/utils.h b/test/unit_tests/utils.h similarity index 100% rename from test/utils.h rename to test/unit_tests/utils.h -- GitLab From b45eb717cf9f64d09258aaac0d8a61affe824aad Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 8 Apr 2020 17:04:21 +0100 Subject: [PATCH 41/62] Add S3 error handling All errors returned from phobos now have at least a little distinct error messages (as far as the return values permit it) and send an S3 readable XML along with the error number. --- doc/Doxyfile.in | 3 +- .../fiphoboserver/FIPhoboServerException.md | 25 ++ doc/Markdown/fiphoboserver/IOException.md | 45 ++++ doc/Markdown/fiphoboserver/S3_header.md | 224 ---------------- doc/Markdown/fiphoboserver/index.md | 2 +- .../fiphoboserver/s3_utilities/S3_header.md | 242 ++++++++++++++++++ .../fiphoboserver/s3_utilities/index.md | 54 ++++ .../s3_utilities/s3_error_info.md | 57 +++++ .../fiphoboserver/storage/PhobosException.md | 45 ++++ .../fiphoboserver/storage/Phobos_file.md | 22 +- doc/openapi.yaml | 98 ++++++- src/fiphoboserver_exception.h | 24 +- src/server/CMakeLists.txt | 3 + src/server/get_request_handler.cc | 30 ++- src/server/get_request_handler.h | 4 +- src/server/getmd_request_handler.cc | 8 +- src/server/getmd_request_handler.h | 4 +- src/server/put_request_handler.cc | 20 +- src/server/put_request_handler.h | 4 +- src/server/s3_utilities/CMakeLists.txt | 6 + .../{S3_header.h => s3_utilities/s3_header.h} | 16 ++ src/server/s3_utilities/s3_utilities.cc | 54 ++++ src/server/s3_utilities/s3_utilities.h | 38 +++ src/storage/phobos_exception.h | 12 +- 24 files changed, 781 insertions(+), 259 deletions(-) delete mode 100644 doc/Markdown/fiphoboserver/S3_header.md create mode 100644 doc/Markdown/fiphoboserver/s3_utilities/S3_header.md create mode 100644 doc/Markdown/fiphoboserver/s3_utilities/index.md create mode 100644 doc/Markdown/fiphoboserver/s3_utilities/s3_error_info.md create mode 100644 src/server/s3_utilities/CMakeLists.txt rename src/server/{S3_header.h => s3_utilities/s3_header.h} (89%) create mode 100644 src/server/s3_utilities/s3_utilities.cc create mode 100644 src/server/s3_utilities/s3_utilities.h diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 6842916..a1dd2bc 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -19,7 +19,7 @@ CASE_SENSE_NAMES = YES # Warning options # Warn on everything possible, and fail on warning -QUIET = NO +QUIET = YES WARNINGS = YES WARN_IF_UNDOCUMENTED = NO WARN_IF_DOC_ERROR = YES @@ -50,4 +50,3 @@ PREDEFINED = __cplusplus SEARCH_INCLUDES = YES GENERATE_XML = YES -XML_NS_MEMB_FILE_SCOPE = YES diff --git a/doc/Markdown/fiphoboserver/FIPhoboServerException.md b/doc/Markdown/fiphoboserver/FIPhoboServerException.md index 65d03cb..a941236 100644 --- a/doc/Markdown/fiphoboserver/FIPhoboServerException.md +++ b/doc/Markdown/fiphoboserver/FIPhoboServerException.md @@ -20,6 +20,7 @@ Is inherited by [fiphoboserver::IOException][fiphoboserver-IOException], [fiphob | [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException) | default constructor | | [FIPhoboServerException](#fiphoboserver-FIPhoboServerException-FIPhoboServerException-1) | constructor for a message with information on the exception | | [what](#fiphoboserver-FIPhoboServerException-what) | get information on this exception | +| [get_inner_error](#fiphoboserver-FIPhoboServerException-get_inner_error) | return the inner error value that caused this exception | @@ -107,6 +108,30 @@ get information on this exception +#### Qualifiers: +* const +* inline +* virtual + + +[Go to Top](#fiphoboserver-FIPhoboServerException) + +### public int fiphoboserver::FIPhoboServerException::get_inner_error () const noexcept + +return the inner error value that caused this exception + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| int | | + + + + + #### Qualifiers: * const * inline diff --git a/doc/Markdown/fiphoboserver/IOException.md b/doc/Markdown/fiphoboserver/IOException.md index a164c7a..a4b18c7 100644 --- a/doc/Markdown/fiphoboserver/IOException.md +++ b/doc/Markdown/fiphoboserver/IOException.md @@ -8,13 +8,34 @@ exceptions specifically for errors in I/O ## Inheritance: Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_inner_value](#fiphoboserver-IOException-m_inner_value) | the value of the error that caused this exception | + + ## Public Functions | Name | Description | | ---- | ---- | | [IOException](#fiphoboserver-IOException-IOException) | constructor | +| [get_inner_error](#fiphoboserver-IOException-get_inner_error) | return the inner error value that caused this exception | + + + +## Private Attributes +### private fiphoboserver::IOException::m_inner_value = 0 + +the value of the error that caused this exception + + + + + +[Go to Top](#fiphoboserver-IOException) + ## Public Functions ### public fiphoboserver::IOException::IOException (const char *caller, const char *function_name, int return_value) @@ -45,6 +66,30 @@ constructor * inline +[Go to Top](#fiphoboserver-IOException) + +### public int fiphoboserver::IOException::get_inner_error () const noexcept override + +return the inner error value that caused this exception + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| int | | + + + + + +#### Qualifiers: +* const +* inline +* virtual + + [Go to Top](#fiphoboserver-IOException) [fiphoboserver-FIPhoboServerException]:./FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/S3_header.md b/doc/Markdown/fiphoboserver/S3_header.md deleted file mode 100644 index 45b3a89..0000000 --- a/doc/Markdown/fiphoboserver/S3_header.md +++ /dev/null @@ -1,224 +0,0 @@ -# public fiphoboserver::S3_header - -class to extract the S3 specific information from proxygens HTTPMessage headers - - - - -## Public Functions -| Name | Description | -| ---- | ---- | -| [set_headers](#fiphoboserver-S3_header-set_headers) | sets the internal header instance to the given header object | -| [get_bucket](#fiphoboserver-S3_header-get_bucket) | extracts the name of the S3 bucket requested in the HTTP message | -| [get_key_without_bucket](#fiphoboserver-S3_header-get_key_without_bucket) | extracts the name of the key id requested in the HTTP message | -| [get_key](#fiphoboserver-S3_header-get_key) | extracts the name of the key id requested in the HTTP message | -| [is_create_bucket_request](#fiphoboserver-S3_header-is_create_bucket_request) | checks if the message we got is one for just creating a bucket | -| [get_body_length](#fiphoboserver-S3_header-get_body_length) | gets the total length of all body chunks that is expected | -| [get_meta_data](#fiphoboserver-S3_header-get_meta_data) | gets the S3 metadata stored in the HTTP headers | - - - -## Public Functions -### public void fiphoboserver::S3_header::set_headers (std::unique_ptr< proxygen::HTTPMessage > new_m_headers) - -sets the internal header instance to the given header object - - - - -#### Parameters: -| Type | Name | Description | -| ---- | ---- | ---- | -| std::unique_ptr< proxygen::HTTPMessage > | new_m_headers | headers that should be read out in this instance | - - - - - - - - - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public std::string fiphoboserver::S3_header::get_bucket () - -extracts the name of the S3 bucket requested in the HTTP message - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| std::string | name of the bucket | - - - - - - - - - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public std::string fiphoboserver::S3_header::get_key_without_bucket () - -extracts the name of the key id requested in the HTTP message - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| std::string | name of the key | - - - - - - - -This will only return the key itself without the leading bucket name! - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public std::string fiphoboserver::S3_header::get_key () - -extracts the name of the key id requested in the HTTP message - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| std::string | name of the key | - - - - - - - -This is the whole path argument without the leading '/' but including the bucket name! - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public bool fiphoboserver::S3_header::is_create_bucket_request () - -checks if the message we got is one for just creating a bucket - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| bool | true if these headers belong to a create bucket request, false for all other requests | - - - - - - - -This is convenient since we don't really support buckets themselves and can easily stop working if this returns true - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public size_t fiphoboserver::S3_header::get_body_length () - -gets the total length of all body chunks that is expected - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| size_t | the length of the total body to be expected | - - - - - - - -This reads out the header "Content-Length" - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - -### public std::map< std::string, std::string > fiphoboserver::S3_header::get_meta_data () - -gets the S3 metadata stored in the HTTP headers - - - - -#### Returns: -| Type | Description | -| ---- | ---- | -| std::map< std::string, std::string > | a map consisting of all key value pairs found in the header | - - - - - - - -In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE for for example a KEY:VALUE pair. - - - - -#### Qualifiers: -* inline - - -[Go to Top](#fiphoboserver-S3_header) - diff --git a/doc/Markdown/fiphoboserver/index.md b/doc/Markdown/fiphoboserver/index.md index 7f52e6e..be787b6 100644 --- a/doc/Markdown/fiphoboserver/index.md +++ b/doc/Markdown/fiphoboserver/index.md @@ -12,13 +12,13 @@ | [HandlerFactory](./HandlerFactory.md) | factory to create the handlers for incoming HTTP requests | | [IOException](./IOException.md) | exceptions specifically for errors in I/O | | [PutRequestHandler](./PutRequestHandler.md) | proxygen class implementation for handling PUT requests | -| [S3_header](./S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | | [UnsupportedRequestHandler](./UnsupportedRequestHandler.md) | proxygen class implementation for handling all requests that we don't support | ## Namespaces | Name | Description | | ---- | ---- | +| [s3_utilities](./s3_utilities/index.md) | namespace for S3 specific utilities | | [storage](./storage/index.md) | namespace for storage classes that belong to / inherit from from [fiphoboserver::storage::Storage][fiphoboserver-storage-Storage] | | [stream](./stream/index.md) | namespace for stream classes that belong to / inherit from from [fiphoboserver::stream::Stream][fiphoboserver-stream-Stream] | diff --git a/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md new file mode 100644 index 0000000..d9ef525 --- /dev/null +++ b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md @@ -0,0 +1,242 @@ +# public fiphoboserver::s3_utilities::S3_header + +class to extract the S3 specific information from proxygens HTTPMessage headers + + + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [set_headers](#fiphoboserver-s3_utilities-S3_header-set_headers) | sets the internal header instance to the given header object | +| [get_bucket](#fiphoboserver-s3_utilities-S3_header-get_bucket) | extracts the name of the S3 bucket requested in the HTTP message | +| [get_key_without_bucket](#fiphoboserver-s3_utilities-S3_header-get_key_without_bucket) | extracts the name of the key id requested in the HTTP message | +| [get_key](#fiphoboserver-s3_utilities-S3_header-get_key) | extracts the name of the key id requested in the HTTP message | +| [is_create_bucket_request](#fiphoboserver-s3_utilities-S3_header-is_create_bucket_request) | checks if the message we got is one for just creating a bucket | +| [get_body_length](#fiphoboserver-s3_utilities-S3_header-get_body_length) | gets the total length of all body chunks that is expected | +| [get_meta_data](#fiphoboserver-s3_utilities-S3_header-get_meta_data) | gets the S3 metadata stored in the HTTP headers | +| [print_all_headers](#fiphoboserver-s3_utilities-S3_header-print_all_headers) | print all headers to std out | + + + +## Public Functions +### public void fiphoboserver::s3_utilities::S3_header::set_headers (std::unique_ptr< proxygen::HTTPMessage > new_m_headers) + +sets the internal header instance to the given header object + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::unique_ptr< proxygen::HTTPMessage > | new_m_headers | headers that should be read out in this instance | + + + + + + + + + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_bucket () + +extracts the name of the S3 bucket requested in the HTTP message + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the bucket | + + + + + + + + + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_key_without_bucket () + +extracts the name of the key id requested in the HTTP message + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the key | + + + + + + + +This will only return the key itself without the leading bucket name! + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_key () + +extracts the name of the key id requested in the HTTP message + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | name of the key | + + + + + + + +This is the whole path argument without the leading '/' but including the bucket name! + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public bool fiphoboserver::s3_utilities::S3_header::is_create_bucket_request () + +checks if the message we got is one for just creating a bucket + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if these headers belong to a create bucket request, false for all other requests | + + + + + + + +This is convenient since we don't really support buckets themselves and can easily stop working if this returns true + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public size_t fiphoboserver::s3_utilities::S3_header::get_body_length () + +gets the total length of all body chunks that is expected + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| size_t | the length of the total body to be expected | + + + + + + + +This reads out the header "Content-Length" + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::map< std::string, std::string > fiphoboserver::s3_utilities::S3_header::get_meta_data () + +gets the S3 metadata stored in the HTTP headers + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::map< std::string, std::string > | a map consisting of all key value pairs found in the header | + + + + + + + +In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE for for example a KEY:VALUE pair. + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public void fiphoboserver::s3_utilities::S3_header::print_all_headers () + +print all headers to std out + + + + + + + + +#### Qualifiers: +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + diff --git a/doc/Markdown/fiphoboserver/s3_utilities/index.md b/doc/Markdown/fiphoboserver/s3_utilities/index.md new file mode 100644 index 0000000..0e86f87 --- /dev/null +++ b/doc/Markdown/fiphoboserver/s3_utilities/index.md @@ -0,0 +1,54 @@ +# fiphoboserver::s3_utilities + +namespace for S3 specific utilities + + + + +## Classes +| Name | Description | +| ---- | ---- | +| [s3_error_info](./s3_error_info.md) | struct to contain all the information needed to send a proper s3 error message | +| [S3_header](./S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | + + +## Functions +| Name | Description | +| ---- | ---- | +| [create_s3_error](#fiphoboserver-s3_utilities-create_s3_error) | create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on the error value | + + + +## Functions +### public [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] fiphoboserver::s3_utilities::create_s3_error (int return_value, std::string request) + +create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on the error value + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| int | return_value | the error code (errno, etc) that the function returned that raised the exception that preceded the call to this function | +| std::string | request | the request path that was being processed | + +#### Returns: +| Type | Description | +| ---- | ---- | +| [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] | an [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] struct containing readable information on the error | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +[fiphoboserver-s3_utilities-s3_error_info]:./s3_error_info.md#fiphoboserver-s3_utilities-s3_error_info diff --git a/doc/Markdown/fiphoboserver/s3_utilities/s3_error_info.md b/doc/Markdown/fiphoboserver/s3_utilities/s3_error_info.md new file mode 100644 index 0000000..456a448 --- /dev/null +++ b/doc/Markdown/fiphoboserver/s3_utilities/s3_error_info.md @@ -0,0 +1,57 @@ +# public fiphoboserver::s3_utilities::s3_error_info + +struct to contain all the information needed to send a proper s3 error message + + + + +## Public Attributes +| Name | Description | +| ---- | ---- | +| [https_error_code](#fiphoboserver-s3_utilities-s3_error_info-https_error_code) | https error code (400s or 500s) | +| [https_error_identifier](#fiphoboserver-s3_utilities-s3_error_info-https_error_identifier) | readable string for the code [https_error_code][fiphoboserver-s3_utilities-s3_error_info-https_error_code] | +| [error_xml](#fiphoboserver-s3_utilities-s3_error_info-error_xml) | XML encoded message containing more information on the error. | + + + +## Public Attributes +### public fiphoboserver::s3_utilities::s3_error_info::https_error_code + +https error code (400s or 500s) + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-s3_error_info) + +### public fiphoboserver::s3_utilities::s3_error_info::https_error_identifier + +readable string for the code [https_error_code][fiphoboserver-s3_utilities-s3_error_info-https_error_code] + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-s3_error_info) + +### public fiphoboserver::s3_utilities::s3_error_info::error_xml + +XML encoded message containing more information on the error. + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-s3_error_info) + +[fiphoboserver-s3_utilities-s3_error_info-https_error_code]:./s3_error_info.md#fiphoboserver-s3_utilities-s3_error_info-https_error_code diff --git a/doc/Markdown/fiphoboserver/storage/PhobosException.md b/doc/Markdown/fiphoboserver/storage/PhobosException.md index 9763107..1877926 100644 --- a/doc/Markdown/fiphoboserver/storage/PhobosException.md +++ b/doc/Markdown/fiphoboserver/storage/PhobosException.md @@ -8,13 +8,34 @@ exceptions specifically for the phobos backend library ## Inheritance: Inherits from [fiphoboserver::FIPhoboServerException][fiphoboserver-FIPhoboServerException]. +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_inner_value](#fiphoboserver-storage-PhobosException-m_inner_value) | the value of the error that caused this exception | + + ## Public Functions | Name | Description | | ---- | ---- | | [PhobosException](#fiphoboserver-storage-PhobosException-PhobosException) | constructor | +| [get_inner_error](#fiphoboserver-storage-PhobosException-get_inner_error) | return the inner error value that caused this exception | + + + +## Private Attributes +### private fiphoboserver::storage::PhobosException::m_inner_value = 0 + +the value of the error that caused this exception + + + + + +[Go to Top](#fiphoboserver-storage-PhobosException) + ## Public Functions ### public fiphoboserver::storage::PhobosException::PhobosException (const char *caller, const char *function_name, int return_value) @@ -45,6 +66,30 @@ constructor * inline +[Go to Top](#fiphoboserver-storage-PhobosException) + +### public int fiphoboserver::storage::PhobosException::get_inner_error () const noexcept override + +return the inner error value that caused this exception + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| int | | + + + + + +#### Qualifiers: +* const +* inline +* virtual + + [Go to Top](#fiphoboserver-storage-PhobosException) [fiphoboserver-FIPhoboServerException]:./../FIPhoboServerException.md diff --git a/doc/Markdown/fiphoboserver/storage/Phobos_file.md b/doc/Markdown/fiphoboserver/storage/Phobos_file.md index 3bd1ab7..3c1a4a9 100644 --- a/doc/Markdown/fiphoboserver/storage/Phobos_file.md +++ b/doc/Markdown/fiphoboserver/storage/Phobos_file.md @@ -84,26 +84,40 @@ default destructor [Go to Top](#fiphoboserver-storage-Phobos_file) -### public fiphoboserver::storage::Phobos_file::Phobos_file (Phobos_file &&)=default +### public fiphoboserver::storage::Phobos_file::Phobos_file (Phobos_file &&input) default move contructor +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| [Phobos_file][fiphoboserver-storage-Phobos_file] && | input | | + +#### Qualifiers: +* inline + + [Go to Top](#fiphoboserver-storage-Phobos_file) -### public [Phobos_file][fiphoboserver-storage-Phobos_file] & fiphoboserver::storage::Phobos_file::operator= (Phobos_file &&)=default +### public [Phobos_file][fiphoboserver-storage-Phobos_file] & fiphoboserver::storage::Phobos_file::operator= (Phobos_file &&input) default assignment operator for rvalue references +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| [Phobos_file][fiphoboserver-storage-Phobos_file] && | input | | + #### Returns: | Type | Description | | ---- | ---- | @@ -113,6 +127,10 @@ default assignment operator for rvalue references +#### Qualifiers: +* inline + + [Go to Top](#fiphoboserver-storage-Phobos_file) ### public fiphoboserver::storage::Phobos_file::Phobos_file (const Phobos_file &)=delete diff --git a/doc/openapi.yaml b/doc/openapi.yaml index b4dcb61..f26758a 100644 --- a/doc/openapi.yaml +++ b/doc/openapi.yaml @@ -26,7 +26,7 @@ paths: get: tags: - bucket - summary: Lists all objects in the bucket + summary: Lists all objects in the bucket (not yet implemented) operationId: getBucket responses: '200': @@ -37,14 +37,27 @@ paths: $ref: '#/components/schemas/ListObjectsV2Output' '403': description: Access denied + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' + '404': description: No such bucket + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '500': description: Internal Server Error + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' put: tags: - bucket - summary: Creates a new bucket + summary: Creates a new bucket (not yet implemented) operationId: putBucket requestBody: description: Information on the bucket that should be created @@ -58,8 +71,16 @@ paths: description: Success '403': description: Access denied + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '500': description: Internal Server Error + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '/{bucket}/{key}': parameters: - name: bucket @@ -93,10 +114,32 @@ paths: type: string '403': description: Access denied + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '404': - description: No such key + description: Not found (No such key) + content: + application/xml: + schema: + allOf: + - $ref: '#/components/schemas/Error' + - example: + Code: NoSuchKey + Message: The resource you requested does not exist! + Resource: key '500': description: Internal Server Error + content: + application/xml: + schema: + allOf: + - $ref: '#/components/schemas/Error' + - example: + Code: InternalServerPhobosError + Message: The Phobos database is not running correctly on the server + Resource: key put: tags: - object @@ -123,25 +166,61 @@ paths: description: Continue '200': description: Success + '400': + description: Bad Request + content: + application/xml: + schema: + allOf: + - $ref: '#/components/schemas/Error' + - example: + Code: InvalidArgument + Message: Some arguments given to the storage were invalid! + Resource: key '403': description: Access denied + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '500': description: Internal Server Error + content: + application/xml: + schema: + allOf: + - $ref: '#/components/schemas/Error' + - example: + Code: InternalServerPhobosError + Message: The Phobos database is not running correctly on the server + Resource: key delete: tags: - object summary: Delete an object operationId: deleteObject - description: Deletes an object from the internal storage + description: Deletes an object from the internal storage (not yet implemented) responses: '204': description: No Content '403': description: Access denied + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '500': description: Internal Server Error + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' '501': description: Not Implemented + content: + application/xml: + schema: + $ref: '#/components/schemas/Error' components: schemas: CreateBucketInfo: @@ -156,6 +235,17 @@ components: type: string xml: name: CreateBucketConfiguration + Error: + type: object + properties: + Code: + type: string + Message: + type: string + Resouce: + type: string + xml: + name: Error ListObjectsV2Output: type: object properties: diff --git a/src/fiphoboserver_exception.h b/src/fiphoboserver_exception.h index 9fd65cb..cc01a9d 100644 --- a/src/fiphoboserver_exception.h +++ b/src/fiphoboserver_exception.h @@ -34,6 +34,11 @@ class FIPhoboServerException : public std::exception { return m_message.c_str(); } + /// + /// @brief return the inner error value that caused this exception + /// + virtual int get_inner_error() const noexcept { return 0; } + protected: /// /// @brief the internal message @@ -49,11 +54,13 @@ class IOException : public FIPhoboServerException { /// /// @brief constructor /// - /// @param caller the function that threw this exception - /// @param function_name the io function that returned an error - /// @param return_value the return value of the io function + /// @param caller the function that threw this exception + /// @param function_name the io function that returned an error + /// @param return_value the return value of the io function /// - IOException(const char* caller, const char* function_name, int return_value) + IOException( + const char* caller, const char* function_name, int return_value) : + m_inner_value(return_value) { std::stringstream ss; ss << "[" << caller << "]: IO Exception occurred: " << function_name @@ -61,6 +68,15 @@ class IOException : public FIPhoboServerException { << ")"; m_message = ss.str(); } + + /// @copydoc FIPhoboServerException::get_inner_error + int get_inner_error() const noexcept override { return m_inner_value; } + + private: + /// + /// @brief the value of the error that caused this exception + /// + int m_inner_value = 0; }; } // namespace fiphoboserver diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index c08cbf6..98feba5 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -1,4 +1,6 @@ +add_subdirectory(s3_utilities) + add_library( server get_request_handler.cc @@ -10,3 +12,4 @@ add_library( #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") target_link_libraries(server PUBLIC proxygen::proxygen) target_link_libraries(server PUBLIC stream) +target_link_libraries(server PUBLIC s3_utilities) diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index 510741f..0640029 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -8,6 +8,7 @@ #include "get_request_handler.h" #include "../fiphoboserver_exception.h" +#include "s3_utilities/s3_utilities.h" #include #include @@ -29,11 +30,17 @@ void GetRequestHandler::onRequest( /* Creating Bucket queries */ m_s3_header.set_headers(std::move(headers)); + // m_s3_header.print_all_headers(); try { /* Send meta data to backend through the stream */ - m_stream->set_storage_meta_data( - m_s3_header.get_meta_data(), m_s3_header.get_bucket()); + // m_stream->set_storage_meta_data( + // m_s3_header.get_meta_data(), m_s3_header.get_bucket()); + + // Check if metadata for file exists + std::map meta_data = + m_stream->get_meta_data(m_s3_header.get_key()); + /* Tell stream to coordinate with backend to prepare for GET operation */ m_stream->start_get(m_s3_header.get_key()); @@ -51,10 +58,14 @@ void GetRequestHandler::onRequest( } catch (const FIPhoboServerException& ex) { std::cerr << "Caught exception: " << ex.what() << '\n'; + + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(ex.what()) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) .sendWithEOM(); + return; } @@ -108,9 +119,14 @@ void GetRequestHandler::read_file(folly::EventBase* evb) } catch (const FIPhoboServerException& ex) { std::cerr << "Read error=" << ex.what() << '\n'; - evb->runInEventBaseThread([this] { - LOG(ERROR) << "Error reading file"; - downstream_->sendAbort(); + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); + evb->runInEventBaseThread([this, error] { + proxygen::ResponseBuilder(downstream_) + .status( + error.https_error_code, error.https_error_identifier) + .body(error.error_xml) + .sendWithEOM(); }); break; } diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 9ef05f6..4a55a42 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -9,7 +9,7 @@ #pragma once #include "../stream/stream.h" -#include "S3_header.h" +#include "s3_utilities/s3_header.h" #include #include @@ -99,7 +99,7 @@ class GetRequestHandler : public proxygen::RequestHandler { bool check_for_completion(); std::unique_ptr m_stream; - S3_header m_s3_header; + s3_utilities::S3_header m_s3_header; bool m_read_file_scheduled{false}; std::atomic m_paused{false}; bool m_finished{false}; diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index b95d5f6..24b33c9 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -8,6 +8,7 @@ #include "getmd_request_handler.h" #include "../fiphoboserver_exception.h" +#include "s3_utilities/s3_utilities.h" #include @@ -43,9 +44,12 @@ void GetmdRequestHandler::onRequest( } catch (const FIPhoboServerException& ex) { std::cerr << "Caught exception: " << ex.what() << '\n'; + + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(ex.what()) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) .sendWithEOM(); return; } diff --git a/src/server/getmd_request_handler.h b/src/server/getmd_request_handler.h index f3a1d16..aa0b991 100644 --- a/src/server/getmd_request_handler.h +++ b/src/server/getmd_request_handler.h @@ -9,7 +9,7 @@ #pragma once #include "../stream/stream.h" -#include "S3_header.h" +#include "s3_utilities/s3_header.h" #include #include @@ -65,7 +65,7 @@ class GetmdRequestHandler : public proxygen::RequestHandler { private: std::unique_ptr m_stream; - S3_header m_s3_header; + s3_utilities::S3_header m_s3_header; std::map m_meta_data; }; diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index a58d674..f7ddbd9 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -13,6 +13,7 @@ #include #include "../fiphoboserver_exception.h" +#include "s3_utilities/s3_utilities.h" namespace fiphoboserver { @@ -26,6 +27,7 @@ void PutRequestHandler::onRequest( /* Creating Bucket queries */ m_s3_header.set_headers(std::move(headers)); + // m_s3_header.print_all_headers(); if (m_s3_header.is_create_bucket_request()) { /* * Ignore, since we don't really have buckets. @@ -60,9 +62,11 @@ void PutRequestHandler::onRequest( } catch (const FIPhoboServerException& ex) { std::cerr << "Caught exception: " << ex.what() << '\n'; + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(ex.what()) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) .sendWithEOM(); return; } @@ -76,9 +80,11 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept } catch (const FIPhoboServerException& ex) { std::cerr << "Caught an exception in put: " << ex.what() << '\n'; + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(ex.what()) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) .sendWithEOM(); return; } @@ -94,9 +100,11 @@ void PutRequestHandler::onEOM() noexcept } catch (const FIPhoboServerException& ex) { std::cerr << "Caught an exception in finish put: " << ex.what() << '\n'; + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); proxygen::ResponseBuilder(downstream_) - .status(409, "Conflict") - .body(ex.what()) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) .sendWithEOM(); return; } diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index d05aeed..1ddb4bf 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -9,7 +9,7 @@ #pragma once #include "../stream/stream.h" -#include "S3_header.h" +#include "s3_utilities/s3_header.h" #include #include @@ -63,7 +63,7 @@ class PutRequestHandler : public proxygen::RequestHandler { } private: - S3_header m_s3_header; + s3_utilities::S3_header m_s3_header; std::unique_ptr m_stream; }; diff --git a/src/server/s3_utilities/CMakeLists.txt b/src/server/s3_utilities/CMakeLists.txt new file mode 100644 index 0000000..b4dc9ce --- /dev/null +++ b/src/server/s3_utilities/CMakeLists.txt @@ -0,0 +1,6 @@ + +add_library( + s3_utilities + s3_utilities.cc +) + diff --git a/src/server/S3_header.h b/src/server/s3_utilities/s3_header.h similarity index 89% rename from src/server/S3_header.h rename to src/server/s3_utilities/s3_header.h index e1dfe79..19f4635 100644 --- a/src/server/S3_header.h +++ b/src/server/s3_utilities/s3_header.h @@ -10,6 +10,7 @@ #include namespace fiphoboserver { +namespace s3_utilities { /// /// @brief class to extract the S3 specific information from proxygens @@ -143,8 +144,23 @@ class S3_header { return metadata; } + /// + /// @brief print all headers to std out + /// + void print_all_headers() + { + std::cout << "Printing all headers for " << m_headers->getPath() + << "\n"; + m_headers->getHeaders().forEach( + [&](const std::string& header, const std::string& val) { + std::cout << header << ": " << val << "\n"; + }); + std::cout << std::flush; + } + private: std::unique_ptr m_headers; }; +} // namespace s3_utilities } // namespace fiphoboserver diff --git a/src/server/s3_utilities/s3_utilities.cc b/src/server/s3_utilities/s3_utilities.cc new file mode 100644 index 0000000..8e973b0 --- /dev/null +++ b/src/server/s3_utilities/s3_utilities.cc @@ -0,0 +1,54 @@ +#include "s3_utilities.h" + +#include + +namespace fiphoboserver { +namespace s3_utilities { + +s3_error_info create_s3_error(int return_value, std::string request) +{ + s3_error_info new_info; + + std::stringstream error_stream; + error_stream << "\n"; + error_stream << "\n\t"; + + if (return_value == -2) { + new_info.https_error_code = 404; + new_info.https_error_identifier = "Not found"; + error_stream << "NoSuchKey"; + error_stream << "\n\t"; + error_stream << "The resource you requested does not exist!"; + } + else if (return_value == -17) { + new_info.https_error_code = 400; + new_info.https_error_identifier = "Bad Request"; + error_stream << "InvalidArgument"; + error_stream << "\n\t"; + error_stream << "Some arguments given to the storage were invalid!"; + } + else if (return_value == -107) { + new_info.https_error_code = 500; + new_info.https_error_identifier = "Internal Server Error"; + error_stream << "InternalServerPhobosError"; + error_stream << "\n\t"; + error_stream + << "The Phobos database is not running correctly on the server"; + } + else { + new_info.https_error_code = 500; + new_info.https_error_identifier = "Internal Server Error"; + error_stream << "InternalServerError"; + error_stream << "\n\t"; + error_stream << "An error occurred in the underlying storage."; + } + + error_stream << "\n\t/" << request << "\n"; + error_stream << ""; + + new_info.error_xml = error_stream.str(); + return new_info; +} + +} // namespace s3_utilities +} // namespace fiphoboserver diff --git a/src/server/s3_utilities/s3_utilities.h b/src/server/s3_utilities/s3_utilities.h new file mode 100644 index 0000000..581a86a --- /dev/null +++ b/src/server/s3_utilities/s3_utilities.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +namespace fiphoboserver { +/// +/// @brief namespace for S3 specific utilities +/// +namespace s3_utilities { + +/// +/// @brief struct to contain all the information needed to send a proper s3 +/// error message +/// +struct s3_error_info { + /// @brief https error code (400s or 500s) + unsigned short https_error_code; + /// @brief readable string for the code @ref https_error_code + std::string https_error_identifier; + /// @brief XML encoded message containing more information on the error + std::string error_xml; +}; + +/// +/// @brief create a @ref s3_error_info depending on the error value +/// +/// @param return_value the error code (errno, etc) that the function returned +/// that raised the exception that preceded the call to +/// this function +/// @param request the request path that was being processed +/// +/// @return an @ref s3_error_info struct containing readable information on the +/// error +/// +s3_error_info create_s3_error(int return_value, std::string request); + +} // namespace s3_utilities +} // namespace fiphoboserver diff --git a/src/storage/phobos_exception.h b/src/storage/phobos_exception.h index 0199074..9912719 100644 --- a/src/storage/phobos_exception.h +++ b/src/storage/phobos_exception.h @@ -22,7 +22,8 @@ class PhobosException : public FIPhoboServerException { /// @param return_value the return value of the phobos function /// PhobosException( - const char* caller, const char* function_name, int return_value) + const char* caller, const char* function_name, int return_value) : + m_inner_value(return_value) { std::stringstream ss; ss << "[" << caller << "]: Phobos returned an error: " << function_name @@ -30,6 +31,15 @@ class PhobosException : public FIPhoboServerException { << ")"; m_message = ss.str(); } + + /// @copydoc FIPhoboServerException::get_inner_error + int get_inner_error() const noexcept override { return m_inner_value; } + + private: + /// + /// @brief the value of the error that caused this exception + /// + int m_inner_value = 0; }; } // namespace storage -- GitLab From 018c0f78e6d600c96660688f4d9eae3f19b8ba9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Wed, 8 Apr 2020 14:26:33 +0100 Subject: [PATCH 42/62] Improve build process and update README with build instructions Change log: * Dockerfile that inherits from an image with dependencies installed * describe build process in README * refer to superfiphoboserver in README * rename m_file_closed to m_stream_completed * formalise README --- README.md | 56 ++++++++++++++++++++++++++++--- src/server/get_request_handler.cc | 6 ++-- src/server/get_request_handler.h | 2 +- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a1ea366..1f8d82a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,22 @@ # FIPhoboServer +## Dependencies Installation Instructions + +FIPhoboServer has many dependencies that require much patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as; + +``` +FROM ciaranorourke/fiphoboserver:debian + +RUN apt-get -y update && apt-get -y upgrade +RUN apt-get -y install \ + [personal installation preferences] +CMD /home/superfiphoboserver/tools/build_phobos.sh -r && [favourite shell] +``` + +Alternatively, see [SuperFIPhoboServer](oilgas/ltfs/superfiphoboserver) for instructions on how to create your own build environment. + +It is recommended to run this Docker image by mounting your local branch of the FIPhoboServer project with `docker run -it -v /path/to/fiphoboserver:/destination [name]:[tag]` + To use the Phobos raid1 layout which is compatible with FIFOs, the following environment variables must be set; ``` @@ -7,16 +24,45 @@ export PHOBOS_STORE_layout=raid1 export PHOBOS_LAYOUT_RAID1_repl_count={1,2,3,...} ``` -Build with +## Build Instructions + +To configure and build FIPhoboServer, a C++14 compatible compiler is required, and CMake >= 3.0. ``` -cmake -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/ +# Make a directory to hold the temporary build files +mkdir -p build + +# Change directory to the build directory +cd build + +# Configure the FIPhoboServer build using CMake +cmake \ + -DCMAKE_PREFIX_PATH=/path/to/proxygen \ + [more options below] \ + /path/to/fiphoboserver/project +# Build the FIPhoboServer project make ``` -from any directory you want and hope for the best! ;) +### Useful Options + +#### Setting Compile/Link Flags +Option | Effect +------ | ------ +`-DCMAKE_CXX_COMPILER=...` | Set the C++ compiler. +`-DCMAKE_CXX_FLAGS=...` | Set the flags to pass to the C++ compiler. Overrides the default flags. + +#### Enabling / Disabling Sections of FIPhoboServer +Option | Effect +------ | ------ +`-DFIPHOBOSERVER_BUILD_TESTS=...` | Set to `ON` to build FIPhoboServer tests and enable the `make test` target, or `OFF` to skip (Default `OFF`). +`-DCUSTOM_DOCU_PATH=...` | Set to `doc/` to build FIPhoboServer documentation or don't set to skip (Default `not set`). + +## Testing -Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there +Units test can be run with `make -C build tests`, given `FIPHOBOSERVER_BUILD_TESTS` was set to `ON` during the build process. +Integration tests can be run from the `test/integration_tests/` directory with `python -m unittest client`. -The Doxygen documentation can be found [here](doc/Markdown/fiphoboserver) +## Documentation +The Doxygen documentation can be found [here](doc/Markdown/fiphoboserver). diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index 0640029..4dccb34 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -45,7 +45,7 @@ void GetRequestHandler::onRequest( */ m_stream->start_get(m_s3_header.get_key()); - m_file_closed = false; // TODO: Better way of communicating this + m_stream_completed = false; // TODO: Better way of communicating this } catch (const std::system_error& ex) { proxygen::ResponseBuilder(downstream_) @@ -81,7 +81,7 @@ void GetRequestHandler::onRequest( void GetRequestHandler::read_file(folly::EventBase* evb) { folly::IOBufQueue buf; - while (!m_file_closed && !m_paused) { + while (!m_stream_completed && !m_paused) { /* read 4k-ish chunks and foward each one to the client */ auto data = buf.preallocate(4000, 4000); /* @@ -100,7 +100,7 @@ void GetRequestHandler::read_file(folly::EventBase* evb) } else if (rc == 0) { // done - m_file_closed = true; + m_stream_completed = true; m_stream->finish_io(); VLOG(4) << "Read EOF"; evb->runInEventBaseThread([this] { diff --git a/src/server/get_request_handler.h b/src/server/get_request_handler.h index 4a55a42..9cf5cf6 100644 --- a/src/server/get_request_handler.h +++ b/src/server/get_request_handler.h @@ -103,7 +103,7 @@ class GetRequestHandler : public proxygen::RequestHandler { bool m_read_file_scheduled{false}; std::atomic m_paused{false}; bool m_finished{false}; - bool m_file_closed{false}; + bool m_stream_completed{false}; }; } // namespace fiphoboserver -- GitLab From 6b31b789ad2d16a6cc3c388e13a63b76ef5a90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Tue, 7 Apr 2020 16:28:58 +0100 Subject: [PATCH 43/62] Stress tests for the server request handlers Change log: * remove empty get file * add function for PUT, GETMD GET with random key and get_file_name * add stress test involving multiple parallel requests --- test/integration_tests/client.py | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/integration_tests/client.py b/test/integration_tests/client.py index f4b7790..98733c3 100644 --- a/test/integration_tests/client.py +++ b/test/integration_tests/client.py @@ -3,6 +3,7 @@ import unittest import subprocess import signal import os +from multiprocessing import Process import utils.s3_client as s3_client import utils.compare as compare @@ -23,6 +24,7 @@ class Server_test_case(unittest.TestCase): @classmethod def tearDownClass(cls): os.remove(cls.get_file_name) + os.remove(cls.get_empty_file_name) cls.server.terminate() # kill the fiphoboserver process @@ -98,3 +100,40 @@ class Server_test_case(unittest.TestCase): self.assertTrue( put_meta_data == get_meta_data ) + + def client_process(self): + key = random_string.random_string('key_', 12) + get_file_name = random_string.random_string('data/', 11) + + self.client.put( + self.put_file_name, + self.bucket_name, + key + ) + self.client.get( + get_file_name, + self.bucket_name, + key + ) + get_meta_data = self.client.get_md( + self.bucket_name, + key + ) + self.assertTrue( + compare.files_are_equal(self.put_file_name, get_file_name) + ) + + # clean up files + os.remove(get_file_name) + + return 0 + + def test_stress(self): + processes = [] + for i in range(10): + process = Process(target=self.client_process()) + processes.append(process) + process.start() + + for process in processes: + process.join() -- GitLab From 8a63ca21810516090cd55fe5c9ac1216776527f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Tue, 14 Apr 2020 15:14:31 +0100 Subject: [PATCH 44/62] Formatting for Python integration tests Commit log: * add yapf execution to formatting script * run formatting script * rename formatting and linting jobs * install yapf in Format job --- .gitlab-ci.yml | 5 +- test/integration_tests/client.py | 92 +++++-------------- test/integration_tests/utils/compare.py | 1 + test/integration_tests/utils/random_string.py | 9 +- test/integration_tests/utils/s3_client.py | 33 +++---- tools/run_format.sh | 4 + 6 files changed, 52 insertions(+), 92 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 54ddbc1..350f5b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,7 +53,7 @@ stages: - build - test -Clang Format: +Format: <<: *default_job stage: static analysis variables: @@ -64,13 +64,14 @@ Clang Format: - apt-get install -y git clang-format + - pip3 install yapf script: - ./tools/run_format.sh && git status && git diff-index --quiet HEAD || { echo "This commit contains unformatted files! Run tools/run_format.sh on the project to format them correctly."; false; } -Clang Tidy: +Lint: <<: *default_job stage: static analysis variables: diff --git a/test/integration_tests/client.py b/test/integration_tests/client.py index 98733c3..697d409 100644 --- a/test/integration_tests/client.py +++ b/test/integration_tests/client.py @@ -9,12 +9,15 @@ import utils.s3_client as s3_client import utils.compare as compare import utils.random_string as random_string + class Server_test_case(unittest.TestCase): @classmethod def setUpClass(cls): cls.bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0" cls.client = s3_client.S3_client('http://localhost:11000/') - cls.server = subprocess.Popen('../../build/fiphoboserver', shell=True, preexec_fn=os.setsid) + cls.server = subprocess.Popen('../../build/fiphoboserver', + shell=True, + preexec_fn=os.setsid) cls.put_file_name = 'data/EmperorWu.txt' cls.get_file_name = 'data/EmperorWu_get.txt' cls.put_empty_file_name = 'data/empty.txt' @@ -27,101 +30,56 @@ class Server_test_case(unittest.TestCase): os.remove(cls.get_empty_file_name) cls.server.terminate() - # kill the fiphoboserver process + # kill the fiphoboserver process os.killpg(os.getpgid(cls.server.pid), signal.SIGTERM) def test_file_put_and_get(self): - self.client.put( - self.put_file_name, - self.bucket_name, - self.key - ) - self.client.get( - self.get_file_name, - self.bucket_name, - self.key - ) + self.client.put(self.put_file_name, self.bucket_name, self.key) + self.client.get(self.get_file_name, self.bucket_name, self.key) self.assertTrue( - compare.files_are_equal(self.put_file_name, self.get_file_name) - ) + compare.files_are_equal(self.put_file_name, self.get_file_name)) + def test_empty_file_put_and_get(self): key = random_string.random_string('key_', 12) - self.client.put( - self.put_empty_file_name, - self.bucket_name, - key - ) - self.client.get( - self.get_empty_file_name, - self.bucket_name, - key - ) + self.client.put(self.put_empty_file_name, self.bucket_name, key) + self.client.get(self.get_empty_file_name, self.bucket_name, key) self.assertTrue( - compare.files_are_equal(self.put_empty_file_name, self.get_empty_file_name) - ) + compare.files_are_equal(self.put_empty_file_name, + self.get_empty_file_name)) @unittest.expectedFailure def test_put_with_used_key(self): self.assertTrue( - self.client.put( - self.put_file_name, - self.bucket_name, - self.key - ) - ) + self.client.put(self.put_file_name, self.bucket_name, self.key)) @unittest.expectedFailure def test_get_with_bad_key(self): self.assertTrue( - self.client.get( - self.get_file_name, - self.bucket_name, - 'bad_key' - ) - ) + self.client.get(self.get_file_name, self.bucket_name, 'bad_key')) def test_get_meta_data(self): key = random_string.random_string('key_', 12) put_meta_data = random_string.random_map(5, 8) - self.client.put( - self.put_file_name, - self.bucket_name, - key, - meta_data=put_meta_data - ) + self.client.put(self.put_file_name, + self.bucket_name, + key, + meta_data=put_meta_data) - get_meta_data = self.client.get_md( - self.bucket_name, - key - ) + get_meta_data = self.client.get_md(self.bucket_name, key) - self.assertTrue( - put_meta_data == get_meta_data - ) + self.assertTrue(put_meta_data == get_meta_data) def client_process(self): key = random_string.random_string('key_', 12) get_file_name = random_string.random_string('data/', 11) - self.client.put( - self.put_file_name, - self.bucket_name, - key - ) - self.client.get( - get_file_name, - self.bucket_name, - key - ) - get_meta_data = self.client.get_md( - self.bucket_name, - key - ) + self.client.put(self.put_file_name, self.bucket_name, key) + self.client.get(get_file_name, self.bucket_name, key) + get_meta_data = self.client.get_md(self.bucket_name, key) self.assertTrue( - compare.files_are_equal(self.put_file_name, get_file_name) - ) + compare.files_are_equal(self.put_file_name, get_file_name)) # clean up files os.remove(get_file_name) diff --git a/test/integration_tests/utils/compare.py b/test/integration_tests/utils/compare.py index 985109a..db920c9 100644 --- a/test/integration_tests/utils/compare.py +++ b/test/integration_tests/utils/compare.py @@ -1,5 +1,6 @@ import os + def files_are_equal(filename1, filename2): if not os.path.isfile(filename1): return False, "Error: File not found: " + filename1 diff --git a/test/integration_tests/utils/random_string.py b/test/integration_tests/utils/random_string.py index f4ba7c3..4f6cf39 100644 --- a/test/integration_tests/utils/random_string.py +++ b/test/integration_tests/utils/random_string.py @@ -1,8 +1,13 @@ import random import string -def random_string(prefix = '', length = 16): - return prefix + ''.join([random.choice(string.ascii_letters + string.digits) for i in range(length)]) + +def random_string(prefix='', length=16): + return prefix + ''.join([ + random.choice(string.ascii_letters + string.digits) + for i in range(length) + ]) + def random_map(size, length): dictionary = {random_string(): random_string()} diff --git a/test/integration_tests/utils/s3_client.py b/test/integration_tests/utils/s3_client.py index af36c4b..28b18f8 100644 --- a/test/integration_tests/utils/s3_client.py +++ b/test/integration_tests/utils/s3_client.py @@ -2,33 +2,24 @@ import boto3 import uuid import sys + class S3_client: def __init__(self, server_url): self.session = boto3.session.Session() - self.client = self.session.client( - service_name='s3', - endpoint_url=server_url - ) + self.client = self.session.client(service_name='s3', + endpoint_url=server_url) - def put(self, filename, bucket_name, key, meta_data = {}): - self.client.upload_file( - Filename = filename, - Bucket = bucket_name, - Key = key, - ExtraArgs={"Metadata": meta_data} - ) + def put(self, filename, bucket_name, key, meta_data={}): + self.client.upload_file(Filename=filename, + Bucket=bucket_name, + Key=key, + ExtraArgs={"Metadata": meta_data}) def get(self, filename, bucket_name, key): - self.client.download_file( - Filename = filename, - Bucket = bucket_name, - Key = key - ) + self.client.download_file(Filename=filename, + Bucket=bucket_name, + Key=key) def get_md(self, bucket_name, key): - response = self.client.head_object( - Bucket = bucket_name, - Key = key - ) + response = self.client.head_object(Bucket=bucket_name, Key=key) return response["Metadata"] - diff --git a/tools/run_format.sh b/tools/run_format.sh index cd01c94..4235e89 100755 --- a/tools/run_format.sh +++ b/tools/run_format.sh @@ -33,4 +33,8 @@ do | xargs -n 1 -0 -P "${nprocs}" ${FORMAT_EXECUTABLE} fi done + if [ ! -z "$(find $dir -iname "*.py")" ] + then + yapf -ir $dir + fi done -- GitLab From 18b7825f41a70ead5ee9ece96e7e7a426ada805f Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Wed, 15 Apr 2020 11:16:40 +0100 Subject: [PATCH 45/62] Add Amazon Authorisation The signature of amazon users is checked against a text file of known users and requests are accepted or rejected depending on the signature --- .clang-tidy | 1 - doc/Doxyfile.in | 2 +- .../s3_utilities/S3_authorisation.md | 641 ++++++++++++++++++ .../fiphoboserver/s3_utilities/S3_header.md | 108 ++- .../fiphoboserver/s3_utilities/index.md | 195 +++++- src/CMakeLists.txt | 7 + src/server/get_request_handler.cc | 10 + src/server/getmd_request_handler.cc | 11 + src/server/put_request_handler.cc | 65 +- src/server/put_request_handler.h | 4 + src/server/s3_utilities/CMakeLists.txt | 6 + src/server/s3_utilities/s3_authorisation.cc | 294 ++++++++ src/server/s3_utilities/s3_authorisation.h | 221 ++++++ src/server/s3_utilities/s3_header.h | 35 +- src/server/s3_utilities/s3_utilities.cc | 109 ++- src/server/s3_utilities/s3_utilities.h | 53 +- src/storage/phobos_file.cc | 5 + test/unit_tests/CMakeLists.txt | 1 + test/unit_tests/s3_utilities.cc | 334 +++++++++ test/users.txt | 4 + 20 files changed, 2066 insertions(+), 40 deletions(-) create mode 100644 doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md create mode 100644 src/server/s3_utilities/s3_authorisation.cc create mode 100644 src/server/s3_utilities/s3_authorisation.h create mode 100644 test/unit_tests/s3_utilities.cc create mode 100644 test/users.txt diff --git a/.clang-tidy b/.clang-tidy index 298375f..6de8aca 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -6,7 +6,6 @@ Checks: clang-diagnostic-*, -clang-analyzer-security.insecureAPI.rand, mpi-*, readability-braces-around-statements, - readability-avoid-const-params-in-decls, readability-redundant-string-init, readability-container-size-empty, readability-implicit-bool-conversion, diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index a1dd2bc..ee70717 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -7,7 +7,7 @@ PROJECT_NAME = FiPhoboServer # Extraction options -EXTRACT_PRIVATE = NO +EXTRACT_PRIVATE = YES EXTRACT_PACKAGE = YES EXTRACT_STATIC = YES EXTRACT_LOCAL_METHODS = YES diff --git a/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md b/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md new file mode 100644 index 0000000..9f5e21a --- /dev/null +++ b/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md @@ -0,0 +1,641 @@ +# public fiphoboserver::s3_utilities::S3_authorisation + +class to perform an authorisation of an S3 request. + + + +The implementation of this class follows the guidelines from [https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) + + + + + + +## Helper functions +| Name | Description | +| ---- | ---- | +| [search_for_user](#fiphoboserver-s3_utilities-S3_authorisation-search_for_user) | checks whether the user id saved in [m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier] exists in the user database | +| [is_chunked](#fiphoboserver-s3_utilities-S3_authorisation-is_chunked) | check if this message has a content body | +| [print_info](#fiphoboserver-s3_utilities-S3_authorisation-print_info) | print current state of member variables | + + +## Extract header information +| Name | Description | +| ---- | ---- | +| [set_authorisation_info](#fiphoboserver-s3_utilities-S3_authorisation-set_authorisation_info) | get the information about authorisation from the header and save it to member variables | +| [extract_string_part](#fiphoboserver-s3_utilities-S3_authorisation-extract_string_part) | Extract information from a string. | +| [split_signed_headers](#fiphoboserver-s3_utilities-S3_authorisation-split_signed_headers) | split up the "SignedHeaders" value of the HTTP request | +| [split_credentials](#fiphoboserver-s3_utilities-S3_authorisation-split_credentials) | split up the "Credential" value of the HTTP request | + + +## Signature creation +| Name | Description | +| ---- | ---- | +| [check](#fiphoboserver-s3_utilities-S3_authorisation-check) | checks the signature of the message | +| [create_canonical_request](#fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request) | create a canonical request | +| [create_string_to_sign](#fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign) | create a string to sign | +| [get_signature](#fiphoboserver-s3_utilities-S3_authorisation-get_signature) | create the signature | + + +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_status](#fiphoboserver-s3_utilities-S3_authorisation-m_status) | the current status of the authorisation | +| [m_user_identifier](#fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier) | the public key / identifier of the user that send the request | +| [m_user_key](#fiphoboserver-s3_utilities-S3_authorisation-m_user_key) | the key that belongs to [m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier] | +| [m_date](#fiphoboserver-s3_utilities-S3_authorisation-m_date) | the date as saved in the credentials of the request | +| [m_region](#fiphoboserver-s3_utilities-S3_authorisation-m_region) | the S3 region as saved in the credentials of the request | +| [m_signed_headers](#fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers) | a list of the headers that have been used to sign the request | +| [m_signature](#fiphoboserver-s3_utilities-S3_authorisation-m_signature) | the signature stored in the request | +| [m_payload](#fiphoboserver-s3_utilities-S3_authorisation-m_payload) | the payload of the request | + + +## Public Functions +| Name | Description | +| ---- | ---- | +| [authorise](#fiphoboserver-s3_utilities-S3_authorisation-authorise) | main method to run the authorisation algorithm | +| [add_chunk](#fiphoboserver-s3_utilities-S3_authorisation-add_chunk) | add a chunk of data to the payload | +| [is_valid](#fiphoboserver-s3_utilities-S3_authorisation-is_valid) | checks if the authorisation was successful | + + + +## Helper functions +### private bool fiphoboserver::s3_utilities::S3_authorisation::search_for_user () + +checks whether the user id saved in [m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier] exists in the user database + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if the user has been found, false otherwise | + + + + + + + + + +Sets +[m_user_key][fiphoboserver-s3_utilities-S3_authorisation-m_user_key] on success + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private bool fiphoboserver::s3_utilities::S3_authorisation::is_chunked (const S3_header &headers) const + +check if this message has a content body + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | the headers of the message to check | + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if the message contains a body, false otherwise | + + + + + + + + + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private void fiphoboserver::s3_utilities::S3_authorisation::print_info () const + +print current state of member variables + + + + + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +## Extract header information +### private bool fiphoboserver::s3_utilities::S3_authorisation::set_authorisation_info (const S3_header &headers) + +get the information about authorisation from the header and save it to member variables + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | the header to extract the information from | + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if all information was found, false if an error occurred | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private std::string fiphoboserver::s3_utilities::S3_authorisation::extract_string_part (std::string complete, std::string identifier, std::string delimiter) const + +Extract information from a string. + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | complete | the complete string will all information | +| std::string | identifier | the identifier of the specific piece of information required | +| std::string | delimiter | the delimiter between the parts of information | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | required information as string | + + + + + + + + + + + +If there is for example a string of the form "id=info;id2=info2" then ';' is the delimiter, 'id' is the identifier and this function will return "info". + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private void fiphoboserver::s3_utilities::S3_authorisation::split_signed_headers (std::string all_signed_headers) + +split up the "SignedHeaders" value of the HTTP request + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | all_signed_headers | the value of the SignedHeaders header | + + + + + + + + +This function saves the headers into +[m_signed_headers][fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers] + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private void fiphoboserver::s3_utilities::S3_authorisation::split_credentials (std::string credentials) + +split up the "Credential" value of the HTTP request + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | credentials | the value of the Credential header | + + + + + + + + +This function saves the credentials into +[m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier], +[m_date][fiphoboserver-s3_utilities-S3_authorisation-m_date] and +[m_region][fiphoboserver-s3_utilities-S3_authorisation-m_region] + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +## Signature creation +### private bool fiphoboserver::s3_utilities::S3_authorisation::check (const S3_header &headers) const + +checks the signature of the message + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | the headers of the message to check | + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if the signature was valid, false otherwise | + + + + + + + + + + + +This function splits the message's contents into the needed parts, creates a signature and compares it with the one saved in the headers + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private std::string fiphoboserver::s3_utilities::S3_authorisation::create_canonical_request (const S3_header &headers) const + +create a canonical request + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | the headers of the message to create the request from | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the canonical request | + + + + + + + + + + + +For more information what this means, see: +[https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html) + + + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private std::string fiphoboserver::s3_utilities::S3_authorisation::create_string_to_sign (const S3_header &headers, std::string canonical_request) const + +create a string to sign + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | the headers of the message to create the request from | +| std::string | canonical_request | the canonical request from [create_canonical_request][fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request] | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the string to sign | + + + + + + + + + + + +For more information what this means, see: +[https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html) + + + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private std::string fiphoboserver::s3_utilities::S3_authorisation::get_signature (std::string string_to_sign) const + +create the signature + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | string_to_sign | the string to sign from [create_string_to_sign][fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign] | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the signature | + + + + + + + + + + + +For more information what this means, see: +[https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html) + + + + + + +#### Qualifiers: +* const + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +## Private Attributes +### private fiphoboserver::s3_utilities::S3_authorisation::m_status = + +the current status of the authorisation + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_user_identifier + +the public key / identifier of the user that send the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_user_key + +the key that belongs to [m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier] + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_date + +the date as saved in the credentials of the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_region + +the S3 region as saved in the credentials of the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_signed_headers + +a list of the headers that have been used to sign the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_signature + +the signature stored in the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_payload = "" + +the payload of the request + + + + + + + + + + +This is the whole body since we do not support multiple chunked signatures + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +## Public Functions +### public [Authorisation_status][fiphoboserver-s3_utilities-Authorisation_status] fiphoboserver::s3_utilities::S3_authorisation::authorise (const S3_header &headers) + +main method to run the authorisation algorithm + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const [S3_header][fiphoboserver-s3_utilities-S3_header] & | headers | of the message to authenticate | + +#### Returns: +| Type | Description | +| ---- | ---- | +| [Authorisation_status][fiphoboserver-s3_utilities-Authorisation_status] | status of the authorisation | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### public void fiphoboserver::s3_utilities::S3_authorisation::add_chunk (std::string chunk) + +add a chunk of data to the payload + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | chunk | the string to add | + + + + + + + + +Since the whole body has to be hashed in order to verify the signature of the message each chunk of data has to be added here in order. + +Otherwise the authentication will fail! + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### public bool fiphoboserver::s3_utilities::S3_authorisation::is_valid () const + +checks if the authorisation was successful + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| bool | true if it has been valid, false if it hasn't started, it is in progress, or if an error occurred. | + + + + + + + + + + + + +#### Qualifiers: +* const +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +[fiphoboserver-s3_utilities-Authorisation_status]:./index.md#fiphoboserver-s3_utilities-Authorisation_status +[fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request +[fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign +[fiphoboserver-s3_utilities-S3_authorisation-m_date]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_date +[fiphoboserver-s3_utilities-S3_authorisation-m_region]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_region +[fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers +[fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier +[fiphoboserver-s3_utilities-S3_authorisation-m_user_key]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_user_key +[fiphoboserver-s3_utilities-S3_header]:./S3_header.md diff --git a/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md index d9ef525..ad9ff88 100644 --- a/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md +++ b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md @@ -5,6 +5,12 @@ class to extract the S3 specific information from proxygens HTTPMessage headers +## Private Attributes +| Name | Description | +| ---- | ---- | +| [m_headers](#fiphoboserver-s3_utilities-S3_header-m_headers) | inner pointer to the proxygen header object | + + ## Public Functions | Name | Description | | ---- | ---- | @@ -16,9 +22,25 @@ class to extract the S3 specific information from proxygens HTTPMessage headers | [get_body_length](#fiphoboserver-s3_utilities-S3_header-get_body_length) | gets the total length of all body chunks that is expected | | [get_meta_data](#fiphoboserver-s3_utilities-S3_header-get_meta_data) | gets the S3 metadata stored in the HTTP headers | | [print_all_headers](#fiphoboserver-s3_utilities-S3_header-print_all_headers) | print all headers to std out | +| [get_header_by_name](#fiphoboserver-s3_utilities-S3_header-get_header_by_name) | extract one header value from the headers | +| [get_method](#fiphoboserver-s3_utilities-S3_header-get_method) | get the HTTP method the header belongs to | + + + +## Private Attributes +### private fiphoboserver::s3_utilities::S3_header::m_headers + +inner pointer to the proxygen header object + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + ## Public Functions ### public void fiphoboserver::s3_utilities::S3_header::set_headers (std::unique_ptr< proxygen::HTTPMessage > new_m_headers) @@ -49,7 +71,7 @@ sets the internal header instance to the given header object [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public std::string fiphoboserver::s3_utilities::S3_header::get_bucket () +### public std::string fiphoboserver::s3_utilities::S3_header::get_bucket () const extracts the name of the S3 bucket requested in the HTTP message @@ -73,12 +95,13 @@ extracts the name of the S3 bucket requested in the HTTP message #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public std::string fiphoboserver::s3_utilities::S3_header::get_key_without_bucket () +### public std::string fiphoboserver::s3_utilities::S3_header::get_key_without_bucket () const extracts the name of the key id requested in the HTTP message @@ -102,12 +125,13 @@ This will only return the key itself without the leading bucket name! #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public std::string fiphoboserver::s3_utilities::S3_header::get_key () +### public std::string fiphoboserver::s3_utilities::S3_header::get_key () const extracts the name of the key id requested in the HTTP message @@ -131,12 +155,13 @@ This is the whole path argument without the leading '/' but including the bucket #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public bool fiphoboserver::s3_utilities::S3_header::is_create_bucket_request () +### public bool fiphoboserver::s3_utilities::S3_header::is_create_bucket_request () const checks if the message we got is one for just creating a bucket @@ -160,12 +185,13 @@ This is convenient since we don't really support buckets themselves and can easi #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public size_t fiphoboserver::s3_utilities::S3_header::get_body_length () +### public size_t fiphoboserver::s3_utilities::S3_header::get_body_length () const gets the total length of all body chunks that is expected @@ -189,12 +215,13 @@ This reads out the header "Content-Length" #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public std::map< std::string, std::string > fiphoboserver::s3_utilities::S3_header::get_meta_data () +### public std::map< std::string, std::string > fiphoboserver::s3_utilities::S3_header::get_meta_data () const gets the S3 metadata stored in the HTTP headers @@ -218,12 +245,13 @@ In S3 arbitrary metadata can be defined. This has the form x-amx-meta-KEY: VALUE #### Qualifiers: +* const * inline [Go to Top](#fiphoboserver-s3_utilities-S3_header) -### public void fiphoboserver::s3_utilities::S3_header::print_all_headers () +### public void fiphoboserver::s3_utilities::S3_header::print_all_headers () const print all headers to std out @@ -235,6 +263,72 @@ print all headers to std out #### Qualifiers: +* const +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_header_by_name (const std::string header_name) const + +extract one header value from the headers + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const std::string | header_name | the name of the header that should be extracted | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the value of the requested header or an empty_string if it is not found | + + + + + + + + + + + + +#### Qualifiers: +* const +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_method () const + +get the HTTP method the header belongs to + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the method as a string | + + + + + + + + + + + + +#### Qualifiers: +* const * inline diff --git a/doc/Markdown/fiphoboserver/s3_utilities/index.md b/doc/Markdown/fiphoboserver/s3_utilities/index.md index 0e86f87..960e603 100644 --- a/doc/Markdown/fiphoboserver/s3_utilities/index.md +++ b/doc/Markdown/fiphoboserver/s3_utilities/index.md @@ -8,19 +8,53 @@ namespace for S3 specific utilities ## Classes | Name | Description | | ---- | ---- | +| [S3_authorisation](./S3_authorisation.md) | class to perform an authorisation of an S3 request. | | [s3_error_info](./s3_error_info.md) | struct to contain all the information needed to send a proper s3 error message | | [S3_header](./S3_header.md) | class to extract the S3 specific information from proxygens HTTPMessage headers | +## Enumerations +| Name | Description | +| ---- | ---- | +| [Authorisation_status](#fiphoboserver-s3_utilities-Authorisation_status) | enum to store all the different states a [fiphoboserver::s3_utilities::S3_authorisation][fiphoboserver-s3_utilities-S3_authorisation] object can be in | + + ## Functions | Name | Description | | ---- | ---- | | [create_s3_error](#fiphoboserver-s3_utilities-create_s3_error) | create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on the error value | +| [uri_encode](#fiphoboserver-s3_utilities-uri_encode) | create a uri encoding of a string | +| [openssl_sha256](#fiphoboserver-s3_utilities-openssl_sha256) | create a sha256 hash from the data | +| [openssl_sha256](#fiphoboserver-s3_utilities-openssl_sha256-1) | shortcut for openssl_sha256(void*, unsigned int) | +| [openssl_hmac](#fiphoboserver-s3_utilities-openssl_hmac) | create a key hashed message | +| [openssl_hmac_hex](#fiphoboserver-s3_utilities-openssl_hmac_hex) | create a key hashed message | + + + +## Enumerations +### public fiphoboserver::s3_utilities::Authorisation_status + +enum to store all the different states a [fiphoboserver::s3_utilities::S3_authorisation][fiphoboserver-s3_utilities-S3_authorisation] object can be in + + + +#### Enum Values: +| Name | Description | Value | +| ---- | ---- | ---- | +| undefined | authorisation has not yet started | | +| waiting_for_payload | members have been set up and payload can be added | | +| failed | the authorisation was invalid or some error occurred | | +| valid | authorisation successful | | + + + +[Go to Top](#fiphoboserver-s3_utilities) + ## Functions -### public [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] fiphoboserver::s3_utilities::create_s3_error (int return_value, std::string request) +### public [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] fiphoboserver::s3_utilities::create_s3_error (const int return_value, const std::string request) create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on the error value @@ -30,8 +64,8 @@ create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on #### Parameters: | Type | Name | Description | | ---- | ---- | ---- | -| int | return_value | the error code (errno, etc) that the function returned that raised the exception that preceded the call to this function | -| std::string | request | the request path that was being processed | +| const int | return_value | the error code (errno, etc) that the function returned that raised the exception that preceded the call to this function | +| const std::string | request | the request path that was being processed | #### Returns: | Type | Description | @@ -51,4 +85,159 @@ create a [s3_error_info][fiphoboserver-s3_utilities-s3_error_info] depending on [Go to Top](#fiphoboserver-s3_utilities) +### public std::string fiphoboserver::s3_utilities::uri_encode (const std::string string_to_encode, const bool encode_slash) + +create a uri encoding of a string + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const std::string | string_to_encode | the string that should be encoded | +| const bool | encode_slash | bool to signalise if the / should be encoded | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the URI encoded string | + + + + + + + +for more information see [https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +### public std::string fiphoboserver::s3_utilities::openssl_sha256 (const void *data, const unsigned int length) + +create a sha256 hash from the data + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const void * | data | buffer to the data to hash | +| const unsigned int | length | number of byte in the buffer | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the resulting hash as string or "" in case of error | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +### public std::string fiphoboserver::s3_utilities::openssl_sha256 (const std::string msg) + +shortcut for openssl_sha256(void*, unsigned int) + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const std::string | msg | the string to hash | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the resulting hash as string or "" in case of error | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +### public std::string fiphoboserver::s3_utilities::openssl_hmac (const std::string key, const std::string msg) + +create a key hashed message + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const std::string | key | the key to sign the message with | +| const std::string | msg | the message to sign | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | a string representation of the signed message (unreadable) | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +### public std::string fiphoboserver::s3_utilities::openssl_hmac_hex (const std::string key, const std::string msg) + +create a key hashed message + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| const std::string | key | the key to sign the message with | +| const std::string | msg | the message to sign | + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the signed message in HEX format | + + + + + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities) + +[fiphoboserver-s3_utilities-S3_authorisation]:./S3_authorisation.md [fiphoboserver-s3_utilities-s3_error_info]:./s3_error_info.md#fiphoboserver-s3_utilities-s3_error_info diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 438978c..5b9e746 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,3 +20,10 @@ target_compile_features(fiphoboserver PUBLIC cxx_std_14) target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen) target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver) target_link_libraries(fiphoboserver PUBLIC server) + +add_custom_command( + TARGET fiphoboserver POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/../test/users.txt + /tmp/users.txt +) \ No newline at end of file diff --git a/src/server/get_request_handler.cc b/src/server/get_request_handler.cc index 4dccb34..450903d 100644 --- a/src/server/get_request_handler.cc +++ b/src/server/get_request_handler.cc @@ -8,6 +8,7 @@ #include "get_request_handler.h" #include "../fiphoboserver_exception.h" +#include "s3_utilities/s3_authorisation.h" #include "s3_utilities/s3_utilities.h" #include @@ -31,6 +32,15 @@ void GetRequestHandler::onRequest( /* Creating Bucket queries */ m_s3_header.set_headers(std::move(headers)); // m_s3_header.print_all_headers(); + + s3_utilities::S3_authorisation auth; + if (auth.authorise(m_s3_header) + != s3_utilities::Authorisation_status::valid) { + proxygen::ResponseBuilder(downstream_) + .status(403, "Access Denied") + .sendWithEOM(); + return; + } try { /* Send meta data to backend through the stream */ diff --git a/src/server/getmd_request_handler.cc b/src/server/getmd_request_handler.cc index 24b33c9..c93bb3d 100644 --- a/src/server/getmd_request_handler.cc +++ b/src/server/getmd_request_handler.cc @@ -8,6 +8,7 @@ #include "getmd_request_handler.h" #include "../fiphoboserver_exception.h" +#include "s3_utilities/s3_authorisation.h" #include "s3_utilities/s3_utilities.h" #include @@ -28,6 +29,16 @@ void GetmdRequestHandler::onRequest( /* Creating Bucket queries */ m_s3_header.set_headers(std::move(headers)); + // m_s3_header.print_all_headers(); + + s3_utilities::S3_authorisation auth; + if (auth.authorise(m_s3_header) + != s3_utilities::Authorisation_status::valid) { + proxygen::ResponseBuilder(downstream_) + .status(403, "Access Denied") + .sendWithEOM(); + return; + } try { /* Get meta data from backend through the stream */ diff --git a/src/server/put_request_handler.cc b/src/server/put_request_handler.cc index f7ddbd9..0bd2d62 100644 --- a/src/server/put_request_handler.cc +++ b/src/server/put_request_handler.cc @@ -28,6 +28,16 @@ void PutRequestHandler::onRequest( /* Creating Bucket queries */ m_s3_header.set_headers(std::move(headers)); // m_s3_header.print_all_headers(); + + if (m_auth.authorise(m_s3_header) + != s3_utilities::Authorisation_status::waiting_for_payload) { + proxygen::ResponseBuilder(downstream_) + .status(403, "Access Denied") + .sendWithEOM(); + m_stop_processing = true; + return; + } + if (m_s3_header.is_create_bucket_request()) { /* * Ignore, since we don't really have buckets. @@ -35,6 +45,7 @@ void PutRequestHandler::onRequest( * requested bucket doesn't exist? */ proxygen::ResponseBuilder(downstream_).status(200, "Ok").sendWithEOM(); + m_stop_processing = true; return; } try { @@ -51,15 +62,6 @@ void PutRequestHandler::onRequest( m_stream->start_put( m_s3_header.get_body_length(), m_s3_header.get_key()); } - catch (const std::system_error& ex) { - proxygen::ResponseBuilder(downstream_) - .status(404, "Not Found") - .body(folly::to( - "Could not find ", headers->getPathAsStringPiece(), - " ex=", folly::exceptionStr(ex))) - .sendWithEOM(); - return; - } catch (const FIPhoboServerException& ex) { std::cerr << "Caught exception: " << ex.what() << '\n'; s3_utilities::s3_error_info error = s3_utilities::create_s3_error( @@ -68,13 +70,23 @@ void PutRequestHandler::onRequest( .status(error.https_error_code, error.https_error_identifier) .body(error.error_xml) .sendWithEOM(); + m_stop_processing = true; return; } } void PutRequestHandler::onBody(std::unique_ptr body) noexcept { + if (m_stop_processing) { + return; + } + try { + // Add body as string to authentication object as payload + std::string body_as_string = std::string( + reinterpret_cast(body->data()), body->length()); + m_auth.add_chunk(body_as_string); + /* Hand message body over to stream for PUT operation */ m_stream->put(std::move(body->data()), std::move(body->length())); } @@ -86,6 +98,7 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept .status(error.https_error_code, error.https_error_identifier) .body(error.error_xml) .sendWithEOM(); + m_stop_processing = true; return; } @@ -94,20 +107,36 @@ void PutRequestHandler::onBody(std::unique_ptr body) noexcept void PutRequestHandler::onEOM() noexcept { - try { - /* Tell stream it's time to clean up */ - m_stream->finish_io(); + if (m_stop_processing) { + return; } - catch (const FIPhoboServerException& ex) { - std::cerr << "Caught an exception in finish put: " << ex.what() << '\n'; - s3_utilities::s3_error_info error = s3_utilities::create_s3_error( - ex.get_inner_error(), m_s3_header.get_key()); + + if (m_auth.authorise(m_s3_header) + != s3_utilities::Authorisation_status::valid) { + // TODO: Delete object from phobos or revert put! (Issue #45) proxygen::ResponseBuilder(downstream_) - .status(error.https_error_code, error.https_error_identifier) - .body(error.error_xml) + .status(403, "Access Denied") .sendWithEOM(); + return; } + else { + try { + /* Tell stream it's time to clean up */ + m_stream->finish_io(); + } + catch (const FIPhoboServerException& ex) { + std::cerr << "Caught an exception in finish put: " << ex.what() + << '\n'; + s3_utilities::s3_error_info error = s3_utilities::create_s3_error( + ex.get_inner_error(), m_s3_header.get_key()); + proxygen::ResponseBuilder(downstream_) + .status(error.https_error_code, error.https_error_identifier) + .body(error.error_xml) + .sendWithEOM(); + return; + } + } proxygen::ResponseBuilder(downstream_).status(200, "OK").sendWithEOM(); } diff --git a/src/server/put_request_handler.h b/src/server/put_request_handler.h index 1ddb4bf..f410e54 100644 --- a/src/server/put_request_handler.h +++ b/src/server/put_request_handler.h @@ -9,6 +9,7 @@ #pragma once #include "../stream/stream.h" +#include "s3_utilities/s3_authorisation.h" #include "s3_utilities/s3_header.h" #include @@ -65,6 +66,9 @@ class PutRequestHandler : public proxygen::RequestHandler { private: s3_utilities::S3_header m_s3_header; std::unique_ptr m_stream; + s3_utilities::S3_authorisation m_auth; + + bool m_stop_processing = false; }; } // namespace fiphoboserver diff --git a/src/server/s3_utilities/CMakeLists.txt b/src/server/s3_utilities/CMakeLists.txt index b4dc9ce..7cd127f 100644 --- a/src/server/s3_utilities/CMakeLists.txt +++ b/src/server/s3_utilities/CMakeLists.txt @@ -1,6 +1,12 @@ +find_package(OpenSSL REQUIRED) + add_library( s3_utilities s3_utilities.cc + s3_authorisation.cc ) +target_link_libraries(s3_utilities PUBLIC proxygen::proxygen) +target_link_libraries(s3_utilities PUBLIC OpenSSL::Crypto) + diff --git a/src/server/s3_utilities/s3_authorisation.cc b/src/server/s3_utilities/s3_authorisation.cc new file mode 100644 index 0000000..8f0bb60 --- /dev/null +++ b/src/server/s3_utilities/s3_authorisation.cc @@ -0,0 +1,294 @@ + +#include "s3_authorisation.h" + +#include "s3_utilities.h" + +#include +#include +#include +#include +#include + +namespace fiphoboserver { +namespace s3_utilities { + +Authorisation_status S3_authorisation::authorise(const S3_header& headers) +{ + if (m_status + == Authorisation_status::undefined) // start authorization procedure: + { + if (!set_authorisation_info(headers) || !search_for_user()) { + m_status = Authorisation_status::failed; + return m_status; + } + + if (is_chunked(headers)) { + m_status = Authorisation_status::waiting_for_payload; + } + else { + if (check(headers)) { + m_status = Authorisation_status::valid; + } + else { + m_status = Authorisation_status::failed; + } + } + } + else if (m_status == Authorisation_status::waiting_for_payload) { + if (check(headers)) { + m_status = Authorisation_status::valid; + } + else { + m_status = Authorisation_status::failed; + } + } + + return m_status; +} + +void S3_authorisation::add_chunk(const std::string chunk) +{ + m_payload += chunk; +} + + +/* + * *************************************************************************** + * ** Split information of headers into member variables + * *************************************************************************** + */ + +bool S3_authorisation::set_authorisation_info(const S3_header& headers) +{ + std::string full_authorisation = + headers.get_header_by_name("Authorization"); + + if (full_authorisation.rfind("AWS4-HMAC-SHA256", 0) != 0) { + std::cerr << "Could not find AWS4-HMAC-SHA256 at the beginning of "; + std::cerr << full_authorisation << std::endl; + return false; + } + + std::string credentials = + extract_string_part(full_authorisation, "Credential", ","); + std::string all_signed_headers = + extract_string_part(full_authorisation, "SignedHeaders", ","); + m_signature = extract_string_part(full_authorisation, "Signature", ""); + + if (credentials.empty() || all_signed_headers.empty() + || m_signature.empty()) { + return false; + } + + split_signed_headers(all_signed_headers); + split_credentials(credentials); + + // print_info(); + + return true; +} + +std::string S3_authorisation::extract_string_part( + std::string complete, std::string identifier, std::string delimiter) const +{ + size_t index = complete.find(identifier); + if (index == complete.npos) { + return ""; + } + + index = complete.find('=', index) + 1; + size_t delimiter_index = complete.npos; + if (!delimiter.empty()) { + delimiter_index = complete.find(delimiter, index); + } + + return complete.substr(index, delimiter_index - index); +} + +void S3_authorisation::split_signed_headers(std::string all_signed_headers) +{ + size_t index = 0; + while (index != all_signed_headers.npos) { + index = all_signed_headers.find(';'); + m_signed_headers.push_back(all_signed_headers.substr(0, index)); + all_signed_headers = all_signed_headers.substr(index + 1); + } +} + +void S3_authorisation::split_credentials(std::string credentials) +{ + m_user_identifier = credentials.substr(0, credentials.find('/')); + credentials = credentials.substr(credentials.find('/') + 1); + m_date = credentials.substr(0, credentials.find('/')); + credentials = credentials.substr(credentials.find('/') + 1); + m_region = credentials.substr(0, credentials.find('/')); + + // Note: The rest of the credentials must in our case always be + // /s3/aws4_request +} + +/* + * *************************************************************************** + * ** Helper functions + * *************************************************************************** + */ + +bool S3_authorisation::search_for_user() +{ + // TODO: Proper database: Issue #41 + std::ifstream user_database("/tmp/users.txt"); + if (!user_database.good()) { + return false; + } + std::string user; + while (user_database >> user) { + int colon = user.find(':'); + if (colon == std::string::npos) { + return false; // Database corrupted + } + std::string user_id = user.substr(0, colon); + + if (user_id == m_user_identifier) { + m_user_key = user.substr(colon + 1, user.size()); + return true; + } + } + return false; +} + +bool S3_authorisation::is_chunked(const S3_header& headers) const +{ + std::string content_md5 = headers.get_header_by_name("content-md5"); + std::string content_length = headers.get_header_by_name("Content-Length"); + + if (!content_md5.empty() || !content_length.empty()) { + return true; + } + + return false; +} + +void S3_authorisation::print_info() const +{ + std::cout << "Authorization object:\n status = "; + if (m_status == Authorisation_status::failed) { + std::cout << "failed\n"; + } + else if (m_status == Authorisation_status::undefined) { + std::cout << "undefined\n"; + } + else if (m_status == Authorisation_status::valid) { + std::cout << "valid\n"; + } + else if (m_status == Authorisation_status::waiting_for_payload) { + std::cout << "waiting_for_payload\n"; + } + + std::cout << "user_identifier = " << m_user_identifier << '\n'; + std::cout << "user_key = " << m_user_key << '\n'; + std::cout << "date = " << m_date << '\n'; + std::cout << "region = " << m_region << '\n'; + + std::cout << "signed_headers = "; + std::for_each( + m_signed_headers.begin(), m_signed_headers.end(), + [&](std::string header) { std::cout << header << ';'; }); + std::cout << '\n'; + std::cout << "signature = " << m_signature << '\n'; + std::cout << "payload = " << m_payload << '\n'; + + std::cout << std::flush; +} + +/* + * *************************************************************************** + * ** Create signature and compare with header + * *************************************************************************** + */ + +bool S3_authorisation::check(const S3_header& headers) const +{ + std::string request = create_canonical_request(headers); + std::string string_to_sign = create_string_to_sign(headers, request); + std::string signature = get_signature(string_to_sign); + + // std::cout << "request: " << request << std::endl; + // std::cout << "string_to_sign: " << string_to_sign << std::endl; + // std::cout << "Calculated signature: " << signature << std::endl; + + return m_signature == signature; +} + +std::string S3_authorisation::create_canonical_request( + const S3_header& headers) const +{ + std::stringstream result_stream; + result_stream << headers.get_method() << '\n'; + result_stream << '/' << uri_encode(headers.get_key(), false) << '\n'; + + // TODO insert queries! Issue #46 + result_stream << '\n'; + + for (auto it = m_signed_headers.begin(); it != m_signed_headers.end(); + ++it) { + std::string header_value = headers.get_header_by_name(*it); + while (isspace(header_value[0]) != 0) { + header_value.erase(0, 1); + } + while (isspace(header_value[header_value.size() - 1]) != 0) { + header_value.erase(header_value.size() - 2, 1); + } + + result_stream << *it << ':' << header_value << '\n'; + } + result_stream << '\n'; + + if (!m_signed_headers.empty()) { + for (size_t i = 0; i < m_signed_headers.size() - 1; ++i) { + result_stream << m_signed_headers[i] << ';'; + } + result_stream << m_signed_headers[m_signed_headers.size() - 1]; + } + result_stream << '\n'; + + result_stream << openssl_sha256(m_payload); + + return result_stream.str(); +} + +std::string S3_authorisation::create_string_to_sign( + const S3_header& headers, const std::string canonical_request) const +{ + std::stringstream result_stream; + result_stream << "AWS4-HMAC-SHA256\n"; + std::string timestamp = headers.get_header_by_name("x-amz-date"); + if (timestamp.empty()) { + // I don't have to check this return value anywhere because it will + // let the signature fail in the end + return ""; + } + result_stream << timestamp << '\n'; + result_stream << m_date << '/' << m_region << "/s3/aws4_request\n"; + result_stream << openssl_sha256(canonical_request); + + return result_stream.str(); +} + +std::string S3_authorisation::get_signature( + const std::string string_to_sign) const +{ + std::string key = "AWS4" + m_user_key; + std::string service = "s3"; + std::string request = "aws4_request"; + + std::string date_key = openssl_hmac(key, m_date); + std::string date_region_key = openssl_hmac(date_key, m_region); + std::string date_region_service_key = + openssl_hmac(date_region_key, service); + std::string signing_key = openssl_hmac(date_region_service_key, request); + + return openssl_hmac_hex(signing_key, string_to_sign); +} + +} // namespace s3_utilities +} // namespace fiphoboserver diff --git a/src/server/s3_utilities/s3_authorisation.h b/src/server/s3_utilities/s3_authorisation.h new file mode 100644 index 0000000..1a3d8e0 --- /dev/null +++ b/src/server/s3_utilities/s3_authorisation.h @@ -0,0 +1,221 @@ +#pragma once + +#include "s3_header.h" + +#include +#include + +namespace fiphoboserver { +namespace s3_utilities { + +/// +/// @brief enum to store all the different states a +/// @ref fiphoboserver::s3_utilities::S3_authorisation object can be in +enum class Authorisation_status { + /// @brief authorisation has not yet started + undefined, + /// @brief members have been set up and payload can be added + waiting_for_payload, + /// @brief the authorisation was invalid or some error occurred + failed, + /// @brief authorisation successful + valid +}; + +/// +/// @brief class to perform an authorisation of an S3 request. +/// +/// The implementation of this class follows the guidelines from +/// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +/// +class S3_authorisation { + public: + /// + /// @brief main method to run the authorisation algorithm + /// + /// @param headers of the message to authenticate + /// + /// @return status of the authorisation + /// + Authorisation_status authorise(const S3_header& headers); + + /// + /// @brief add a chunk of data to the payload + /// + /// @param chunk the string to add + /// + /// Since the whole body has to be hashed in order to verify the signature + /// of the message each chunk of data has to be added here in order. + /// + /// Otherwise the authentication will fail! + /// + void add_chunk(std::string chunk); + + /// + /// @brief checks if the authorisation was successful + /// + /// @return true if it has been valid, false if it hasn't started, it is + /// in progress, or if an error occurred. + /// + bool is_valid() const { return m_status == Authorisation_status::valid; } + + private: + /// @name Helper functions + /// + /// @{ + + /// + /// @brief checks whether the user id saved in @ref m_user_identifier + /// exists in the user database + /// + /// @return true if the user has been found, false otherwise + /// + /// Sets @ref m_user_key on success + /// + bool search_for_user(); + /// + /// @brief check if this message has a content body + /// + /// @param headers the headers of the message to check + /// + /// @returns true if the message contains a body, false otherwise + /// + bool is_chunked(const S3_header& headers) const; + + /// + /// @brief print current state of member variables + /// + void print_info() const; + + /// @} + /// + /// @name Extract header information + /// + /// @{ + + /// + /// @brief get the information about authorisation from the header and save + /// it to member variables + /// + /// @param headers the header to extract the information from + /// + /// @return true if all information was found, false if an error occurred + /// + bool set_authorisation_info(const S3_header& headers); + /// + /// @brief Extract information from a string + /// + /// @param complete the complete string will all information + /// @param identifier the identifier of the specific piece of information + /// required + /// @param delimiter the delimiter between the parts of information + /// + /// @return required information as string + /// + /// If there is for example a string of the form "id=info;id2=info2" then + /// ';' is the delimiter, 'id' is the identifier and this function + /// will return "info". + /// + std::string extract_string_part( + std::string complete, + std::string identifier, + std::string delimiter) const; + /// + /// @brief split up the "SignedHeaders" value of the HTTP request + /// + /// @param all_signed_headers the value of the SignedHeaders header + /// + /// This function saves the headers into @ref m_signed_headers + /// + void split_signed_headers(std::string all_signed_headers); + /// + /// @brief split up the "Credential" value of the HTTP request + /// + /// @param credentials the value of the Credential header + /// + /// This function saves the credentials into @ref m_user_identifier, + /// @ref m_date and @ref m_region + /// + void split_credentials(std::string credentials); + + /// @} + /// + /// @name Signature creation + /// + /// @{ + + /// + /// @brief checks the signature of the message + /// + /// @param headers the headers of the message to check + /// + /// @returns true if the signature was valid, false otherwise + /// + /// This function splits the message's contents into the needed parts, + /// creates a signature and compares it with the one saved in the headers + /// + bool check(const S3_header& headers) const; + + /// + /// @brief create a canonical request + /// + /// @param headers the headers of the message to create the request from + /// + /// @returns the canonical request + /// + /// For more information what this means, see: + /// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + /// + std::string create_canonical_request(const S3_header& headers) const; + /// + /// @brief create a string to sign + /// + /// @param headers the headers of the message to create the request + /// from + /// @param canonical_request the canonical request from + /// @ref create_canonical_request + /// + /// @returns the string to sign + /// + /// For more information what this means, see: + /// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + /// + std::string create_string_to_sign( + const S3_header& headers, std::string canonical_request) const; + /// + /// @brief create the signature + /// + /// @param string_to_sign the string to sign from @ref create_string_to_sign + /// + /// @returns the signature + /// + /// For more information what this means, see: + /// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + /// + std::string get_signature(std::string string_to_sign) const; + + /// @} + + /// @brief the current status of the authorisation + Authorisation_status m_status = Authorisation_status::undefined; + /// @brief the public key / identifier of the user that send the request + std::string m_user_identifier; + /// @brief the key that belongs to @ref m_user_identifier + std::string m_user_key; + /// @brief the date as saved in the credentials of the request + std::string m_date; + /// @brief the S3 region as saved in the credentials of the request + std::string m_region; + /// @brief a list of the headers that have been used to sign the request + std::vector m_signed_headers; + /// @brief the signature stored in the request + std::string m_signature; + /// @brief the payload of the request + /// + /// This is the whole body since we do not support multiple chunked + /// signatures + std::string m_payload = ""; +}; + +} // namespace s3_utilities +} // namespace fiphoboserver diff --git a/src/server/s3_utilities/s3_header.h b/src/server/s3_utilities/s3_header.h index 19f4635..04b0f07 100644 --- a/src/server/s3_utilities/s3_header.h +++ b/src/server/s3_utilities/s3_header.h @@ -33,7 +33,7 @@ class S3_header { /// /// @returns name of the bucket /// - std::string get_bucket() + std::string get_bucket() const { if (!m_headers) { return ""; @@ -51,7 +51,7 @@ class S3_header { /// /// @returns name of the key /// - std::string get_key_without_bucket() + std::string get_key_without_bucket() const { if (!m_headers) { return ""; @@ -71,7 +71,7 @@ class S3_header { /// /// @returns name of the key /// - std::string get_key() + std::string get_key() const { if (!m_headers) { return ""; @@ -94,7 +94,7 @@ class S3_header { /// @returns true if these headers belong to a create bucket request, /// false for all other requests /// - bool is_create_bucket_request() + bool is_create_bucket_request() const { if (!m_headers) { return false; @@ -112,7 +112,7 @@ class S3_header { /// /// @returns the length of the total body to be expected /// - size_t get_body_length() + size_t get_body_length() const { std::string content_length = m_headers->getHeaders().getSingleOrEmpty("Content-Length"); @@ -128,7 +128,7 @@ class S3_header { /// /// @returns a map consisting of all key value pairs found in the header /// - std::map get_meta_data() + std::map get_meta_data() const { std::map metadata; std::string meta_search_string = "x-amz-meta-"; @@ -147,7 +147,7 @@ class S3_header { /// /// @brief print all headers to std out /// - void print_all_headers() + void print_all_headers() const { std::cout << "Printing all headers for " << m_headers->getPath() << "\n"; @@ -158,7 +158,28 @@ class S3_header { std::cout << std::flush; } + /// + /// @brief extract one header value from the headers + /// + /// @param header_name the name of the header that should be extracted + /// + /// @returns the value of the requested header or an empty_string if it is + /// not found + /// + std::string get_header_by_name(const std::string header_name) const + { + return m_headers->getHeaders().getSingleOrEmpty(header_name); + } + + /// + /// @brief get the HTTP method the header belongs to + /// + /// @returns the method as a string + /// + std::string get_method() const { return m_headers->getMethodString(); } + private: + /// @brief inner pointer to the proxygen header object std::unique_ptr m_headers; }; diff --git a/src/server/s3_utilities/s3_utilities.cc b/src/server/s3_utilities/s3_utilities.cc index 8e973b0..f1f73ab 100644 --- a/src/server/s3_utilities/s3_utilities.cc +++ b/src/server/s3_utilities/s3_utilities.cc @@ -1,11 +1,16 @@ #include "s3_utilities.h" +#include + +#include +#include #include +#include namespace fiphoboserver { namespace s3_utilities { -s3_error_info create_s3_error(int return_value, std::string request) +s3_error_info create_s3_error(const int return_value, const std::string request) { s3_error_info new_info; @@ -18,7 +23,7 @@ s3_error_info create_s3_error(int return_value, std::string request) new_info.https_error_identifier = "Not found"; error_stream << "NoSuchKey"; error_stream << "\n\t"; - error_stream << "The resource you requested does not exist!"; + error_stream << "The resource you requested does not exist"; } else if (return_value == -17) { new_info.https_error_code = 400; @@ -50,5 +55,105 @@ s3_error_info create_s3_error(int return_value, std::string request) return new_info; } +std::string uri_encode( + const std::string string_to_encode, const bool encode_slash) +{ + std::stringstream result_stream; + for (size_t i = 0; i < string_to_encode.size(); ++i) { + char current_char = string_to_encode[i]; + if ((current_char >= 'A' && current_char <= 'Z') + || (current_char >= 'a' && current_char <= 'z') + || (current_char >= '0' && current_char <= '9') + || current_char == '_' || current_char == '-' || current_char == '~' + || current_char == '.') { + result_stream << current_char; + } + else if (current_char == '/') { + if (encode_slash) { + result_stream << "%2F"; + } + else { + result_stream << current_char; + } + } + else if (current_char == ' ') { + result_stream << "%20"; + } + else { + result_stream << '%' << std::hex << std::uppercase << std::setw(2) + << std::setfill('0') << int(current_char); + result_stream << std::dec; + } + } + return result_stream.str(); +} + + +std::string openssl_sha256(const void* data, const unsigned int length) +{ + unsigned char* hash = new unsigned char[EVP_MAX_MD_SIZE]; + unsigned int result_size; + + int result = + EVP_Digest(data, length, hash, &result_size, EVP_sha256(), NULL); + + if (result != 1) { + delete[] hash; + return ""; + } + + std::stringstream ss; + ss << std::hex << std::setfill('0'); + for (int i = 0; i < result_size; i++) { + ss << std::hex << std::setw(2) << (unsigned int)hash[i]; + } + + delete[] hash; + return (ss.str()); +} + +std::string openssl_sha256(const std::string msg) +{ + return openssl_sha256(&msg[0], msg.size()); +} + +std::string openssl_hmac(const std::string key, const std::string msg) +{ + unsigned char* hash; + unsigned int result_size; + + hash = HMAC( + EVP_sha256(), &key[0], key.length(), (unsigned char*)&msg[0], + msg.length(), NULL, &result_size); + + + std::stringstream ss; + ss << std::setfill('0'); + for (int i = 0; i < result_size; i++) { + ss << hash[i]; + } + + return (ss.str()); +} + +std::string openssl_hmac_hex(const std::string key, const std::string msg) +{ + unsigned char* hash; + unsigned int result_size; + + hash = HMAC( + EVP_sha256(), &key[0], key.length(), (unsigned char*)&msg[0], + msg.length(), NULL, &result_size); + + std::stringstream ss; + ss << std::hex << std::setfill('0'); + for (int i = 0; i < result_size; i++) { + ss << std::hex << std::setw(2) << (unsigned int)hash[i]; + } + + return (ss.str()); +} + + } // namespace s3_utilities } // namespace fiphoboserver diff --git a/src/server/s3_utilities/s3_utilities.h b/src/server/s3_utilities/s3_utilities.h index 581a86a..cd463e2 100644 --- a/src/server/s3_utilities/s3_utilities.h +++ b/src/server/s3_utilities/s3_utilities.h @@ -32,7 +32,58 @@ struct s3_error_info { /// @return an @ref s3_error_info struct containing readable information on the /// error /// -s3_error_info create_s3_error(int return_value, std::string request); +s3_error_info create_s3_error( + const int return_value, const std::string request); + +/// +/// @brief create a uri encoding of a string +/// +/// for more information see +/// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +/// +/// @param string_to_encode the string that should be encoded +/// @param encode_slash bool to signalise if the / should be encoded +/// +/// @returns the URI encoded string +/// +std::string uri_encode( + const std::string string_to_encode, const bool encode_slash); + +/// +/// @brief create a sha256 hash from the data +/// +/// @param data buffer to the data to hash +/// @param length number of byte in the buffer +/// +/// @return the resulting hash as string or "" in case of error +/// +std::string openssl_sha256(const void* data, const unsigned int length); + +/// +/// @brief shortcut for openssl_sha256(void*, unsigned int) +/// +/// @param msg the string to hash +/// +/// @return the resulting hash as string or "" in case of error +/// +std::string openssl_sha256(const std::string msg); + +/// +/// @brief create a key hashed message +/// +/// @param key the key to sign the message with +/// @param msg the message to sign +/// +/// @return a string representation of the signed message (unreadable) +/// +std::string openssl_hmac(const std::string key, const std::string msg); + +/// +/// @copydoc openssl_hmac +/// +/// @return the signed message in HEX format +/// +std::string openssl_hmac_hex(const std::string key, const std::string msg); } // namespace s3_utilities } // namespace fiphoboserver diff --git a/src/storage/phobos_file.cc b/src/storage/phobos_file.cc index d023d40..0de2019 100644 --- a/src/storage/phobos_file.cc +++ b/src/storage/phobos_file.cc @@ -58,6 +58,11 @@ ssize_t Phobos_file::db_put(size_t size) */ void Phobos_file::prepare_get(int file_descriptor, std::string object_id) { + if (m_descriptor.xd_objid != nullptr) { + delete[] m_descriptor.xd_objid; + m_descriptor.xd_objid = NULL; + } + m_descriptor.xd_fd = file_descriptor; char* unconsted_object_id = new char[object_id.length() + 1]; strcpy(unconsted_object_id, object_id.c_str()); // NOLINT diff --git a/test/unit_tests/CMakeLists.txt b/test/unit_tests/CMakeLists.txt index 675f823..229a6e4 100644 --- a/test/unit_tests/CMakeLists.txt +++ b/test/unit_tests/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable( tests-main.cc phobos_file.cc fifo.cc + s3_utilities.cc storage/disk.cc utils.cc diff --git a/test/unit_tests/s3_utilities.cc b/test/unit_tests/s3_utilities.cc new file mode 100644 index 0000000..60aecb1 --- /dev/null +++ b/test/unit_tests/s3_utilities.cc @@ -0,0 +1,334 @@ +#include + +#include +#include +#include + +#include "../../src/server/s3_utilities/s3_authorisation.h" +#include "../../src/server/s3_utilities/s3_header.h" +#include "../../src/server/s3_utilities/s3_utilities.h" + +#include + +namespace fiphoboserver { +namespace s3_utilities { + +SCENARIO("S3 error handling", "[errors]") +{ + GIVEN("an error code from an underlying function") + { + std::string key_name = "text_key"; + WHEN("The key does not exist in the databse") + { + int return_value = -2; + + THEN("an error object is created") + { + s3_error_info new_info = + create_s3_error(return_value, key_name); + std::stringstream stream; + stream << "\n"; + stream << "\n"; + stream << "\tNoSuchKey\n"; + stream + << "\tThe resource you requested does not exist\n"; + stream << "\t/" << key_name << "\n"; + stream << ""; + REQUIRE(new_info.https_error_code == 404); + REQUIRE(new_info.https_error_identifier == "Not found"); + REQUIRE(new_info.error_xml == stream.str()); + } + } + WHEN("The key has already been added") + { + int return_value = -17; + + THEN("an error object is created") + { + s3_error_info new_info = + create_s3_error(return_value, key_name); + std::stringstream stream; + stream << "\n"; + stream << "\n"; + stream << "\tInvalidArgument\n"; + stream + << "\tSome arguments given to the storage were invalid!\n"; + stream << "\t/" << key_name << "\n"; + stream << ""; + REQUIRE(new_info.https_error_code == 400); + REQUIRE(new_info.https_error_identifier == "Bad Request"); + REQUIRE(new_info.error_xml == stream.str()); + } + } + WHEN("The Phobos database is not running correctly") + { + int return_value = -107; + + THEN("an error object is created") + { + s3_error_info new_info = + create_s3_error(return_value, key_name); + std::stringstream stream; + stream << "\n"; + stream << "\n"; + stream << "\tInternalServerPhobosError\n"; + stream + << "\tThe Phobos database is not running correctly on the server\n"; + stream << "\t/" << key_name << "\n"; + stream << ""; + REQUIRE(new_info.https_error_code == 500); + REQUIRE( + new_info.https_error_identifier == "Internal Server Error"); + REQUIRE(new_info.error_xml == stream.str()); + } + } + WHEN("An unanticipated error occurrs") + { + int return_value = -1; + + THEN("an error object is created") + { + s3_error_info new_info = + create_s3_error(return_value, key_name); + std::stringstream stream; + stream << "\n"; + stream << "\n"; + stream << "\tInternalServerError\n"; + stream + << "\tAn error occurred in the underlying storage.\n"; + stream << "\t/" << key_name << "\n"; + stream << ""; + REQUIRE(new_info.https_error_code == 500); + REQUIRE( + new_info.https_error_identifier == "Internal Server Error"); + REQUIRE(new_info.error_xml == stream.str()); + } + } + } +} + +SCENARIO("String encoding", "[encoding]") +{ + GIVEN("a string to URI encode") + { + WHEN("The string contains only allowed characters") + { + std::string string_to_encode = "This-is_1.Normal~string"; + THEN("Nothing will be changed") + { + std::string encoded = uri_encode(string_to_encode, false); + REQUIRE(encoded == string_to_encode); + } + } + WHEN("The string contains a space") + { + std::string string_to_encode = "Before after"; + THEN("The space will be replaced by %20") + { + std::string encoded = uri_encode(string_to_encode, false); + REQUIRE(encoded == "Before%20after"); + } + } + WHEN("The string contains special characters") + { + std::string string_to_encode = "text^*:text"; + THEN("The characters will be replaced by '%' and their hex values") + { + std::string encoded = uri_encode(string_to_encode, false); + REQUIRE(encoded == "text%5E%2A%3Atext"); + } + } + WHEN("A containting slash should be encoded") + { + std::string string_to_encode = "Before/After"; + THEN("The '/' will be replaced by %2F") + { + std::string encoded = uri_encode(string_to_encode, true); + REQUIRE(encoded == "Before%2FAfter"); + } + } + WHEN("A containting slash should not be encoded") + { + std::string string_to_encode = "Before/After"; + THEN("Nothing will be changed") + { + std::string encoded = uri_encode(string_to_encode, false); + REQUIRE(encoded == string_to_encode); + } + } + } + + GIVEN("A string to hash") + { + std::string string_to_hash = "This is a content string"; + WHEN("The string should be hashed with sha256 as a string") + { + std::string hash = openssl_sha256(string_to_hash); + REQUIRE( + hash + == "3545d7357e3a5d7d8077f14d26d8f51e6aafccfe78652388f7599b7f88b238b4"); + } + WHEN("The string should be hashed with sha256 as raw data") + { + std::string hash = + openssl_sha256(string_to_hash.data(), string_to_hash.size()); + REQUIRE( + hash + == "3545d7357e3a5d7d8077f14d26d8f51e6aafccfe78652388f7599b7f88b238b4"); + } + WHEN("The string should be signed with a key using HMAC") + { + std::string key = "My Key"; + std::string hash = openssl_hmac_hex(key, string_to_hash); + REQUIRE( + hash + == "5c6474a2b4e4344142469233286da96a3eb7ceedf44da6c43ac05677cc216e2d"); + } + } +} + +SCENARIO("S3 header authorisation", "[authorisation]") +{ + // Those are the examples from + // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + std::unique_ptr message = + std::make_unique(); + proxygen::HTTPHeaders& headers = message->getHeaders(); + headers.add("Host", "examplebucket.s3.amazonaws.com"); + headers.add("x-amz-date", "20130524T000000Z"); + + S3_header s3_header; + S3_authorisation auth; + + GIVEN("a GET request") + { + message->setMethod(proxygen::HTTPMethod::GET); + message->setURL("/test.txt"); + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"); + headers.add("Range", "bytes=0-9"); + + WHEN("The request is valid") + { + headers.add( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + s3_header.set_headers(std::move(message)); + + THEN("The authorisation is valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::valid); + } + } + WHEN("The request is NOT valid") + { + headers.add( + "x-amz-content-sha256", + "e3b0544298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + s3_header.set_headers(std::move(message)); + + THEN("The authorisation is not valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::failed); + } + } + WHEN("The signing user does not exist") + { + headers.add( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + s3_header.set_headers(std::move(message)); + + // empty user database + std::ifstream user_in_stream("/tmp/users.txt"); + std::stringstream user_text; + user_text << user_in_stream.rdbuf(); + user_in_stream.close(); + + std::ofstream user_of_stream("/tmp/users.txt", std::fstream::trunc); + user_of_stream.close(); + + THEN("The authorisation is not valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::failed); + } + + // restore user database + user_of_stream = std::ofstream("/tmp/users.txt"); + user_of_stream << user_text.str(); + user_of_stream.close(); + } + } + GIVEN("a PUT request") + { + message->setMethod(proxygen::HTTPMethod::PUT); + message->setURL("/test$file.text"); + headers.add("Date", "Fri, 24 May 2013 00:00:00 GMT"); + headers.add("x-amz-storage-class", "REDUCED_REDUNDANCY"); + headers.add( + "x-amz-content-sha256", + "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072"); + headers.add("Content-Length", "21"); + + WHEN("The authorization header is not valid") + { + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"); + + s3_header.set_headers(std::move(message)); + + THEN("The authorisation will fail before payload is added") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::failed); + } + } + WHEN("The request is valid") + { + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"); + s3_header.set_headers(std::move(message)); + + THEN( + "After the first call to authorise() the status is 'waiting_for_payload'") + { + REQUIRE( + auth.authorise(s3_header) + == Authorisation_status::waiting_for_payload); + } + + THEN("After adding the payload, the authorisation is valid") + { + auth.authorise(s3_header); + auth.add_chunk("Welcome to Amazon S3."); + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::valid); + } + } + WHEN("The payload is NOT valid") + { + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"); + s3_header.set_headers(std::move(message)); + auth.authorise(s3_header); + + auth.add_chunk("Goodbye from Amazon S3."); + THEN("The authorisation is not valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::failed); + } + } + } +} + + +} // namespace s3_utilities +} // namespace fiphoboserver diff --git a/test/users.txt b/test/users.txt new file mode 100644 index 0000000..2330a73 --- /dev/null +++ b/test/users.txt @@ -0,0 +1,4 @@ +OPEN_KEY:SECRET_KEY +User2:Key_of_user_2 +Carl:CarlsKey +AKIAIOSFODNN7EXAMPLE:wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ No newline at end of file -- GitLab From ee649fa45420cb5bb39bea244629766946d29be5 Mon Sep 17 00:00:00 2001 From: Sophie Wenzel-Teuber Date: Fri, 24 Apr 2020 13:34:30 +0100 Subject: [PATCH 46/62] Add queries to authentication --- .../s3_utilities/S3_authorisation.md | 47 +++++++++++++++++- .../fiphoboserver/s3_utilities/S3_header.md | 31 ++++++++++++ src/server/s3_utilities/s3_authorisation.cc | 49 +++++++++++++++++-- src/server/s3_utilities/s3_authorisation.h | 14 +++++- src/server/s3_utilities/s3_header.h | 11 +++++ test/unit_tests/s3_utilities.cc | 44 +++++++++++++++++ 6 files changed, 190 insertions(+), 6 deletions(-) diff --git a/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md b/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md index 9f5e21a..16c339f 100644 --- a/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md +++ b/doc/Markdown/fiphoboserver/s3_utilities/S3_authorisation.md @@ -25,6 +25,7 @@ The implementation of this class follows the guidelines from [https://docs.aws.a | [set_authorisation_info](#fiphoboserver-s3_utilities-S3_authorisation-set_authorisation_info) | get the information about authorisation from the header and save it to member variables | | [extract_string_part](#fiphoboserver-s3_utilities-S3_authorisation-extract_string_part) | Extract information from a string. | | [split_signed_headers](#fiphoboserver-s3_utilities-S3_authorisation-split_signed_headers) | split up the "SignedHeaders" value of the HTTP request | +| [split_queries](#fiphoboserver-s3_utilities-S3_authorisation-split_queries) | split up the queries string of the HTTP request (thats the part of the request path after '?') | | [split_credentials](#fiphoboserver-s3_utilities-S3_authorisation-split_credentials) | split up the "Credential" value of the HTTP request | @@ -45,7 +46,8 @@ The implementation of this class follows the guidelines from [https://docs.aws.a | [m_user_key](#fiphoboserver-s3_utilities-S3_authorisation-m_user_key) | the key that belongs to [m_user_identifier][fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier] | | [m_date](#fiphoboserver-s3_utilities-S3_authorisation-m_date) | the date as saved in the credentials of the request | | [m_region](#fiphoboserver-s3_utilities-S3_authorisation-m_region) | the S3 region as saved in the credentials of the request | -| [m_signed_headers](#fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers) | a list of the headers that have been used to sign the request | +| [m_signed_headers](#fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers) | a (sorted) list of the headers that have been used to sign the request | +| [m_queries](#fiphoboserver-s3_utilities-S3_authorisation-m_queries) | a (sorted) list of all queries split up into key and value | | [m_signature](#fiphoboserver-s3_utilities-S3_authorisation-m_signature) | the signature stored in the request | | [m_payload](#fiphoboserver-s3_utilities-S3_authorisation-m_payload) | the payload of the request | @@ -235,6 +237,33 @@ This function saves the headers into +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private void fiphoboserver::s3_utilities::S3_authorisation::split_queries (std::string queries) + +split up the queries string of the HTTP request (thats the part of the request path after '?') + + + + +#### Parameters: +| Type | Name | Description | +| ---- | ---- | ---- | +| std::string | queries | the complete query string | + + + + + + + + +This function saves each query into +[m_queries][fiphoboserver-s3_utilities-S3_authorisation-m_queries] + + + + [Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) ### private void fiphoboserver::s3_utilities::S3_authorisation::split_credentials (std::string credentials) @@ -497,7 +526,20 @@ the S3 region as saved in the credentials of the request ### private fiphoboserver::s3_utilities::S3_authorisation::m_signed_headers -a list of the headers that have been used to sign the request +a (sorted) list of the headers that have been used to sign the request + + + + + + + + +[Go to Top](#fiphoboserver-s3_utilities-S3_authorisation) + +### private fiphoboserver::s3_utilities::S3_authorisation::m_queries + +a (sorted) list of all queries split up into key and value @@ -634,6 +676,7 @@ checks if the authorisation was successful [fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-create_canonical_request [fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-create_string_to_sign [fiphoboserver-s3_utilities-S3_authorisation-m_date]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_date +[fiphoboserver-s3_utilities-S3_authorisation-m_queries]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_queries [fiphoboserver-s3_utilities-S3_authorisation-m_region]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_region [fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_signed_headers [fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier]:./S3_authorisation.md#fiphoboserver-s3_utilities-S3_authorisation-m_user_identifier diff --git a/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md index ad9ff88..17600f2 100644 --- a/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md +++ b/doc/Markdown/fiphoboserver/s3_utilities/S3_header.md @@ -23,6 +23,7 @@ class to extract the S3 specific information from proxygens HTTPMessage headers | [get_meta_data](#fiphoboserver-s3_utilities-S3_header-get_meta_data) | gets the S3 metadata stored in the HTTP headers | | [print_all_headers](#fiphoboserver-s3_utilities-S3_header-print_all_headers) | print all headers to std out | | [get_header_by_name](#fiphoboserver-s3_utilities-S3_header-get_header_by_name) | extract one header value from the headers | +| [get_queries](#fiphoboserver-s3_utilities-S3_header-get_queries) | extract the query string from the headers | | [get_method](#fiphoboserver-s3_utilities-S3_header-get_method) | get the HTTP method the header belongs to | @@ -297,6 +298,36 @@ extract one header value from the headers +#### Qualifiers: +* const +* inline + + +[Go to Top](#fiphoboserver-s3_utilities-S3_header) + +### public std::string fiphoboserver::s3_utilities::S3_header::get_queries () const + +extract the query string from the headers + + + + +#### Returns: +| Type | Description | +| ---- | ---- | +| std::string | the query string of the request or an empty_string if it doesn't have one | + + + + + + + + + + + + #### Qualifiers: * const * inline diff --git a/src/server/s3_utilities/s3_authorisation.cc b/src/server/s3_utilities/s3_authorisation.cc index 8f0bb60..8a4828d 100644 --- a/src/server/s3_utilities/s3_authorisation.cc +++ b/src/server/s3_utilities/s3_authorisation.cc @@ -81,6 +81,7 @@ bool S3_authorisation::set_authorisation_info(const S3_header& headers) } split_signed_headers(all_signed_headers); + split_queries(headers.get_queries()); split_credentials(credentials); // print_info(); @@ -109,10 +110,45 @@ void S3_authorisation::split_signed_headers(std::string all_signed_headers) { size_t index = 0; while (index != all_signed_headers.npos) { - index = all_signed_headers.find(';'); - m_signed_headers.push_back(all_signed_headers.substr(0, index)); + index = all_signed_headers.find(';'); + std::string header_lowercase = all_signed_headers.substr(0, index); + std::transform( + header_lowercase.begin(), header_lowercase.end(), + header_lowercase.begin(), ::tolower); + m_signed_headers.push_back(header_lowercase); all_signed_headers = all_signed_headers.substr(index + 1); } + // in all tests this was already sorted, but calling sort here as precaution + // it should be rather fast because this list is not long even if unsorted + std::sort(m_signed_headers.begin(), m_signed_headers.end()); +} + +void S3_authorisation::split_queries(std::string queries) +{ + if (queries.empty()) { + return; + } + + size_t index = 0; + while (index != queries.npos) { + index = queries.find('&'); + std::string query = queries.substr(0, index); + size_t equality_sign_index = query.find('='); + if (equality_sign_index != query.npos) { + m_queries.push_back(std::make_pair( + query.substr(0, equality_sign_index), + query.substr(equality_sign_index + 1))); + } + else { + m_queries.push_back( + std::make_pair(query.c_str(), "")); + } + + queries = queries.substr(index + 1); + } + // in all tests this was already sorted, but calling sort here as precaution + // it should be rather fast because this list is not long even if unsorted + std::sort(m_queries.begin(), m_queries.end()); } void S3_authorisation::split_credentials(std::string credentials) @@ -226,7 +262,14 @@ std::string S3_authorisation::create_canonical_request( result_stream << headers.get_method() << '\n'; result_stream << '/' << uri_encode(headers.get_key(), false) << '\n'; - // TODO insert queries! Issue #46 + if (!m_queries.empty()) { + for (size_t i = 0; i < m_queries.size() - 1; ++i) { + result_stream << m_queries[i].first << '=' << m_queries[i].second + << '&'; + } + result_stream << m_queries[m_queries.size() - 1].first << '=' + << m_queries[m_queries.size() - 1].second; + } result_stream << '\n'; for (auto it = m_signed_headers.begin(); it != m_signed_headers.end(); diff --git a/src/server/s3_utilities/s3_authorisation.h b/src/server/s3_utilities/s3_authorisation.h index 1a3d8e0..a580ef7 100644 --- a/src/server/s3_utilities/s3_authorisation.h +++ b/src/server/s3_utilities/s3_authorisation.h @@ -129,6 +129,15 @@ class S3_authorisation { /// void split_signed_headers(std::string all_signed_headers); /// + /// @brief split up the queries string of the HTTP request (thats the part + /// of the request path after '?') + /// + /// @param queries the complete query string + /// + /// This function saves each query into @ref m_queries + /// + void split_queries(std::string queries); + /// /// @brief split up the "Credential" value of the HTTP request /// /// @param credentials the value of the Credential header @@ -206,8 +215,11 @@ class S3_authorisation { std::string m_date; /// @brief the S3 region as saved in the credentials of the request std::string m_region; - /// @brief a list of the headers that have been used to sign the request + /// @brief a (sorted) list of the headers that have been used to sign the + /// request std::vector m_signed_headers; + /// @brief a (sorted) list of all queries split up into key and value + std::vector> m_queries; /// @brief the signature stored in the request std::string m_signature; /// @brief the payload of the request diff --git a/src/server/s3_utilities/s3_header.h b/src/server/s3_utilities/s3_header.h index 04b0f07..79fe2fc 100644 --- a/src/server/s3_utilities/s3_header.h +++ b/src/server/s3_utilities/s3_header.h @@ -151,6 +151,9 @@ class S3_header { { std::cout << "Printing all headers for " << m_headers->getPath() << "\n"; + if (!m_headers->getQueryString().empty()) { + std::cout << "Queries: " << m_headers->getQueryString() << '\n'; + } m_headers->getHeaders().forEach( [&](const std::string& header, const std::string& val) { std::cout << header << ": " << val << "\n"; @@ -171,6 +174,14 @@ class S3_header { return m_headers->getHeaders().getSingleOrEmpty(header_name); } + /// + /// @brief extract the query string from the headers + /// + /// @returns the query string of the request or an empty_string if it + /// doesn't have one + /// + std::string get_queries() const { return m_headers->getQueryString(); } + /// /// @brief get the HTTP method the header belongs to /// diff --git a/test/unit_tests/s3_utilities.cc b/test/unit_tests/s3_utilities.cc index 60aecb1..65771b8 100644 --- a/test/unit_tests/s3_utilities.cc +++ b/test/unit_tests/s3_utilities.cc @@ -263,6 +263,50 @@ SCENARIO("S3 header authorisation", "[authorisation]") user_of_stream.close(); } } + GIVEN("A GET bucket request with queries") + { + message->setMethod(proxygen::HTTPMethod::GET); + message->setURL("/"); + + WHEN("The GET request contains one empty valued query") + { + message->setQueryString("lifecycle"); + + headers.add( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543"); + + s3_header.set_headers(std::move(message)); + + THEN("The authorisation is valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::valid); + } + } + WHEN("The GET request contains two queries") + { + message->setQueryString("max-keys=2&prefix=J"); + + headers.add( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + headers.add( + "Authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"); + + s3_header.set_headers(std::move(message)); + + THEN("The authorisation is valid") + { + REQUIRE( + auth.authorise(s3_header) == Authorisation_status::valid); + } + } + } GIVEN("a PUT request") { message->setMethod(proxygen::HTTPMethod::PUT); -- GitLab From ba82dd3672ee19119adda86159cedc645d2aeb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20=C3=93=20Rourke?= Date: Tue, 5 May 2020 15:32:54 +0100 Subject: [PATCH 47/62] Support command line parsing and configuration files Change log: * src/utils directory for configuration, logging, etc * Config class for handling configuration (command line parsing, etc) * include Catch2 and CLI11 as system files so clang-tidy will ignore them * support basic command line options: - hostname (positional, required) - number of threads - http_port - https_port * add configuration to fiphoboserver executable main function * return on exception for incompatible hostname * return after parsing command line arguments if arguments are bad or --help is supplied * don't pass command line arguments to folly * Config returns struct of command line options * accept configuration options from config file * option to write current configuration to file * integration tests: reflect changes to fiphoboserver executable --- CMakeLists.txt | 5 + include/CLI/CLI11.hpp | 8223 ++++++++++++++ include/catch2/catch.hpp | 17618 +++++++++++++++++++++++++++++ src/CMakeLists.txt | 4 +- src/main.cc | 51 +- src/utils/config.cc | 58 + src/utils/config.h | 126 + test/integration_tests/client.py | 2 +- test/unit_tests/tests-main.cc | 2 +- 9 files changed, 26063 insertions(+), 26 deletions(-) create mode 100644 include/CLI/CLI11.hpp create mode 100644 include/catch2/catch.hpp create mode 100644 src/utils/config.cc create mode 100644 src/utils/config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8973e69..36a1ea2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,11 @@ option( add_subdirectory(src) +target_include_directories( + server SYSTEM PUBLIC + $ +) + if(CUSTOM_DOCU_PATH) add_subdirectory(doc) endif(CUSTOM_DOCU_PATH) diff --git a/include/CLI/CLI11.hpp b/include/CLI/CLI11.hpp new file mode 100644 index 0000000..8f95807 --- /dev/null +++ b/include/CLI/CLI11.hpp @@ -0,0 +1,8223 @@ +#pragma once + +// CLI11: Version 1.9.0 +// Originally designed by Henry Schreiner +// https://github.com/CLIUtils/CLI11 +// +// This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts +// from: v1.9.0 +// +// From LICENSE: +// +// CLI11 1.8 Copyright (c) 2017-2019 University of Cincinnati, developed by Henry +// Schreiner under NSF AWARD 1414736. All rights reserved. +// +// Redistribution and use in source and binary forms of CLI11, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Standard combined includes: + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// Verbatim copy from CLI/Version.hpp: + + +#define CLI11_VERSION_MAJOR 1 +#define CLI11_VERSION_MINOR 9 +#define CLI11_VERSION_PATCH 0 +#define CLI11_VERSION "1.9.0" + + + + +// Verbatim copy from CLI/Macros.hpp: + + +// The following version macro is very similar to the one in PyBind11 +#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) +#if __cplusplus >= 201402L +#define CLI11_CPP14 +#if __cplusplus >= 201703L +#define CLI11_CPP17 +#if __cplusplus > 201703L +#define CLI11_CPP20 +#endif +#endif +#endif +#elif defined(_MSC_VER) && __cplusplus == 199711L +// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully implemented) +// Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 or newer +#if _MSVC_LANG >= 201402L +#define CLI11_CPP14 +#if _MSVC_LANG > 201402L && _MSC_VER >= 1910 +#define CLI11_CPP17 +#if __MSVC_LANG > 201703L && _MSC_VER >= 1910 +#define CLI11_CPP20 +#endif +#endif +#endif +#endif + +#if defined(CLI11_CPP14) +#define CLI11_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(_MSC_VER) +#define CLI11_DEPRECATED(reason) __declspec(deprecated(reason)) +#else +#define CLI11_DEPRECATED(reason) __attribute__((deprecated(reason))) +#endif + + + + +// Verbatim copy from CLI/Validators.hpp: + + +// C standard library +// Only needed for existence checking +#if defined CLI11_CPP17 && defined __has_include && !defined CLI11_HAS_FILESYSTEM +#if __has_include() +// Filesystem cannot be used if targeting macOS < 10.15 +#if defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 +#define CLI11_HAS_FILESYSTEM 0 +#else +#include +#if defined __cpp_lib_filesystem && __cpp_lib_filesystem >= 201703 +#define CLI11_HAS_FILESYSTEM 1 +#else +#define CLI11_HAS_FILESYSTEM 0 +#endif +#endif +#endif +#endif + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include // NOLINT(build/include) +#else +#include +#include +#endif + + + +// From CLI/Version.hpp: + + + +// From CLI/Macros.hpp: + + + +// From CLI/StringTools.hpp: + +namespace CLI { + +/// Include the items in this namespace to get free conversion of enums to/from streams. +/// (This is available inside CLI as well, so CLI11 will use this without a using statement). +namespace enums { + +/// output streaming for enumerations +template ::value>::type> +std::ostream &operator<<(std::ostream &in, const T &item) { + // make sure this is out of the detail namespace otherwise it won't be found when needed + return in << static_cast::type>(item); +} + +} // namespace enums + +/// Export to CLI namespace +using enums::operator<<; + +namespace detail { +/// a constant defining an expected max vector size defined to be a big number that could be multiplied by 4 and not +/// produce overflow for some expected uses +constexpr int expected_max_vector_size{1 << 29}; +// Based on http://stackoverflow.com/questions/236129/split-a-string-in-c +/// Split a string by a delim +inline std::vector split(const std::string &s, char delim) { + std::vector elems; + // Check to see if empty string, give consistent result + if(s.empty()) + elems.emplace_back(); + else { + std::stringstream ss; + ss.str(s); + std::string item; + while(std::getline(ss, item, delim)) { + elems.push_back(item); + } + } + return elems; +} + +/// Simple function to join a string +template std::string join(const T &v, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + if(beg != end) + s << *beg++; + while(beg != end) { + s << delim << *beg++; + } + return s.str(); +} + +/// Simple function to join a string from processed elements +template ::value>::type> +std::string join(const T &v, Callable func, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + if(beg != end) + s << func(*beg++); + while(beg != end) { + s << delim << func(*beg++); + } + return s.str(); +} + +/// Join a string in reverse order +template std::string rjoin(const T &v, std::string delim = ",") { + std::ostringstream s; + for(std::size_t start = 0; start < v.size(); start++) { + if(start > 0) + s << delim; + s << v[v.size() - start - 1]; + } + return s.str(); +} + +// Based roughly on http://stackoverflow.com/questions/25829143/c-trim-whitespace-from-a-string + +/// Trim whitespace from left of string +inline std::string <rim(std::string &str) { + auto it = std::find_if(str.begin(), str.end(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(str.begin(), it); + return str; +} + +/// Trim anything from left of string +inline std::string <rim(std::string &str, const std::string &filter) { + auto it = std::find_if(str.begin(), str.end(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(str.begin(), it); + return str; +} + +/// Trim whitespace from right of string +inline std::string &rtrim(std::string &str) { + auto it = std::find_if(str.rbegin(), str.rend(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(it.base(), str.end()); + return str; +} + +/// Trim anything from right of string +inline std::string &rtrim(std::string &str, const std::string &filter) { + auto it = + std::find_if(str.rbegin(), str.rend(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(it.base(), str.end()); + return str; +} + +/// Trim whitespace from string +inline std::string &trim(std::string &str) { return ltrim(rtrim(str)); } + +/// Trim anything from string +inline std::string &trim(std::string &str, const std::string filter) { return ltrim(rtrim(str, filter), filter); } + +/// Make a copy of the string and then trim it +inline std::string trim_copy(const std::string &str) { + std::string s = str; + return trim(s); +} + +/// remove quotes at the front and back of a string either '"' or '\'' +inline std::string &remove_quotes(std::string &str) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\'')) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +/// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered) +inline std::string trim_copy(const std::string &str, const std::string &filter) { + std::string s = str; + return trim(s, filter); +} +/// Print a two part "help" string +inline std::ostream &format_help(std::ostream &out, std::string name, std::string description, std::size_t wid) { + name = " " + name; + out << std::setw(static_cast(wid)) << std::left << name; + if(!description.empty()) { + if(name.length() >= wid) + out << "\n" << std::setw(static_cast(wid)) << ""; + for(const char c : description) { + out.put(c); + if(c == '\n') { + out << std::setw(static_cast(wid)) << ""; + } + } + } + out << "\n"; + return out; +} + +/// Verify the first character of an option +template bool valid_first_char(T c) { + return std::isalnum(c, std::locale()) || c == '_' || c == '?' || c == '@'; +} + +/// Verify following characters of an option +template bool valid_later_char(T c) { return valid_first_char(c) || c == '.' || c == '-'; } + +/// Verify an option name +inline bool valid_name_string(const std::string &str) { + if(str.empty() || !valid_first_char(str[0])) + return false; + for(auto c : str.substr(1)) + if(!valid_later_char(c)) + return false; + return true; +} + +/// Verify that str consists of letters only +inline bool isalpha(const std::string &str) { + return std::all_of(str.begin(), str.end(), [](char c) { return std::isalpha(c, std::locale()); }); +} + +/// Return a lower case version of a string +inline std::string to_lower(std::string str) { + std::transform(std::begin(str), std::end(str), std::begin(str), [](const std::string::value_type &x) { + return std::tolower(x, std::locale()); + }); + return str; +} + +/// remove underscores from a string +inline std::string remove_underscore(std::string str) { + str.erase(std::remove(std::begin(str), std::end(str), '_'), std::end(str)); + return str; +} + +/// Find and replace a substring with another substring +inline std::string find_and_replace(std::string str, std::string from, std::string to) { + + std::size_t start_pos = 0; + + while((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); + } + + return str; +} + +/// check if the flag definitions has possible false flags +inline bool has_default_flag_values(const std::string &flags) { + return (flags.find_first_of("{!") != std::string::npos); +} + +inline void remove_default_flag_values(std::string &flags) { + auto loc = flags.find_first_of('{'); + while(loc != std::string::npos) { + auto finish = flags.find_first_of("},", loc + 1); + if((finish != std::string::npos) && (flags[finish] == '}')) { + flags.erase(flags.begin() + static_cast(loc), + flags.begin() + static_cast(finish) + 1); + } + loc = flags.find_first_of('{', loc + 1); + } + flags.erase(std::remove(flags.begin(), flags.end(), '!'), flags.end()); +} + +/// Check if a string is a member of a list of strings and optionally ignore case or ignore underscores +inline std::ptrdiff_t find_member(std::string name, + const std::vector names, + bool ignore_case = false, + bool ignore_underscore = false) { + auto it = std::end(names); + if(ignore_case) { + if(ignore_underscore) { + name = detail::to_lower(detail::remove_underscore(name)); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(detail::remove_underscore(local_name)) == name; + }); + } else { + name = detail::to_lower(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(local_name) == name; + }); + } + + } else if(ignore_underscore) { + name = detail::remove_underscore(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::remove_underscore(local_name) == name; + }); + } else + it = std::find(std::begin(names), std::end(names), name); + + return (it != std::end(names)) ? (it - std::begin(names)) : (-1); +} + +/// Find a trigger string and call a modify callable function that takes the current string and starting position of the +/// trigger and returns the position in the string to search for the next trigger string +template inline std::string find_and_modify(std::string str, std::string trigger, Callable modify) { + std::size_t start_pos = 0; + while((start_pos = str.find(trigger, start_pos)) != std::string::npos) { + start_pos = modify(str, start_pos); + } + return str; +} + +/// Split a string '"one two" "three"' into 'one two', 'three' +/// Quote characters can be ` ' or " +inline std::vector split_up(std::string str, char delimiter = '\0') { + + const std::string delims("\'\"`"); + auto find_ws = [delimiter](char ch) { + return (delimiter == '\0') ? (std::isspace(ch, std::locale()) != 0) : (ch == delimiter); + }; + trim(str); + + std::vector output; + bool embeddedQuote = false; + char keyChar = ' '; + while(!str.empty()) { + if(delims.find_first_of(str[0]) != std::string::npos) { + keyChar = str[0]; + auto end = str.find_first_of(keyChar, 1); + while((end != std::string::npos) && (str[end - 1] == '\\')) { // deal with escaped quotes + end = str.find_first_of(keyChar, end + 1); + embeddedQuote = true; + } + if(end != std::string::npos) { + output.push_back(str.substr(1, end - 1)); + str = str.substr(end + 1); + } else { + output.push_back(str.substr(1)); + str = ""; + } + } else { + auto it = std::find_if(std::begin(str), std::end(str), find_ws); + if(it != std::end(str)) { + std::string value = std::string(str.begin(), it); + output.push_back(value); + str = std::string(it + 1, str.end()); + } else { + output.push_back(str); + str = ""; + } + } + // transform any embedded quotes into the regular character + if(embeddedQuote) { + output.back() = find_and_replace(output.back(), std::string("\\") + keyChar, std::string(1, keyChar)); + embeddedQuote = false; + } + trim(str); + } + return output; +} + +/// Add a leader to the beginning of all new lines (nothing is added +/// at the start of the first line). `"; "` would be for ini files +/// +/// Can't use Regex, or this would be a subs. +inline std::string fix_newlines(const std::string &leader, std::string input) { + std::string::size_type n = 0; + while(n != std::string::npos && n < input.size()) { + n = input.find('\n', n); + if(n != std::string::npos) { + input = input.substr(0, n + 1) + leader + input.substr(n + 1); + n += leader.size(); + } + } + return input; +} + +/// This function detects an equal or colon followed by an escaped quote after an argument +/// then modifies the string to replace the equality with a space. This is needed +/// to allow the split up function to work properly and is intended to be used with the find_and_modify function +/// the return value is the offset+1 which is required by the find_and_modify function. +inline std::size_t escape_detect(std::string &str, std::size_t offset) { + auto next = str[offset + 1]; + if((next == '\"') || (next == '\'') || (next == '`')) { + auto astart = str.find_last_of("-/ \"\'`", offset - 1); + if(astart != std::string::npos) { + if(str[astart] == ((str[offset] == '=') ? '-' : '/')) + str[offset] = ' '; // interpret this as a space so the split_up works properly + } + } + return offset + 1; +} + +/// Add quotes if the string contains spaces +inline std::string &add_quotes_if_needed(std::string &str) { + if((str.front() != '"' && str.front() != '\'') || str.front() != str.back()) { + char quote = str.find('"') < str.find('\'') ? '\'' : '"'; + if(str.find(' ') != std::string::npos) { + str.insert(0, 1, quote); + str.append(1, quote); + } + } + return str; +} + +} // namespace detail + +} // namespace CLI + +// From CLI/Error.hpp: + +namespace CLI { + +// Use one of these on all error classes. +// These are temporary and are undef'd at the end of this file. +#define CLI11_ERROR_DEF(parent, name) \ + protected: \ + name(std::string ename, std::string msg, int exit_code) : parent(std::move(ename), std::move(msg), exit_code) {} \ + name(std::string ename, std::string msg, ExitCodes exit_code) \ + : parent(std::move(ename), std::move(msg), exit_code) {} \ + \ + public: \ + name(std::string msg, ExitCodes exit_code) : parent(#name, std::move(msg), exit_code) {} \ + name(std::string msg, int exit_code) : parent(#name, std::move(msg), exit_code) {} + +// This is added after the one above if a class is used directly and builds its own message +#define CLI11_ERROR_SIMPLE(name) \ + explicit name(std::string msg) : name(#name, msg, ExitCodes::name) {} + +/// These codes are part of every error in CLI. They can be obtained from e using e.exit_code or as a quick shortcut, +/// int values from e.get_error_code(). +enum class ExitCodes { + Success = 0, + IncorrectConstruction = 100, + BadNameString, + OptionAlreadyAdded, + FileError, + ConversionError, + ValidationError, + RequiredError, + RequiresError, + ExcludesError, + ExtrasError, + ConfigError, + InvalidError, + HorribleError, + OptionNotFound, + ArgumentMismatch, + BaseClass = 127 +}; + +// Error definitions + +/// @defgroup error_group Errors +/// @brief Errors thrown by CLI11 +/// +/// These are the errors that can be thrown. Some of them, like CLI::Success, are not really errors. +/// @{ + +/// All errors derive from this one +class Error : public std::runtime_error { + int actual_exit_code; + std::string error_name{"Error"}; + + public: + int get_exit_code() const { return actual_exit_code; } + + std::string get_name() const { return error_name; } + + Error(std::string name, std::string msg, int exit_code = static_cast(ExitCodes::BaseClass)) + : runtime_error(msg), actual_exit_code(exit_code), error_name(std::move(name)) {} + + Error(std::string name, std::string msg, ExitCodes exit_code) : Error(name, msg, static_cast(exit_code)) {} +}; + +// Note: Using Error::Error constructors does not work on GCC 4.7 + +/// Construction errors (not in parsing) +class ConstructionError : public Error { + CLI11_ERROR_DEF(Error, ConstructionError) +}; + +/// Thrown when an option is set to conflicting values (non-vector and multi args, for example) +class IncorrectConstruction : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, IncorrectConstruction) + CLI11_ERROR_SIMPLE(IncorrectConstruction) + static IncorrectConstruction PositionalFlag(std::string name) { + return IncorrectConstruction(name + ": Flags cannot be positional"); + } + static IncorrectConstruction Set0Opt(std::string name) { + return IncorrectConstruction(name + ": Cannot set 0 expected, use a flag instead"); + } + static IncorrectConstruction SetFlag(std::string name) { + return IncorrectConstruction(name + ": Cannot set an expected number for flags"); + } + static IncorrectConstruction ChangeNotVector(std::string name) { + return IncorrectConstruction(name + ": You can only change the expected arguments for vectors"); + } + static IncorrectConstruction AfterMultiOpt(std::string name) { + return IncorrectConstruction( + name + ": You can't change expected arguments after you've changed the multi option policy!"); + } + static IncorrectConstruction MissingOption(std::string name) { + return IncorrectConstruction("Option " + name + " is not defined"); + } + static IncorrectConstruction MultiOptionPolicy(std::string name) { + return IncorrectConstruction(name + ": multi_option_policy only works for flags and exact value options"); + } +}; + +/// Thrown on construction of a bad name +class BadNameString : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, BadNameString) + CLI11_ERROR_SIMPLE(BadNameString) + static BadNameString OneCharName(std::string name) { return BadNameString("Invalid one char name: " + name); } + static BadNameString BadLongName(std::string name) { return BadNameString("Bad long name: " + name); } + static BadNameString DashesOnly(std::string name) { + return BadNameString("Must have a name, not just dashes: " + name); + } + static BadNameString MultiPositionalNames(std::string name) { + return BadNameString("Only one positional name allowed, remove: " + name); + } +}; + +/// Thrown when an option already exists +class OptionAlreadyAdded : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, OptionAlreadyAdded) + explicit OptionAlreadyAdded(std::string name) + : OptionAlreadyAdded(name + " is already added", ExitCodes::OptionAlreadyAdded) {} + static OptionAlreadyAdded Requires(std::string name, std::string other) { + return OptionAlreadyAdded(name + " requires " + other, ExitCodes::OptionAlreadyAdded); + } + static OptionAlreadyAdded Excludes(std::string name, std::string other) { + return OptionAlreadyAdded(name + " excludes " + other, ExitCodes::OptionAlreadyAdded); + } +}; + +// Parsing errors + +/// Anything that can error in Parse +class ParseError : public Error { + CLI11_ERROR_DEF(Error, ParseError) +}; + +// Not really "errors" + +/// This is a successful completion on parsing, supposed to exit +class Success : public ParseError { + CLI11_ERROR_DEF(ParseError, Success) + Success() : Success("Successfully completed, should be caught and quit", ExitCodes::Success) {} +}; + +/// -h or --help on command line +class CallForHelp : public ParseError { + CLI11_ERROR_DEF(ParseError, CallForHelp) + CallForHelp() : CallForHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Usually something like --help-all on command line +class CallForAllHelp : public ParseError { + CLI11_ERROR_DEF(ParseError, CallForAllHelp) + CallForAllHelp() + : CallForAllHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Does not output a diagnostic in CLI11_PARSE, but allows to return from main() with a specific error code. +class RuntimeError : public ParseError { + CLI11_ERROR_DEF(ParseError, RuntimeError) + explicit RuntimeError(int exit_code = 1) : RuntimeError("Runtime error", exit_code) {} +}; + +/// Thrown when parsing an INI file and it is missing +class FileError : public ParseError { + CLI11_ERROR_DEF(ParseError, FileError) + CLI11_ERROR_SIMPLE(FileError) + static FileError Missing(std::string name) { return FileError(name + " was not readable (missing?)"); } +}; + +/// Thrown when conversion call back fails, such as when an int fails to coerce to a string +class ConversionError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConversionError) + CLI11_ERROR_SIMPLE(ConversionError) + ConversionError(std::string member, std::string name) + : ConversionError("The value " + member + " is not an allowed value for " + name) {} + ConversionError(std::string name, std::vector results) + : ConversionError("Could not convert: " + name + " = " + detail::join(results)) {} + static ConversionError TooManyInputsFlag(std::string name) { + return ConversionError(name + ": too many inputs for a flag"); + } + static ConversionError TrueFalse(std::string name) { + return ConversionError(name + ": Should be true/false or a number"); + } +}; + +/// Thrown when validation of results fails +class ValidationError : public ParseError { + CLI11_ERROR_DEF(ParseError, ValidationError) + CLI11_ERROR_SIMPLE(ValidationError) + explicit ValidationError(std::string name, std::string msg) : ValidationError(name + ": " + msg) {} +}; + +/// Thrown when a required option is missing +class RequiredError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiredError) + explicit RequiredError(std::string name) : RequiredError(name + " is required", ExitCodes::RequiredError) {} + static RequiredError Subcommand(std::size_t min_subcom) { + if(min_subcom == 1) { + return RequiredError("A subcommand"); + } + return RequiredError("Requires at least " + std::to_string(min_subcom) + " subcommands", + ExitCodes::RequiredError); + } + static RequiredError + Option(std::size_t min_option, std::size_t max_option, std::size_t used, const std::string &option_list) { + if((min_option == 1) && (max_option == 1) && (used == 0)) + return RequiredError("Exactly 1 option from [" + option_list + "]"); + if((min_option == 1) && (max_option == 1) && (used > 1)) { + return RequiredError("Exactly 1 option from [" + option_list + "] is required and " + std::to_string(used) + + " were given", + ExitCodes::RequiredError); + } + if((min_option == 1) && (used == 0)) + return RequiredError("At least 1 option from [" + option_list + "]"); + if(used < min_option) { + return RequiredError("Requires at least " + std::to_string(min_option) + " options used and only " + + std::to_string(used) + "were given from [" + option_list + "]", + ExitCodes::RequiredError); + } + if(max_option == 1) + return RequiredError("Requires at most 1 options be given from [" + option_list + "]", + ExitCodes::RequiredError); + + return RequiredError("Requires at most " + std::to_string(max_option) + " options be used and " + + std::to_string(used) + "were given from [" + option_list + "]", + ExitCodes::RequiredError); + } +}; + +/// Thrown when the wrong number of arguments has been received +class ArgumentMismatch : public ParseError { + CLI11_ERROR_DEF(ParseError, ArgumentMismatch) + CLI11_ERROR_SIMPLE(ArgumentMismatch) + ArgumentMismatch(std::string name, int expected, std::size_t recieved) + : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name + + ", got " + std::to_string(recieved)) + : ("Expected at least " + std::to_string(-expected) + " arguments to " + name + + ", got " + std::to_string(recieved)), + ExitCodes::ArgumentMismatch) {} + + static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch AtMost(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At Most " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch TypedAtLeast(std::string name, int num, std::string type) { + return ArgumentMismatch(name + ": " + std::to_string(num) + " required " + type + " missing"); + } + static ArgumentMismatch FlagOverride(std::string name) { + return ArgumentMismatch(name + " was given a disallowed flag override"); + } +}; + +/// Thrown when a requires option is missing +class RequiresError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiresError) + RequiresError(std::string curname, std::string subname) + : RequiresError(curname + " requires " + subname, ExitCodes::RequiresError) {} +}; + +/// Thrown when an excludes option is present +class ExcludesError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExcludesError) + ExcludesError(std::string curname, std::string subname) + : ExcludesError(curname + " excludes " + subname, ExitCodes::ExcludesError) {} +}; + +/// Thrown when too many positionals or options are found +class ExtrasError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExtrasError) + explicit ExtrasError(std::vector args) + : ExtrasError((args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} + ExtrasError(const std::string &name, std::vector args) + : ExtrasError(name, + (args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} +}; + +/// Thrown when extra values are found in an INI file +class ConfigError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConfigError) + CLI11_ERROR_SIMPLE(ConfigError) + static ConfigError Extras(std::string item) { return ConfigError("INI was not able to parse " + item); } + static ConfigError NotConfigurable(std::string item) { + return ConfigError(item + ": This option is not allowed in a configuration file"); + } +}; + +/// Thrown when validation fails before parsing +class InvalidError : public ParseError { + CLI11_ERROR_DEF(ParseError, InvalidError) + explicit InvalidError(std::string name) + : InvalidError(name + ": Too many positional arguments with unlimited expected args", ExitCodes::InvalidError) { + } +}; + +/// This is just a safety check to verify selection and parsing match - you should not ever see it +/// Strings are directly added to this error, but again, it should never be seen. +class HorribleError : public ParseError { + CLI11_ERROR_DEF(ParseError, HorribleError) + CLI11_ERROR_SIMPLE(HorribleError) +}; + +// After parsing + +/// Thrown when counting a non-existent option +class OptionNotFound : public Error { + CLI11_ERROR_DEF(Error, OptionNotFound) + explicit OptionNotFound(std::string name) : OptionNotFound(name + " not found", ExitCodes::OptionNotFound) {} +}; + +#undef CLI11_ERROR_DEF +#undef CLI11_ERROR_SIMPLE + +/// @} + +} // namespace CLI + +// From CLI/TypeTools.hpp: + +namespace CLI { + +// Type tools + +// Utilities for type enabling +namespace detail { +// Based generally on https://rmf.io/cxx11/almost-static-if +/// Simple empty scoped class +enum class enabler {}; + +/// An instance to use in EnableIf +constexpr enabler dummy = {}; +} // namespace detail + +/// A copy of enable_if_t from C++14, compatible with C++11. +/// +/// We could check to see if C++14 is being used, but it does not hurt to redefine this +/// (even Google does this: https://github.com/google/skia/blob/master/include/private/SkTLogic.h) +/// It is not in the std namespace anyway, so no harm done. +template using enable_if_t = typename std::enable_if::type; + +/// A copy of std::void_t from C++17 (helper for C++11 and C++14) +template struct make_void { using type = void; }; + +/// A copy of std::void_t from C++17 - same reasoning as enable_if_t, it does not hurt to redefine +template using void_t = typename make_void::type; + +/// A copy of std::conditional_t from C++14 - same reasoning as enable_if_t, it does not hurt to redefine +template using conditional_t = typename std::conditional::type; + +/// Check to see if something is a vector (fail check by default) +template struct is_vector : std::false_type {}; + +/// Check to see if something is a vector (true if actually a vector) +template struct is_vector> : std::true_type {}; + +/// Check to see if something is a vector (true if actually a const vector) +template struct is_vector> : std::true_type {}; + +/// Check to see if something is bool (fail check by default) +template struct is_bool : std::false_type {}; + +/// Check to see if something is bool (true if actually a bool) +template <> struct is_bool : std::true_type {}; + +/// Check to see if something is a shared pointer +template struct is_shared_ptr : std::false_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is copyable pointer +template struct is_copyable_ptr { + static bool const value = is_shared_ptr::value || std::is_pointer::value; +}; + +/// This can be specialized to override the type deduction for IsMember. +template struct IsMemberType { using type = T; }; + +/// The main custom type needed here is const char * should be a string. +template <> struct IsMemberType { using type = std::string; }; + +namespace detail { + +// These are utilities for IsMember and other transforming objects + +/// Handy helper to access the element_type generically. This is not part of is_copyable_ptr because it requires that +/// pointer_traits be valid. + +/// not a pointer +template struct element_type { using type = T; }; + +template struct element_type::value>::type> { + using type = typename std::pointer_traits::element_type; +}; + +/// Combination of the element type and value type - remove pointer (including smart pointers) and get the value_type of +/// the container +template struct element_value_type { using type = typename element_type::type::value_type; }; + +/// Adaptor for set-like structure: This just wraps a normal container in a few utilities that do almost nothing. +template struct pair_adaptor : std::false_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } +}; + +/// Adaptor for map-like structure (true version, must have key_type and mapped_type). +/// This wraps a mapped container in a few utilities access it in a general way. +template +struct pair_adaptor< + T, + conditional_t, void>> + : std::true_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::get<0>(std::forward(pair_value))) { + return std::get<0>(std::forward(pair_value)); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::get<1>(std::forward(pair_value))) { + return std::get<1>(std::forward(pair_value)); + } +}; + +// Warning is suppressed due to "bug" in gcc<5.0 and gcc 7.0 with c++17 enabled that generates a Wnarrowing warning +// in the unevaluated context even if the function that was using this wasn't used. The standard says narrowing in +// brace initialization shouldn't be allowed but for backwards compatibility gcc allows it in some contexts. It is a +// little fuzzy what happens in template constructs and I think that was something GCC took a little while to work out. +// But regardless some versions of gcc generate a warning when they shouldn't from the following code so that should be +// suppressed +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif +// check for constructibility from a specific type and copy assignable used in the parse detection +template class is_direct_constructible { + template + static auto test(int, std::true_type) -> decltype( +// NVCC warns about narrowing conversions here +#ifdef __CUDACC__ +#pragma diag_suppress 2361 +#endif + TT { std::declval() } +#ifdef __CUDACC__ +#pragma diag_default 2361 +#endif + , + std::is_move_assignable()); + + template static auto test(int, std::false_type) -> std::false_type; + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0, typename std::is_constructible::type()))::value; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +// Check for output streamability +// Based on https://stackoverflow.com/questions/22758291/how-can-i-detect-if-a-type-can-be-streamed-to-an-stdostream + +template class is_ostreamable { + template + static auto test(int) -> decltype(std::declval() << std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for input streamability +template class is_istreamable { + template + static auto test(int) -> decltype(std::declval() >> std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Templated operation to get a value from a stream +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string &istring, T &obj) { + std::istringstream is; + is.str(istring); + is >> obj; + return !is.fail() && !is.rdbuf()->in_avail(); +} + +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string & /*istring*/, T & /*obj*/) { + return false; +} + +// Check for tuple like types, as in classes with a tuple_size type trait +template class is_tuple_like { + template + // static auto test(int) + // -> decltype(std::conditional<(std::tuple_size::value > 0), std::true_type, std::false_type>::type()); + static auto test(int) -> decltype(std::tuple_size::value, std::true_type{}); + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Convert an object to a string (directly forward if this can become a string) +template ::value, detail::enabler> = detail::dummy> +auto to_string(T &&value) -> decltype(std::forward(value)) { + return std::forward(value); +} + +/// Convert an object to a string (streaming must be supported for that type) +template ::value && is_ostreamable::value, detail::enabler> = + detail::dummy> +std::string to_string(T &&value) { + std::stringstream stream; + stream << value; + return stream.str(); +} + +/// If conversion is not supported, return an empty string (streaming is not supported for that type) +template ::value && !is_ostreamable::value && + !is_vector::type>::type>::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&) { + return std::string{}; +} + +/// convert a vector to a string +template ::value && !is_ostreamable::value && + is_vector::type>::type>::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&variable) { + std::vector defaults; + defaults.reserve(variable.size()); + auto cval = variable.begin(); + auto end = variable.end(); + while(cval != end) { + defaults.emplace_back(CLI::detail::to_string(*cval)); + ++cval; + } + return std::string("[" + detail::join(defaults) + "]"); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +auto checked_to_string(T &&value) -> decltype(to_string(std::forward(value))) { + return to_string(std::forward(value)); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +std::string checked_to_string(T &&) { + return std::string{}; +} +/// get a string as a convertible value for arithmetic types +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(value); +} +/// get a string as a convertible value for enumerations +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(static_cast::type>(value)); +} +/// for other types just use the regular to_string function +template ::value && !std::is_arithmetic::value, detail::enabler> = detail::dummy> +auto value_string(const T &value) -> decltype(to_string(value)) { + return to_string(value); +} + +/// This will only trigger for actual void type +template struct type_count { static const int value{0}; }; + +/// Set of overloads to get the type size of an object +template struct type_count::value>::type> { + static constexpr int value{std::tuple_size::value}; +}; +/// Type size for regular object types that do not look like a tuple +template +struct type_count< + T, + typename std::enable_if::value && !is_tuple_like::value && !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// Type size of types that look like a vector +template struct type_count::value>::type> { + static constexpr int value{is_vector::value ? expected_max_vector_size + : type_count::value}; +}; + +/// This will only trigger for actual void type +template struct expected_count { static const int value{0}; }; + +/// For most types the number of expected items is 1 +template +struct expected_count::value && !std::is_void::value>::type> { + static constexpr int value{1}; +}; +/// number of expected items in a vector +template struct expected_count::value>::type> { + static constexpr int value{expected_max_vector_size}; +}; + +// Enumeration of the different supported categorizations of objects +enum class object_category : int { + integral_value = 2, + unsigned_integral = 4, + enumeration = 6, + boolean_value = 8, + floating_point = 10, + number_constructible = 12, + double_constructible = 14, + integer_constructible = 16, + vector_value = 30, + tuple_value = 35, + // string assignable or greater used in a condition so anything string like must come last + string_assignable = 50, + string_constructible = 60, + other = 200, + +}; + +/// some type that is not otherwise recognized +template struct classify_object { + static constexpr object_category value{object_category::other}; +}; + +/// Set of overloads to classify an object according to type +template +struct classify_object::value && std::is_signed::value && + !is_bool::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::integral_value}; +}; + +/// Unsigned integers +template +struct classify_object< + T, + typename std::enable_if::value && std::is_unsigned::value && !is_bool::value>::type> { + static constexpr object_category value{object_category::unsigned_integral}; +}; + +/// Boolean values +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::boolean_value}; +}; + +/// Floats +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::floating_point}; +}; + +/// String and similar direct assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + std::is_assignable::value && !is_vector::value>::type> { + static constexpr object_category value{object_category::string_assignable}; +}; + +/// String and similar constructible and copy assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && + std::is_constructible::value && !is_vector::value>::type> { + static constexpr object_category value{object_category::string_constructible}; +}; + +/// Enumerations +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::enumeration}; +}; + +/// Handy helper to contain a bunch of checks that rule out many common types (integers, string like, floating point, +/// vectors, and enumerations +template struct uncommon_type { + using type = typename std::conditional::value && !std::is_integral::value && + !std::is_assignable::value && + !std::is_constructible::value && !is_vector::value && + !std::is_enum::value, + std::true_type, + std::false_type>::type; + static constexpr bool value = type::value; +}; + +/// Assignable from double or int +template +struct classify_object::value && type_count::value == 1 && + is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::number_constructible}; +}; + +/// Assignable from int +template +struct classify_object::value && type_count::value == 1 && + !is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::integer_constructible}; +}; + +/// Assignable from double +template +struct classify_object::value && type_count::value == 1 && + is_direct_constructible::value && + !is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::double_constructible}; +}; + +/// Tuple type +template +struct classify_object::value >= 2 && !is_vector::value) || + (is_tuple_like::value && uncommon_type::value && + !is_direct_constructible::value && + !is_direct_constructible::value)>::type> { + static constexpr object_category value{object_category::tuple_value}; +}; + +/// Vector type +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::vector_value}; +}; + +// Type name print + +/// Was going to be based on +/// http://stackoverflow.com/questions/1055452/c-get-name-of-type-in-template +/// But this is cleaner and works better in this case + +template ::value == object_category::integral_value || + classify_object::value == object_category::integer_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "INT"; +} + +template ::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "UINT"; +} + +template ::value == object_category::floating_point || + classify_object::value == object_category::number_constructible || + classify_object::value == object_category::double_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "FLOAT"; +} + +/// Print name for enumeration types +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "ENUM"; +} + +/// Print name for enumeration types +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "BOOLEAN"; +} + +/// Print for all other types +template ::value >= object_category::string_assignable, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "TEXT"; +} + +/// Print name for single element tuple types +template ::value == object_category::tuple_value && type_count::value == 1, + detail::enabler> = detail::dummy> +inline std::string type_name() { + return type_name::type>(); +} + +/// Empty string if the index > tuple size +template +inline typename std::enable_if::value, std::string>::type tuple_name() { + return std::string{}; +} + +/// Recursively generate the tuple type name +template + inline typename std::enable_if < I::value, std::string>::type tuple_name() { + std::string str = std::string(type_name::type>()) + ',' + tuple_name(); + if(str.back() == ',') + str.pop_back(); + return str; +} + +/// Print type name for tuples with 2 or more elements +template ::value == object_category::tuple_value && type_count::value >= 2, + detail::enabler> = detail::dummy> +std::string type_name() { + auto tname = std::string(1, '[') + tuple_name(); + tname.push_back(']'); + return tname; +} + +/// This one should not be used normally, since vector types print the internal type +template ::value == object_category::vector_value, detail::enabler> = detail::dummy> +inline std::string type_name() { + return type_name(); +} + +// Lexical cast + +/// Convert a flag into an integer value typically binary flags +inline int64_t to_flag_value(std::string val) { + static const std::string trueString("true"); + static const std::string falseString("false"); + if(val == trueString) { + return 1; + } + if(val == falseString) { + return -1; + } + val = detail::to_lower(val); + int64_t ret; + if(val.size() == 1) { + if(val[0] >= '1' && val[0] <= '9') { + return (static_cast(val[0]) - '0'); + } + switch(val[0]) { + case '0': + case 'f': + case 'n': + case '-': + ret = -1; + break; + case 't': + case 'y': + case '+': + ret = 1; + break; + default: + throw std::invalid_argument("unrecognized character"); + } + return ret; + } + if(val == trueString || val == "on" || val == "yes" || val == "enable") { + ret = 1; + } else if(val == falseString || val == "off" || val == "no" || val == "disable") { + ret = -1; + } else { + ret = std::stoll(val); + } + return ret; +} + +/// Signed integers +template ::value == object_category::integral_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + try { + std::size_t n = 0; + std::int64_t output_ll = std::stoll(input, &n, 0); + output = static_cast(output_ll); + return n == input.size() && static_cast(output) == output_ll; + } catch(const std::invalid_argument &) { + return false; + } catch(const std::out_of_range &) { + return false; + } +} + +/// Unsigned integers +template ::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(!input.empty() && input.front() == '-') + return false; // std::stoull happily converts negative values to junk without any errors. + + try { + std::size_t n = 0; + std::uint64_t output_ll = std::stoull(input, &n, 0); + output = static_cast(output_ll); + return n == input.size() && static_cast(output) == output_ll; + } catch(const std::invalid_argument &) { + return false; + } catch(const std::out_of_range &) { + return false; + } +} + +/// Boolean values +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + try { + auto out = to_flag_value(input); + output = (out > 0); + return true; + } catch(const std::invalid_argument &) { + return false; + } catch(const std::out_of_range &) { + // if the number is out of the range of a 64 bit value then it is still a number and for this purpose is still + // valid all we care about the sign + output = (input[0] != '-'); + return true; + } +} + +/// Floats +template ::value == object_category::floating_point, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + try { + std::size_t n = 0; + output = static_cast(std::stold(input, &n)); + return n == input.size(); + } catch(const std::invalid_argument &) { + return false; + } catch(const std::out_of_range &) { + return false; + } +} + +/// String and similar direct assignment +template ::value == object_category::string_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = input; + return true; +} + +/// String and similar constructible and copy assignment +template < + typename T, + enable_if_t::value == object_category::string_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T(input); + return true; +} + +/// Enumerations +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename std::underlying_type::type val; + bool retval = detail::lexical_cast(input, val); + if(!retval) { + return false; + } + output = static_cast(val); + return true; +} + +/// Assignable from double or int +template < + typename T, + enable_if_t::value == object_category::number_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val; + if(lexical_cast(input, val)) { + output = T(val); + return true; + } else { + double dval; + if(lexical_cast(input, dval)) { + output = T{dval}; + return true; + } + } + return from_stream(input, output); +} + +/// Assignable from int +template < + typename T, + enable_if_t::value == object_category::integer_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val; + if(lexical_cast(input, val)) { + output = T(val); + return true; + } + return from_stream(input, output); +} + +/// Assignable from double +template < + typename T, + enable_if_t::value == object_category::double_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + double val; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Non-string parsable by a stream +template ::value == object_category::other, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + static_assert(is_istreamable::value, + "option object type must have a lexical cast overload or streaming input operator(>>) defined, if it " + "is convertible from another type use the add_option(...) with XC being the known type"); + return from_stream(input, output); +} + +/// Assign a value through lexical cast operations +template < + typename T, + typename XC, + enable_if_t::value && (classify_object::value == object_category::string_assignable || + classify_object::value == object_category::string_constructible), + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, T &output) { + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && classify_object::value != object_category::string_assignable && + classify_object::value != object_category::string_constructible, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, T &output) { + if(input.empty()) { + output = T{}; + return true; + } + return lexical_cast(input, output); +} + +/// Assign a value converted from a string in lexical cast to the output value directly +template < + typename T, + typename XC, + enable_if_t::value && std::is_assignable::value, detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, T &output) { + XC val{}; + bool parse_result = (!input.empty()) ? lexical_cast(input, val) : true; + if(parse_result) { + output = val; + } + return parse_result; +} + +/// Assign a value from a lexical cast through constructing a value and move assigning it +template ::value && !std::is_assignable::value && + std::is_move_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, T &output) { + XC val{}; + bool parse_result = input.empty() ? true : lexical_cast(input, val); + if(parse_result) { + output = T(val); // use () form of constructor to allow some implicit conversions + } + return parse_result; +} +/// Lexical conversion if there is only one element +template < + typename T, + typename XC, + enable_if_t::value && !is_tuple_like::value && !is_vector::value && !is_vector::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + return lexical_assign(strings[0], output); +} + +/// Lexical conversion if there is only one element but the conversion type is for two call a two element constructor +template ::value == 1 && type_count::value == 2, detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + typename std::tuple_element<0, XC>::type v1; + typename std::tuple_element<1, XC>::type v2; + bool retval = lexical_assign(strings[0], v1); + if(strings.size() > 1) { + retval = retval && lexical_assign(strings[1], v2); + } + if(retval) { + output = T{v1, v2}; + } + return retval; +} + +/// Lexical conversion of a vector types +template ::value == expected_max_vector_size && + expected_count::value == expected_max_vector_size && type_count::value == 1, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + output.clear(); + output.reserve(strings.size()); + for(const auto &elem : strings) { + + output.emplace_back(); + bool retval = lexical_assign(elem, output.back()); + if(!retval) { + return false; + } + } + return (!output.empty()); +} + +/// Lexical conversion of a vector types with type size of two +template ::value == expected_max_vector_size && + expected_count::value == expected_max_vector_size && type_count::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + output.clear(); + for(std::size_t ii = 0; ii < strings.size(); ii += 2) { + + typename std::tuple_element<0, typename XC::value_type>::type v1; + typename std::tuple_element<1, typename XC::value_type>::type v2; + bool retval = lexical_assign(strings[ii], v1); + if(strings.size() > ii + 1) { + retval = retval && lexical_assign(strings[ii + 1], v2); + } + if(retval) { + output.emplace_back(v1, v2); + } else { + return false; + } + } + return (!output.empty()); +} + +/// Conversion to a vector type using a particular single type as the conversion type +template ::value == expected_max_vector_size) && (expected_count::value == 1) && + (type_count::value == 1), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + bool retval = true; + output.clear(); + output.reserve(strings.size()); + for(const auto &elem : strings) { + + output.emplace_back(); + retval = retval && lexical_assign(elem, output.back()); + } + return (!output.empty()) && retval; +} +// This one is last since it can call other lexical_conversion functions +/// Lexical conversion if there is only one element but the conversion type is a vector +template ::value && !is_vector::value && is_vector::value, detail::enabler> = + detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + + if(strings.size() > 1 || (!strings.empty() && !(strings.front().empty()))) { + XC val; + auto retval = lexical_conversion(strings, val); + output = T{val}; + return retval; + } + output = T{}; + return true; +} + +/// function template for converting tuples if the static Index is greater than the tuple size +template +inline typename std::enable_if= type_count::value, bool>::type tuple_conversion(const std::vector &, + T &) { + return true; +} +/// Tuple conversion operation +template + inline typename std::enable_if < + I::value, bool>::type tuple_conversion(const std::vector &strings, T &output) { + bool retval = true; + if(strings.size() > I) { + retval = retval && lexical_assign::type, + typename std::conditional::value, + typename std::tuple_element::type, + XC>::type>(strings[I], std::get(output)); + } + retval = retval && tuple_conversion(strings, output); + return retval; +} + +/// Conversion for tuples +template ::value, detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + static_assert( + !is_tuple_like::value || type_count::value == type_count::value, + "if the conversion type is defined as a tuple it must be the same size as the type you are converting to"); + return tuple_conversion(strings, output); +} + +/// Lexical conversion of a vector types with type_size >2 +template ::value == expected_max_vector_size && + expected_count::value == expected_max_vector_size && (type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, T &output) { + bool retval = true; + output.clear(); + std::vector temp; + std::size_t ii = 0; + std::size_t icount = 0; + std::size_t xcm = type_count::value; + while(ii < strings.size()) { + temp.push_back(strings[ii]); + ++ii; + ++icount; + if(icount == xcm || temp.back().empty()) { + if(static_cast(xcm) == expected_max_vector_size) { + temp.pop_back(); + } + output.emplace_back(); + retval = retval && lexical_conversion(temp, output.back()); + temp.clear(); + if(!retval) { + return false; + } + icount = 0; + } + } + return retval; +} +/// Sum a vector of flag representations +/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is +/// by +/// "-1" an if numbers are passed by some fashion they are captured as well so the function just checks for the most +/// common true and false strings then uses stoll to convert the rest for summing +template ::value && std::is_unsigned::value, detail::enabler> = detail::dummy> +void sum_flag_vector(const std::vector &flags, T &output) { + int64_t count{0}; + for(auto &flag : flags) { + count += detail::to_flag_value(flag); + } + output = (count > 0) ? static_cast(count) : T{0}; +} + +/// Sum a vector of flag representations +/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is +/// by +/// "-1" an if numbers are passed by some fashion they are captured as well so the function just checks for the most +/// common true and false strings then uses stoll to convert the rest for summing +template ::value && std::is_signed::value, detail::enabler> = detail::dummy> +void sum_flag_vector(const std::vector &flags, T &output) { + int64_t count{0}; + for(auto &flag : flags) { + count += detail::to_flag_value(flag); + } + output = static_cast(count); +} + +} // namespace detail +} // namespace CLI + +// From CLI/Split.hpp: + +namespace CLI { +namespace detail { + +// Returns false if not a short option. Otherwise, sets opt name and rest and returns true +inline bool split_short(const std::string ¤t, std::string &name, std::string &rest) { + if(current.size() > 1 && current[0] == '-' && valid_first_char(current[1])) { + name = current.substr(1, 1); + rest = current.substr(2); + return true; + } + return false; +} + +// Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true +inline bool split_long(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 2 && current.substr(0, 2) == "--" && valid_first_char(current[2])) { + auto loc = current.find_first_of('='); + if(loc != std::string::npos) { + name = current.substr(2, loc - 2); + value = current.substr(loc + 1); + } else { + name = current.substr(2); + value = ""; + } + return true; + } + return false; +} + +// Returns false if not a windows style option. Otherwise, sets opt name and value and returns true +inline bool split_windows_style(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 1 && current[0] == '/' && valid_first_char(current[1])) { + auto loc = current.find_first_of(':'); + if(loc != std::string::npos) { + name = current.substr(1, loc - 1); + value = current.substr(loc + 1); + } else { + name = current.substr(1); + value = ""; + } + return true; + } + return false; +} + +// Splits a string into multiple long and short names +inline std::vector split_names(std::string current) { + std::vector output; + std::size_t val; + while((val = current.find(",")) != std::string::npos) { + output.push_back(trim_copy(current.substr(0, val))); + current = current.substr(val + 1); + } + output.push_back(trim_copy(current)); + return output; +} + +/// extract default flag values either {def} or starting with a ! +inline std::vector> get_default_flag_values(const std::string &str) { + std::vector flags = split_names(str); + flags.erase(std::remove_if(flags.begin(), + flags.end(), + [](const std::string &name) { + return ((name.empty()) || (!(((name.find_first_of('{') != std::string::npos) && + (name.back() == '}')) || + (name[0] == '!')))); + }), + flags.end()); + std::vector> output; + output.reserve(flags.size()); + for(auto &flag : flags) { + auto def_start = flag.find_first_of('{'); + std::string defval = "false"; + if((def_start != std::string::npos) && (flag.back() == '}')) { + defval = flag.substr(def_start + 1); + defval.pop_back(); + flag.erase(def_start, std::string::npos); + } + flag.erase(0, flag.find_first_not_of("-!")); + output.emplace_back(flag, defval); + } + return output; +} + +/// Get a vector of short names, one of long names, and a single name +inline std::tuple, std::vector, std::string> +get_names(const std::vector &input) { + + std::vector short_names; + std::vector long_names; + std::string pos_name; + + for(std::string name : input) { + if(name.length() == 0) { + continue; + } + if(name.length() > 1 && name[0] == '-' && name[1] != '-') { + if(name.length() == 2 && valid_first_char(name[1])) + short_names.emplace_back(1, name[1]); + else + throw BadNameString::OneCharName(name); + } else if(name.length() > 2 && name.substr(0, 2) == "--") { + name = name.substr(2); + if(valid_name_string(name)) + long_names.push_back(name); + else + throw BadNameString::BadLongName(name); + } else if(name == "-" || name == "--") { + throw BadNameString::DashesOnly(name); + } else { + if(pos_name.length() > 0) + throw BadNameString::MultiPositionalNames(name); + pos_name = name; + } + } + + return std::tuple, std::vector, std::string>( + short_names, long_names, pos_name); +} + +} // namespace detail +} // namespace CLI + +// From CLI/ConfigFwd.hpp: + +namespace CLI { + +class App; + +/// Holds values to load into Options +struct ConfigItem { + /// This is the list of parents + std::vector parents{}; + + /// This is the name + std::string name{}; + + /// Listing of inputs + std::vector inputs{}; + + /// The list of parents and name joined by "." + std::string fullname() const { + std::vector tmp = parents; + tmp.emplace_back(name); + return detail::join(tmp, "."); + } +}; + +/// This class provides a converter for configuration files. +class Config { + protected: + std::vector items{}; + + public: + /// Convert an app into a configuration + virtual std::string to_config(const App *, bool, bool, std::string) const = 0; + + /// Convert a configuration into an app + virtual std::vector from_config(std::istream &) const = 0; + + /// Get a flag value + virtual std::string to_flag(const ConfigItem &item) const { + if(item.inputs.size() == 1) { + return item.inputs.at(0); + } + throw ConversionError::TooManyInputsFlag(item.fullname()); + } + + /// Parse a config file, throw an error (ParseError:ConfigParseError or FileError) on failure + std::vector from_file(const std::string &name) { + std::ifstream input{name}; + if(!input.good()) + throw FileError::Missing(name); + + return from_config(input); + } + + /// Virtual destructor + virtual ~Config() = default; +}; + +/// This converter works with INI/TOML files; to write proper TOML files use ConfigTOML +class ConfigBase : public Config { + protected: + /// the character used for comments + char commentChar = ';'; + /// the character used to start an array '\0' is a default to not use + char arrayStart = '\0'; + /// the character used to end an array '\0' is a default to not use + char arrayEnd = '\0'; + /// the character used to separate elements in an array + char arraySeparator = ' '; + /// the character used separate the name from the value + char valueDelimiter = '='; + + public: + std::string + to_config(const App * /*app*/, bool default_also, bool write_description, std::string prefix) const override; + + std::vector from_config(std::istream &input) const override; + /// Specify the configuration for comment characters + ConfigBase *comment(char cchar) { + commentChar = cchar; + return this; + } + /// Specify the start and end characters for an array + ConfigBase *arrayBounds(char aStart, char aEnd) { + arrayStart = aStart; + arrayEnd = aEnd; + return this; + } + /// Specify the delimiter character for an array + ConfigBase *arrayDelimiter(char aSep) { + arraySeparator = aSep; + return this; + } + /// Specify the delimiter between a name and value + ConfigBase *valueSeparator(char vSep) { + valueDelimiter = vSep; + return this; + } +}; + +/// the default Config is the INI file format +using ConfigINI = ConfigBase; + +/// ConfigTOML generates a TOML compliant output +class ConfigTOML : public ConfigINI { + + public: + ConfigTOML() { + commentChar = '#'; + arrayStart = '['; + arrayEnd = ']'; + arraySeparator = ','; + valueDelimiter = '='; + } +}; +} // namespace CLI + +// From CLI/Validators.hpp: + +namespace CLI { + +class Option; + +/// @defgroup validator_group Validators + +/// @brief Some validators that are provided +/// +/// These are simple `std::string(const std::string&)` validators that are useful. They return +/// a string if the validation fails. A custom struct is provided, as well, with the same user +/// semantics, but with the ability to provide a new type name. +/// @{ + +/// +class Validator { + protected: + /// This is the description function, if empty the description_ will be used + std::function desc_function_{[]() { return std::string{}; }}; + + /// This is the base function that is to be called. + /// Returns a string error message if validation fails. + std::function func_{[](std::string &) { return std::string{}; }}; + /// The name for search purposes of the Validator + std::string name_{}; + /// A Validator will only apply to an indexed value (-1 is all elements) + int application_index_ = -1; + /// Enable for Validator to allow it to be disabled if need be + bool active_{true}; + /// specify that a validator should not modify the input + bool non_modifying_{false}; + + public: + Validator() = default; + /// Construct a Validator with just the description string + explicit Validator(std::string validator_desc) : desc_function_([validator_desc]() { return validator_desc; }) {} + /// Construct Validator from basic information + Validator(std::function op, std::string validator_desc, std::string validator_name = "") + : desc_function_([validator_de