Commit 88a6e21c authored by Aaron Dees's avatar Aaron Dees
Browse files

Merge branch '8-ci' into 'devel'

Resolve "CI"

See merge request oilgas/ltfs/fiphoboserver!22
parents 070ed9d2 1392c1f9
Pipeline #1199 passed with stages
in 4 minutes and 32 seconds
image: ciaranorourke/fiphoboserver:debian
.snippets:
- &start_phobos
/home/superfiphoboserver/tools/build_phobos.sh -r
- &configure
mkdir build
&& pushd build
&& (
set -x;
cmake
-DCMAKE_VERBOSE_MAKEFILE=ON
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DFIPHOBOSERVER_BUILD_TESTS="${FIPHOBOSERVER_BUILD_TESTS}"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}"
-DCUSTOM_DOCU_PATH="${FIPHOBOSERVER_DOC_DIR}"
..
)
&& popd
- &build
make -C build
- &test
make -C build test ARGS=-V
.cmake_variables:
default_cmake_variables: &default_cmake_variables
DEPS_DIR: /home/superfiphoboserver/build/install
FIPHOBOSERVER_DOC_DIR: ""
FIPHOBOSERVER_BUILD_TESTS: "ON"
.default_job: &default_job
tags:
- docker
image: ciaranorourke/fiphoboserver:debian
before_script:
- apt-get update
- apt-get install -y
cmake
clang clang-tidy
script:
- *configure
- *build
- *test
stages:
- static analysis
- build
- test
Clang Format:
<<: *default_job
stage: static analysis
variables:
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
git
clang-format
script:
- ./tools/run_format.sh
&& git status
&& git diff-index --quiet HEAD
|| { echo "This commit contains unformatted files! Run tools/run_format.sh on the project to format them correctly."; false; }
Clang Tidy:
<<: *default_job
stage: static analysis
variables:
<<: *default_cmake_variables
CC: clang
CXX: clang++
script:
- *configure
- *build
- make -C build/test
- ./tools/run_lint.sh build
Doxygen:
<<: *default_job
stage: build
variables:
<<: *default_cmake_variables
FIPHOBOSERVER_DOC_DIR: /home/superfiphoboserver/external/doxygen-xml-parser/src
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
cmake
doxygen
graphviz
clang
script:
- *configure
- make -C build/doc
Unit Tests:
<<: *default_job
stage: test
variables:
<<: *default_cmake_variables
FIPHOBOSERVER_BUILD_TESTS: "ON"
CC: gcc
CXX: g++
script:
- *start_phobos
- *configure
- *build
- *test
#pragma once
#include <cstring>
#include <exception>
#include <sstream>
#include <cstring>
namespace fiphoboserver {
......@@ -10,8 +10,7 @@ namespace fiphoboserver {
/// @brief exception class for all user defined exceptions in FiPhboServer
///
class FIPhoboServerException : public std::exception {
public:
public:
///
/// @brief default constructor
///
......@@ -20,7 +19,7 @@ public:
///
/// @brief constructor for a message with information on the exception
///
/// @param message message explaining what went wrong such that this
/// @param message message explaining what went wrong such that this
/// exception was thrown.
///
FIPhoboServerException(const char* message) : m_message(message) {}
......@@ -35,8 +34,8 @@ public:
return m_message.c_str();
}
protected:
///
protected:
///
/// @brief the internal message
///
std::string m_message;
......@@ -46,7 +45,7 @@ protected:
/// @brief exceptions specifically for errors in I/O
///
class IOException : public FIPhoboServerException {
public:
public:
///
/// @brief constructor
///
......
......@@ -33,10 +33,11 @@ int main(int argc, char* argv[])
{
folly::init(&argc, &argv, true);
std::vector<proxygen::HTTPServer::IPConfig> IPs = {
{SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP},
{SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2},
};
std::vector<proxygen::HTTPServer::IPConfig> IPs // NOLINT
= {
{SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP},
{SocketAddress(FLAGS_ip, FLAGS_h2_port, true), Protocol::HTTP2},
};
if (FLAGS_threads <= 0) {
FLAGS_threads = sysconf(_SC_NPROCESSORS_ONLN);
......@@ -48,9 +49,9 @@ int main(int argc, char* argv[])
options.idleTimeout = std::chrono::milliseconds(60000);
options.shutdownOn = {SIGINT, SIGTERM};
options.enableContentCompression = false;
options.handlerFactories =
proxygen::RequestHandlerChain().addThen<fiphoboserver::HandlerFactory>()
.build();
options.handlerFactories = proxygen::RequestHandlerChain()
.addThen<fiphoboserver::HandlerFactory>()
.build();
options.h2cEnabled = true;
auto disk_io_thread_pool = std::make_shared<folly::CPUThreadPoolExecutor>(
......
......@@ -4,20 +4,19 @@
#include <algorithm>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <map>
namespace fiphoboserver {
///
/// @brief class to extract the S3 specific information from proxygens
/// @brief class to extract the S3 specific information from proxygens
/// HTTPMessage headers
///
class S3_header {
public:
///
/// @brief sets the internal header instance to the given header object
///
......@@ -65,8 +64,8 @@ class S3_header {
///
/// @brief extracts the name of the key id requested in the HTTP message
///
/// This is the whole path argument without the leading '/' but including
///
/// This is the whole path argument without the leading '/' but including
/// the bucket name!
///
/// @returns name of the key
......@@ -78,8 +77,7 @@ class S3_header {
}
std::string path = m_headers->getPath();
if(path.rfind('/', 0) == 0)
{
if (path.rfind('/', 0) == 0) {
path = path.substr(1, path.size());
}
......@@ -89,10 +87,10 @@ class S3_header {
///
/// @brief checks if the message we got is one for just creating a bucket
///
/// This is convenient since we don't really support buckets themselves and
/// This is convenient since we don't really support buckets themselves and
/// can easily stop working if this returns true
///
/// @returns true if these headers belong to a create bucket request,
/// @returns true if these headers belong to a create bucket request,
/// false for all other requests
///
bool is_create_bucket_request()
......@@ -123,7 +121,7 @@ class S3_header {
///
/// @brief gets the S3 metadata stored in the HTTP headers
///
/// In S3 arbitrary metadata can be defined. This has the form
/// In S3 arbitrary metadata can be defined. This has the form
/// x-amx-meta-KEY: VALUE
/// for for example a KEY:VALUE pair.
///
......
......@@ -21,7 +21,6 @@ namespace fiphoboserver {
///
class GetRequestHandler : public proxygen::RequestHandler {
public:
///
/// @brief first function to be called when a new request comes in
///
......@@ -33,7 +32,7 @@ class GetRequestHandler : public proxygen::RequestHandler {
///
/// @brief function called on every body chunk belonging to this message
///
/// This is not used in this case because GET requests don't usually have a
/// This is not used in this case because GET requests don't usually have a
/// body
///
/// @param body buffer for the body
......@@ -87,7 +86,7 @@ class GetRequestHandler : public proxygen::RequestHandler {
///
/// @brief Constructor for stream class initialization
///
/// @param input_stream @ref stream::Stream class instance to initialize the
/// @param input_stream @ref stream::Stream class instance to initialize the
/// server
///
GetRequestHandler(std::unique_ptr<stream::Stream> input_stream) :
......
......@@ -55,8 +55,9 @@ void GetmdRequestHandler::onRequest(
std::for_each(
m_meta_data.begin(), m_meta_data.end(),
[&](std::pair<std::string, std::string> pair) {
if(pair.first != "creation_time" && pair.first != "bucket")
if (pair.first != "creation_time" && pair.first != "bucket") {
response.header("x-amz-meta-" + pair.first, pair.second);
}
});
response.sendWithEOM();
}
......
......@@ -24,7 +24,6 @@ namespace fiphoboserver {
///
class GetmdRequestHandler : public proxygen::RequestHandler {
public:
///
/// @copydoc GetRequestHandler::onRequest
///
......
......@@ -20,12 +20,11 @@ namespace fiphoboserver {
/// Depending on the method (GET, PUT, HEAD) this class creates an object of the
/// corresponding request handler.
///
/// All other requests result in an object of @ref UnsupportedRequestHandler,
/// All other requests result in an object of @ref UnsupportedRequestHandler,
/// that will then send an 400 Bad request response.
///
///
class HandlerFactory : public proxygen::RequestHandlerFactory {
public:
/// @brief function that gets called when the server starts
void onServerStart(folly::EventBase* /*evb*/) noexcept override {}
......@@ -50,19 +49,16 @@ class HandlerFactory : public proxygen::RequestHandlerFactory {
proxygen::HTTPMessage* headers) noexcept override
{
if (headers->getMethod() == proxygen::HTTPMethod::GET) {
return new GetRequestHandler(
std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
return new GetRequestHandler(std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
}
else if (headers->getMethod() == proxygen::HTTPMethod::PUT) {
return new PutRequestHandler(
std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
return new PutRequestHandler(std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
}
else if (headers->getMethod() == proxygen::HTTPMethod::HEAD) {
return new GetmdRequestHandler(
std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
return new GetmdRequestHandler(std::make_unique<stream::Fifo>(
std::make_unique<storage::Phobos_file>()));
}
else {
return new UnsupportedRequestHandler();
......
......@@ -36,13 +36,12 @@ void PutRequestHandler::onRequest(
return;
}
try {
/*
* Add timestamp to meta data and send it to backend through the stream
/*
* Add timestamp to meta data and send it to backend through the stream
*/
auto metadata = m_s3_header.get_meta_data();
auto metadata = m_s3_header.get_meta_data();
metadata["creation_time"] = std::to_string(std::time(nullptr));
m_stream->set_storage_meta_data(
metadata, m_s3_header.get_bucket());
m_stream->set_storage_meta_data(metadata, m_s3_header.get_bucket());
/*
* Tell stream to coordinate with backend to prepare for PUT operation
......
......@@ -21,7 +21,6 @@ namespace fiphoboserver {
///
class PutRequestHandler : public proxygen::RequestHandler {
public:
///
/// @copydoc GetRequestHandler::onRequest
///
......@@ -59,7 +58,7 @@ class PutRequestHandler : public proxygen::RequestHandler {
/// @copydoc GetRequestHandler::GetRequestHandler
///
PutRequestHandler(std::unique_ptr<stream::Stream> input_stream) :
m_stream(std::move(input_stream))
m_stream(std::move(input_stream))
{
}
......
......@@ -13,7 +13,7 @@ namespace storage {
/// @brief exceptions specifically for the phobos backend library
///
class PhobosException : public FIPhoboServerException {
public:
public:
///
/// @brief constructor
///
......
......@@ -19,8 +19,10 @@ Phobos_file::Phobos_file()
/* Destroy Phobos' m_descriptor struct */
Phobos_file::~Phobos_file()
{
if(m_descriptor.xd_objid)
if (m_descriptor.xd_objid != nullptr) {
delete[] m_descriptor.xd_objid;
m_descriptor.xd_objid = NULL;
}
pho_xfer_desc_destroy_cpp(&m_descriptor);
}
......@@ -32,7 +34,7 @@ void Phobos_file::prepare_put(int file_descriptor, std::string object_id)
m_descriptor.xd_fd = file_descriptor;
char* unconsted_object_id = new char[object_id.length() + 1];
strcpy(unconsted_object_id, object_id.c_str());
strcpy(unconsted_object_id, object_id.c_str()); // NOLINT
m_descriptor.xd_objid = unconsted_object_id;
m_descriptor.xd_op = PHO_XFER_OP_PUT;
......@@ -56,9 +58,9 @@ ssize_t Phobos_file::db_put(size_t size)
*/
void Phobos_file::prepare_get(int file_descriptor, std::string object_id)
{
m_descriptor.xd_fd = file_descriptor;
m_descriptor.xd_fd = file_descriptor;
char* unconsted_object_id = new char[object_id.length() + 1];
strcpy(unconsted_object_id, object_id.c_str());
strcpy(unconsted_object_id, object_id.c_str()); // NOLINT
m_descriptor.xd_objid = unconsted_object_id;
m_descriptor.xd_op = PHO_XFER_OP_GET;
......@@ -93,8 +95,7 @@ void Phobos_file::set_bucket_name(std::string bucket_name)
/* Pass s3 meta data to Phobos */
void Phobos_file::set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name)
std::map<std::string, std::string> meta_data, std::string bucket_name)
{
set_bucket_name(bucket_name);
std::for_each(
......@@ -124,7 +125,7 @@ std::map<std::string, std::string> Phobos_file::get_meta_data(
std::map<std::string, std::string> meta_data;
char* unconsted_object_id = new char[object_id.length() + 1];
strcpy(unconsted_object_id, object_id.c_str());
strcpy(unconsted_object_id, object_id.c_str()); // NOLINT
m_descriptor.xd_objid = unconsted_object_id;
m_descriptor.xd_op = PHO_XFER_OP_GETMD;
......
......@@ -4,71 +4,81 @@ extern "C" {
#include "phobos_cpp_wrapper/phobos_cpp_wrapper.h"
}
#include <string>
#include <map>
#include <string>
namespace fiphoboserver {
namespace storage {
///
///
/// @brief Phobos specific mplementation of the @ref Storage class
///
/// This class uses the phobos library as a backend to store and retrieve files
/// in chunks.
///
class Phobos_file : public Storage {
public:
/// @copydoc Storage::Storage
Phobos_file();
/// @copydoc Storage::~Storage
~Phobos_file();
public:
/// @copydoc Storage::Storage
Phobos_file();
/// @copydoc Storage::~Storage
~Phobos_file();
/// @brief default move contructor
Phobos_file(Phobos_file&& input) :
m_descriptor(std::move(input.m_descriptor))
{
input.m_descriptor.xd_objid = NULL;
}
/// @brief default assignment operator for rvalue references
Phobos_file& operator=(Phobos_file&& input)
{
m_descriptor = std::move(input.m_descriptor);
input.m_descriptor.xd_objid = NULL;
return *this;
}
/// @brief copy constructor deleted
Phobos_file(const Phobos_file&) = delete;
/// @brief assignment operator deleted
Phobos_file& operator=(const Phobos_file&) = delete;
/// @brief default move contructor
Phobos_file(Phobos_file&&) = default;
/// @brief default assignment operator for rvalue references
Phobos_file& operator=(Phobos_file&&) = default;
/// @brief copy constructor deleted
Phobos_file(const Phobos_file&) = delete;
/// @brief assignment operator deleted
Phobos_file& operator=(const Phobos_file&) = delete;
/// @copydoc Storage::set_meta_data
void set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) override;
/// @copydoc Storage::get_meta_data
std::map<std::string, std::string> get_meta_data(
std::string object_id) override;
/// @copydoc Storage::set_meta_data
void set_meta_data(
std::map<std::string, std::string> meta_data,
std::string bucket_name) override;
/// @copydoc Storage::get_meta_data
std::map<std::string, std::string> get_meta_data(
std::string object_id) override;
/// @copydoc Storage::db_put
ssize_t db_put(size_t size) override;
/// @copydoc Storage::db_get
ssize_t db_get() override;
/// @copydoc Storage::db_put
ssize_t db_put(size_t size) override;
/// @copydoc Storage::db_get
ssize_t db_get() override;
/// @copydoc Storage::prepare_put
void prepare_put(int file_descriptor, std::string object_id) override;
/// @copydoc Storage::prepare_get
void prepare_get(int file_descriptor, std::string object_id) override;
/// @copydoc Storage::prepare_put
void prepare_put(int file_descriptor, std::string object_id) override;
/// @copydoc Storage::prepare_get
void prepare_get(int file_descriptor, std::string object_id) override;
int get_fd() const { return m_descriptor.xd_fd; }
int get_fd() const { return m_descriptor.xd_fd; }
private:
/* Implementation specific variables */
private:
/* Implementation specific variables */
/// @brief struct to contain information for phobos
struct pho_xfer_desc m_descriptor = {0};
/// @brief struct to contain information for phobos
struct pho_xfer_desc m_descriptor = {0};
///
/// @brief set the name for the bucket an object blongs to
///
/// @param bucket_name name of the bucket
///
void set_bucket_name(std::string bucket_name);
///
/// @brief set the name for the bucket an object blongs to
///
/// @param bucket_name name of the bucket
///
void set_bucket_name(std::string bucket_name);
///
/// @brief close the file opened for phobos
///
void close_file();
};
///
/// @brief close the file opened for phobos
///
void close_file();
};
} // namespace storage
} // namespace fiphoboserver
} // namespace storage
} // namespace fiphoboserver
#pragma once
#include <string>
#include <map>
#include <string>
namespace fiphoboserver {
///
/// @brief namespace for storage classes that belong to / inherit from from
/// @brief namespace for storage classes that belong to / inherit from from
/// @ref fiphoboserver::storage::Storage
///
namespace storage {
......@@ -14,95 +14,93 @@ namespace storage {
/// @brief virtual storage class to be implemented by backend storage
///
/// This class is a prototype for every class that handles to internal database
/// that should be accessed by the FiPhoboServer.
/// that should be accessed by the FiPhoboServer.
/// This class represents exactly one object file in the internal database
///
class Storage {
public:
public:
///
/// @brief default constructor
///
Storage(){};
///
/// @brief default destructor
///
virtual ~Storage(){};
///
/// @brief default constructor
///
Storage() {};
///
/// @brief default destructor
///
virtual ~Storage() {};
/// @name Metadata functions
///
/// @{
/// @name Metadata functions
///
/// @{
///
/// @brief set the metadata that an object that is added to the
/// database should get
///
/// @param meta_data a map of all key:value pairs that should be