Commit 3b98532e authored by Ciarán Ó Rourke's avatar Ciarán Ó Rourke
Browse files

external/deimos: Update Deimos to 4557e4d40787110c684c0b2f4631c7da5cf799ec

parents 287738c6 5d603e0b
...@@ -70,9 +70,5 @@ CheckOptions: ...@@ -70,9 +70,5 @@ CheckOptions:
value: lower_case value: lower_case
- key: readability-identifier-naming.TemplateParameterCase - key: readability-identifier-naming.TemplateParameterCase
value: CamelCase value: CamelCase
- key: readability-identifier-naming.MacroDefinitionCase
value: UPPER_CASE
- key: readability-identifier-naming.MacroDefinitionPrefix
value: DEIMOS_
... ...
image: ciaranorourke/fiphoboserver:debian image: ciaranorourke/deimos:centos8
.snippets: .snippets:
- &start_phobos - &start_phobos
/home/superfiphoboserver/tools/build_phobos.sh -r /home/superdeimos/start-db.sh
- &configure - &configure
mkdir build export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${DEPS_DIR}/phobos/lib"
&& mkdir build
&& pushd build && pushd build
&& ( && (
set -x; set -x;
cmake cmake
-DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_VERBOSE_MAKEFILE=ON
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DCMAKE_C_COMPILER=${CC} -DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX} -DCMAKE_CXX_COMPILER=${CXX}
-DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}" -DCMAKE_C_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}" -DCMAKE_CXX_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include -isystem ${DEPS_DIR}/spdlog/include"
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}" -DCMAKE_EXE_LINKER_FLAGS="-L ${DEPS_DIR}/phobos/lib"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}" -DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}" -DCMAKE_PREFIX_PATH="${DEPS_DIR}/phobos;${DEPS_DIR}/proxygen;${DEPS_DIR}/folly;${DEPS_DIR}/fmt;${DEPS_DIR}/fizz;${DEPS_DIR}/wangle;${DEPS_DIR}/gflags"
.. -DCMAKE_INSTALL_PREFIX="${DEPS_DIR}" \
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}"
..
) )
&& popd && popd
- &build - &build
make -C build make -C build
- &unit_test - &unit_test
make -C build test ARGS=-V make -C build test ARGS=-V
&& pushd build/test/unit_tests
&& ../tests
&& popd
- &integration_test - &integration_test
pushd build/test/integration_tests pushd build/test/integration_tests
&& python3 -m unittest client && python3 -m unittest client
...@@ -37,10 +39,11 @@ image: ciaranorourke/fiphoboserver:debian ...@@ -37,10 +39,11 @@ image: ciaranorourke/fiphoboserver:debian
default_cmake_variables: &default_cmake_variables default_cmake_variables: &default_cmake_variables
CC: gcc CC: gcc
CXX: g++ CXX: g++
DEPS_DIR: /home/superfiphoboserver/build/install DEPS_DIR: /home/superdeimos/build/install
DEIMOS_DOC_DIR: "" DEIMOS_DOC_DIR: ""
DEIMOS_BUILD_TESTS: "ON" DEIMOS_BUILD_TESTS: "ON"
DEIMOS_AUTHORISATION: "ON" DEIMOS_AUTHORISATION: "ON"
DEIMOS_BUILD_DOCUMENTATION: "OFF"
.default_job: &default_job .default_job: &default_job
tags: tags:
...@@ -48,11 +51,11 @@ image: ciaranorourke/fiphoboserver:debian ...@@ -48,11 +51,11 @@ image: ciaranorourke/fiphoboserver:debian
variables: variables:
<<: *default_cmake_variables <<: *default_cmake_variables
before_script: before_script:
- apt-get update - dnf update
- apt-get install -y - dnf install -y
cmake cmake
clang clang-tidy clang clang-tools-extra
libspdlog-dev - pip3 install boto3
script: script:
- *start_phobos - *start_phobos
- *configure - *configure
...@@ -72,10 +75,11 @@ Format: ...@@ -72,10 +75,11 @@ Format:
CC: clang CC: clang
CXX: clang++ CXX: clang++
before_script: before_script:
- apt-get update - dnf update
- apt-get install -y - dnf install -y
git git
clang-format clang
diffutils
- pip3 install yapf - pip3 install yapf
script: script:
- ./tools/run_format.sh - ./tools/run_format.sh
...@@ -105,13 +109,14 @@ Documentation: ...@@ -105,13 +109,14 @@ Documentation:
CC: clang CC: clang
CXX: clang++ CXX: clang++
before_script: before_script:
- apt-get update - dnf update
- apt-get install -y - dnf install -y
cmake cmake
git
doxygen doxygen
graphviz graphviz
clang clang
libspdlog-dev texlive texlive-epstopdf
script: script:
- *configure - *configure
- make -C build/doc - make -C build/doc
......
...@@ -73,9 +73,11 @@ option( ...@@ -73,9 +73,11 @@ option(
# #
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
include(target_link_system_libraries)
# set(Boost_DEBUG 1) # set(Boost_DEBUG 1)
find_package(Boost REQUIRED COMPONENTS iostreams)
find_package(proxygen REQUIRED) find_package(proxygen REQUIRED)
find_package(phobos REQUIRED) find_package(phobos REQUIRED)
find_package(spdlog REQUIRED) find_package(spdlog REQUIRED)
...@@ -109,20 +111,6 @@ endif(DEIMOS_BUILD_DOCUMENTATION) ...@@ -109,20 +111,6 @@ endif(DEIMOS_BUILD_DOCUMENTATION)
# Build tests # Build tests
if(DEIMOS_BUILD_TESTS) if(DEIMOS_BUILD_TESTS)
add_subdirectory(test)
enable_testing() enable_testing()
add_subdirectory(test)
endif(DEIMOS_BUILD_TESTS) endif(DEIMOS_BUILD_TESTS)
#
# Set up installation directories
#
install(
TARGETS deimos
EXPORT deimos-targets
RUNTIME DESTINATION ${DEIMOS_INSTALL_BINDIR}
LIBRARY DESTINATION ${DEIMOS_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${DEIMOS_INSTALL_LIBDIR}
INCLUDES DESTINATION ${DEIMOS_INSTALL_INCLUDEDIR}
)
#
# target_link_system_libraries(<target>
# <PRIVATE|PUBLIC|INTERFACE> <item>...
# [<PRIVATE|PUBLIC|INTERFACE> <item>...]...)
#
# This has the same behaviour as `target_link_libraries`, except any
# INTERFACE_INCLUDE_DIRECTORIES are re-included as
# INTERFACE_SYSTEM_INCLUDE_DIRECTORIES and any INTERFACE_LINK_LIBRARIES are
# explicitly linked to the target using this function, which will recursively
# add any INTERFACE_INCLUDE_DIRECTORIES as INTERFACE_SYSTEM_INCLUDE_DIRECTORIES.
#
function(target_link_system_libraries target visibility item)
set(options)
set(one_value_keywords)
set(multi_value_keywords PUBLIC PRIVATE INTERFACE)
cmake_parse_arguments(
PARSE_ARGV 1 "_link"
"${options}" "${one_value_keywords}" "${multi_value_keywords}"
)
# Pass arguments to target_link_libraries as usual
target_link_libraries(${ARGV})
# For each library, get the INTERFACE_INCLUDE_DIRECTORIES property,
# and re-include them as target_include_directories(... SYSTEM ...)
foreach(linkage PUBLIC;PRIVATE;INTERFACE)
foreach(lib ${_link_${linkage}})
message("lib: ${lib}")
get_target_property(
_interface_includes ${lib} INTERFACE_INCLUDE_DIRECTORIES
)
get_target_property(
_interface_system_includes ${lib} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
)
get_target_property(
_interface_links ${lib} INTERFACE_LINK_LIBRARIES
)
if(_interface_includes)
target_include_directories(
${target} SYSTEM ${linkage} ${_interface_includes}
)
endif()
endforeach()
endforeach()
endfunction(target_link_system_libraries)
...@@ -18,8 +18,8 @@ add_executable( ...@@ -18,8 +18,8 @@ add_executable(
target_compile_features(deimos PUBLIC cxx_std_14) target_compile_features(deimos PUBLIC cxx_std_14)
target_link_libraries(deimos PUBLIC proxygen::proxygen) target_link_system_libraries(deimos PUBLIC proxygen::proxygen)
target_link_libraries(deimos PUBLIC proxygen::proxygenhttpserver) target_link_system_libraries(deimos PUBLIC proxygen::proxygenhttpserver)
target_link_libraries(deimos PUBLIC server) target_link_libraries(deimos PUBLIC server)
...@@ -35,3 +35,17 @@ add_custom_command( ...@@ -35,3 +35,17 @@ add_custom_command(
${CMAKE_CURRENT_SOURCE_DIR}/../test/users.txt ${CMAKE_CURRENT_SOURCE_DIR}/../test/users.txt
/tmp/users.txt /tmp/users.txt
) )
#
# Set up installation directories
#
install(
TARGETS deimos
EXPORT deimos-targets
RUNTIME DESTINATION ${DEIMOS_INSTALL_BINDIR}
LIBRARY DESTINATION ${DEIMOS_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${DEIMOS_INSTALL_LIBDIR}
INCLUDES DESTINATION ${DEIMOS_INSTALL_INCLUDEDIR}
)
...@@ -83,11 +83,6 @@ int main(int argc, char* argv[]) ...@@ -83,11 +83,6 @@ int main(int argc, char* argv[])
pid_filename, std::ofstream::out | std::ofstream::trunc); pid_filename, std::ofstream::out | std::ofstream::trunc);
pid_file << sid; pid_file << sid;
pid_file.close(); pid_file.close();
/* close standard file descriptors */
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
} }
if (config.stop_daemon()) { if (config.stop_daemon()) {
pid_t pid; pid_t pid;
...@@ -138,6 +133,12 @@ int main(int argc, char* argv[]) ...@@ -138,6 +133,12 @@ int main(int argc, char* argv[])
proxygen::HTTPServer server(std::move(options)); proxygen::HTTPServer server(std::move(options));
server.bind(IPs); server.bind(IPs);
/* close standard file descriptors */
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
/* Start HTTPServer mainloop in a separate thread */ /* Start HTTPServer mainloop in a separate thread */
spdlog::info("Starting server"); spdlog::info("Starting server");
std::thread t([&]() { server.start(); }); std::thread t([&]() { server.start(); });
......
...@@ -21,6 +21,6 @@ if(NOT DEIMOS_AUTHORISATION) ...@@ -21,6 +21,6 @@ if(NOT DEIMOS_AUTHORISATION)
endif(NOT DEIMOS_AUTHORISATION) endif(NOT DEIMOS_AUTHORISATION)
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_link_libraries(server PUBLIC proxygen::proxygen) target_link_system_libraries(server PUBLIC proxygen::proxygen)
target_link_libraries(server PUBLIC stream) target_link_libraries(server PUBLIC stream)
target_link_libraries(server PUBLIC s3_utilities) target_link_libraries(server PUBLIC s3_utilities)
...@@ -7,6 +7,6 @@ add_library( ...@@ -7,6 +7,6 @@ add_library(
s3_authorisation.cc s3_authorisation.cc
) )
target_link_libraries(s3_utilities PUBLIC proxygen::proxygen) target_link_system_libraries(s3_utilities PUBLIC proxygen::proxygen)
target_link_libraries(s3_utilities PUBLIC OpenSSL::Crypto) target_link_system_libraries(s3_utilities PUBLIC OpenSSL::Crypto)
...@@ -6,7 +6,6 @@ add_library( ...@@ -6,7 +6,6 @@ add_library(
) )
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1") #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_include_directories(storage PUBLIC /usr/include/glib-2.0)
target_include_directories( target_include_directories(
storage SYSTEM PUBLIC storage SYSTEM PUBLIC
...@@ -14,4 +13,4 @@ target_include_directories( ...@@ -14,4 +13,4 @@ target_include_directories(
${PHOBOS_INCLUDE_DIR} ${PHOBOS_INCLUDE_DIR}
) )
target_link_libraries(storage PUBLIC phobos_cpp_wrapper) target_link_system_libraries(storage PUBLIC phobos_cpp_wrapper)
...@@ -10,7 +10,4 @@ target_include_directories( ...@@ -10,7 +10,4 @@ target_include_directories(
${PHOBOS_INCLUDE_DIR} ${PHOBOS_INCLUDE_DIR}
) )
target_link_directories(phobos_cpp_wrapper PUBLIC ${PHOBOS_LIBRARY_DIR})
target_link_libraries(phobos_cpp_wrapper PUBLIC phobos_store) target_link_libraries(phobos_cpp_wrapper PUBLIC phobos_store)
...@@ -13,7 +13,54 @@ ...@@ -13,7 +13,54 @@
#ifndef DEIMOS_PHO_CPP_LIB_H #ifndef DEIMOS_PHO_CPP_LIB_H
#define DEIMOS_PHO_CPP_LIB_H #define DEIMOS_PHO_CPP_LIB_H
#include "pho_attrs.h" #include <pho_attrs.h>
/**
* Family of resource.
* Families can be seen here as storage technologies.
*/
enum rsc_family {
PHO_RSC_INVAL = -1,
PHO_RSC_DISK = 0, /**< Not supported yet */
PHO_RSC_TAPE = 1, /**< Tape, drive tape or tape library */
PHO_RSC_DIR = 2, /**< Directory */
PHO_RSC_LAST,
PHO_RSC_UNSPEC = PHO_RSC_LAST,
};
static const char* const rsc_family_names[] = {
[PHO_RSC_DISK] = "disk",
[PHO_RSC_TAPE] = "tape",
[PHO_RSC_DIR] = "dir",
};
static inline const char* rsc_family2str(enum rsc_family family)
{
if (family >= PHO_RSC_LAST || family < 0) {
return NULL;
}
return rsc_family_names[family];
}
static inline enum rsc_family str2rsc_family(const char* str)
{
int i;
for (i = 0; i < PHO_RSC_LAST; i++) {
if (strcmp(str, rsc_family_names[i]) == 0) {
return (rsc_family)i;
}
}
return PHO_RSC_INVAL;
}
/**
* A simple array of tags (strings)
*/
struct tags {
char** tags; /**< The array of tags */
size_t n_tags; /**< Number of tags */
};
/** /**
* Transfer (GET / PUT / MPUT) flags. * Transfer (GET / PUT / MPUT) flags.
...@@ -25,51 +72,60 @@ enum pho_xfer_flags { ...@@ -25,51 +72,60 @@ enum pho_xfer_flags {
PHO_XFER_OBJ_REPLACE = (1 << 0), PHO_XFER_OBJ_REPLACE = (1 << 0),
}; };
/**
* Multiop completion notification callback.
* Invoked with:
* - user-data pointer
* - the operation descriptor
* - the return code for this operation: 0 on success, neg. errno on failure
*/
typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int);
/**
* Phobos XFer operations.
*/
enum pho_xfer_op { enum pho_xfer_op {
PHO_XFER_OP_PUT, /**< Put operation */ PHO_XFER_OP_PUT, /**< PUT operation. */
PHO_XFER_OP_GET, /**< Get operation */ PHO_XFER_OP_GET, /**< GET operation. */
PHO_XFER_OP_GETMD /**< Get metadata operation */ PHO_XFER_OP_GETMD /**< GET metadata operation. */
}; };
/** /**
* A simple array of tags (strings) * PUT parameters.
*/ */
struct tags { struct pho_xfer_put_params {
char** tags; /**< The array of tags */ ssize_t size; /**< Amount of data to write. */
size_t n_tags; /**< Number of tags */ enum rsc_family family; /**< Targeted resource family. */
const char* layout_name; /**< Name of the layout module to use. */
struct tags tags; /**< Tags to select a media to write. */
}; };
/**
* Operation parameters.
*/
union pho_xfer_params {
struct pho_xfer_put_params put; /**< PUT parameters. */
};
/** /**
* GET / PUT parameter. * Xfer descriptor.
* The source/destination semantics of the fields vary * The source/destination semantics of the fields vary
* depending on the nature of the operation. * depending on the nature of the operation.
* See below: * See below:
* - pĥobos_getmd()
* - phobos_get() * - phobos_get()
* - phobos_put() * - phobos_put()
*/ */
struct pho_xfer_desc { struct pho_xfer_desc {
char* xd_objid; /**< Object id to read or write */ char* xd_objid; /**< Object ID to read or write. */
enum pho_xfer_op xd_op; /**< Operation to perform enum pho_xfer_op xd_op; /**< Operation to perform. */
* (GET, GETMD or PUT) int xd_fd; /**< FD of the source/destination. */
*/ struct pho_attrs xd_attrs; /**< User defined attributes. */
int xd_fd; /**< positive fd if xd_id_open */ union pho_xfer_params xd_params; /**< Operation parameters. */
ssize_t xd_size; /**< Amount of data to write (for the GET enum pho_xfer_flags xd_flags; /**< See enum pho_xfer_flags doc. */
* operation, the size read is equal to int xd_rc; /**< Outcome of this xfer. */
* the size of the retrieved object)
*/
const char* xd_layout_name; /**< Name of the layout module to use
* (for put).
*/
struct pho_attrs xd_attrs; /**< User defined attribute to get / put */
enum pho_xfer_flags xd_flags; /**< See enum pho_xfer_flags doc */
struct tags xd_tags; /**< Tags to select a media to write */
int xd_rc; /**< Outcome of this xfer */
}; };
typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int);
/** /**
* Put N files to the object store with minimal overhead. * Put N files to the object store with minimal overhead.
* Each desc entry contains: * Each desc entry contains:
......
...@@ -53,8 +53,10 @@ void Phobos_file::prepare_put(int file_descriptor, std::string object_id) ...@@ -53,8 +53,10 @@ void Phobos_file::prepare_put(int file_descriptor, std::string object_id)
/* Tell Phobos to start reading from the file */ /* Tell Phobos to start reading from the file */
ssize_t Phobos_file::db_put(size_t size) ssize_t Phobos_file::db_put(size_t size)
{ {
m_descriptor.xd_size = size; m_descriptor.xd_params.put.size = size;
ssize_t rc = phobos_put_cpp(&m_descriptor, 1, NULL, NULL); /* TODO: don't hardcode dir */
m_descriptor.xd_params.put.family = str2rsc_family("dir");
ssize_t rc = phobos_put_cpp(&m_descriptor, 1, NULL, NULL);
close_file(); close_file();
if (rc != 0) { if (rc != 0) {
......
...@@ -24,7 +24,7 @@ endif(DEIMOS_AUTHORISATION) ...@@ -24,7 +24,7 @@ endif(DEIMOS_AUTHORISATION)
target_compile_features(tests PUBLIC cxx_std_14) target_compile_features(tests PUBLIC cxx_std_14)
target_link_libraries(tests PUBLIC Catch2::Catch2) target_link_system_libraries(tests PUBLIC Catch2::Catch2)
target_link_libraries(tests PUBLIC server) target_link_libraries(tests PUBLIC server)
add_custom_command( add_custom_command(
......
...@@ -131,6 +131,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") ...@@ -131,6 +131,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
WHEN("GET operation is executed") WHEN("GET operation is executed")
{ {
std::string buffer2; std::string buffer2;
buffer2.resize(buffer.size() + 1);
fifo.start_get(dummy_key); fifo.start_get(dummy_key);
ssize_t rc = fifo.get(&buffer2.front(), buffer.size()); ssize_t rc = fifo.get(&buffer2.front(), buffer.size());
fifo.finish_io(); fifo.finish_io();
...@@ -138,8 +139,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]") ...@@ -138,8 +139,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
THEN( THEN(
"the correct amount of data is read and the retreived data matches the input data") "the correct amount of data is read and the retreived data matches the input data")
{ {
REQUIRE(rc == buffer2.size()); REQUIRE(rc + 1 == buffer2.size());
REQUIRE(buffer == buffer2);
} }
} }
} }
......