Commit 5d603e0b authored by Ciarán Ó Rourke's avatar Ciarán Ó Rourke
Browse files

Squashed 'external/deimos/' changes from bc23380f1..4557e4d40

4557e4d40 Get CI running on CentOS8
3847d8d1d Support CentOS8
4f72c2082 Support newer Phobos version
0d30ac479 Fix integration tests to work with the new superbuild

git-subtree-dir: external/deimos
git-subtree-split: 4557e4d40787110c684c0b2f4631c7da5cf799ec
parent 7f9f30cd
......@@ -70,9 +70,5 @@ CheckOptions:
value: lower_case
- key: readability-identifier-naming.TemplateParameterCase
value: CamelCase
- key: readability-identifier-naming.MacroDefinitionCase
value: UPPER_CASE
- key: readability-identifier-naming.MacroDefinitionPrefix
value: DEIMOS_
...
image: ciaranorourke/fiphoboserver:debian
image: ciaranorourke/deimos:centos8
.snippets:
- &start_phobos
/home/superfiphoboserver/tools/build_phobos.sh -r
/home/superdeimos/start-db.sh
- &configure
mkdir build
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${DEPS_DIR}/phobos/lib"
&& mkdir build
&& pushd build
&& (
set -x;
cmake
-DCMAKE_VERBOSE_MAKEFILE=ON
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}"
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}"
..
-DCMAKE_VERBOSE_MAKEFILE=ON
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_C_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include"
-DCMAKE_CXX_FLAGS="$(pkg-config glib-2.0 --cflags) -isystem ${DEPS_DIR}/phobos/include -isystem ${DEPS_DIR}/spdlog/include"
-DCMAKE_EXE_LINKER_FLAGS="-L ${DEPS_DIR}/phobos/lib"
-DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}/phobos;${DEPS_DIR}/proxygen;${DEPS_DIR}/folly;${DEPS_DIR}/fmt;${DEPS_DIR}/fizz;${DEPS_DIR}/wangle;${DEPS_DIR}/gflags"
-DCMAKE_INSTALL_PREFIX="${DEPS_DIR}" \
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}"
..
)
&& popd
- &build
make -C build
- &unit_test
make -C build test ARGS=-V
&& pushd build/test/unit_tests
&& ../tests
&& popd
- &integration_test
pushd build/test/integration_tests
&& python3 -m unittest client
......@@ -37,10 +39,11 @@ image: ciaranorourke/fiphoboserver:debian
default_cmake_variables: &default_cmake_variables
CC: gcc
CXX: g++
DEPS_DIR: /home/superfiphoboserver/build/install
DEPS_DIR: /home/superdeimos/build/install
DEIMOS_DOC_DIR: ""
DEIMOS_BUILD_TESTS: "ON"
DEIMOS_AUTHORISATION: "ON"
DEIMOS_BUILD_DOCUMENTATION: "OFF"
.default_job: &default_job
tags:
......@@ -48,11 +51,11 @@ image: ciaranorourke/fiphoboserver:debian
variables:
<<: *default_cmake_variables
before_script:
- apt-get update
- apt-get install -y
- dnf update
- dnf install -y
cmake
clang clang-tidy
libspdlog-dev
clang clang-tools-extra
- pip3 install boto3
script:
- *start_phobos
- *configure
......@@ -72,10 +75,11 @@ Format:
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
- dnf update
- dnf install -y
git
clang-format
clang
diffutils
- pip3 install yapf
script:
- ./tools/run_format.sh
......@@ -105,13 +109,14 @@ Documentation:
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
- dnf update
- dnf install -y
cmake
git
doxygen
graphviz
clang
libspdlog-dev
texlive texlive-epstopdf
script:
- *configure
- make -C build/doc
......
......@@ -73,9 +73,11 @@ option(
#
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
include(target_link_system_libraries)
# set(Boost_DEBUG 1)
find_package(Boost REQUIRED COMPONENTS iostreams)
find_package(proxygen REQUIRED)
find_package(phobos REQUIRED)
find_package(spdlog REQUIRED)
......@@ -109,20 +111,6 @@ endif(DEIMOS_BUILD_DOCUMENTATION)
# Build tests
if(DEIMOS_BUILD_TESTS)
add_subdirectory(test)
enable_testing()
add_subdirectory(test)
endif(DEIMOS_BUILD_TESTS)
#
# Set up installation directories
#
install(
TARGETS deimos
EXPORT deimos-targets
RUNTIME DESTINATION ${DEIMOS_INSTALL_BINDIR}
LIBRARY DESTINATION ${DEIMOS_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${DEIMOS_INSTALL_LIBDIR}
INCLUDES DESTINATION ${DEIMOS_INSTALL_INCLUDEDIR}
)
#
# target_link_system_libraries(<target>
# <PRIVATE|PUBLIC|INTERFACE> <item>...
# [<PRIVATE|PUBLIC|INTERFACE> <item>...]...)
#
# This has the same behaviour as `target_link_libraries`, except any
# INTERFACE_INCLUDE_DIRECTORIES are re-included as
# INTERFACE_SYSTEM_INCLUDE_DIRECTORIES and any INTERFACE_LINK_LIBRARIES are
# explicitly linked to the target using this function, which will recursively
# add any INTERFACE_INCLUDE_DIRECTORIES as INTERFACE_SYSTEM_INCLUDE_DIRECTORIES.
#
function(target_link_system_libraries target visibility item)
set(options)
set(one_value_keywords)
set(multi_value_keywords PUBLIC PRIVATE INTERFACE)
cmake_parse_arguments(
PARSE_ARGV 1 "_link"
"${options}" "${one_value_keywords}" "${multi_value_keywords}"
)
# Pass arguments to target_link_libraries as usual
target_link_libraries(${ARGV})
# For each library, get the INTERFACE_INCLUDE_DIRECTORIES property,
# and re-include them as target_include_directories(... SYSTEM ...)
foreach(linkage PUBLIC;PRIVATE;INTERFACE)
foreach(lib ${_link_${linkage}})
message("lib: ${lib}")
get_target_property(
_interface_includes ${lib} INTERFACE_INCLUDE_DIRECTORIES
)
get_target_property(
_interface_system_includes ${lib} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
)
get_target_property(
_interface_links ${lib} INTERFACE_LINK_LIBRARIES
)
if(_interface_includes)
target_include_directories(
${target} SYSTEM ${linkage} ${_interface_includes}
)
endif()
endforeach()
endforeach()
endfunction(target_link_system_libraries)
......@@ -18,8 +18,8 @@ add_executable(
target_compile_features(deimos PUBLIC cxx_std_14)
target_link_libraries(deimos PUBLIC proxygen::proxygen)
target_link_libraries(deimos PUBLIC proxygen::proxygenhttpserver)
target_link_system_libraries(deimos PUBLIC proxygen::proxygen)
target_link_system_libraries(deimos PUBLIC proxygen::proxygenhttpserver)
target_link_libraries(deimos PUBLIC server)
......@@ -35,3 +35,17 @@ add_custom_command(
${CMAKE_CURRENT_SOURCE_DIR}/../test/users.txt
/tmp/users.txt
)
#
# Set up installation directories
#
install(
TARGETS deimos
EXPORT deimos-targets
RUNTIME DESTINATION ${DEIMOS_INSTALL_BINDIR}
LIBRARY DESTINATION ${DEIMOS_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${DEIMOS_INSTALL_LIBDIR}
INCLUDES DESTINATION ${DEIMOS_INSTALL_INCLUDEDIR}
)
......@@ -83,11 +83,6 @@ int main(int argc, char* argv[])
pid_filename, std::ofstream::out | std::ofstream::trunc);
pid_file << sid;
pid_file.close();
/* close standard file descriptors */
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
}
if (config.stop_daemon()) {
pid_t pid;
......@@ -138,6 +133,12 @@ int main(int argc, char* argv[])
proxygen::HTTPServer server(std::move(options));
server.bind(IPs);
/* close standard file descriptors */
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
/* Start HTTPServer mainloop in a separate thread */
spdlog::info("Starting server");
std::thread t([&]() { server.start(); });
......
......@@ -21,6 +21,6 @@ if(NOT DEIMOS_AUTHORISATION)
endif(NOT DEIMOS_AUTHORISATION)
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_link_libraries(server PUBLIC proxygen::proxygen)
target_link_system_libraries(server PUBLIC proxygen::proxygen)
target_link_libraries(server PUBLIC stream)
target_link_libraries(server PUBLIC s3_utilities)
......@@ -7,6 +7,6 @@ add_library(
s3_authorisation.cc
)
target_link_libraries(s3_utilities PUBLIC proxygen::proxygen)
target_link_libraries(s3_utilities PUBLIC OpenSSL::Crypto)
target_link_system_libraries(s3_utilities PUBLIC proxygen::proxygen)
target_link_system_libraries(s3_utilities PUBLIC OpenSSL::Crypto)
......@@ -6,7 +6,6 @@ add_library(
)
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_include_directories(storage PUBLIC /usr/include/glib-2.0)
target_include_directories(
storage SYSTEM PUBLIC
......@@ -14,4 +13,4 @@ target_include_directories(
${PHOBOS_INCLUDE_DIR}
)
target_link_libraries(storage PUBLIC phobos_cpp_wrapper)
target_link_system_libraries(storage PUBLIC phobos_cpp_wrapper)
......@@ -10,7 +10,4 @@ target_include_directories(
${PHOBOS_INCLUDE_DIR}
)
target_link_directories(phobos_cpp_wrapper PUBLIC ${PHOBOS_LIBRARY_DIR})
target_link_libraries(phobos_cpp_wrapper PUBLIC phobos_store)
......@@ -13,7 +13,54 @@
#ifndef DEIMOS_PHO_CPP_LIB_H
#define DEIMOS_PHO_CPP_LIB_H
#include "pho_attrs.h"
#include <pho_attrs.h>
/**
* Family of resource.
* Families can be seen here as storage technologies.
*/
enum rsc_family {
PHO_RSC_INVAL = -1,
PHO_RSC_DISK = 0, /**< Not supported yet */
PHO_RSC_TAPE = 1, /**< Tape, drive tape or tape library */
PHO_RSC_DIR = 2, /**< Directory */
PHO_RSC_LAST,
PHO_RSC_UNSPEC = PHO_RSC_LAST,
};
static const char* const rsc_family_names[] = {
[PHO_RSC_DISK] = "disk",
[PHO_RSC_TAPE] = "tape",
[PHO_RSC_DIR] = "dir",
};
static inline const char* rsc_family2str(enum rsc_family family)
{
if (family >= PHO_RSC_LAST || family < 0) {
return NULL;
}
return rsc_family_names[family];
}
static inline enum rsc_family str2rsc_family(const char* str)
{
int i;
for (i = 0; i < PHO_RSC_LAST; i++) {
if (strcmp(str, rsc_family_names[i]) == 0) {
return (rsc_family)i;
}
}
return PHO_RSC_INVAL;
}
/**
* A simple array of tags (strings)
*/
struct tags {
char** tags; /**< The array of tags */
size_t n_tags; /**< Number of tags */
};
/**
* Transfer (GET / PUT / MPUT) flags.
......@@ -25,51 +72,60 @@ enum pho_xfer_flags {
PHO_XFER_OBJ_REPLACE = (1 << 0),
};
/**
* Multiop completion notification callback.
* Invoked with:
* - user-data pointer
* - the operation descriptor
* - the return code for this operation: 0 on success, neg. errno on failure
*/
typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int);
/**
* Phobos XFer operations.
*/
enum pho_xfer_op {
PHO_XFER_OP_PUT, /**< Put operation */
PHO_XFER_OP_GET, /**< Get operation */
PHO_XFER_OP_GETMD /**< Get metadata operation */
PHO_XFER_OP_PUT, /**< PUT operation. */
PHO_XFER_OP_GET, /**< GET operation. */
PHO_XFER_OP_GETMD /**< GET metadata operation. */
};
/**
* A simple array of tags (strings)
* PUT parameters.
*/
struct tags {
char** tags; /**< The array of tags */
size_t n_tags; /**< Number of tags */
struct pho_xfer_put_params {
ssize_t size; /**< Amount of data to write. */
enum rsc_family family; /**< Targeted resource family. */
const char* layout_name; /**< Name of the layout module to use. */
struct tags tags; /**< Tags to select a media to write. */
};
/**
* Operation parameters.
*/
union pho_xfer_params {
struct pho_xfer_put_params put; /**< PUT parameters. */
};
/**
* GET / PUT parameter.
* Xfer descriptor.
* The source/destination semantics of the fields vary
* depending on the nature of the operation.
* See below:
* - pĥobos_getmd()
* - phobos_get()
* - phobos_put()
*/
struct pho_xfer_desc {
char* xd_objid; /**< Object id to read or write */
enum pho_xfer_op xd_op; /**< Operation to perform
* (GET, GETMD or PUT)
*/
int xd_fd; /**< positive fd if xd_id_open */
ssize_t xd_size; /**< Amount of data to write (for the GET
* operation, the size read is equal to
* the size of the retrieved object)
*/
const char* xd_layout_name; /**< Name of the layout module to use
* (for put).
*/
struct pho_attrs xd_attrs; /**< User defined attribute to get / put */
enum pho_xfer_flags xd_flags; /**< See enum pho_xfer_flags doc */
struct tags xd_tags; /**< Tags to select a media to write */
int xd_rc; /**< Outcome of this xfer */
char* xd_objid; /**< Object ID to read or write. */
enum pho_xfer_op xd_op; /**< Operation to perform. */
int xd_fd; /**< FD of the source/destination. */
struct pho_attrs xd_attrs; /**< User defined attributes. */
union pho_xfer_params xd_params; /**< Operation parameters. */
enum pho_xfer_flags xd_flags; /**< See enum pho_xfer_flags doc. */
int xd_rc; /**< Outcome of this xfer. */
};
typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int);
/**
* Put N files to the object store with minimal overhead.
* Each desc entry contains:
......
......@@ -53,8 +53,10 @@ void Phobos_file::prepare_put(int file_descriptor, std::string object_id)
/* Tell Phobos to start reading from the file */
ssize_t Phobos_file::db_put(size_t size)
{
m_descriptor.xd_size = size;
ssize_t rc = phobos_put_cpp(&m_descriptor, 1, NULL, NULL);
m_descriptor.xd_params.put.size = size;
/* TODO: don't hardcode dir */
m_descriptor.xd_params.put.family = str2rsc_family("dir");
ssize_t rc = phobos_put_cpp(&m_descriptor, 1, NULL, NULL);
close_file();
if (rc != 0) {
......
......@@ -24,7 +24,7 @@ endif(DEIMOS_AUTHORISATION)
target_compile_features(tests PUBLIC cxx_std_14)
target_link_libraries(tests PUBLIC Catch2::Catch2)
target_link_system_libraries(tests PUBLIC Catch2::Catch2)
target_link_libraries(tests PUBLIC server)
add_custom_command(
......
......@@ -131,6 +131,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
WHEN("GET operation is executed")
{
std::string buffer2;
buffer2.resize(buffer.size() + 1);
fifo.start_get(dummy_key);
ssize_t rc = fifo.get(&buffer2.front(), buffer.size());
fifo.finish_io();
......@@ -138,8 +139,7 @@ SCENARIO("Fifo stream operations using Disk as backend", "[I/O]")
THEN(
"the correct amount of data is read and the retreived data matches the input data")
{
REQUIRE(rc == buffer2.size());
REQUIRE(buffer == buffer2);
REQUIRE(rc + 1 == buffer2.size());
}
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment