Commit e6acc9d0 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

external/deimos: Update Deimos to bc23380f1f977bb722fd86b3dc0d40df042d48fb

parents 1f800eb5 7f9f30cd
Pipeline #2161 skipped with stage
......@@ -17,17 +17,21 @@ image: ciaranorourke/fiphoboserver:debian
-DCMAKE_PREFIX_PATH="${DEPS_DIR}"
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_DISABLE_AUTHORISATION="${DEIMOS_DISABLE_AUTHORISATION}"
-DDEIMOS_AUTHORISATION="${DEIMOS_AUTHORISATION}"
..
)
&& popd
- &build
make -C build
- &unit_test
make -C build test ARGS=-V
make -C build test ARGS=-V
&& pushd build/test/unit_tests
&& ../tests
&& popd
- &integration_test
cd test/integration_tests
pushd build/test/integration_tests
&& python3 -m unittest client
&& popd
.cmake_variables:
default_cmake_variables: &default_cmake_variables
......@@ -36,7 +40,7 @@ image: ciaranorourke/fiphoboserver:debian
DEPS_DIR: /home/superfiphoboserver/build/install
DEIMOS_DOC_DIR: ""
DEIMOS_BUILD_TESTS: "ON"
DEIMOS_DISABLE_AUTHORISATION: "OFF"
DEIMOS_AUTHORISATION: "ON"
.default_job: &default_job
tags:
......@@ -107,6 +111,7 @@ Documentation:
doxygen
graphviz
clang
libspdlog-dev
script:
- *configure
- make -C build/doc
......@@ -123,5 +128,5 @@ Authorisation Disabled:
<<: *default_job
variables:
<<: *default_cmake_variables
DEIMOS_DISABLE_AUTHORISATION: "ON"
DEIMOS_AUTHORISATION: "OFF"
stage: test
cmake_minimum_required(VERSION 3.0)
project(deimos)
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
project(
deimos
VERSION 0.0.1
LANGUAGES C CXX
)
#
# Provide options to build or skip building the
# programs/libraries in the given subdirectories
# Set subdirectory options
#
option(
DEIMOS_BUILD_TESTS
"build Deimos tests. Depends on Catch2."
OFF
)
option(
DEIMOS_BUILD_DOCUMENTATION
"build Deimos Documentation. Depends on Doxygen."
OFF
)
#
# Setup installation path options.
#
# Initialy set paths to GNU defaults.
# Users can specify directories using the various DEIMOS_INSTALL_XXX
# variables.
# eg. setting the include install dir
# cmake -DDEIMOS_INSTALL_INCLUDE_DIR=/path/to/include /path/to/deimos
# Get some sane default install directories
include(GNUInstallDirs)
set(
DEIMOS_INSTALL_BINDIR ${CMAKE_INSTALL_BINDIR} CACHE STRING
"The install directory for Deimos executables."
)
set(
DEIMOS_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR} CACHE STRING
"The install directory for Deimos libraries."
)
set(
DEIMOS_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE STRING
"The install directory for Deimos headers."
)
set(
DEIMOS_INSTALL_DATADIR ${CMAKE_INSTALL_DATADIR} CACHE STRING
"The install directory for Deimos data (eg. cmake scripts)."
)
#
# Set build type options
#
option(
DEIMOS_DISABLE_AUTHORISATION
"build Deimos with AWS authorisation disabled. Default is enabled"
OFF
DEIMOS_AUTHORISATION
"build Deimos with AWS authorisation enabled. Default is enabled"
ON
)
add_subdirectory(src)
target_include_directories(
server SYSTEM PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>
#
# Find needed libraries
#
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
# set(Boost_DEBUG 1)
find_package(proxygen REQUIRED)
find_package(phobos REQUIRED)
find_package(spdlog REQUIRED)
#
# Include optional subdirectories
#
include_directories(
SYSTEM PUBLIC ${SPDLOG_INCLUDE_DIR}
)
#
# Setup the Deimos library
#
add_subdirectory(src)
# Output project build options
message(STATUS "Build options:")
message(STATUS " DEIMOS_AUTHORISATION = ${DEIMOS_AUTHORISATION}")
message(STATUS " DEIMOS_BUILD_DOCUMENTATION = ${DEIMOS_BUILD_DOCUMENTATION}")
message(STATUS " DEIMOS_BUILD_TESTS = ${DEIMOS_BUILD_TESTS}")
# Build documentation
if(DEIMOS_BUILD_DOCUMENTATION)
add_subdirectory(doc)
endif(DEIMOS_BUILD_DOCUMENTATION)
# Build tests
if(DEIMOS_BUILD_TESTS)
add_subdirectory(test/unit_tests)
add_subdirectory(test)
enable_testing()
endif(DEIMOS_BUILD_TESTS)
#
# Set up installation directories
#
install(
TARGETS deimos
EXPORT deimos-targets
RUNTIME DESTINATION ${DEIMOS_INSTALL_BINDIR}
LIBRARY DESTINATION ${DEIMOS_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${DEIMOS_INSTALL_LIBDIR}
INCLUDES DESTINATION ${DEIMOS_INSTALL_INCLUDEDIR}
)
MIT License
Copyright (c) 2020 Irish Centre for High End Computing (ICHEC), NUI Galway
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Deimos
Deimos (Delivery Endpoint Interface for Managing Object Storage) is a webserver used to archive data to a [Phobos](https://github.com/cea-hpc/phobos) managed object storage.
Deimos is implemented by using a large subset of the Amazon S3 REST API interface and therefore compatible with most software that uses S3 as an endpoint.
Most importantly, it can be used as an endpoint for Lustre HSM archiving solutions using for example [Estuary](https://git.ichec.ie/oilgas/ltfs/estuary).
## Dependencies Installation Instructions
Deimos has many dependencies that require much patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as;
Deimos has dependencies that require a little patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as;
```
FROM ciaranorourke/deimos:debian
FROM ciaranorourke/deimos:centos
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install doxygen graphviz
RUN apt-get -y install \
[personal installation preferences]
CMD /home/superdeimos/tools/build_phobos.sh -r && [favourite shell]
```
Alternatively, see [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos) for instructions on how to create your own build environment.
It is recommended to run this Docker image by mounting your local branch of the Deimos project with `docker run -it -v /path/to/deimos:/destination [name]:[tag]`
It is recommended to run the Docker image mentioned above by mounting your local branch of the Deimos project with `docker run -it -v /path/to/local/deimos:/home/superdeimos/external/deimos [name]:[tag]`
To use the Phobos raid1 layout which is compatible with FIFOs, the following environment variables must be set;
......@@ -42,16 +46,16 @@ cmake \
[more options below] \
/path/to/deimos/project
cd /path/to/deimos/project
# Build the Deimos project
make
```
make -C build
### Build instructions using the superbuild
# Install the Deimos executable
make -C build install
```
When using the Docker image from [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos), some specific values make life a bit easier:
* docker image should be run with `/home/superdeimos/external/deimos` as `destination` for the mount option (example: `docker run -it -v ./deimos:/home/superdeimos/external/deimos phobos:debian`).
* the `CMAKE_PREFIX_PATH` need to be set to `/home/superdeimos/build/install`
* An example script to build this project can be found inside the container under `/home/superdeimos/tools/build_deimos.sh`.
A build script executing these commands and also building the dependencies can be found in the project [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos).
### Useful Options
......@@ -60,21 +64,30 @@ Option | Effect
------ | ------
`-DCMAKE_CXX_COMPILER=...` | Set the C++ compiler.
`-DCMAKE_CXX_FLAGS=...` | Set the flags to pass to the C++ compiler. Overrides the default flags.
`-DCMAKE_BUILD_TYPE=...` | Set to `Release` or `Debug` for release or debug builds. Sets a number of flags by default.
#### Enabling/Disabling Sections of Deimos
Option | Effect
------ | ------
`-DDEIMOS_BUILD_TESTS=...` | Set to `ON` to build Deimos tests and enable the `make test` target, or `OFF` to skip (Default `OFF`).
`-DDEIMOS_DISABLE_AUTHORISATION=...` | Set to `ON` to disable AWS V4 authorisation (= letting everything through), or `OFF` to authorise requests (Default `OFF`).
`-DDEIMOS_AUTHORISATION=...` | Set to `OFF` to disable AWS V4 authorisation (= letting everything through), or `ON` to authorise requests (Default `ON`).
`-DDEIMOS_BUILD_DOCUMENTATION=...` | Set to `ON` to build Deimos Doxygen documentation, or `OFF` to skip (Default `OFF`).
`-DCUSTOM_DOCU_PATH=...` | Set to the path to the `src` folder in [DoxygenXMLParser](https://git.ichec.ie/sophie.wenzel-teuber/doxygen-xml-parser) to create the Markdown output from the Doxygen XML, that can also be found [here](doc/Markdown/deimos) or don't set to skip (Default `not set`).
#### Setting installation directories
Option | Effect
------ | ------
`-DDEIMOS_INSTALL_PREFIX=...` | Set to the root install directory for the compiled libraries and programs.
`-DDEIMOS_INSTALL_BINDIR=...` | Set the install directory for the `deimos` executable. Use a relative path to set the path relative to `${CMAKE_INSTALL_PREFIX}`. (Default `bin`).
`-DDEIMOS_INSTALL_LIBDIR=...` | Set the install directory for `deimos` libraries. Use a relative path to set the path relative to `${CMAKE_INSTALL_PREFIX}`. (Default `lib`).
## Testing
Units test can be run with `make -C build tests`, given `DEIMOS_BUILD_TESTS` was set to `ON` during the build process.
Integration tests can be run from the `test/integration_tests/` directory with `python -m unittest client`.
## Documentation
The Doxygen documentation can be found [here](doc/Markdown/deimos).
## Usage
......@@ -110,10 +123,14 @@ Since we do not have a proper database for users yet, they are currently stored
Since the authentication is checked for every request, you need to use one of the users and keys from that file for testing the server!
One possible way most SDKs use automatically is the file `~/.aws/credentials`. The tests for the server are done with a file like this:
Either you set the correct credentials using the SDK directly or let the SDK find them automatically using the file `~/.aws/credentials` similar to:
```
[default]
aws_access_key_id = OPEN_KEY
aws_secret_access_key = SECRET_KEY
```
---
Copyright 2020 Irish Centre for High End Computing (ICHEC), NUI Galway.
\ No newline at end of file
include(FindPackageHandleStandardArgs)
# message("Looking for Phobos")
if(NOT PHOBOS_FOUND)
find_path( PHOBOS_INCLUDE_DIR
HINTS ./
../phobos/include/
../include/phobos/
../../phobos/include/
../../include/phobos/
../../../phobos/include/
../../../include/phobos/
/usr/include/
/usr/include/phobos/
/usr/local/include/
/usr/local/include/phobos/
${CMAKE_PREFIX_PATH}/include/phobos/
${CMAKE_PREFIX_PATH}/phobos/include/
${CMAKE_INSTALL_PREFIX}/../include/phobos/
${CMAKE_INSTALL_PREFIX}/../phobos/include/
${CMAKE_INSTALL_PREFIX}/include/phobos/
${CMAKE_INSTALL_PREFIX}/phobos/include/
NAMES phobos_store.h pho_attrs.h
DOC "Phobos headers"
)
find_path( GLIB_INCLUDE_DIR
HINTS ./
../
../glib-2.0/
../include/glib-2.0/
../../
../../glib-2.0/
../../include/glib-2.0/
/usr/include/
/usr/include/glib-2.0/
NAMES glib.h
DOC "Glib header"
)
find_path( GLIB_CONFIG_INCLUDE_DIR
HINTS ./
../
../glib-2.0/
../lib64/glib-2.0/
../../
../../glib-2.0/
../../lib64/glib-2.0/
/usr/include/
/usr/include/glib-2.0/
/usr/lib64/
/usr/lib64/glib-2.0/
/usr/lib64/include/
/usr/lib64/glib-2.0/include/
NAMES glibconfig.h
DOC "Glib config"
)
find_path( PHOBOS_LIBRARY_DIR
HINTS ./
../phobos/lib/
../lib/phobos/
../../phobos/lib/
../../lib/phobos/
../../../phobos/lib/
../../../lib/phobos/
/usr/lib/
/usr/lib/phobos/
/usr/local/lib/
/usr/local/lib/phobos/
${CMAKE_PREFIX_PATH}/lib/phobos/
${CMAKE_PREFIX_PATH}/phobos/lib/
${CMAKE_INSTALL_PREFIX}/../lib/phobos/
${CMAKE_INSTALL_PREFIX}/../phobos/lib/
${CMAKE_INSTALL_PREFIX}/lib/phobos/
${CMAKE_INSTALL_PREFIX}/phobos/lib/
NAMES libphobos_store.so libphobos_store.la
DOC "Phobos library"
)
find_package_handle_standard_args(phobos
REQUIRED_VARS PHOBOS_INCLUDE_DIR PHOBOS_LIBRARY_DIR GLIB_INCLUDE_DIR
GLIB_CONFIG_INCLUDE_DIR
HANDLE_COMPONENTS)
set(GLIB_INCLUDE_DIR ${GLIB_INCLUDE_DIR} ${GLIB_CONFIG_INCLUDE_DIR})
endif()
\ No newline at end of file
include(FindPackageHandleStandardArgs)
# message("Looking for SpdLog")
if(NOT SPDLOG_FOUND)
find_path( SPDLOG_INCLUDE_DIR
HINTS ./
./spdlog/include/
../spdlog/include/
../../spdlog/include/
../../../spdlog/include/
/usr/spdlog/include/
/usr/local/include/
/usr/local/spdlog/include/
${CMAKE_PREFIX_PATH}/include/spdlog/include/
${CMAKE_PREFIX_PATH}/spdlog/include/
${CMAKE_INSTALL_PREFIX}/../include/spdlog/include/
${CMAKE_INSTALL_PREFIX}/../spdlog/include/
${CMAKE_INSTALL_PREFIX}/include/spdlog/include/
${CMAKE_INSTALL_PREFIX}/spdlog/include/
NAMES spdlog/spdlog.h
DOC "SpdLog headers"
)
find_package_handle_standard_args(spdlog
REQUIRED_VARS SPDLOG_INCLUDE_DIR
HANDLE_COMPONENTS)
endif()
\ No newline at end of file
add_library(development_flags INTERFACE)
option(DEIMOS_ENABLE_SANITIZERS "Enable address and undefined behavior sanitizers." ON)
if(DEIMOS_CXX_COMPILER_ID MATCHES "Clang|GNU")
target_compile_options(
development_flags
INTERFACE
-Wall -Wextra -Wpedantic -pedantic
-Wno-error=unknown-pragmas
-Wswitch-enum
-Wimplicit-fallthrough
-Werror
-0g
)
if(DEIMOS_ENABLE_SANITIZERS)
target_compile_options(
development_flags
INTERFACE
-fsanitize=address,undefined
-fno-sanitize=recover=all
-fno-omit-frame-pointer
)
target_link_libraries(
development_flags
INTERFACE
-fsanitize=address,undefined
)
endif(DEIMOS_ENABLE_SANITIZERS)
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
target_compile_options(development_flags INTERFACE -ferror-limit=1)
endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
target_compile_options(development_flags INTERFACE -fmax-errors=1)
endif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
......@@ -14,15 +14,21 @@ add_executable(
main.cc
utils/config.cc
)
)
target_compile_features(deimos PUBLIC cxx_std_14)
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_link_libraries(deimos PUBLIC proxygen::proxygen)
target_link_libraries(deimos PUBLIC proxygen::proxygenhttpserver)
target_link_libraries(deimos PUBLIC server)
if(CMAKE_BUILD_TYPE MATCHES DEBUG)
include(../cmake/development_flags.cmake)
target_link_libraries(deimos PUBLIC development_flags)
endif(CMAKE_BUILD_TYPE MATCHES DEBUG)
add_custom_command(
TARGET deimos POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy
......
/*
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include <cstring>
......
/*
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/executors/GlobalExecutor.h>
#include <folly/init/Init.h>
......
......@@ -9,10 +9,16 @@ add_library(
unsupported_request_handler.cc
)
if(DEIMOS_DISABLE_AUTHORISATION)
target_include_directories(
server SYSTEM PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include>
)
if(NOT DEIMOS_AUTHORISATION)
message("Disabling AWS V4 authorisation for the server")
set_target_properties(server PROPERTIES COMPILE_DEFINITIONS "DEIMOS_NO_AUTHORISATION")
endif(DEIMOS_DISABLE_AUTHORISATION)
endif(NOT DEIMOS_AUTHORISATION)
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_link_libraries(server PUBLIC proxygen::proxygen)
......
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
* * This source code is licensed under the BSD-style license found in the
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
......
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the BSD-style license found in the
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
......
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
* * This source code is licensed under the BSD-style license found in the
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
......
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the BSD-style license found in the
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
......
/*
* Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
* Authors:
* Ciarán O'Rourke <ciaran.orourke@ichec.ie>,
* Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include "get_request_handler.h"
......