Commit bc23380f authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Update Deimos to new superbuild

Changelog:
* Add Find scripts to CMake to search for phobos and spdlog
* Add integrations tests to cmake by just copying the python code to the build directory
* Update Readme to reflect new SuperDeimos layout and add brief description
parent 24f8b59e
Pipeline #1978 passed with stages
in 8 minutes and 9 seconds
......@@ -24,10 +24,14 @@ image: ciaranorourke/fiphoboserver:debian
- &build
make -C build
- &unit_test
make -C build test ARGS=-V
make -C build test ARGS=-V
&& pushd build/test/unit_tests
&& ../tests
&& popd
- &integration_test
cd test/integration_tests
pushd build/test/integration_tests
&& python3 -m unittest client
&& popd
.cmake_variables:
default_cmake_variables: &default_cmake_variables
......@@ -107,6 +111,7 @@ Documentation:
doxygen
graphviz
clang
libspdlog-dev
script:
- *configure
- make -C build/doc
......
......@@ -28,7 +28,7 @@ option(
#
# Initialy set paths to GNU defaults.
# Users can specify directoryies using the various DEIMOS_INSTALL_XXX
# Users can specify directories using the various DEIMOS_INSTALL_XXX
# variables.
# eg. setting the include install dir
# cmake -DDEIMOS_INSTALL_INCLUDE_DIR=/path/to/include /path/to/deimos
......@@ -72,25 +72,30 @@ option(
# Find needed libraries
#
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
# set(Boost_DEBUG 1)
find_package(proxygen REQUIRED)
find_package(phobos REQUIRED)
find_package(spdlog REQUIRED)
#
# Setup the Deimos library
# Include optional subdirectories
#
add_subdirectory(src)
target_include_directories(
server SYSTEM PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>
include_directories(
SYSTEM PUBLIC ${SPDLOG_INCLUDE_DIR}
)
#
# Include optional subdirectories
# Setup the Deimos library
#
add_subdirectory(src)
# Output project build options
message(STATUS "Build options:")
message(STATUS " DEIMOS_AUTHORISATION = ${DEIMOS_AUTHORISATION}")
......@@ -104,7 +109,7 @@ endif(DEIMOS_BUILD_DOCUMENTATION)
# Build tests
if(DEIMOS_BUILD_TESTS)
add_subdirectory(test/unit_tests)
add_subdirectory(test)
enable_testing()
endif(DEIMOS_BUILD_TESTS)
......
# Deimos
Deimos (Delivery Endpoint Interface for Managing Object Storage) is a webserver used to archive data to a [Phobos](https://github.com/cea-hpc/phobos) managed object storage.
Deimos is implemented by using a large subset of the Amazon S3 REST API interface and therefore compatible with most software that uses S3 as an endpoint.
Most importantly, it can be used as an endpoint for Lustre HSM archiving solutions using for example [Estuary](https://git.ichec.ie/oilgas/ltfs/estuary).
## Dependencies Installation Instructions
Deimos has many dependencies that require much patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as;
Deimos has dependencies that require a little patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as;
```
FROM ciaranorourke/deimos:debian
FROM ciaranorourke/deimos:centos
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install doxygen graphviz libspdlog-dev
RUN apt-get -y install \
[personal installation preferences]
CMD /home/superdeimos/tools/build_phobos.sh -r && [favourite shell]
```
Alternatively, see [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos) for instructions on how to create your own build environment.
It is recommended to run this Docker image by mounting your local branch of the Deimos project with `docker run -it -v /path/to/deimos:/destination [name]:[tag]`
It is recommended to run the Docker image mentioned above by mounting your local branch of the Deimos project with `docker run -it -v /path/to/local/deimos:/home/superdeimos/external/deimos [name]:[tag]`
To use the Phobos raid1 layout which is compatible with FIFOs, the following environment variables must be set;
......@@ -45,18 +49,13 @@ cmake \
cd /path/to/deimos/project
# Build the Deimos project
make -C build install
make -C build
# Install the Deimos executable
make -C build install
```
### Build instructions using the superbuild
When using the Docker image from [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos), some specific values make life a bit easier:
* docker image should be run with `/home/superdeimos/external/deimos` as `destination` for the mount option (example: `docker run -it -v ./deimos:/home/superdeimos/external/deimos phobos:debian`).
* the `CMAKE_PREFIX_PATH` need to be set to `/home/superdeimos/build/install`
* An example script to build this project can be found inside the container under `/home/superdeimos/tools/build_deimos.sh`.
A build script executing these commands and also building the dependencies can be found in the project [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos).
### Useful Options
......@@ -124,7 +123,7 @@ Since we do not have a proper database for users yet, they are currently stored
Since the authentication is checked for every request, you need to use one of the users and keys from that file for testing the server!
One possible way most SDKs use automatically is the file `~/.aws/credentials`. The tests for the server are done with a file like this:
Either you set the correct credentials using the SDK directly or let the SDK find them automatically using the file `~/.aws/credentials` similar to:
```
[default]
......@@ -132,5 +131,6 @@ aws_access_key_id = OPEN_KEY
aws_secret_access_key = SECRET_KEY
```
---
Copyright 2020 Irish Centre for High End Computing (ICHEC), NUI Galway.
\ No newline at end of file
include(FindPackageHandleStandardArgs)
# message("Looking for Phobos")
if(NOT PHOBOS_FOUND)
find_path( PHOBOS_INCLUDE_DIR
HINTS ./
../phobos/include/
../include/phobos/
../../phobos/include/
../../include/phobos/
../../../phobos/include/
../../../include/phobos/
/usr/include/
/usr/include/phobos/
/usr/local/include/
/usr/local/include/phobos/
${CMAKE_PREFIX_PATH}/include/phobos/
${CMAKE_PREFIX_PATH}/phobos/include/
${CMAKE_INSTALL_PREFIX}/../include/phobos/
${CMAKE_INSTALL_PREFIX}/../phobos/include/
${CMAKE_INSTALL_PREFIX}/include/phobos/
${CMAKE_INSTALL_PREFIX}/phobos/include/
NAMES phobos_store.h pho_attrs.h
DOC "Phobos headers"
)
find_path( GLIB_INCLUDE_DIR
HINTS ./
../
../glib-2.0/
../include/glib-2.0/
../../
../../glib-2.0/
../../include/glib-2.0/
/usr/include/
/usr/include/glib-2.0/
NAMES glib.h
DOC "Glib header"
)
find_path( GLIB_CONFIG_INCLUDE_DIR
HINTS ./
../
../glib-2.0/
../lib64/glib-2.0/
../../
../../glib-2.0/
../../lib64/glib-2.0/
/usr/include/
/usr/include/glib-2.0/
/usr/lib64/
/usr/lib64/glib-2.0/
/usr/lib64/include/
/usr/lib64/glib-2.0/include/
NAMES glibconfig.h
DOC "Glib config"
)
find_path( PHOBOS_LIBRARY_DIR
HINTS ./
../phobos/lib/
../lib/phobos/
../../phobos/lib/
../../lib/phobos/
../../../phobos/lib/
../../../lib/phobos/
/usr/lib/
/usr/lib/phobos/
/usr/local/lib/
/usr/local/lib/phobos/
${CMAKE_PREFIX_PATH}/lib/phobos/
${CMAKE_PREFIX_PATH}/phobos/lib/
${CMAKE_INSTALL_PREFIX}/../lib/phobos/
${CMAKE_INSTALL_PREFIX}/../phobos/lib/
${CMAKE_INSTALL_PREFIX}/lib/phobos/
${CMAKE_INSTALL_PREFIX}/phobos/lib/
NAMES libphobos_store.so libphobos_store.la
DOC "Phobos library"
)
find_package_handle_standard_args(phobos
REQUIRED_VARS PHOBOS_INCLUDE_DIR PHOBOS_LIBRARY_DIR GLIB_INCLUDE_DIR
GLIB_CONFIG_INCLUDE_DIR
HANDLE_COMPONENTS)
set(GLIB_INCLUDE_DIR ${GLIB_INCLUDE_DIR} ${GLIB_CONFIG_INCLUDE_DIR})
endif()
\ No newline at end of file
include(FindPackageHandleStandardArgs)
# message("Looking for SpdLog")
if(NOT SPDLOG_FOUND)
find_path( SPDLOG_INCLUDE_DIR
HINTS ./
./spdlog/include/
../spdlog/include/
../../spdlog/include/
../../../spdlog/include/
/usr/spdlog/include/
/usr/local/include/
/usr/local/spdlog/include/
${CMAKE_PREFIX_PATH}/include/spdlog/include/
${CMAKE_PREFIX_PATH}/spdlog/include/
${CMAKE_INSTALL_PREFIX}/../include/spdlog/include/
${CMAKE_INSTALL_PREFIX}/../spdlog/include/
${CMAKE_INSTALL_PREFIX}/include/spdlog/include/
${CMAKE_INSTALL_PREFIX}/spdlog/include/
NAMES spdlog/spdlog.h
DOC "SpdLog headers"
)
find_package_handle_standard_args(spdlog
REQUIRED_VARS SPDLOG_INCLUDE_DIR
HANDLE_COMPONENTS)
endif()
\ No newline at end of file
......@@ -9,6 +9,12 @@ add_library(
unsupported_request_handler.cc
)
target_include_directories(
server SYSTEM PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include>
)
if(NOT DEIMOS_AUTHORISATION)
message("Disabling AWS V4 authorisation for the server")
set_target_properties(server PROPERTIES COMPILE_DEFINITIONS "DEIMOS_NO_AUTHORISATION")
......
......@@ -8,4 +8,10 @@ add_library(
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
target_include_directories(storage PUBLIC /usr/include/glib-2.0)
target_include_directories(
storage SYSTEM PUBLIC
${GLIB_INCLUDE_DIR}
${PHOBOS_INCLUDE_DIR}
)
target_link_libraries(storage PUBLIC phobos_cpp_wrapper)
......@@ -4,6 +4,13 @@ add_library(
phobos_cpp_wrapper.c
)
target_include_directories(phobos_cpp_wrapper PUBLIC /usr/include/glib-2.0)
target_include_directories(
phobos_cpp_wrapper SYSTEM PUBLIC
${GLIB_INCLUDE_DIR}
${PHOBOS_INCLUDE_DIR}
)
target_link_directories(phobos_cpp_wrapper PUBLIC ${PHOBOS_LIBRARY_DIR})
target_link_libraries(phobos_cpp_wrapper PUBLIC phobos_store)
# Unit tests
add_subdirectory(unit_tests)
# Integration tests
add_custom_target(integration_tests ALL)
add_custom_command(TARGET integration_tests POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
${CMAKE_CURRENT_SOURCE_DIR}/integration_tests ${CMAKE_CURRENT_BINARY_DIR}/integration_tests)
\ No newline at end of file
......@@ -24,7 +24,7 @@ class Server_test_case(unittest.TestCase):
cls.client = s3_client.S3_client('http://localhost:11000/')
# start the server daemon
subprocess.run("../../build/deimos start localhost", shell=True)
subprocess.run("../../deimos start localhost", shell=True)
cls.put_file_name = 'data/EmperorWu.txt'
cls.get_file_name = 'data/EmperorWu_get.txt'
......@@ -37,8 +37,8 @@ class Server_test_case(unittest.TestCase):
os.remove(cls.get_file_name)
os.remove(cls.get_empty_file_name)
# start the server daemon
subprocess.run("../../build/deimos stop", shell=True)
# stop the server daemon
subprocess.run("../../deimos stop", shell=True)
def test_file_put_and_get(self):
self.client.put(self.put_file_name, self.bucket_name, self.key)
......
......@@ -15,11 +15,11 @@ def files_are_equal(filename1, filename2):
if not os.path.isfile(filename2):
return False, "Error: File not found: " + filename2
file1 = open(filename1)
file1 = open(filename1, encoding="utf-8")
data1 = file1.read()
file1.close()
file2 = open(filename2)
file2 = open(filename2, encoding="utf-8")
data2 = file2.read()
file2.close()
......
......@@ -13,7 +13,8 @@ import sys
class S3_client:
def __init__(self, server_url):
self.session = boto3.session.Session()
self.session = boto3.session.Session(
aws_access_key_id="OPEN_KEY", aws_secret_access_key="SECRET_KEY")
self.client = self.session.client(service_name='s3',
endpoint_url=server_url)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment