Commit febbe8e2 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber Committed by Pádraig Ó Conbhuí
Browse files

TODO: Sophie rename this commit and change authorship

parent 6fa6f9c5
Pipeline #2150 skipped with stage
# Ignore build directory
build/
......@@ -18,7 +18,7 @@ Build image:
- docker
stage: build
script:
- docker build -t superfiphoboserver:debian -f ci.Dockerfile .
- docker tag superfiphoboserver:debian ciaranorourke/fiphoboserver:debian
- docker push ciaranorourke/fiphoboserver:debian
- docker build -t superdeimos:centos -f Dockerfile .
- docker tag superdeimos:centos ciaranorourke/deimos:centos
- docker push ciaranorourke/deimos:centos
when: manual
cmake_minimum_required(VERSION 3.10)
project(
superfiphoboserver
)
if(NOT DEFINED FIPHOBOSERVER_PREFIX_PATH)
set(FIPHOBOSERVER_PREFIX_PATH "./external/fiphoboserver")
endif()
get_filename_component(PHOBOS_INCLUDE_DIRECTORY ./external/phobos/src/include ABSOLUTE
${CMAKE_CURRENT_SOURCE_DIR})
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
add_subdirectory(${FIPHOBOSERVER_PREFIX_PATH} build/build)
FROM centos:7
RUN yum -y update
COPY . /home/superdeimos
RUN cd /home/superdeimos && ./src/phobos/build.sh -j 1
RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --skip-stages=test --skip=deimos
# RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --only=deimos
# superFIPhoboServer
# SuperDeimos
The easiest (and maybe most convenient) way to build fiphoboserver's dependencies is by creating a Docker image.
When the following code is added to a Dockerfile in the same directory as the superfiphoboserver directory, you can build it with `docker build -t superfiphoboserver:debian .` from that directory.
SuperDeimos is the superbuild project to [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos).
This project contains all dependencies and scripts to build and install them on CentOS 7.
```
FROM debian
## Building
The main targets to be build are [Phobos](https://github.com/cea-hpc/phobos), [Proxygen](https://github.com/facebook/proxygen) and [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos).
In the folder `src` there exist build scripts for each of these targets, while `src/deimos/build.sh` will build and install everything.
The following options exist for all build scripts:
ENV DEBIAN_FRONTEND=noninteractive
| option | default behaviour | description |
|:---:|:---:|:---:|
| `--dockerize` / `--no-dockerize` | no-dockerize | rerun the script in a docker container |
| `--image=` | centos:7 | image for the docker container if `dockerize` is set |
| `-j` | 1 | number of threads for `make`|
| `--skip=` / `--only=` | build all | Comma separated list of targets to skip / build (options are catch2, deimos, fizz, folly, fmt, gflags, googletest, openssl, phobos, proxygen, spdlog, wangle) |
| `--stages=` / `--skip-stages=` | execute all stages | Comma seperated list of stages to execute / skip (Stages are `build` and `test`) |
| `--verbose` / `--no-verbose` | verbose | Set verbosity of build script |
| `--prefix=` | `$PWD/build/install` | installation directory |
| `--namespaced-install` / `--no-namespaced-install` | namespaced install | create named directories in the install directory or install everything directly |
| `--build-static` / `--build-shared` | shared | Build static / shared libraries |
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install git sudo python3-pip
## Development
COPY /path_to_superfiphoboserver /home/superfiphoboserver
RUN /home/superfiphoboserver/tools/build_phobos.sh -c
RUN /home/superfiphoboserver/tools/build_proxygen.sh
RUN /home/superfiphoboserver/tools/install_catch2.sh
For development it is advised to build a docker image with this project and then mount a current Deimos directory in the container when executing the image.
An example [Dockerfile](/Dockerfile) for this purpose is provided in this project.
An image can be built with the command
```
docker build -t deimos:centos .
```
RUN pip3 install boto3
RUN mkdir ~/.aws
RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config
from within the SuperDeimos directory and the container executed with
CMD /home/superfiphoboserver/tools/build_phobos.sh -r
```
docker run -it -v /path/to/local/deimos/dir:/home/superdeimos/external/deimos deimos:centos
```
This will also set up amazon AWS for the s3 tools in the tests folder. The resulting image can then be used as the base image for the [FIPhoboServer](oilgas/ltfs/fiphoboserver) project.
In the container the same scripts can be used to build deimos. An example command can be found in the Dockerfile (it is commented out but can be added when convenient). The build and install directories are `/home/superdeimos/build/build` and `/home/superdeimos/build/install`.
Subtrees are added using the following commands:
For convenience, the path variables should be set to
```
git remote add [repo] [repo-url]
git fetch [repo]
git subtree add --squash -P external/[repo] [repo] [ref]
export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
```
when working in the container interactively. These can also be found in the file [envs.list](/envs.list) and added to the docker container using the [`--env-file`](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file) option.
To setup a Phobos-Postgres database, the functions in the script [`src/phobos/testdb-setup.sh`](/src/phobos/testdb-setup.sh) should be used. An example of how to use that is given in the file [`start-db.sh`](/start-db.sh).
\ No newline at end of file
FROM debian:10.3
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install git cmake sudo wget curl python3-pip \
man clang-tidy clang-format doxygen
COPY . /home/superfiphoboserver
RUN /home/superfiphoboserver/tools/build_phobos.sh -c
RUN /home/superfiphoboserver/tools/build_proxygen.sh
RUN /home/superfiphoboserver/tools/install_catch2.sh
RUN pip3 install boto3
RUN mkdir ~/.aws
RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config
PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
\ No newline at end of file
......@@ -43,6 +43,11 @@ then
yum install -y \
glib2 glib2-devel
yum install -y \
python3-pip
python3 -m pip install boto3
# EPEL Packages
yum install -y \
epel-release
......@@ -199,71 +204,37 @@ then
then
set +o errexit
mkdir -p phobos-scripts
cp -r ${external_dir}/phobos/scripts/ phobos-scripts
ls phobos-scripts
mkdir -p phobos-scripts/src/tests
cp -r ${external_dir}/phobos/src/tests/*.sh phobos-scripts/src/tests/
mkdir -p phobos-scripts/src/cli/tests
cp -r ${external_dir}/phobos/src/cli/tests/*.sh phobos-scripts/src/cli/tests/
# Patch test scripts
sed -i 's/\$PHO_LAYOUTLIB_PATH/$PHO_LAYOUTLIB_PATH:$LD_LIBRARY_PATH/' phobos-scripts/src/cli/tests/run_tests.sh
sed -i 's/\$PHO_PYTHON_PATH/$PHO_PYTHON_PATH:$PYTHONPATH/' phobos-scripts/src/cli/tests/run_tests.sh
sed -i 's/\$PHO_PYTHON_PATH/$PHO_PYTHON_PATH:$PYTHONPATH/' phobos-scripts/src/tests/test_env.sh
source "${script_dir}"/../phobos/testdb-setup.sh
export PATH=$(install_dir phobos)/bin:$(install_dir phobos/sbin):${PATH}
export PYTHONPATH="$(install_dir phobos)"/lib/python2.7/site-packages:"${PYTHONPATH}"
export LD_LIBRARY_PATH="$(install_dir phobos)"/lib:"${LD_LIBRARY_PATH}"
cp ${external_dir}/phobos/src/tests/phobos.conf /etc/
sed -i '/lrs/ a default_family = dir' /etc/phobos.conf
setup_phobos_scripts
#
# Setup postgres
#
test_db_dir=${deimos_build_dir}/test-db
rm -rf "${test_db_dir}"
mkdir -p "${test_db_dir}"
chown -R postgres:postgres "${test_db_dir}"
chmod 0700 "${test_db_dir}"
su postgres -c "initdb ${test_db_dir}"
cp "${script_dir}"/../phobos/pg_hba.conf "${test_db_dir}"/pg_hba.conf
cp "${script_dir}"/../phobos/postgresql.conf "${test_db_dir}"/postgresql.conf
create_database ${deimos_build_dir}
# Boot postgres
su postgres -c "pg_ctl -D ${test_db_dir} start"
setup_phobos_db
export PHOBOS_STORE_layout=simple
export PHOBOS_LAYOUT_RAID1_repl_count=2
# Initialize phobos db
su postgres -c "bash ./phobos-scripts/scripts/phobos_db_local setup_db -s -p phobos"
phobosd
test_phobos_dir=${deimos_build_dir}/test-phobos-dir
rm -rf ${test_phobos_dir}
mkdir -p ${test_phobos_dir}
phobos dir add --unlock ${test_phobos_dir}
phobos dir format --fs posix ${test_phobos_dir}
start_phobos ${deimos_build_dir}
ls
ls test
pushd test/unit_tests
../tests
popd
pushd test/integration_tests
python3 -m unittest client
popd
#make test ARGS=-V
# Shutdown postgres
su postgres -c "pg_ctl -D ${test_db_dir} stop"
stop_postgres
set -o errexit
fi
make install
# make install
popd # deimos_build_dir
fi
......@@ -103,7 +103,7 @@ then
#
# Make phobos
#
make ${MAKE_FLAGS[@]}
make ${MAKE_FLAGS[@]} -j 1
#
# Some pre-test installation
......@@ -111,21 +111,6 @@ then
cp src/tests/phobos.conf /etc/
sed -i '/lrs/ a default_family = dir' /etc/phobos.conf
function setup_postgres_dir() {
local db_dir="$1"
# Setup db dir
mkdir -p "${db_dir}"
chown -R postgres:postgres "${db_dir}"
chmod 0700 "${db_dir}"
set +o errexit
su postgres -c "initdb ${db_dir}"
set -o errexit
cp "${script_dir}"/pg_hba.conf "${db_dir}"/pg_hba.conf
cp "${script_dir}"/postgresql.conf "${db_dir}"/postgresql.conf
}
#
# Test phobos
......@@ -136,22 +121,13 @@ then
sed -i 's/\$PHO_LAYOUTLIB_PATH/$PHO_LAYOUTLIB_PATH:$LD_LIBRARY_PATH/' src/cli/tests/run_tests.sh
sed -i 's/\$PHO_PYTHON_PATH/$PHO_PYTHON_PATH:$PYTHONPATH/' src/cli/tests/run_tests.sh
source "${script_dir}"/testdb-setup.sh
# Setup test db dir
phobos_test_dir="${build_dir}"/phobos/test
test_db_dir="${phobos_test_dir}"/db
rm -rf "${test_db_dir}"
mkdir -p "${test_db_dir}"
setup_postgres_dir "${test_db_dir}"
# Boot postgres
su postgres -c "pg_ctl -D ${test_db_dir} start"
export PHOBOS_STORE_layout=raid1
export PHOBOS_LAYOUT_RAID1_repl_count=2
create_database ${phobos_test_dir}
# Initialize phobos db
su postgres -c "bash ./scripts/phobos_db_local setup_db -s -p phobos"
setup_phobos_db ./scripts
# Run tests
# TODO: enable errexit
......@@ -160,7 +136,7 @@ then
set -o errexit
# Shutdown postgres
su postgres -c "pg_ctl -D ${test_db_dir} stop"
stop_postgres
fi
......
# The functions in this file will setup a testing database with postgres for
# phobos
# It expects, that the variables external_dir and phobos_install_dir are set
# correctly along with PATH, PYTHONPATH and LD_LIBRARY_PATH
setup_phobos_scripts () {
mkdir -p phobos-scripts
cp -r ${external_dir}/phobos/scripts/ phobos-scripts
phobos_script_dir=phobos-scripts/
export PATH=${phobos_install_dir}/bin:${phobos_install_dir}/sbin:${PATH}
export PYTHONPATH="${phobos_install_dir}"/lib/python2.7/site-packages:"${PYTHONPATH}"
export LD_LIBRARY_PATH="${phobos_install_dir}"/lib:"${LD_LIBRARY_PATH}"
cp ${external_dir}/phobos/src/tests/phobos.conf /etc/
sed -i '/lrs/ a default_family = dir' /etc/phobos.conf
}
create_database () {
test_dir=$1
local_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
test_db_dir=${test_dir}/test-db
rm -rf "${test_db_dir}"
mkdir -p "${test_db_dir}"
chown -R postgres:postgres "${test_db_dir}"
chmod 0700 "${test_db_dir}"
su postgres -c "initdb ${test_db_dir}"
cp "${local_script_dir}"/pg_hba.conf "${test_db_dir}"/pg_hba.conf
cp "${local_script_dir}"/postgresql.conf "${test_db_dir}"/postgresql.conf
}
setup_phobos_db () {
su postgres -c "pg_ctl -D ${test_db_dir} start"
export PHOBOS_STORE_layout=raid1
export PHOBOS_LAYOUT_RAID1_repl_count=1
# Initialize phobos db
su postgres -c "bash ${phobos_script_dir}scripts/phobos_db_local setup_db -s -p phobos"
}
start_phobos () {
test_dir=$1
phobosd
test_phobos_dir=${test_dir}/test-phobos-dir
rm -rf ${test_phobos_dir}
mkdir -p ${test_phobos_dir}
phobos dir add --unlock ${test_phobos_dir}
phobos dir format --fs posix ${test_phobos_dir}
}
stop_postgres () {
su postgres -c "pg_ctl -D ${test_db_dir} stop"
}
\ No newline at end of file
......@@ -62,6 +62,30 @@ BOOST_CMAKE_FLAGS+=(
-DBOOST_LIBRARYDIR=/usr/lib64/boost169
)
if test ! -L /usr/include/boost; then
ln -s /usr/include/boost169/boost /usr/include/boost
fi
if test ! -L /usr/lib/libboost_context.so ; then
ln -s /usr/lib64/boost169/libboost_context.so /usr/lib/libboost_context.so
fi
if test ! -L /usr/lib/libboost_filesystem.so ; then
ln -s /usr/lib64/boost169/libboost_filesystem.so /usr/lib/libboost_filesystem.so
fi
if test ! -L /usr/lib/libboost_iostreams.so ; then
ln -s /usr/lib64/boost169/libboost_iostreams.so /usr/lib/libboost_iostreams.so
fi
if test ! -L /usr/lib/libboost_program_options.so ; then
ln -s /usr/lib64/boost169/libboost_program_options.so /usr/lib/libboost_program_options.so
fi
if test ! -L /usr/lib/libboost_regex.so ; then
ln -s /usr/lib64/boost169/libboost_regex.so /usr/lib/libboost_regex.so
fi
if test ! -L /usr/lib/libboost_system.so ; then
ln -s /usr/lib64/boost169/libboost_system.so /usr/lib/libboost_system.so
fi
if test ! -L /usr/lib/libboost_thread.so ; then
ln -s /usr/lib64/boost169/libboost_thread.so /usr/lib/libboost_thread.so
fi
#
......@@ -341,7 +365,9 @@ then
if should_run_stage fizz:test fizz:run:test
then
set +o errexit
make test
set -o errexit
fi
make install
......@@ -437,7 +463,9 @@ then
if should_run_stage proxygen:test
then
set +o errexit
make test
set -o errexit
fi
make install
......
source ./src/phobos/testdb-setup.sh
set -x
# set variables as in build script
test_dir=/home
external_dir=/home/superdeimos/external
phobos_install_dir=/home/superdeimos/build/install/phobos
export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
# call functions for db
setup_phobos_scripts
create_database $test_dir
setup_phobos_db
start_phobos $test_dir
set +x
\ No newline at end of file
#!/bin/bash
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source_dir="${script_dir}/.."
fiphoboserver_source_dir="${source_dir}/external/fiphoboserver"
build_dir="${source_dir}/build/fiphoboserver"
mkdir ${build_dir}
cd ${build_dir}
cmake \
-DFIPHOBOSERVER_BUILD_TESTS=ON \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DCMAKE_PREFIX_PATH="$source_dir/build/install" \
-DFIPHOBOSERVER_BUILD_DOCUMENTATION=ON \
-DCUSTOM_DOCU_PATH="$source_dir/external/doxygen-xml-parser/src" \
${fiphoboserver_source_dir}
make clean
make
#!/bin/bash
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
phobos_dir="${script_dir}/../external/phobos"
# Use the -c flag to only run the compilation part of the script
# Use the -r flag to only run the run part of the script
COMPILE=true
RUN=true
if [[ $# -eq 1 ]]
then
key="$1"
case $key in
-c|--compile)
RUN=false
;;
-r|--run)
COMPILE=false
;;
*)
;;
esac
fi
if [ "$COMPILE" = true ]
then
apt-get install -y \
libglib2.0-dev \
python-dev python-argparse python-yaml python-psycopg2 \
libjansson-dev \
libini-config5 libini-config-dev \
openssl libssh-dev \
libattr1 libattr1-dev \
libsgutils2-dev \
protobuf-c-compiler libprotobuf-c-dev \
clustershell \
automake autogen autoconf autoconf-archive autotools-dev libtool libev-dev pkg-config \
libconfig-dev \
postgresql-all \
python-pip
pip install ClusterShell
ln -s /usr/lib/x86_64-linux-gnu/glib-2.0/include/glibconfig.h /usr/include/glib-2.0/
echo "# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 md5
# IPv6 local connections:
host all all ::1/128 md5"\
| tee /etc/postgresql/11/main/pg_hba.conf
cd ${phobos_dir}
${phobos_dir}/autogen.sh
${phobos_dir}/configure
make
cd ${phobos_dir} && chmod a+rx . .. ../.. ../../.. ./scripts/phobos_db{,_local}
cp ${phobos_dir}/src/tests/phobos.conf /etc/
sed -i '/lrs/ a default_family = dir' /etc/phobos.conf
make install
cd ${phobos_dir}/src/cli && python setup.py install
fi
if [ "$RUN" = true ]
then
pg_ctlcluster 11 main start
export PHOBOS_STORE_layout=raid1
export PHOBOS_LAYOUT_RAID1_repl_count=2
cd ${phobos_dir} && sudo -u postgres bash ./scripts/phobos_db_local setup_db -s -p phobos
phobosd
mkdir /tmp/phobos_test
phobos dir add --unlock /tmp/phobos_test
phobos dir format --fs posix /tmp/phobos_test
fi
#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
## Run this script to build proxygen and run the tests. If you want to
## install proxygen to use in another C++ project on this machine, run
## the sibling file `reinstall.sh`.
# Useful constants
COLOR_RED="\033[0;31m"
COLOR_GREEN="\033[0;32m"
COLOR_OFF="\033[0m"
function install_dependencies() {
apt-get install -y \
git \
cmake \
g++ \
flex \
bison \
libgflags-dev \
libjson-c-dev \
libjson-glib-dev \
libgoogle-glog-dev \
libkrb5-dev \
libsasl2-dev \
libnuma-dev \
pkg-config \
libssl-dev \
libcap-dev \
gperf \
libevent-dev \
libtool \
libboost-all-dev \
libjemalloc-dev \
libsnappy-dev \
wget \
unzip \
google-mock \
libiberty-dev \
libdwarf-dev \
libaio-dev \
libbz2-dev \
liblzma-dev \
libunwind-dev \
make \
liblz4-dev \
zlib1g-dev \
binutils-dev \
libsodium-dev \
libzstd-dev \
libdouble-conversion-dev \
libfmt-dev
}
function setup_folly() {
FOLLY_DIR=$BUILD_DIR/folly
FOLLY_BUILD_DIR=$BUILD_DIR/folly/build/
if [ ! -d "$FOLLY_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning folly repo ${COLOR_OFF}"
git clone https://github.com/facebook/folly.git "$FOLLY_DIR"
fi
cd $FOLLY_DIR
git fetch
FOLLY_REV=$(sed 's/Subproject commit //' "$GITHUB_HASH_DIR"/facebook/folly-rev.txt)