Build on CentOS

Changelog:
* Create new build scripts for deimos, proxygen and phobos
* Add more external dependencies and build them
* Update the documentation, CI and help scripts

.dockerignore

+# Ignore build directory
+build/

.gitlab-ci.yml

@@ -18,7 +18,7 @@ Build image:
         - docker
     stage: build
     script:
-        - docker build -t superfiphoboserver:debian -f ci.Dockerfile .
-        - docker tag superfiphoboserver:debian ciaranorourke/fiphoboserver:debian
-        - docker push ciaranorourke/fiphoboserver:debian
+        - docker build -t superdeimos:centos -f Dockerfile .
+        - docker tag superdeimos:centos ciaranorourke/deimos:centos
+        - docker push ciaranorourke/deimos:centos
     when: manual

CMakeLists.txt

-cmake_minimum_required(VERSION 3.10)
-
-project(
-    superfiphoboserver
-)
-
-if(NOT DEFINED FIPHOBOSERVER_PREFIX_PATH)
-    set(FIPHOBOSERVER_PREFIX_PATH "./external/fiphoboserver")
-endif()
-
-get_filename_component(PHOBOS_INCLUDE_DIRECTORY ./external/phobos/src/include ABSOLUTE
-                       ${CMAKE_CURRENT_SOURCE_DIR})
-
-set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-
-find_package(proxygen REQUIRED)
-
-add_subdirectory(${FIPHOBOSERVER_PREFIX_PATH} build/build)

Dockerfile

+FROM centos:7
+
+RUN yum -y update
+
+COPY . /home/superdeimos
+RUN cd /home/superdeimos && ./src/phobos/build.sh -j 1
+RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --skip-stages=test --skip=deimos
+# RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --only=deimos

README.md

-# superFIPhoboServer
+# SuperDeimos
 
-The easiest (and maybe most convenient) way to build fiphoboserver's dependencies is by creating a Docker image.
-When the following code is added to a Dockerfile in the same directory as the superfiphoboserver directory, you can build it with `docker build -t superfiphoboserver:debian .` from that directory.
+SuperDeimos is the superbuild project to [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos). 
+This project contains all dependencies and scripts to build and install them on CentOS 7. 
 
-```
-FROM debian
+## Building
+
+The main targets to be build are [Phobos](https://github.com/cea-hpc/phobos), [Proxygen](https://github.com/facebook/proxygen) and [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos).
+
+In the folder `src` there exist build scripts for each of these targets, while `src/deimos/build.sh` will build and install everything.
+
+The following options exist for all build scripts:
 
-ENV DEBIAN_FRONTEND=noninteractive
+| option | default behaviour | description |
+|:---:|:---:|:---:|
+| `--dockerize` / `--no-dockerize` | no-dockerize | rerun the script in a docker container |
+| `--image=` | centos:7 | image for the docker container if `dockerize` is set |
+| `-j` | 1 | number of threads for `make`|
+| `--skip=` / `--only=` | build all | Comma separated list of targets to skip / build (options are catch2, deimos, fizz, folly, fmt, gflags, googletest, openssl, phobos, proxygen, spdlog, wangle) |
+| `--stages=` / `--skip-stages=` | execute all stages | Comma seperated list of stages to execute / skip (Stages are `build` and `test`) |
+| `--verbose` / `--no-verbose` | verbose | Set verbosity of build script |
+| `--prefix=` | `$PWD/build/install` | installation directory |
+| `--namespaced-install` / `--no-namespaced-install` | namespaced install | create named directories in the install directory or install everything directly |
+| `--build-static` / `--build-shared` | shared | Build static / shared libraries |
 
-RUN apt-get -y update && apt-get -y upgrade
-RUN apt-get -y install git sudo python3-pip
+## Development
 
-COPY /path_to_superfiphoboserver /home/superfiphoboserver
-RUN /home/superfiphoboserver/tools/build_phobos.sh -c
-RUN /home/superfiphoboserver/tools/build_proxygen.sh
-RUN /home/superfiphoboserver/tools/install_catch2.sh
+For development it is advised to build a docker image with this project and then mount a current Deimos directory in the container when executing the image. 
+
+An example [Dockerfile](/Dockerfile) for this purpose is provided in this project. 
+An image can be built with the command
+
+```
+docker build -t deimos:centos .
+```
 
-RUN pip3 install boto3
-RUN mkdir ~/.aws
-RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
-RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config
+from within the SuperDeimos directory and the container executed with
 
-CMD /home/superfiphoboserver/tools/build_phobos.sh -r
+```
+docker run -it -v /path/to/local/deimos/dir:/home/superdeimos/external/deimos deimos:centos
 ```
 
-This will also set up amazon AWS for the s3 tools in the tests folder. The resulting image can then be used as the base image for the [FIPhoboServer](oilgas/ltfs/fiphoboserver) project.
+In the container the same scripts can be used to build deimos. An example command can be found in the Dockerfile (it is commented out but can be added when convenient). The build and install directories are `/home/superdeimos/build/build` and `/home/superdeimos/build/install`.
 
-Subtrees are added using the following commands:
+For convenience, the path variables should be set to
 
 ```
-git remote add [repo] [repo-url]
-git fetch [repo]
-git subtree add --squash -P external/[repo] [repo] [ref]
+export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
+
+export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
+
 ```
+
+when working in the container interactively. These can also be found in the file [envs.list](/envs.list) and added to the docker container using the [`--env-file`](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file) option.
+
+To setup a Phobos-Postgres database, the functions in the script [`src/phobos/testdb-setup.sh`](/src/phobos/testdb-setup.sh) should be used. An example of how to use that is given in the file [`start-db.sh`](/start-db.sh).
\ No newline at end of file

ci.Dockerfile

-FROM debian:10.3
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get -y update && apt-get -y upgrade
-RUN apt-get -y install git cmake sudo wget curl python3-pip \
-               man clang-tidy clang-format doxygen 
-
-COPY . /home/superfiphoboserver
-RUN /home/superfiphoboserver/tools/build_phobos.sh -c
-RUN /home/superfiphoboserver/tools/build_proxygen.sh
-RUN /home/superfiphoboserver/tools/install_catch2.sh
-
-RUN pip3 install boto3
-RUN mkdir ~/.aws
-RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
-RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config

envs.list

+PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
+PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
\ No newline at end of file

external/catch2/.conan/build.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+from cpt.packager import ConanMultiPackager
+from cpt.ci_manager import CIManager
+from cpt.printer import Printer
+
+
+class BuilderSettings(object):
+    @property
+    def username(self):
+        """ Set catchorg as package's owner
+        """
+        return os.getenv("CONAN_USERNAME", "catchorg")
+
+    @property
+    def login_username(self):
+        """ Set Bintray login username
+        """
+        return os.getenv("CONAN_LOGIN_USERNAME", "horenmar")
+
+    @property
+    def upload(self):
+        """ Set Catch2 repository to be used on upload.
+            The upload server address could be customized by env var
+            CONAN_UPLOAD. If not defined, the method will check the branch name.
+            Only master or CONAN_STABLE_BRANCH_PATTERN will be accepted.
+            The master branch will be pushed to testing channel, because it does
+            not match the stable pattern. Otherwise it will upload to stable
+            channel.
+        """
+        return os.getenv("CONAN_UPLOAD", "https://api.bintray.com/conan/catchorg/catch2")
+
+    @property
+    def upload_only_when_stable(self):
+        """ Force to upload when running over tag branch
+        """
+        return os.getenv("CONAN_UPLOAD_ONLY_WHEN_STABLE", "True").lower() in ["true", "1", "yes"]
+
+    @property
+    def stable_branch_pattern(self):
+        """ Only upload the package the branch name is like a tag
+        """
+        return os.getenv("CONAN_STABLE_BRANCH_PATTERN", r"v\d+\.\d+\.\d+")
+
+    @property
+    def reference(self):
+        """ Read project version from branch create Conan reference
+        """
+        return os.getenv("CONAN_REFERENCE", "Catch2/{}".format(self._version))
+
+    @property
+    def channel(self):
+        """ Default Conan package channel when not stable
+        """
+        return os.getenv("CONAN_CHANNEL", "testing")
+
+    @property
+    def _version(self):
+        """ Get version name from cmake file
+        """
+        pattern = re.compile(r"project\(Catch2 LANGUAGES CXX VERSION (\d+\.\d+\.\d+)\)")
+        version = "latest"
+        with open("CMakeLists.txt") as file:
+            for line in file:
+                result = pattern.search(line)
+                if result:
+                    version = result.group(1)
+        return version
+
+    @property
+    def _branch(self):
+        """ Get branch name from CI manager
+        """
+        printer = Printer(None)
+        ci_manager = CIManager(printer)
+        return ci_manager.get_branch()
+
+
+if __name__ == "__main__":
+    settings = BuilderSettings()
+    builder = ConanMultiPackager(
+        reference=settings.reference,
+        channel=settings.channel,
+        upload=settings.upload,
+        upload_only_when_stable=settings.upload_only_when_stable,
+        stable_branch_pattern=settings.stable_branch_pattern,
+        login_username=settings.login_username,
+        username=settings.username,
+        test_folder=os.path.join(".conan", "test_package"))
+    builder.add()
+    builder.run()

external/catch2/.conan/test_package/CMakeLists.txt

+cmake_minimum_required(VERSION 3.2.0)
+project(test_package CXX)
+
+include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
+conan_basic_setup(TARGETS)
+
+find_package(Catch2 REQUIRED CONFIG)
+
+add_executable(${PROJECT_NAME} test_package.cpp)
+target_link_libraries(${PROJECT_NAME} CONAN_PKG::Catch2)
+set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 11)

external/catch2/.conan/test_package/conanfile.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from conans import ConanFile, CMake
+import os
+
+
+class TestPackageConan(ConanFile):
+    settings = "os", "compiler", "build_type", "arch"
+    generators = "cmake"
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def test(self):
+        assert os.path.isfile(os.path.join(self.deps_cpp_info["Catch2"].rootpath, "licenses", "LICENSE.txt"))
+        bin_path = os.path.join("bin", "test_package")
+        self.run("%s -s" % bin_path, run_environment=True)

external/catch2/.conan/test_package/test_package.cpp

+#define CATCH_CONFIG_MAIN
+
+#include <catch2/catch.hpp>
+
+int Factorial( int number ) {
+    return number <= 1 ? 1 : Factorial( number - 1 ) * number;
+}
+
+TEST_CASE( "Factorial Tests", "[single-file]" ) {
+    REQUIRE( Factorial(0) == 1 );
+    REQUIRE( Factorial(1) == 1 );
+    REQUIRE( Factorial(2) == 2 );
+    REQUIRE( Factorial(3) == 6 );
+    REQUIRE( Factorial(10) == 3628800 );
+}
\ No newline at end of file

external/catch2/.gitattributes

+# This sets the default behaviour, overriding core.autocrlf
+* text=auto
+
+# All source files should have unix line-endings in the repository,
+# but convert to native line-endings on checkout
+*.cpp text
+*.h text
+*.hpp text
+
+# Windows specific files should retain windows line-endings
+*.sln text eol=crlf
+
+# Keep executable scripts with LFs so they can be run after being
+# checked out on Windows
+*.py text eol=lf
+
+
+# Keep the single include header with LFs to make sure it is uploaded,
+# hashed etc with LF
+single_include/**/*.hpp eol=lf
+# Also keep the LICENCE file with LFs for the same reason
+LICENCE.txt eol=lf

external/catch2/.github/FUNDING.yml

+patreon: horenmar

external/catch2/.github/ISSUE_TEMPLATE/bug_report.md

+---
+name: Bug report
+about: Create an issue that documents a bug
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Reproduction steps**
+Steps to reproduce the bug.
+<!-- Usually this means a small and self-contained piece of code that uses Catch and specifying compiler flags if relevant. -->
+
+
+**Platform information:**
+<!-- Fill in any extra information that might be important for your issue. -->
+ - OS: **Windows NT**
+ - Compiler+version: **GCC v2.9.5**
+ - Catch version: **v1.2.3**
+
+
+**Additional context**
+Add any other context about the problem here.

external/catch2/.github/ISSUE_TEMPLATE/feature_request.md

+---
+name: Feature request
+about: Create an issue that requests a feature or other improvement
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Description**
+Describe the feature/change you request and why do you want it.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

external/catch2/.github/pull_request_template.md

+<!--
+Please do not submit pull requests changing the `version.hpp`
+or the single-include `catch.hpp` file, these are changed
+only when a new release is made.
+
+Before submitting a PR you should probably read the contributor documentation
+at docs/contributing.md. It will tell you how to properly test your changes.
+-->
+
+
+## Description
+<!--
+Describe the what and the why of your pull request. Remember that these two
+are usually a bit different. As an example, if you have made various changes
+to decrease the number of new strings allocated, that's what. The why probably
+was that you have a large set of tests and found that this speeds them up.
+-->
+
+## GitHub Issues
+<!-- 
+If this PR was motivated by some existing issues, reference them here.
+
+If it is a simple bug-fix, please also add a line like 'Closes #123'
+to your commit message, so that it is automatically closed.
+If it is not, don't, as it might take several iterations for a feature
+to be done properly. If in doubt, leave it open and reference it in the
+PR itself, so that maintainers can decide.
+-->

external/catch2/.gitignore

+*.build
+*.pbxuser
+*.mode1v3
+*.ncb
+*.suo
+Debug
+Release
+*.user
+*.xcuserstate
+.DS_Store
+xcuserdata
+CatchSelfTest.xcscheme
+Breakpoints.xcbkptlist
+projects/VS2010/TestCatch/_UpgradeReport_Files/
+projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters
+projects/VisualStudio/TestCatch/UpgradeLog.XML
+projects/CMake/.idea
+projects/CMake/cmake-build-debug
+UpgradeLog.XML
+Resources/DWARF
+projects/Generated
+*.pyc
+DerivedData
+*.xccheckout
+Build
+.idea
+.vs
+cmake-build-*
+benchmark-dir
+.conan/test_package/build
+bazel-*

external/catch2/.travis.yml

+language: cpp
+
+branches:
+  except:
+  - /dev-appveyor.*/
+
+common_sources: &all_sources
+  - ubuntu-toolchain-r-test
+  - llvm-toolchain-trusty
+  - llvm-toolchain-trusty-3.9
+  - llvm-toolchain-trusty-4.0
+  - llvm-toolchain-xenial-5.0
+  - llvm-toolchain-xenial-6.0
+
+matrix:
+  include:
+
+    # 1/ Linux Clang Builds
+    - os: linux
+      compiler: clang
+      addons:
+        apt:
+          sources: *all_sources
+          packages: ['clang-3.5']
+      env: COMPILER='clang++-3.5'
+
+    - os: linux
+      compiler: clang
+      addons:
+        apt:
+          sources: *all_sources
+          packages: ['clang-3.6']
+      env: COMPILER='clang++-3.6'
+
+    # Clang 3.7 is intentionally skipped as we cannot get it easily on
+    # TravisCI container
+
+    - os: linux
+      compiler: clang
+      addons:
+        apt:
+          sources: *all_sources
+          packages: ['lcov', 'clang-3.8']
+      env: COMPILER='clang++-3.8'
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-3.9']
+      env: COMPILER='clang++-3.9'
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-4.0']
+      env: COMPILER='clang++-4.0'
+
+    - os: linux
+      dist: xenial
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-5.0']
+      env: COMPILER='clang++-5.0'
+
+    - os: linux
+      dist: xenial
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-6.0']
+      env: COMPILER='clang++-6.0'
+
+    # 2/ Linux GCC Builds
+    - os: linux
+      compiler: gcc
+      addons:
+        apt:
+         sources: *all_sources
+         packages: ['g++-4.8']
+      env: COMPILER='g++-4.8'
+
+    - os: linux
+      compiler: gcc
+      addons:
+        apt:
+          sources: *all_sources
+          packages: ['g++-4.9']
+      env: COMPILER='g++-4.9'
+
+    - os: linux
+      compiler: gcc
+      addons:
+        apt:
+          sources: *all_sources
+          packages: ['g++-5']
+      env: COMPILER='g++-5'
+
+    - os: linux
+      compiler: gcc
+      addons: &gcc6
+        apt:
+          sources: *all_sources
+          packages: ['g++-6']
+      env: COMPILER='g++-6'
+
+    - os: linux
+      compiler: gcc
+      addons: &gcc7
+        apt:
+          sources: *all_sources
+          packages: ['g++-7']
+      env: COMPILER='g++-7'
+
+    - os: linux
+      compiler: gcc
+      addons: &gcc8
+        apt:
+          sources: *all_sources
+          packages: ['g++-8']
+      env: COMPILER='g++-8'
+
+    # 3b/ Linux C++14 Clang builds
+    # Note that we need newer libstdc++ for C++14 support
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              packages: ['clang-3.8', 'libstdc++-6-dev']
+              sources:
+                  - ubuntu-toolchain-r-test
+                  - llvm-toolchain-trusty
+      env: COMPILER='clang++-3.8' CPP14=1
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-3.9', 'libstdc++-6-dev']
+      env: COMPILER='clang++-3.9' CPP14=1
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-4.0', 'libstdc++-6-dev']
+      env: COMPILER='clang++-4.0' CPP14=1
+
+    - os: linux
+      dist: xenial
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-5.0', 'libstdc++-6-dev']
+      env: COMPILER='clang++-5.0' CPP14=1
+
+    - os: linux
+      dist: xenial
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-6.0', 'libstdc++-6-dev']
+      env: COMPILER='clang++-6.0' CPP14=1
+
+
+    # 4a/ Linux C++14 GCC builds
+    - os: linux
+      compiler: gcc
+      addons: *gcc6
+      env: COMPILER='g++-6' CPP14=1
+
+    - os: linux
+      compiler: gcc
+      addons: *gcc7
+      env: COMPILER='g++-7' CPP14=1
+
+    - os: linux
+      compiler: gcc
+      addons: *gcc8
+      env: COMPILER='g++-8' CPP14=1