Commit 7efed712 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Build on CentOS

Changelog:
* Create new build scripts for deimos, proxygen and phobos
* Add more external dependencies and build them
* Update the documentation, CI and help scripts
parent 48a1a6c0
Pipeline #1985 skipped with stage
# Ignore build directory
build/
......@@ -18,7 +18,7 @@ Build image:
- docker
stage: build
script:
- docker build -t superfiphoboserver:debian -f ci.Dockerfile .
- docker tag superfiphoboserver:debian ciaranorourke/fiphoboserver:debian
- docker push ciaranorourke/fiphoboserver:debian
- docker build -t superdeimos:centos -f Dockerfile .
- docker tag superdeimos:centos ciaranorourke/deimos:centos
- docker push ciaranorourke/deimos:centos
when: manual
cmake_minimum_required(VERSION 3.10)
project(
superfiphoboserver
)
if(NOT DEFINED FIPHOBOSERVER_PREFIX_PATH)
set(FIPHOBOSERVER_PREFIX_PATH "./external/fiphoboserver")
endif()
get_filename_component(PHOBOS_INCLUDE_DIRECTORY ./external/phobos/src/include ABSOLUTE
${CMAKE_CURRENT_SOURCE_DIR})
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
add_subdirectory(${FIPHOBOSERVER_PREFIX_PATH} build/build)
FROM centos:7
RUN yum -y update
COPY . /home/superdeimos
RUN cd /home/superdeimos && ./src/phobos/build.sh -j 1
RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --skip-stages=test --skip=deimos
# RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4 --only=deimos
# superFIPhoboServer
# SuperDeimos
The easiest (and maybe most convenient) way to build fiphoboserver's dependencies is by creating a Docker image.
When the following code is added to a Dockerfile in the same directory as the superfiphoboserver directory, you can build it with `docker build -t superfiphoboserver:debian .` from that directory.
SuperDeimos is the superbuild project to [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos).
This project contains all dependencies and scripts to build and install them on CentOS 7.
```
FROM debian
## Building
The main targets to be build are [Phobos](https://github.com/cea-hpc/phobos), [Proxygen](https://github.com/facebook/proxygen) and [Deimos](https://git.ichec.ie/oilgas/ltfs/deimos).
In the folder `src` there exist build scripts for each of these targets, while `src/deimos/build.sh` will build and install everything.
The following options exist for all build scripts:
ENV DEBIAN_FRONTEND=noninteractive
| option | default behaviour | description |
|:---:|:---:|:---:|
| `--dockerize` / `--no-dockerize` | no-dockerize | rerun the script in a docker container |
| `--image=` | centos:7 | image for the docker container if `dockerize` is set |
| `-j` | 1 | number of threads for `make`|
| `--skip=` / `--only=` | build all | Comma separated list of targets to skip / build (options are catch2, deimos, fizz, folly, fmt, gflags, googletest, openssl, phobos, proxygen, spdlog, wangle) |
| `--stages=` / `--skip-stages=` | execute all stages | Comma seperated list of stages to execute / skip (Stages are `build` and `test`) |
| `--verbose` / `--no-verbose` | verbose | Set verbosity of build script |
| `--prefix=` | `$PWD/build/install` | installation directory |
| `--namespaced-install` / `--no-namespaced-install` | namespaced install | create named directories in the install directory or install everything directly |
| `--build-static` / `--build-shared` | shared | Build static / shared libraries |
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install git sudo python3-pip
## Development
COPY /path_to_superfiphoboserver /home/superfiphoboserver
RUN /home/superfiphoboserver/tools/build_phobos.sh -c
RUN /home/superfiphoboserver/tools/build_proxygen.sh
RUN /home/superfiphoboserver/tools/install_catch2.sh
For development it is advised to build a docker image with this project and then mount a current Deimos directory in the container when executing the image.
An example [Dockerfile](/Dockerfile) for this purpose is provided in this project.
An image can be built with the command
```
docker build -t deimos:centos .
```
RUN pip3 install boto3
RUN mkdir ~/.aws
RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config
from within the SuperDeimos directory and the container executed with
CMD /home/superfiphoboserver/tools/build_phobos.sh -r
```
docker run -it -v /path/to/local/deimos/dir:/home/superdeimos/external/deimos deimos:centos
```
This will also set up amazon AWS for the s3 tools in the tests folder. The resulting image can then be used as the base image for the [FIPhoboServer](oilgas/ltfs/fiphoboserver) project.
In the container the same scripts can be used to build deimos. An example command can be found in the Dockerfile (it is commented out but can be added when convenient). The build and install directories are `/home/superdeimos/build/build` and `/home/superdeimos/build/install`.
Subtrees are added using the following commands:
For convenience, the path variables should be set to
```
git remote add [repo] [repo-url]
git fetch [repo]
git subtree add --squash -P external/[repo] [repo] [ref]
export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
```
when working in the container interactively. These can also be found in the file [envs.list](/envs.list) and added to the docker container using the [`--env-file`](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file) option.
To setup a Phobos-Postgres database, the functions in the script [`src/phobos/testdb-setup.sh`](/src/phobos/testdb-setup.sh) should be used. An example of how to use that is given in the file [`start-db.sh`](/start-db.sh).
\ No newline at end of file
FROM debian:10.3
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install git cmake sudo wget curl python3-pip \
man clang-tidy clang-format doxygen
COPY . /home/superfiphoboserver
RUN /home/superfiphoboserver/tools/build_phobos.sh -c
RUN /home/superfiphoboserver/tools/build_proxygen.sh
RUN /home/superfiphoboserver/tools/install_catch2.sh
RUN pip3 install boto3
RUN mkdir ~/.aws
RUN echo "[default] \naws_access_key_id = OPEN_KEY \naws_secret_access_key = SECRET_KEY" > ~/.aws/credentials
RUN echo "[default] \noutput = json \nregion = eu-west-1" > ~/.aws/config
PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
\ No newline at end of file
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import re
from cpt.packager import ConanMultiPackager
from cpt.ci_manager import CIManager
from cpt.printer import Printer
class BuilderSettings(object):
@property
def username(self):
""" Set catchorg as package's owner
"""
return os.getenv("CONAN_USERNAME", "catchorg")
@property
def login_username(self):
""" Set Bintray login username
"""
return os.getenv("CONAN_LOGIN_USERNAME", "horenmar")
@property
def upload(self):
""" Set Catch2 repository to be used on upload.
The upload server address could be customized by env var
CONAN_UPLOAD. If not defined, the method will check the branch name.
Only master or CONAN_STABLE_BRANCH_PATTERN will be accepted.
The master branch will be pushed to testing channel, because it does
not match the stable pattern. Otherwise it will upload to stable
channel.
"""
return os.getenv("CONAN_UPLOAD", "https://api.bintray.com/conan/catchorg/catch2")
@property
def upload_only_when_stable(self):
""" Force to upload when running over tag branch
"""
return os.getenv("CONAN_UPLOAD_ONLY_WHEN_STABLE", "True").lower() in ["true", "1", "yes"]
@property
def stable_branch_pattern(self):
""" Only upload the package the branch name is like a tag
"""
return os.getenv("CONAN_STABLE_BRANCH_PATTERN", r"v\d+\.\d+\.\d+")
@property
def reference(self):
""" Read project version from branch create Conan reference
"""
return os.getenv("CONAN_REFERENCE", "Catch2/{}".format(self._version))
@property
def channel(self):
""" Default Conan package channel when not stable
"""
return os.getenv("CONAN_CHANNEL", "testing")
@property
def _version(self):
""" Get version name from cmake file
"""
pattern = re.compile(r"project\(Catch2 LANGUAGES CXX VERSION (\d+\.\d+\.\d+)\)")
version = "latest"
with open("CMakeLists.txt") as file:
for line in file:
result = pattern.search(line)
if result:
version = result.group(1)
return version
@property
def _branch(self):
""" Get branch name from CI manager
"""
printer = Printer(None)
ci_manager = CIManager(printer)
return ci_manager.get_branch()
if __name__ == "__main__":
settings = BuilderSettings()
builder = ConanMultiPackager(
reference=settings.reference,
channel=settings.channel,
upload=settings.upload,
upload_only_when_stable=settings.upload_only_when_stable,
stable_branch_pattern=settings.stable_branch_pattern,
login_username=settings.login_username,
username=settings.username,
test_folder=os.path.join(".conan", "test_package"))
builder.add()
builder.run()
cmake_minimum_required(VERSION 3.2.0)
project(test_package CXX)
include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup(TARGETS)
find_package(Catch2 REQUIRED CONFIG)
add_executable(${PROJECT_NAME} test_package.cpp)
target_link_libraries(${PROJECT_NAME} CONAN_PKG::Catch2)
set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 11)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from conans import ConanFile, CMake
import os
class TestPackageConan(ConanFile):
settings = "os", "compiler", "build_type", "arch"
generators = "cmake"
def build(self):
cmake = CMake(self)
cmake.configure()
cmake.build()
def test(self):
assert os.path.isfile(os.path.join(self.deps_cpp_info["Catch2"].rootpath, "licenses", "LICENSE.txt"))
bin_path = os.path.join("bin", "test_package")
self.run("%s -s" % bin_path, run_environment=True)
#define CATCH_CONFIG_MAIN
#include <catch2/catch.hpp>
int Factorial( int number ) {
return number <= 1 ? 1 : Factorial( number - 1 ) * number;
}
TEST_CASE( "Factorial Tests", "[single-file]" ) {
REQUIRE( Factorial(0) == 1 );
REQUIRE( Factorial(1) == 1 );
REQUIRE( Factorial(2) == 2 );
REQUIRE( Factorial(3) == 6 );
REQUIRE( Factorial(10) == 3628800 );
}
\ No newline at end of file
# This sets the default behaviour, overriding core.autocrlf
* text=auto
# All source files should have unix line-endings in the repository,
# but convert to native line-endings on checkout
*.cpp text
*.h text
*.hpp text
# Windows specific files should retain windows line-endings
*.sln text eol=crlf
# Keep executable scripts with LFs so they can be run after being
# checked out on Windows
*.py text eol=lf
# Keep the single include header with LFs to make sure it is uploaded,
# hashed etc with LF
single_include/**/*.hpp eol=lf
# Also keep the LICENCE file with LFs for the same reason
LICENCE.txt eol=lf
---
name: Bug report
about: Create an issue that documents a bug
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**Expected behavior**
A clear and concise description of what you expected to happen.
**Reproduction steps**
Steps to reproduce the bug.
<!-- Usually this means a small and self-contained piece of code that uses Catch and specifying compiler flags if relevant. -->
**Platform information:**
<!-- Fill in any extra information that might be important for your issue. -->
- OS: **Windows NT**
- Compiler+version: **GCC v2.9.5**
- Catch version: **v1.2.3**
**Additional context**
Add any other context about the problem here.
---
name: Feature request
about: Create an issue that requests a feature or other improvement
title: ''
labels: ''
assignees: ''
---
**Description**
Describe the feature/change you request and why do you want it.
**Additional context**
Add any other context or screenshots about the feature request here.
<!--
Please do not submit pull requests changing the `version.hpp`
or the single-include `catch.hpp` file, these are changed
only when a new release is made.
Before submitting a PR you should probably read the contributor documentation
at docs/contributing.md. It will tell you how to properly test your changes.
-->
## Description
<!--
Describe the what and the why of your pull request. Remember that these two
are usually a bit different. As an example, if you have made various changes
to decrease the number of new strings allocated, that's what. The why probably
was that you have a large set of tests and found that this speeds them up.
-->
## GitHub Issues
<!--
If this PR was motivated by some existing issues, reference them here.
If it is a simple bug-fix, please also add a line like 'Closes #123'
to your commit message, so that it is automatically closed.
If it is not, don't, as it might take several iterations for a feature
to be done properly. If in doubt, leave it open and reference it in the
PR itself, so that maintainers can decide.
-->
*.build
*.pbxuser
*.mode1v3
*.ncb
*.suo
Debug
Release
*.user
*.xcuserstate
.DS_Store
xcuserdata
CatchSelfTest.xcscheme
Breakpoints.xcbkptlist
projects/VS2010/TestCatch/_UpgradeReport_Files/
projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters
projects/VisualStudio/TestCatch/UpgradeLog.XML
projects/CMake/.idea
projects/CMake/cmake-build-debug
UpgradeLog.XML
Resources/DWARF
projects/Generated
*.pyc
DerivedData
*.xccheckout
Build
.idea
.vs
cmake-build-*
benchmark-dir
.conan/test_package/build
bazel-*
language: cpp
branches:
except:
- /dev-appveyor.*/
common_sources: &all_sources
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty
- llvm-toolchain-trusty-3.9
- llvm-toolchain-trusty-4.0
- llvm-toolchain-xenial-5.0
- llvm-toolchain-xenial-6.0
matrix:
include:
# 1/ Linux Clang Builds
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-3.5']
env: COMPILER='clang++-3.5'
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-3.6']
env: COMPILER='clang++-3.6'
# Clang 3.7 is intentionally skipped as we cannot get it easily on
# TravisCI container
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['lcov', 'clang-3.8']
env: COMPILER='clang++-3.8'
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-3.9']
env: COMPILER='clang++-3.9'
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-4.0']
env: COMPILER='clang++-4.0'
- os: linux
dist: xenial
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-5.0']
env: COMPILER='clang++-5.0'
- os: linux
dist: xenial
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-6.0']
env: COMPILER='clang++-6.0'
# 2/ Linux GCC Builds
- os: linux
compiler: gcc
addons:
apt:
sources: *all_sources
packages: ['g++-4.8']
env: COMPILER='g++-4.8'
- os: linux
compiler: gcc
addons:
apt:
sources: *all_sources
packages: ['g++-4.9']
env: COMPILER='g++-4.9'
- os: linux
compiler: gcc
addons:
apt:
sources: *all_sources
packages: ['g++-5']
env: COMPILER='g++-5'
- os: linux
compiler: gcc
addons: &gcc6
apt:
sources: *all_sources
packages: ['g++-6']
env: COMPILER='g++-6'
- os: linux
compiler: gcc
addons: &gcc7
apt:
sources: *all_sources
packages: ['g++-7']
env: COMPILER='g++-7'
- os: linux
compiler: gcc
addons: &gcc8
apt:
sources: *all_sources
packages: ['g++-8']
env: COMPILER='g++-8'
# 3b/ Linux C++14 Clang builds
# Note that we need newer libstdc++ for C++14 support
- os: linux
compiler: clang
addons:
apt:
packages: ['clang-3.8', 'libstdc++-6-dev']
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty
env: COMPILER='clang++-3.8' CPP14=1
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-3.9', 'libstdc++-6-dev']
env: COMPILER='clang++-3.9' CPP14=1
- os: linux
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-4.0', 'libstdc++-6-dev']
env: COMPILER='clang++-4.0' CPP14=1
- os: linux
dist: xenial
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-5.0', 'libstdc++-6-dev']
env: COMPILER='clang++-5.0' CPP14=1
- os: linux
dist: xenial
compiler: clang
addons:
apt:
sources: *all_sources
packages: ['clang-6.0', 'libstdc++-6-dev']
env: COMPILER='clang++-6.0' CPP14=1
# 4a/ Linux C++14 GCC builds
- os: linux
compiler: gcc
addons: *gcc6
env: COMPILER='g++-6' CPP14=1
- os: linux
compiler: gcc
addons: *gcc7
env: COMPILER='g++-7' CPP14=1
- os: linux
compiler: gcc
addons: *gcc8
env: COMPILER='g++-8' CPP14=1