Modularise build scripts

Redesign the Deimos, Phobos, and Proxygen build scripts to make it
easier to add support for new OSs. OS-dependent requirements are
executed as functions to be implemented in sourced OS-specific scripts

Change log:
* replace Centos7 specific code with function calls
* pull implementations of centos7 build modules into sourced script

src/deimos/build.sh

@@ -24,6 +24,18 @@ build_static=true
 
 source "${script_dir}/../build-common.sh"
 
+case "$(os)" in
+    centos:7)
+        source "${script_dir}/centos7.sh"
+        ;;
+    centos:8)
+        source "${script_dir}/centos8.sh"
+        ;;
+    *)
+        echo "Could not detect OS"
+        exit 1
+        ;;
+esac
 
 #
 # Install dependencies
@@ -32,73 +44,9 @@ print_section "Installing system dependencies"
 
 if should_build system_packages
 then
-#     # Regular packages
-#     yum install -y \
-#         make \
-#         autoconf automake libtool \
-#         glib2-devel \
-#         jansson-devel \
-#         libattr-devel \
-#         libini_config-devel \
-#         openssl-devel \
-#         protobuf-c-devel \
-#         python-devel \
-#         sg3_utils-devel \
-#         which \
-#         python-psycopg2 \
-#         PyYAML
-# 
-# 
-    yum install -y \
-        glib2 glib2-devel
-
-    yum install -y \
-        python3-pip
-    
-    python3 -m pip install boto3
-
-    # EPEL Packages
-    yum install -y \
-        epel-release
-
-    yum install -y \
-        cmake3 \
-        boost169-devel
-# 
-#     yum install -y \
-#         clustershell
-# 
-# 
-#     # SCL packages for postgresql
-#     # Centos-7 provides 9.2 by default
-#     # Phobos needs 9.4 for jsonb
-#     yum install -y \
-#         rh-postgresql95-postgresql-devel \
-#         rh-postgresql95-postgresql-libs \
-#         rh-postgresql95-postgresql-server \
-#         rh-postgresql95-postgresql-contrib
-# 
+    install_dependencies
 fi
 
-function cmake() {
-    command cmake3 "$@"
-}
-
-declare -a BOOST_CMAKE_FLAGS
-BOOST_CMAKE_FLAGS+=(
-    -DBOOST_ROOT=/usr/include/boost169
-    -DBOOST_INCLUDEDIR=/usr/include/boost169
-    -DBOOST_LIBRARYDIR=/usr/lib64/boost169
-)
-
-# Enable rh-postgresql
-# Temporarily disable printing and error checking
-set_bash_verbosity false
-set +o errexit
-source scl_source enable rh-postgresql95
-set -o errexit
-set_bash_verbosity
-
 
 #
 # Build Catch2

src/deimos/centos7.sh

+#!/usr/bin/env bash
+
+# Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
+# Authors:
+#    Pádraig Ó Conbhuí <padraig.oconbhui@ichec.ie>,
+#    Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>,
+#    Ciarán O'Rourke <ciaran.orourke@ichec.ie>
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+#
+# Install dependencies
+#
+
+function install_dependencies() {
+    yum install -y \
+        glib2 glib2-devel
+
+    yum install -y \
+        python3-pip
+    
+    python3 -m pip install boto3
+
+    # EPEL Packages
+    yum install -y \
+        epel-release
+
+    yum install -y \
+        cmake3 \
+        boost169-devel
+}
+
+function configure() {
+    function cmake() {
+        command cmake3 "$@"
+    }
+
+    declare -a BOOST_CMAKE_FLAGS
+    BOOST_CMAKE_FLAGS+=(
+        -DBOOST_ROOT=/usr/include/boost169
+        -DBOOST_INCLUDEDIR=/usr/include/boost169
+        -DBOOST_LIBRARYDIR=/usr/lib64/boost169
+    )
+
+    # Enable rh-postgresql
+    # Temporarily disable printing and error checking
+    set_bash_verbosity false
+    set +o errexit
+    source scl_source enable rh-postgresql95
+    set -o errexit
+    set_bash_verbosity
+}
+

src/phobos/build.sh

@@ -17,6 +17,18 @@ external_dir="${script_dir}/../../external"
 
 source "${script_dir}/../build-common.sh"
 
+case "$(os)" in
+    centos:7)
+        source "${script_dir}/centos7.sh"
+        ;;
+    centos:8)
+        source "${script_dir}/centos8.sh"
+        ;;
+    *)
+        echo "Could not detect OS"
+        exit 1
+        ;;
+esac
 
 #
 # Install dependencies
@@ -25,57 +37,11 @@ print_section "Installing system dependencies"
 
 if should_build system_packages
 then
-    # Regular packages
-    yum install -y \
-        make \
-        autoconf automake libtool \
-        glib2-devel \
-        jansson-devel \
-        libattr-devel \
-        libini_config-devel \
-        openssl-devel \
-        protobuf-c-devel \
-        python-devel \
-        sg3_utils-devel \
-        which \
-        python-psycopg2 \
-        PyYAML
-
-
-    # EPEL Packages
-    yum install -y \
-        epel-release
-
-    yum install -y \
-        clustershell
-
-
-    # SCL packages for postgresql
-    # Centos-7 provides 9.2 by default
-    # Phobos needs 9.4 for jsonb
-    yum install -y \
-        rh-postgresql95-postgresql-devel \
-        rh-postgresql95-postgresql-libs \
-        rh-postgresql95-postgresql-server \
-        rh-postgresql95-postgresql-contrib
-
+    install_dependencies
 fi
 
 
-# Enable rh-postgresql
-# Temporarily disable printing and error checking
-set_bash_verbosity false
-set +o errexit
-source scl_source enable rh-postgresql95
-set -o errexit
-set_bash_verbosity
-
-
-#
-# Set build flags
-#
-export LIBPQ_CFLAGS=$(pg_config --cflags)
-export LIBPQ_LIBS=-lpq
+configure
 
 
 #
@@ -165,8 +131,7 @@ then
     popd # source/
 
     popd # phobos_build_dir
+
 fi
 
-export PYTHONPATH="${phobos_install_dir}"/lib/python2.7/site-packages:"${PYTHONPATH}"
-export LD_LIBRARY_PATH="${phobos_install_dir}"/lib:"${LD_LIBRARY_PATH}"
-export PATH="${phobos_install_dir}"/bin:"${phobos_install_dir}"/sbin:${PATH}
+set_environment_variables

src/phobos/centos7.sh

+#!/usr/bin/env bash
+
+# Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
+# Authors:
+#    Pádraig Ó Conbhuí <padraig.oconbhui@ichec.ie>,
+#    Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>,
+#    Ciarán O'Rourke <ciaran.orourke@ichec.ie>
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+#
+# Install dependencies
+#
+
+function install_dependencies() {
+    # Regular packages
+    yum install -y \
+        make \
+        autoconf automake libtool \
+        glib2-devel \
+        jansson-devel \
+        libattr-devel \
+        libini_config-devel \
+        openssl-devel \
+        protobuf-c-devel \
+        python-devel \
+        sg3_utils-devel \
+        which \
+        python-psycopg2 \
+        PyYAML
+
+
+    # EPEL Packages
+    yum install -y \
+        epel-release
+
+    yum install -y \
+        clustershell \
+        python2-tabulate
+
+
+    # SCL packages for postgresql
+    # Centos-7 provides 9.2 by default
+    # Phobos needs 9.4 for jsonb
+    yum install -y \
+        rh-postgresql95-postgresql-devel \
+        rh-postgresql95-postgresql-libs \
+        rh-postgresql95-postgresql-server \
+        rh-postgresql95-postgresql-contrib
+}
+
+function configure() {
+    # Enable rh-postgresql
+
+    # Temporarily disable printing and error checking
+    set_bash_verbosity false
+    set +o errexit
+    source scl_source enable rh-postgresql95
+    set -o errexit
+    set_bash_verbosity
+
+
+    #
+    # Set build flags
+    #
+    export LIBPQ_CFLAGS=$(pg_config --cflags)
+    export LIBPQ_LIBS=-lpq
+}
+
+
+function set_environment_variables() {
+    export PYTHONPATH="${phobos_install_dir}"/lib/python2.7/site-packages:"${PYTHONPATH}"
+    export LD_LIBRARY_PATH="${phobos_install_dir}"/lib:"${LD_LIBRARY_PATH}"
+    export PATH="${phobos_install_dir}"/bin:"${phobos_install_dir}"/sbin:${PATH}
+}

src/proxygen/build.sh

@@ -23,6 +23,19 @@ build_static=true
 
 source "${script_dir}/../build-common.sh"
 
+case "$(os)" in
+    centos:7)
+        source "${script_dir}/centos7.sh"
+        ;;
+    centos:8)
+        source "${script_dir}/centos8.sh"
+        ;;
+    *)
+        echo "Could not detect OS"
+        exit 1
+        ;;
+esac
+
 # Convert input arguments to a CTest compatible regex list,
 # i.e. separate each entry with a \|
 function to_ctest_regex() {
@@ -36,243 +49,15 @@ print_section "Installing system dependencies"
 
 if should_build system_packages
 then
-    # Regular packages
-    yum install -y \
-        make \
-        gperf \
-        perl perl-core \
-        libevent-devel \
-        zlib-devel \
-        openssl-devel openssl-libs
-
-
-    # EPEL Packages
-    yum install -y \
-        epel-release
-
-    yum install -y \
-        cmake3 \
-        boost169-devel \
-        double-conversion-devel \
-        gflags gflags-devel glog-devel \
-        libsodium-devel \
-        libzstd-devel
+    install_dependencies
 fi
 
 # Set compilers etc.
-function cmake() {
-    command cmake3 "$@"
-}
-
-declare -a BOOST_CMAKE_FLAGS
-BOOST_CMAKE_FLAGS+=(
-    -DBOOST_ROOT=/usr/include/boost169
-    -DBOOST_INCLUDEDIR=/usr/include/boost169
-    -DBOOST_LIBRARYDIR=/usr/lib64/boost169
-)
-
-if test ! -L /usr/include/boost; then
-    ln -s /usr/include/boost169/boost /usr/include/boost
-fi
-if test ! -L /usr/lib/libboost_context.so ; then
-ln -s /usr/lib64/boost169/libboost_context.so  /usr/lib/libboost_context.so 
-fi
-if test ! -L /usr/lib/libboost_filesystem.so ; then
-ln -s /usr/lib64/boost169/libboost_filesystem.so  /usr/lib/libboost_filesystem.so 
-fi
-if test ! -L /usr/lib/libboost_iostreams.so ; then
-ln -s /usr/lib64/boost169/libboost_iostreams.so  /usr/lib/libboost_iostreams.so 
-fi
-if test ! -L /usr/lib/libboost_program_options.so ; then
-ln -s /usr/lib64/boost169/libboost_program_options.so  /usr/lib/libboost_program_options.so 
-fi
-if test ! -L /usr/lib/libboost_regex.so ; then
-ln -s /usr/lib64/boost169/libboost_regex.so  /usr/lib/libboost_regex.so 
-fi
-if test ! -L /usr/lib/libboost_system.so ; then
-ln -s /usr/lib64/boost169/libboost_system.so  /usr/lib/libboost_system.so 
-fi
-if test ! -L /usr/lib/libboost_thread.so ; then
-ln -s /usr/lib64/boost169/libboost_thread.so  /usr/lib/libboost_thread.so 
-fi
+configure
 
 
-#
-# Patch GFlags install
-#
-print_section "Patching GFlags"
-
-gflags_install_dir=$(install_dir gflags)
-
-if should_build gflags
-then
-    mkdir -p "${gflags_install_dir}"/lib/cmake/gflags
-
-    gflags_config="${gflags_install_dir}"/lib/cmake/gflags/gflags-config.cmake
-
-	cat > "${gflags_config}" <<-EOF
-		# include directory
-		set (gflags_INCLUDE_DIR "/usr/include")
-		
-		# gflags library
-		set (gflags_LIBRARIES "/usr/lib64/libgflags.so")
-
-		add_library(gflags-shared SHARED IMPORTED)
-		set_target_properties(
-			gflags-shared
-			PROPERTIES
-				INTERFACE_SYSTEM_INCLUDE_DIRECTORIES \${gflags_INCLUDE_DIR}
-				INTERFACE_LINK_LIBRARIES \${gflags_LIBRARIES}
-		)
-		EOF
-
-fi
-
-CMAKE_PREFIX_PATH="${gflags_install_dir}:${CMAKE_PREFIX_PATH}"
-
-# 
-# # Gflags sets library to "gflags" instead of /usr/lib64/libgflags.so
-# gflags_config_cmake=/usr/lib64/cmake/gflags/gflags-config.cmake
-# sed -i 's#set (gflags_LIBRARIES gflags)#set (gflags_LIBRARIES /usr/lib64/libgflags.so.2.1)#' "${gflags_config_cmake}"
-
-
-case "$(os)" in
-# centos:7)
-#     #
-#     # Build OpenSSL
-#     # OpenSSL on Centos 7 is 1.1.1c, Folly et al. needs 1.1.1f
-#     #
-#     openssl_source_dir=${external_dir}/openssl
-#     openssl_build_dir=${build_dir}/openssl
-#     openssl_install_dir=$(install_dir openssl)
-# 
-#     if should_build openssl
-#     then
-#         mkdir -p "${openssl_build_dir}"
-#         pushd "${openssl_build_dir}"
-# 
-#         declare -a OPENSSL_CONFIGURE_OPTIONS
-#         if should_run_stage openssl:test openssl:configure:test
-#         then
-#             OPENSSL_CONFIGURE_OPTIONS+=( enable-unit-test )
-#         fi
-# 
-#         if should_run_stage openssl:build
-#         then
-#             "${openssl_source_dir}"/config \
-#                 --prefix="${openssl_install_dir}" \
-#                 "${OPENSSL_CONFIGURE_OPTIONS[@]}" \
-#                 enable-shared \
-#                 --release
-# 
-#             make
-#         fi
-# 
-#         if should_run_stage openssl:test openssl:run:test
-#         then
-#             # Intentionally break `nm` to disable test_symbol_presence.
-#             # It's an "internal" test for openssl.
-#             mkdir -p tmp-bin
-#             cat > tmp-bin/nm <<-EOF
-# 				#!/usr/bin/env bash
-# 				exit 1
-# 				EOF
-#             chmod 0755 tmp-bin/nm
-# 
-#             PATH="${PWD}"/tmp-bin:${PATH} make test
-#         fi
-# 
-#         make install_sw
-# 
-#         popd
-#     fi
-# 
-#     # Set libcrypto to v1.1.1f
-#     declare -a OPENSSL_CMAKE_FLAGS
-#     export LD_LIBRARY_PATH="${openssl_install_dir}"/lib:"${LD_LIBRARY_PATH}"
-#     OPENSSL_CMAKE_FLAGS+=(
-#         -DOPENSSL_CRYPTO_LIBRARY="${openssl_install_dir}"/lib/libcrypto.so
-#         -DOPENSSL_SSL_LIBRARY="${openssl_install_dir}"/lib/libssl.so
-#         -DOPENSSL_INCLUDE_DIR="${openssl_install_dir}"/include/
-#     )
-#     ;;
-
-centos:*)
-    declare -a OPENSSL_CMAKE_FLAGS
-    export LD_LIBRARY_PATH="${openssl_install_dir}"/lib:"${LD_LIBRARY_PATH}"
-    OPENSSL_CMAKE_FLAGS+=(
-        -DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/libcrypto.so
-        -DOPENSSL_SSL_LIBRARY=/usr/lib64/libssl.so
-        -DOPENSSL_INCLUDE_DIR=/usr/include/
-    )
-    ;;
-esac
-    
-
-case "$(os)" in
-centos:7)
-    #
-    # Build fmt
-    # Fmt on CentOS 7 is a bit too old for Folly
-    #
-    print_section "Build fmt"
-
-    fmt_source_dir=${external_dir}/fmt
-    fmt_build_dir=${build_dir}/fmt
-    fmt_install_dir=$(install_dir fmt)
-
-    if should_build fmt
-    then
-        mkdir -p "${fmt_build_dir}"
-        pushd "${fmt_build_dir}"
-
-        cmake "${fmt_source_dir}" \
-            "${CMAKE_FLAGS[@]}" \
-            -DCMAKE_INSTALL_PREFIX="${fmt_install_dir}" \
-            -DCMAKE_INSTALL_LIBDIR="lib" \
-            -DFMT_DOC=OFF \
-            -DFMT_TEST=OFF
-        make "${MAKE_FLAGS[@]}"
-        make install
-
-        popd
-    fi
-
-    export CMAKE_PREFIX_PATH="${fmt_install_dir}:${CMAKE_PREFIX_PATH}"
-    ;;
-esac
-
-
-#
-# Build GoogleTest
-# Release 1.8.0 needed for Facebook projects
-#
-print_section "Build GoogleTest"
-
-googletest_source_dir=${external_dir}/googletest
-googletest_build_dir=${build_dir}/googletest
-googletest_install_dir=$(install_dir googletest)
-
-if should_build googletest
-then
-
-    mkdir -p "${googletest_build_dir}"
-    pushd "${googletest_build_dir}"
-
-    cmake "${googletest_source_dir}" \
-        "${CMAKE_FLAGS[@]}" \
-        -DCMAKE_INSTALL_PREFIX="${googletest_install_dir}" \
-        -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \
-        -DBUILD_GMOCK=ON \
-        -DBUILD_GTEST=ON
-
-    make
-    make install
-
-    popd
-fi
-
-CMAKE_PREFIX_PATH="${googletest_install_dir}:${CMAKE_PREFIX_PATH}"
+# Build stuff that are not in repos
+build_dependencies
 
 
 #

src/proxygen/centos7.sh

+#!/usr/bin/env bash
+
+# Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
+# Authors:
+#    Pádraig Ó Conbhuí <padraig.oconbhui@ichec.ie>,
+#    Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>,
+#    Ciarán O'Rourke <ciaran.orourke@ichec.ie>
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+#
+# Install dependencies
+#
+function install_dependencies() {
+    # Regular packages
+    yum install -y \
+        make \
+        gperf \
+        perl perl-core \
+        libevent-devel \
+        zlib-devel \
+        openssl-devel openssl-libs
+
+
+    # EPEL Packages
+    yum install -y \
+        epel-release
+
+    yum install -y \
+        cmake3 \
+        boost169-devel \
+        double-conversion-devel \
+        gflags gflags-devel glog-devel \
+        libsodium-devel \
+        libzstd-devel
+}
+
+
+# Set compilers etc.
+function configure() {
+
+    function cmake() {
+        command cmake3 "$@"
+    }
+
+    declare -a BOOST_CMAKE_FLAGS
+    BOOST_CMAKE_FLAGS+=(
+        -DBOOST_ROOT=/usr/include/boost169
+        -DBOOST_INCLUDEDIR=/usr/include/boost169
+        -DBOOST_LIBRARYDIR=/usr/lib64/boost169
+    )
+
+    if test ! -L /usr/include/boost; then
+        ln -s /usr/include/boost169/boost /usr/include/boost
+    fi
+    if test ! -L /usr/lib/libboost_context.so ; then
+    ln -s /usr/lib64/boost169/libboost_context.so  /usr/lib/libboost_context.so