Skip to content
GitLab
Commit 6fa6f9c5 authored by Pádraig Ó Conbhuí's avatar Pádraig Ó Conbhuí
Browse files

Add Phobos, Proxygen, and Deimos build in src/

parent 5f3981f9
Hide whitespace changes
Inline Side-by-side
src/build-common.sh 0 → 100644
View file @ 6fa6f9c5
#!/usr/bin/env bash
# Common command line parsing and functions for build.sh scripts
# Process args in function to prevent modifying
# the script's "$@"
function process_args() {
while [[ $# -gt 0 ]]
do
case $1 in
--dockerize)
dockerize=true
shift
;;
--no-dockerize)
dockerize=false
shift
;;
--image=*)
image=${1#--image=}
shift
;;
-j) # -j 4
make_threads=${2#-j=}
shift 2
;;
--skip=*)
skip_packages=${1#--skip=}
shift
;;
--only=*)
only_packages=${1#--only=}
shift
;;
--stages=*)
stages=${1#--stages=}
shift
;;
--skip-stages=*)
skip_stages=${1#--skip-stages=*}
shift
;;
--verbose)
verbose=true
shift
;;
--no-verbose)
verbose=false
shift
;;
--prefix=*)
install_root=${1#--prefix=}
shift
;;
--namespaced-install)
namespaced_install=true
shift
;;
--no-namespaced-install)
namespaced_install=false
shift
;;
--build-static)
build_static=true
shift
;;
--build-shared)
build_static=false
shift
;;
*)
echo >&2 "Unknown argument: $1"
exit 1
;;
esac
done;
}
process_args "$@"
# Argument defaults
: ${dockerize:=false}
: ${image:="centos:7"}
: ${make_threads:=1}
: ${build_root:=${PWD}/build}
: ${build_dir:=${build_root}/build}
: ${yum_cachedir:=${build_root}/yum-cache}
: ${install_root:=${build_root}/install}
: ${namespaced_install:=true}
: ${build_static:=false}
: ${skip_packages:=""}
: ${only_packages:=""}
: ${stages:="build,test"}
: ${verbose:=true}
# Rerun this script in a docker container
if [[ "${dockerize}" = "true" ]]
then
echo "Dockerizing: $0 $@"
exec docker run -v ${PWD}:/mnt -it "${image}" bash -c 'cd /mnt && exec $0 "$@" --no-dockerize' $0 "$@"
fi
function is_in_list() {
local list="$1"
local test_item="$2"
local return_value=1
local IFS=","
for item in ${list}
do
item=$(echo ${item} | tr -d '[:space:]')
if [[ "${item}" = "${test_item}" ]]
then
return_value=0
break
fi
done
return ${return_value}
}
# Determine if a given package should be built.
#
# Checks if --only has been set tries to match
# the package.
# Next, checks if --skip has been set and tries to
# match the package
#
function should_build() {
local test_pkg="$1"
# Check "onlys"
if [[ ! -z "${only_packages}" ]]
then
if is_in_list "${only_packages}" "${test_pkg}"
then
return 0
fi
return 1
fi
# Check skips
if is_in_list "${skip_packages}" "${test_pkg}"
then
return 1
fi
return 0
}
function should_run_stage() {
# By default, assume not to run
local return_value=1
for arg in "$@"
do
local tag=$(echo "${arg}" | sed 's/:[^:]*$//')
local stage=$(echo "${arg}" | sed 's/^.*://')
if [[ -z "${tag}" ]]
then
tag="${stage}"
fi
# Check skips (skip if any tag is in the skip list)
if is_in_list "${skip_stages}" "${tag}:${stage}"
then
# Skipping, return early
return 1
fi
if is_in_list "${skip_stages}" "${stage}"
then
# Skipping, return early
return 1
fi
# Check stages
# (run if any tag is in the stages list, and none in skip)
if is_in_list "${stages}" "${tag}:${stage}"
then
return_value=0
fi
if is_in_list "${stages}" "${stage}"
then
return_value=0
fi
done
return ${return_value}
}
# Get install directory for the project
function install_dir() {
if [[ "${namespaced_install}" = "true" ]]
then
local project=$1
echo "${install_root}/${project}"
else
echo "${install_root}"
fi
}
# Output script and OS info
function echo_with_quotes() {
for arg in "$@"
do
echo -n "\"${arg}\" "
done
}
echo
echo PWD: $(echo_with_quotes ${PWD})
echo Script: $(echo_with_quotes ${0} "$@")
grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=/OS: /'
echo
# Which OS are we in?
function os() {
if grep -q "CentOS Linux 7" /etc/os-release
then
echo "centos:7"
return
fi
if grep -q "CentOS Linux 8" /etc/os-release
then
echo "centos:8"
return
fi
echo "unknown"
return
}
# Set the verbosity of Bash using the $verbose
# value by default
function set_bash_verbosity() {
local local_verbose=$1
if [[ -z "${local_verbose}" ]]
then
local_verbose="${verbose}"
fi
if [[ "${local_verbose}" = "true" ]]
then
set -x
else
set +x
fi
}
# Print a banner for each section
function print_section() {
set_bash_verbosity false
echo
echo ----------
echo "$@"
echo ----------
echo
set_bash_verbosity
}
# Print commands (added after defining all the functions)
set_bash_verbosity
# Setup yum and install dev tools
yum_build_dir="${build_dir}/yum"
if should_build system_packages
then
# Setup yum cachedir to point to ${yum_cachedir} and enable caching
mkdir -p "${yum_cachedir}"
yum_build_dir="$(cd "${yum_cachedir}" && pwd)"
mkdir -p "${yum_cachedir}/cache"
cp /etc/yum.conf "${yum_cachedir}/yum.conf"
yum_config="${yum_cachedir}/yum.conf"
sed -i "s#^cachedir=.*\$#cachedir=${yum_cachedir}/cache#" "${yum_config}"
sed -i "s#^keepcache=0#keepcache=1#" "${yum_config}"
echo "skip_missing_names_on_install=False" >> "${yum_config}"
fi
# Wrap Yum to add custom config flag
function yum() {
command yum --config="${yum_cachedir}/yum.conf" "$@"
}
if should_build system_packages
then
# Install SCL and install gcc-7
yum install -y \
centos-release-scl
yum install -y \
devtoolset-7-gcc devtoolset-7-gcc-c++
fi
# Enable gcc-9
# Temporarily disable printing and error checking
set_bash_verbosity false
set +o errexit
source scl_source enable devtoolset-7
set -o errexit
set_bash_verbosity
#
# Common build flags
#
: ${CC:=gcc}
export CC=${CC}
: ${CXX:=g++}
export CXX=${CXX}
echo CC: "${CC}" $(${CC} --version)
echo CXX: "${CXX}" $(${CXX} --version)
declare -a CONFIGURE_FLAGS
declare -a CMAKE_FLAGS
declare -a MAKE_FLAGS
CMAKE_FLAGS+=( "-Wno-dev" )
CMAKE_FLAGS+=( "-DCMAKE_BUILD_TYPE=RelWithDebInfo" )
MAKE_FLAGS+=( "-j" "${make_threads}" )
if [[ "${build_static}" = "true" ]]
then
CMAKE_FLAGS+=( "-DBUILD_SHARED_LIBS=OFF" )
CONFIGURE_FLAGS+=( "--enable-static" "--disable-shared" )
else
CMAKE_FLAGS+=(
"-DBUILD_SHARED_LIBS=ON"
"-DCMAKE_POSITION_INDEPENDENT_CODE=ON" )
CONFIGURE_FLAGS+=( "--enable-shared" "--disable-static" )
fi
if [[ "${verbose}" = "true" ]]
then
CMAKE_FLAGS+=( "-DCMAKE_VERBOSE_MAKEFILE=ON" )
CONFIGURE_FLAGS+=( "--disable-silent-rules" )
MAKE_FLAGS+=( "VERBOSE=1" )
else
CMAKE_FLAGS+=( "-DCMAKE_VERBOSE_MAKEFILES=OFF" )
CONFIGURE_FLAGS+=( "--enable-silent-rules" )
MAKE_FLAGS+=( "VERBOSE=0" )
fi
src/deimos/build.sh 0 → 100755
View file @ 6fa6f9c5
#!/usr/bin/env bash
# Exit on any errors
set -o errexit
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
external_dir="${script_dir}/../../external"
source "${script_dir}/../phobos/build.sh"
source "${script_dir}/../proxygen/build.sh"
# Default options
build_static=true
source "${script_dir}/../build-common.sh"
#
# Install dependencies
#
print_section "Installing system dependencies"
if should_build system_packages
then
# # Regular packages
# yum install -y \
# make \
# autoconf automake libtool \
# glib2-devel \
# jansson-devel \
# libattr-devel \
# libini_config-devel \
# openssl-devel \
# protobuf-c-devel \
# python-devel \
# sg3_utils-devel \
# which \
# python-psycopg2 \
# PyYAML
#
#
yum install -y \
glib2 glib2-devel
# EPEL Packages
yum install -y \
epel-release
yum install -y \
cmake3 \
boost169-devel
#
# yum install -y \
# clustershell
#
#
# # SCL packages for postgresql
# # Centos-7 provides 9.2 by default
# # Phobos needs 9.4 for jsonb
# yum install -y \
# rh-postgresql95-postgresql-devel \
# rh-postgresql95-postgresql-libs \
# rh-postgresql95-postgresql-server \
# rh-postgresql95-postgresql-contrib
#
fi
function cmake() {
command cmake3 "$@"
}
declare -a BOOST_CMAKE_FLAGS
BOOST_CMAKE_FLAGS+=(
-DBOOST_ROOT=/usr/include/boost169
-DBOOST_INCLUDEDIR=/usr/include/boost169
-DBOOST_LIBRARYDIR=/usr/lib64/boost169
)
# Enable rh-postgresql
# Temporarily disable printing and error checking
set_bash_verbosity false
set +o errexit
source scl_source enable rh-postgresql95
set -o errexit
set_bash_verbosity
#
# Build Catch2
#
print_section "Build Catch2"
catch2_source_dir=${external_dir}/catch2
catch2_build_dir=${build_dir}/catch2
catch2_install_dir=$(install_dir catch2)
if should_build catch2
then
mkdir -p "${catch2_build_dir}"
pushd "${catch2_build_dir}"
cmake "${catch2_source_dir}" \
"${CMAKE_FLAGS[@]}" \
-DCMAKE_INSTALL_PREFIX="${catch2_install_dir}" \
-DBUILD_TESTING=OFF
make "${MAKE_FLAGS[@]}"
make install
popd
fi
export CMAKE_PREFIX_PATH="$(install_dir catch2):${CMAKE_PREFIX_PATH}"
#
# Build spdlog
#
print_section "Build spdlog"
spdlog_source_dir=${external_dir}/spdlog
spdlog_build_dir=${build_dir}/spdlog
spdlog_install_dir=$(install_dir spdlog)
if should_build spdlog
then
mkdir -p "${spdlog_build_dir}"
pushd "${spdlog_build_dir}"
cmake "${spdlog_source_dir}" \
"${CMAKE_FLAGS[@]}" \
-DCMAKE_INSTALL_PREFIX="${spdlog_install_dir}" \
-DSPDLOG_BUILD_TESTS=OFF
make "${MAKE_FLAGS[@]}"
make install
popd
fi
export CMAKE_PREFIX_PATH="$(install_dir spdlog):${CMAKE_PREFIX_PATH}"
#
# Build Deimos
#
print_section "Build Deimos"
deimos_source_dir=${external_dir}/deimos
deimos_build_dir=${build_dir}/deimos
deimos_install_dir=$(install_dir deimos)
echo deimos_install_dir: ${deimos_install_dir}
if should_build deimos
then
mkdir -p "${deimos_build_dir}"
pushd "${deimos_build_dir}"
declare -a DEIMOS_CMAKE_FLAGS
if should_run_stage deimos:test
then
DEIMOS_CMAKE_FLAGS+=(
-DDEIMOS_BUILD_TESTS=ON )
fi
CMAKE_PREFIX_PATH=$(install_dir proxygen):$(install_dir folly):$(install_dir fmt):$(install_dir fizz):$(install_dir wangle):$(install_dir gflags):"${CMAKE_PREFIX_PATH}"
# Add glib-2.0
CMAKE_C_FLAGS="${CMAKE_C_FLAGS} $(pkg-config glib-2.0 --cflags)"
CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} $(pkg-config glib-2.0 --cflags)"
# Add phobos
CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -isystem $(install_dir phobos)/include"
CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -isystem $(install_dir phobos)/include -isystem $(install_dir spdlog)/include"
CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -L $(install_dir phobos)/lib"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$(install_dir phobos)/lib"
if should_run_stage deimos:build
then
cmake "${deimos_source_dir}" \
"${CMAKE_FLAGS[@]}" \
"${BOOST_CMAKE_FLAGS[@]}" \
"${DEIMOS_CMAKE_FLAGS[@]}" \
-DCMAKE_INSTALL_PREFIX="${deimos_install_dir}" \
-DBUILD_TESTING=ON \
-DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \
-DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
-DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
-DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
make ${MAKE_FLAGS[@]}
fi
if should_run_stage deimos:test deimos:run:test
then
set +o errexit
mkdir -p phobos-scripts
cp -r ${external_dir}/phobos/scripts/ phobos-scripts
ls phobos-scripts
mkdir -p phobos-scripts/src/tests
cp -r ${external_dir}/phobos/src/tests/*.sh phobos-scripts/src/tests/
mkdir -p phobos-scripts/src/cli/tests
cp -r ${external_dir}/phobos/src/cli/tests/*.sh phobos-scripts/src/cli/tests/
# Patch test scripts
sed -i 's/\$PHO_LAYOUTLIB_PATH/$PHO_LAYOUTLIB_PATH:$LD_LIBRARY_PATH/' phobos-scripts/src/cli/tests/run_tests.sh
sed -i 's/\$PHO_PYTHON_PATH/$PHO_PYTHON_PATH:$PYTHONPATH/' phobos-scripts/src/cli/tests/run_tests.sh
sed -i 's/\$PHO_PYTHON_PATH/$PHO_PYTHON_PATH:$PYTHONPATH/' phobos-scripts/src/tests/test_env.sh
export PATH=$(install_dir phobos)/bin:$(install_dir phobos/sbin):${PATH}
export PYTHONPATH="$(install_dir phobos)"/lib/python2.7/site-packages:"${PYTHONPATH}"
export LD_LIBRARY_PATH="$(install_dir phobos)"/lib:"${LD_LIBRARY_PATH}"
cp ${external_dir}/phobos/src/tests/phobos.conf /etc/
sed -i '/lrs/ a default_family = dir' /etc/phobos.conf
#
# Setup postgres
#
test_db_dir=${deimos_build_dir}/test-db
rm -rf "${test_db_dir}"
mkdir -p "${test_db_dir}"
chown -R postgres:postgres "${test_db_dir}"
chmod 0700 "${test_db_dir}"
su postgres -c "initdb ${test_db_dir}"
cp "${script_dir}"/../phobos/pg_hba.conf "${test_db_dir}"/pg_hba.conf
cp "${script_dir}"/../phobos/postgresql.conf "${test_db_dir}"/postgresql.conf
# Boot postgres
su postgres -c "pg_ctl -D ${test_db_dir} start"
export PHOBOS_STORE_layout=simple
export PHOBOS_LAYOUT_RAID1_repl_count=2
# Initialize phobos db
su postgres -c "bash ./phobos-scripts/scripts/phobos_db_local setup_db -s -p phobos"
phobosd
test_phobos_dir=${deimos_build_dir}/test-phobos-dir
rm -rf ${test_phobos_dir}
mkdir -p ${test_phobos_dir}
phobos dir add --unlock ${test_phobos_dir}
phobos dir format --fs posix ${test_phobos_dir}
ls
ls test
pushd test/unit_tests
../tests
popd
#make test ARGS=-V
# Shutdown postgres
su postgres -c "pg_ctl -D ${test_db_dir} stop"
set -o errexit
fi
make install