Commit 8ee724a5 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Update build for Phobos 1.92

Changelog:
* Revert changes to Phobos from the past and leave only strictly necessary
* Move dependencies to seperate files
* Update to Python 3 (for Phobos) and devtoolset 8
parent 7cb8a361
FROM centos:7
FROM centos:8
RUN yum -y update
RUN dnf -y update
COPY . /home/superdeimos
RUN cd /home/superdeimos && ./src/phobos/build.sh -j 1
RUN cd /home/superdeimos && ./src/phobos/build.sh -j 1 --skip-stages=test
RUN cd /home/superdeimos && ./src/proxygen/build.sh -j 4 --skip-stages=test
#RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4
RUN cd /home/superdeimos && ./src/deimos/build.sh -j 4
......@@ -45,11 +45,11 @@ In the container the same scripts can be used to build deimos. An example comman
For convenience, the path variables should be set to
```
export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-8/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
export LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:
export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages
export PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python3.6/site-packages:/opt/rh/devtoolset-8/root/usr/lib64/python3.6/site-packages
```
......
PATH=/home/superdeimos/build/install/phobos/sbin:/home/superdeimos/build/install/phobos/bin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages
PATH=/home/superdeimos/build/install/phobos/sbin:/home/superdeimos/build/install/phobos/bin:/opt/rh/rh-postgresql95/root/usr/bin:/opt/rh/devtoolset8/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:/opt/rh/rh-postgresql95/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python3.6/site-packages:/opt/rh/devtoolset-8/root/usr/lib64/python3.6/site-packages
PATH=/home/superdeimos/build/install/phobos/bin:/home/superdeimos/build/install/phobos/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib:$LD_LIBRARY_PATH
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python2.7/site-packages:$PYTHONPATH
LD_LIBRARY_PATH=:/home/superdeimos/build/install/phobos/lib/phobos:/home/superdeimos/build/install/phobos/lib:$LD_LIBRARY_PATH
PYTHONPATH=/home/superdeimos/build/install/phobos/lib/python3.6/site-packages:$PYTHONPATH
......@@ -14,7 +14,7 @@ AC_CONFIG_AUX_DIR([autotools])
AC_CONFIG_MACRO_DIR([autotools/m4])
AC_CONFIG_HEADER([src/include/config.h])
AM_INIT_AUTOMAKE([-Wall -Werror tar-pax foreign])
AM_INIT_AUTOMAKE([-Wall -Werror tar-pax foreign subdir-objects])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
AC_PREFIX_DEFAULT([/usr])
......
......@@ -437,8 +437,8 @@ class SyslogTest(BasicExecutionTest):
"""Partially test syslog logging feature"""
# Messages in /var/log/messages or journalctl can't be retrieved as
# an unprivileged user
# self.pho_execute(['--syslog', 'debug', 'dir', 'list'])
# self.pho_execute(['--syslog', 'info', '-vvv', 'dir', 'list'])
self.pho_execute(['--syslog', 'debug', 'dir', 'list'])
self.pho_execute(['--syslog', 'info', '-vvv', 'dir', 'list'])
if __name__ == '__main__':
unittest.main(buffer=True)
......@@ -376,10 +376,10 @@ int str2timeval(const char *tv_str, struct timeval *tv)
int timeval2str(const struct timeval *tv, char *tv_str)
{
char buf[PHO_TIMEVAL_MAX_LEN];
char buf[PHO_TIMEVAL_MAX_LEN-7];
strftime(buf, sizeof(buf), "%Y-%m-%d %T", localtime(&tv->tv_sec));
snprintf(tv_str, sizeof(buf), "%s.%06ld", buf, tv->tv_usec);
snprintf(tv_str, PHO_TIMEVAL_MAX_LEN, "%s.%06ld", buf, tv->tv_usec);
return 0;
}
......
......@@ -2,7 +2,7 @@ SUFFIXES = .pb-c.h
protodir=../proto
proto_headers=pho_proto_common.pb-c.h pho_proto_lrs.pb-c.h
noinst_HEADERS=pho_cfg.h pho_comm.h pho_dss.h pho_io.h \
noinst_HEADERS=pho_cfg.h pho_comm.h pho_common.h pho_dss.h pho_io.h \
pho_layout.h pho_ldm.h pho_mapper.h pho_srl_common.h \
pho_srl_lrs.h pho_type_utils.h slist.h \
$(proto_headers)
......@@ -11,8 +11,7 @@ noinst_HEADERS=pho_cfg.h pho_comm.h pho_dss.h pho_io.h \
# - phobos_store.h main program interface
# - phobos_admin.h admin command interface
# - pho_attrs.h access and manipulate object metadata
include_HEADERS=pho_attrs.h phobos_store.h phobos_admin.h \
pho_types.h pho_common.h
include_HEADERS=pho_attrs.h phobos_store.h phobos_admin.h pho_types.h
$(proto_headers): $(protodir)/gen-proto.stamp
......
......@@ -208,38 +208,38 @@ static int pho_posix_set_addr(const char *extent_key, const char *extent_desc,
*
* support for pipe I/O
*/
static int pho_posix_read_write(int tgt_fd, int src_fd, size_t count)
{
ssize_t r = 0;
ssize_t w = 0;
char *buf = malloc(count);
while (count > 0) {
r = read(src_fd, buf, count);
if (r < 0)
LOG_RETURN(-errno, "read failure");
else {
w = write(tgt_fd, buf, r);
if (w < 0)
LOG_RETURN(-errno, "write failure");
}
if (count && (r == 0 || w == 0))
LOG_RETURN(-ENOBUFS,
"read/write failure, reached source fd eof too soon");
if (r == w) {
pho_debug("read/write returned after copying %zd bytes. %zd bytes left",
r, count - r);
}
else {
LOG_RETURN(-ENOBUFS,
"read/write failure, read and write count mismatch");
}
count -=r;
}
free(buf);
return 0;
}
// static int pho_posix_read_write(int tgt_fd, int src_fd, size_t count)
// {
// ssize_t r = 0;
// ssize_t w = 0;
// char *buf = malloc(count);
// while (count > 0) {
// r = read(src_fd, buf, count);
// if (r < 0)
// LOG_RETURN(-errno, "read failure");
// else {
// w = write(tgt_fd, buf, r);
// if (w < 0)
// LOG_RETURN(-errno, "write failure");
// }
// if (count && (r == 0 || w == 0))
// LOG_RETURN(-ENOBUFS,
// "read/write failure, reached source fd eof too soon");
// if (r == w) {
// pho_debug("read/write returned after copying %zd bytes. %zd bytes left",
// r, count - r);
// }
// else {
// LOG_RETURN(-ENOBUFS,
// "read/write failure, read and write count mismatch");
// }
// count -=r;
// }
// free(buf);
// return 0;
// }
/**
* Sendfile wrapper
......@@ -257,12 +257,12 @@ static int pho_posix_sendfile(int tgt_fd, int src_fd, size_t count)
while (count > 0) {
rw = sendfile(tgt_fd, src_fd, NULL, count);
if (rw < 0) {
if (errno == EINVAL || errno == ENOSYS) {
errno = 0;
rw = pho_posix_read_write(tgt_fd, src_fd, count);
return rw;
}
else
// if (errno == EINVAL || errno == ENOSYS) {
// errno = 0;
// rw = pho_posix_read_write(tgt_fd, src_fd, count);
// return rw;
// }
// else
LOG_RETURN(-errno, "sendfile failure");
}
......
......@@ -44,8 +44,7 @@ static int dir_lookup(const char *dev_id, char *dev_path, size_t path_size)
if (!sep)
return -EINVAL;
strncpy(dev_path, sep + 1, path_size - 1);
dev_path[path_size - 1] = '\0';
strncpy(dev_path, sep + 1, path_size);
return 0;
}
......
......@@ -298,8 +298,7 @@ static int _ltfs_mount_check(const struct mntent *mntent, void *cb_data)
LOG_RETURN(-EMEDIUMTYPE, "Device '%s' is mounted with unexpected "
"FS type '%s'", mntent->mnt_fsname, mntent->mnt_type);
strncpy(check_info->mnt_dir, mntent->mnt_dir, check_info->mnt_size - 1);
check_info->mnt_dir[check_info->mnt_size - 1] = '\0';
strncpy(check_info->mnt_dir, mntent->mnt_dir, check_info->mnt_size);
/* found it! */
return 1;
......
......@@ -49,9 +49,9 @@ static int dir_present(const char *dev_path, char *mnt_path,
if (!S_ISDIR(st.st_mode))
LOG_RETURN(-ENOTDIR, "'%s' is not a directory", dev_path);
strncpy(mnt_path, dev_path, mnt_path_size - 1);
strncpy(mnt_path, dev_path, mnt_path_size);
/* make sure mnt_path is null terminated */
mnt_path[mnt_path_size - 1] = '\0';
mnt_path[mnt_path_size-1] = '\0';
return 0;
}
......
......@@ -147,8 +147,7 @@ static int query_drive_sn(struct lib_descriptor *lib)
for (i = 0; i < count; i++)
strncpy(lib->drives.items[i].dev_id, items[i].dev_id,
sizeof(lib->drives.items[i].dev_id) - 1);
lib->drives.items[i].dev_id[sizeof(lib->drives.items[i].dev_id) - 1] = '\0';
sizeof(lib->drives.items[i].dev_id));
err_free:
free(items);
......
......@@ -243,7 +243,7 @@ static int read_next_element_status(const struct element_descriptor *elmt,
}
if (page->pvoltag) {
strncpy(elem_out->vol, elmt->pvti, VOL_ID_LEN - 1);
strncpy(elem_out->vol, elmt->pvti, VOL_ID_LEN);
elem_out->vol[VOL_ID_LEN - 1] = '\0';
rstrip(elem_out->vol);
}
......@@ -266,8 +266,8 @@ static int read_next_element_status(const struct element_descriptor *elmt,
id_len = DEV_ID_LEN - 1;
if (id_len > 0) {
strncpy(elem_out->dev_id, dev_info->devid, id_len - 1);
elem_out->dev_id[id_len - 1] = '\0';
strncpy(elem_out->dev_id, dev_info->devid, id_len);
elem_out->dev_id[id_len] = '\0';
rstrip(elem_out->dev_id);
}
}
......
*.trs
test_attrs
test_common
test_dev_tape
test_log
test_mapper
test_store
test_store_retry
test_dss
test_cfg
test_ldm
test_lintape
test_scsi
test_scsi_scan
test_type_utils
......@@ -285,57 +285,6 @@ function print_section() {
# Print commands (added after defining all the functions)
set_bash_verbosity
# TODO: generalise this
case "$(os)" in
centos:7)
# Setup yum and install dev tools
yum_build_dir="${build_dir}/yum"
if should_build system_packages
then
# Setup yum cachedir to point to ${yum_cachedir} and enable caching
mkdir -p "${yum_cachedir}"
yum_build_dir="$(cd "${yum_cachedir}" && pwd)"
mkdir -p "${yum_cachedir}/cache"
cp /etc/yum.conf "${yum_cachedir}/yum.conf"
yum_config="${yum_cachedir}/yum.conf"
sed -i "s#^cachedir=.*\$#cachedir=${yum_cachedir}/cache#" "${yum_config}"
sed -i "s#^keepcache=0#keepcache=1#" "${yum_config}"
echo "skip_missing_names_on_install=False" >> "${yum_config}"
fi
# Wrap Yum to add custom config flag
function yum() {
command yum --config="${yum_cachedir}/yum.conf" "$@"
}
if should_build system_packages
then
# Install SCL and install gcc-7
yum install -y \
centos-release-scl
yum install -y \
devtoolset-7-gcc devtoolset-7-gcc-c++
fi
# Enable gcc-7
# Temporarily disable printing and error checking
set_bash_verbosity false
set +o errexit
source scl_source enable devtoolset-7
set -o errexit
set_bash_verbosity
;;
centos:8)
dnf install -y \
gcc gcc-c++
;;
esac
#
# Common build flags
#
......
......@@ -108,6 +108,8 @@ then
then
set +o errexit
set_environment_variables
source "${script_dir}"/../phobos/testdb-setup.sh
setup_phobos_scripts ${deimos_build_dir}/../phobos/source/
......
......@@ -9,6 +9,9 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${script_dir}/../install_dependencies_centos7.sh"
#
# Install dependencies
#
......@@ -18,21 +21,10 @@ function cmake() {
}
function install_dependencies() {
yum install -y \
glib2 glib2-devel
yum install -y \
python3-pip
python3 -m pip install boto3
# EPEL Packages
yum install -y \
epel-release
yum install -y \
cmake3 \
boost169-devel
common_dependencies
phobos_dependencies
proxygen_dependencies
deimos_dependencies
#
# Build Catch2
......
......@@ -9,35 +9,26 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${script_dir}/../install_dependencies_centos8.sh"
#
# Install dependencies
#
function install_dependencies() {
# Enable repositories
dnf install -y \
dnf-plugins-core \
epel-release
dnf config-manager --set-enabled powertools
dnf install -y \
glib2 glib2-devel
dnf install -y \
python3-pip
python3 -m pip install boto3
dnf install -y \
cmake \
boost-devel \
catch-devel \
spdlog-devel
common_dependencies
phobos_dependencies
proxygen_dependencies
deimos_dependencies
}
function configure() {
phobos_install_dir=$(install_dir phobos)
}
function set_environment_variables() {
export PYTHONPATH="${phobos_install_dir}"/lib/python3.6/site-packages:"${PYTHONPATH}"
export LD_LIBRARY_PATH="${phobos_install_dir}"/lib:"${LD_LIBRARY_PATH}"
export PATH="${phobos_install_dir}"/bin:"${phobos_install_dir}"/sbin:${PATH}
}
\ No newline at end of file
#!/usr/bin/env bash
# Copyright (c) 2020, Irish Centre for High End Computing (ICHEC), NUI Galway
# Authors:
# Pádraig Ó Conbhuí <padraig.oconbhui@ichec.ie>,
# Sophie Wenzel-Teuber <sophie.wenzel-teuber@ichec.ie>,
# Ciarán O'Rourke <ciaran.orourke@ichec.ie>
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
function common_dependencies() {
yum install -y \
centos-release-scl
yum install -y \
devtoolset-8-gcc devtoolset-8-gcc-c++
# Enable gcc-8
# Temporarily disable printing and error checking
set_bash_verbosity false
set +o errexit
source scl_source enable devtoolset-8
set -o errexit
set_bash_verbosity
yum install -y \
make \
autoconf automake libtool
}
function phobos_dependencies() {
# Regular packages
yum install -y \
make \
autoconf automake libtool \
glib2-devel \
jansson-devel \
libattr-devel \
libini_config-devel \
openssl-devel \
protobuf-c-devel \
python-devel \
sg3_utils-devel \
which \
python-psycopg2 \
PyYAML \
python3-devel
# EPEL Packages
yum install -y \
epel-release
yum install -y \
clustershell \
python3-tabulate
# SCL packages for postgresql
# Centos-7 provides 9.2 by default
# Phobos needs 9.4 for jsonb
yum install -y \
rh-postgresql95-postgresql-devel \
rh-postgresql95-postgresql-libs \
rh-postgresql95-postgresql-server \
rh-postgresql95-postgresql-contrib
}
function proxygen_dependencies() {
# Regular packages
yum install -y \
make \
gperf \
perl perl-core \
libevent-devel \
zlib-devel \
openssl-devel openssl-libs
# EPEL Packages
yum install -y \
epel-release
yum install -y \
cmake3 \
boost169-devel \
double-conversion-devel \
gflags gflags-devel glog-devel \
libsodium-devel \
libzstd-devel
}
function deimos_dependencies() {
yum install -y \
glib2 glib2-devel
yum install -y \
python3-pip
python3 -m pip install boto3
# EPEL Packages
yum install -y \
epel-release
yum install -y \
cmake3 \
boost169-devel
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment