Commit 7cb8a361 authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

external/phobos: Update Phobos to dc4ddb6cb705e007186aab43aacc75f2179a2f2d

parents 0281365e 977c7ecf
......@@ -38,7 +38,7 @@ load-plugins=
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
#disable=
disable=I, W, R, import-error, no-name-in-module
#disable=I, W, R, import-error, no-name-in-module
[REPORTS]
......@@ -113,9 +113,6 @@ additional-builtins=
[BASIC]
# Required attributes for module, separated by a comma
required-attributes=
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input,file
......@@ -238,7 +235,7 @@ ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis
ignored-modules=
ignored-modules=phobos.core.const,phobos.core.glue
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
......@@ -256,10 +253,6 @@ generated-members=REQUEST,acl_users,aq_parent
[CLASSES]
# List of interface methods to ignore, separated by a comma. This is used for
# instance to not check methods defines in Zope's Interface base class.
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
......
......@@ -210,7 +210,7 @@ define valgrind_tool_rule
check-valgrind-$(1)-am:
ifeq ($$(VALGRIND_ENABLED)-$$(ENABLE_VALGRIND_$(1)),yes-yes)
ifneq ($$(TESTS),)
$$(valgrind_v_use)$$(MAKE) check-TESTS \
$$(valgrind_v_use)$$(MAKE) check \
TESTS_ENVIRONMENT="$$(VALGRIND_TESTS_ENVIRONMENT)" \
LOG_COMPILER="$$(VALGRIND_LOG_COMPILER)" \
LOG_FLAGS="$$(valgrind_$(1)_flags)" \
......
......@@ -4,7 +4,7 @@
AC_PREREQ(2.59)
AC_INIT([phobos], [1.90.0], [st-hpc@cea.fr])
AC_INIT([phobos], [1.92], [st-hpc@cea.fr])
RELEASE="1"
AC_DEFINE_UNQUOTED(RELEASE, "$RELEASE", [release info])
......@@ -67,11 +67,10 @@ CFLAGS="$CFLAGS -I\$(top_srcdir)/src/include"
AC_ARG_ENABLE( [cli], AS_HELP_STRING([--disable-cli],
[Do not generate python cli and modules]),
[enable_cli="$enableval"],[enable_cli="yes"] )
# force using python2
PYTHON=$(which python2)
PYTHON=$(which python3)
AS_IF( [test "x$enable_cli" != "xno"], [
# Enable python bindings
AX_PYTHON_DEVEL([>='2.6'])
AX_PYTHON_DEVEL([>='3.6'])
])
# Checks for header files.
......@@ -85,6 +84,9 @@ AC_CHECK_HEADERS([scsi/sg_io_linux.h], [], [AC_MSG_ERROR(missing sg3_utils heade
AC_CHECK_HEADERS([protobuf-c/protobuf-c.h], [], [AC_MSG_ERROR(missing protobuf-c header files)])
PKG_CHECK_MODULES(JANSSON, [jansson >= 2.5])
PKG_CHECK_MODULES(LIBPQ, [libpq >= 9.4])
PKG_CHECK_MODULES(CMOCKA, [cmocka >= 1.1], ,
[AC_MSG_WARN([missing package libcmocka-devel, tests will fail to compile])]
)
# Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
......@@ -111,6 +113,8 @@ AC_CONFIG_FILES([Makefile
src/layout-modules/Makefile
src/admin/Makefile
src/tests/Makefile
src/tests/unit/Makefile
src/tests/integration/Makefile
src/cli/Makefile
phobos.spec
])
......
......@@ -13,6 +13,8 @@ families = tape,dir
lib_device = /dev/changer
# path of the LRS-server socket
server_socket = /run/phobosd/lrs
# path of the daemon lock file
lock_file = /run/phobosd/phobosd.lock
[scsi]
# retry count for SCSI requests
......@@ -42,6 +44,16 @@ default_layout = simple
# default resource family for put operations
default_family = tape
[io]
# block size used for writing data to multiple media (negative values or 0 will
# be interpreted as an absence of parameter, and the use of default value
# sysconf(_SC_PAGESIZE))
io_block_size = -1
[layout_raid1]
# number of data replicates for raid1 layout (default is 2)
repl_count = 2
######### Tape/drive support and compatibility rules ########
# You should not modify the following configuration unless:
# * You want to add support for a new device or tape model
......
#!/bin/sh
function noisy_cmd
{
echo "$*" >&2
sh -c "$*"
}
echo "# Starting the daemon"
noisy_cmd systemctl start phobosd
phobos_db drop_tables
echo "# Setup database"
noisy_cmd phobos_db setup_tables
echo
rm -rf /tmp/test_get
echo "# Create testing directory"
noisy_cmd mkdir -p /tmp/test_get
echo
echo "# Add directory to phobos"
noisy_cmd phobos dir add /tmp/test_get
noisy_cmd phobos dir format --unlock --fs posix /tmp/test_get
echo
echo "# Create an object to get"
noisy_cmd phobos put --family dir --metadata "a=b" /etc/hosts oid1
echo
echo "# Get the uuid of the current generation"
uuid1=$(noisy_cmd phobos object list -o uuid oid1)
echo "-> $uuid1"
echo
echo "# Get an alive object"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --uuid "$uuid1" oid1 /tmp/out && rm /tmp/out
echo
echo "# Overwrite the current object to create a new version"
noisy_cmd phobos put --family dir --overwrite --metadata "b=c" /etc/hosts oid1
echo
echo "# Get the alive version"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
echo
echo "# Get the deprecated version"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Get the most recent version"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Delete the object"
noisy_cmd phobos delete oid1
echo
echo "# The get with just oid fails as it only targets alive objects"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
echo
echo "# But the deprecated objects can still be accessed"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Create a new generation by reusing the last oid"
noisy_cmd phobos put --family dir --metadata "c=d" /etc/hosts oid1
echo
echo "# Get the uuid of the current generation"
uuid2=$(noisy_cmd phobos object list -o uuid oid1)
echo "-> $uuid2"
echo
echo "# Now we can target the old generation with its uuid"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Create a new version of the current object"
noisy_cmd phobos put --overwrite --family dir /etc/hosts oid1
echo
echo "# Target the current generation's old version"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Target the old generation"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
#!/bin/sh
# This script must be run on a phobos admin node.
# This script shows some of the list features offered by phobos by executing
# the following steps:
# - starts the phobos daemon,
# - creates and add 3 directories to phobos,
# - executes various list commands,
# - stops the phobos daemon and deletes the 3 directories.
# (TO BE DONE: delete the 3 added directories from phobos)
echo "# START DEMO"
echo "# Start Phobos daemon"
echo "systemctl start phobosd"
systemctl start phobosd
sleep 5
echo
echo "# Add directories, format some of them, and unlock one"
mkdir /tmp/aries /tmp/taurus /tmp/gemini
echo "phobos dir add /tmp/aries /tmp/taurus /tmp/gemini"
phobos dir add /tmp/aries /tmp/taurus /tmp/gemini
sleep 5
echo "phobos dir format --fs posix /tmp/aries /tmp/gemini"
phobos dir format --fs posix /tmp/aries /tmp/gemini
sleep 5
echo "phobos dir unlock /tmp/aries"
phobos dir unlock /tmp/aries
sleep 10
echo
echo "# List directories (default behavior)"
echo "phobos dir list"
phobos dir list
sleep 10
echo
echo "# Detailed list"
echo "phobos dir list --output all"
phobos dir list --output all
sleep 10
echo
echo "# Customized list"
echo "phobos dir list --output name,path,adm_status,fs.status"
phobos dir list --output name,adm_status,fs.status
sleep 10
echo
echo "# Other list formats"
echo "phobos dir list --output name,path,adm_status,fs.status --format csv"
phobos dir list --output name,adm_status,fs.status --format csv
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format json"
phobos dir list --output name,adm_status,fs.status --format json
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format yaml"
phobos dir list --output name,adm_status,fs.status --format yaml
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format xml"
phobos dir list --output name,adm_status,fs.status --format xml
sleep 10
echo
# TO BE DONE: delete 3 added directories from phobos.
systemctl stop phobosd
rm -r /tmp/aries /tmp/taurus /tmp/gemini
echo "# END DEMO"
#!/bin/bash
# This scripts shows how phobos could be used in parallel on two servers by
# executing the following steps:
# - create a file on each server,
# - put the two files into phobos in parallel by asking separately each server
# to put its own file into phobos,
# - getting back the two file in parallel by asking separately each server to
# to get back its own file,
# - focus on the need of a future "relocate" call by asking to server1 to get
# back the file put on server0,
# - deletes files on each server.
# (TO BE DONE: delete example files from phobos)
# This scripts needs a running phobos on two servers sharing the same phobos
# database. The two phobos servers need dir devices ready to store a 1k file.
# This script must be run on a node able to connect by ssh to the both servers.
SERVER0="${SERVER0}"
SERVER1="${SERVER1}"
READER=5
echo "# 2 IO servers in parallel"
echo "# both connected to the same phobos system"
echo "# metadata are shared through a common db"
sleep $READER
sleep $READER
sleep $READER
echo
echo "# First phobos IO server : ${SERVER0}"
sleep $READER
echo
echo "ssh ${SERVER0} sudo head -3 /etc/phobos.conf"
ssh ${SERVER0} sudo head -3 /etc/phobos.conf
sleep $READER
echo
echo "ssh ${SERVER0} sudo phobos dir list"
ssh ${SERVER0} sudo phobos dir list
sleep $READER
echo
echo "# Second phobos IO server : ${SERVER1}"
sleep $READER
echo
echo "ssh ${SERVER1} sudo head -3 /etc/phobos.conf"
ssh ${SERVER1} sudo head -3 /etc/phobos.conf
sleep $READER
echo
echo "ssh ${SERVER1} phobos dir list"
ssh ${SERVER1} sudo phobos dir list
sleep $READER
echo
echo "# Prepare on both an input object"
sleep $READER
sleep $READER
echo
echo "ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1"
ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1
echo "ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1"
ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1
sleep $READER
echo
echo "# Put simultaneously the two objects"
echo "ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &"
echo "ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object"
ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &
ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object
sleep $READER
sleep $READER
echo
echo "# List object from both IO servers"
echo "ssh ${SERVER0} sudo phobos extent list -o oid,media_name"
ssh ${SERVER0} sudo phobos extent list -o oid,media_name
echo
echo "ssh ${SERVER1} sudo phobos extent list -o oid,media_name"
ssh ${SERVER1} sudo phobos extent list -o oid,media_name
sleep $READER
sleep $READER
echo
echo "# Get back simultaneously the two objects"
echo "ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &"
echo "ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back"
ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &
ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back
sleep $READER
sleep $READER
echo
echo "# Next phasis : implement relocate call between IO servers"
echo "# (currently : ${SERVER1} has no access to ${SERVER0} data)"
sleep $READER
sleep $READER
echo "ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back"
ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back
sleep $READER
sleep $READER
echo
echo "# Clean"
set -x
ssh ${SERVER0} sudo rm /tmp/${SERVER0}_file
ssh ${SERVER0} sudo rm /tmp/${SERVER0}_object_back
ssh ${SERVER1} sudo rm /tmp/${SERVER1}_file
ssh ${SERVER1} sudo rm /tmp/${SERVER1}_object_back
# TO BE DONE: delete files from phobos
#!/bin/bash
# This script shows raid1 layout features offered by phobos as robustness facing
# media failures and flexibility facing nearly full media.
# It must be run on a node ready to execute a phobos server.
# WARNING : the postgres phobos database is dropped.
READER=5
echo "################################"
echo "# Phobos RAID1 layout quick demo"
echo "################################"
sleep $READER
echo
echo "# Clean DB"
sleep $READER
echo "sudo -u postgres phobos_db drop_db || true"
sudo -u postgres phobos_db drop_db || true
sleep $READER
echo "sudo -u postgres phobos_db setup_db -s -p phobos"
sudo -u postgres phobos_db setup_db -s -p phobos
sleep $READER
echo
echo "# Start phobos daemon"
echo "systemctl start phobosd"
systemctl start phobosd
sleep $READER
echo
echo "################################"
echo "# First test : raid1 resilience"
echo "# - store 2 copies of an object on two different media"
echo "# - retrieve this object even if one medium is unavailable"
echo "################################"
sleep $READER
echo
echo "# Create a file as input object"
echo "echo \"Input object example\" > /tmp/input_object"
echo "Input object example" > /tmp/input_object
sleep $READER
echo
echo "# Add a first directory to phobos ressource"
echo "mkdir /tmp/dir1"
mkdir /tmp/dir1
sleep $READER
echo "phobos dir add /tmp/dir1"
phobos dir add /tmp/dir1
sleep $READER
echo "phobos dir format --fs posix --unlock /tmp/dir1"
phobos dir format --fs posix --unlock /tmp/dir1
sleep $READER
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Store input_object using simple layout"
echo "phobos put -f dir -l simple /tmp/input_object simple_layout_object"
phobos put -f dir -l simple /tmp/input_object simple_layout_object
sleep $READER
echo
echo "# Add a second directory to phobos ressource"
echo "mkdir /tmp/dir2"
mkdir /tmp/dir2
sleep $READER
echo "phobos dir add /tmp/dir2"
phobos dir add /tmp/dir2
sleep $READER
echo "phobos dir format --fs posix --unlock /tmp/dir2"
phobos dir format --fs posix --unlock /tmp/dir2
sleep $READER
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Set raid1 layout replicat count to 2"
echo "export PHOBOS_LAYOUT_RAID1_repl_count=2"
export PHOBOS_LAYOUT_RAID1_repl_count=2
sleep $READER
echo
echo "# Store input_object using raid1 layout"
echo "phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object"
phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object
sleep $READER
echo
echo "# List new objects extents"
echo "phobos extent list -o oid,media_name,layout"
phobos extent list -o oid,media_name,layout
sleep $READER
echo
echo "# Get back simple and raid1 object"
echo "phobos get simple_layout_object /tmp/simple_back"
phobos get simple_layout_object /tmp/simple_object_back
echo "phobos get raid1_2replica_object /tmp/raid1_back"
phobos get raid1_2replica_object /tmp/raid1_back
sleep $READER
echo
echo "# Simulate a failure on first medium"
echo "phobos dir lock /tmp/dir1"
phobos dir lock /tmp/dir1
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Due to first medium failure, simple layout object is unavailable."
echo "phobos get simple_layout_object /tmp/simple_back_with_failure"
phobos get simple_layout_object /tmp/simple_object_back_with_failure
sleep $READER
echo
echo "# Get raid1 layout object even if first medium is failed."
echo "phobos get raid1_2replica_object /tmp/raid1_back_failure"
phobos get raid1_2replica_object /tmp/raid1_back_failure
sleep $READER
echo
echo "# Lock media"
echo
echo "phobos dir lock /tmp/dir1 /tmp/dir2"
phobos dir lock /tmp/dir1 /tmp/dir2
sleep $READER
echo
echo "################################"
echo "# Second test : raid1 management of media allocation"
echo "# - store 2 copies of an object on three different media"
echo "# - first medium is wide enough to store one copy of the object"
echo "# - second and third media are smaller and can't store the object"
echo "# - the second copy is splitted between second and third media"
echo "# - retrieve this object even if one medium is unavailable"
sleep $READER
sleep $READER
sleep $READER
echo
echo "# Create first medium of 128M"
sleep $READER
echo
echo "dd if=/dev/zero of=/tmp/128M_file bs=1M count=128"
dd if=/dev/zero of=/tmp/128M_device_file bs=1M count=128
echo
echo "losetup -f /tmp/128M_device_file"
losetup -f /tmp/128M_device_file
echo
echo "mkfs.ext4 /dev/loop0"
mkfs.ext4 /dev/loop0
echo
echo "mkdir /tmp/128M_dir"
mkdir /tmp/128M_dir
echo
echo "mount /dev/loop0 /tmp/128M_dir"
mount /dev/loop0 /tmp/128M_dir
sleep $READER
echo
echo "phobos dir add /tmp/128M_dir"
phobos dir add /tmp/128M_dir
echo
echo "phobos dir format --fs posix --unlock /tmp/128M_dir"
phobos dir format --fs posix --unlock /tmp/128M_dir
sleep $READER
echo
echo "# Create second and third medium of 64M each"
sleep $READER
echo
echo "dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64"
dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64
echo "dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64"
dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64
echo
echo "losetup -f /tmp/64M_file_1"
losetup -f /tmp/64M_file_1
echo "losetup -f /tmp/64M_file_2"
losetup -f /tmp/64M_file_2
echo
echo "mkfs.ext4 /dev/loop1"
mkfs.ext4 /dev/loop1
echo "mkfs.ext4 /dev/loop2"
mkfs.ext4 /dev/loop2
echo
echo "mkdir /tmp/64M_dir_1"
mkdir /tmp/64M_dir_1
echo "mkdir /tmp/64M_dir_2"
mkdir /tmp/64M_dir_2
echo
echo "mount /dev/loop1 /tmp/64M_dir_1"
mount /dev/loop1 /tmp/64M_dir_1
echo "mount /dev/loop2 /tmp/64M_dir_2"
mount /dev/loop2 /tmp/64M_dir_2
sleep $READER
echo
echo "phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2"
phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2
echo
echo "phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2"
phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2
echo
echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
phobos dir list -o name,adm_status,stats.phys_spc_free
sleep $READER
echo
echo "# Create a 100M file as input object"
sleep $READER
echo
echo "dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100"
dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100
sleep $READER
echo
echo "# Store the 100M file using raid1 layout"
echo
echo "phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object"
phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object
sleep $READER
echo
echo "phobos object list -o all"
phobos object list -o all
echo
echo "phobos extent list -o oid,layout,media_name,size"
phobos extent list -o oid,layout,media_name,size
sleep $READER
echo
echo "# Get back object"
echo "phobos get 100M_raid1_object /tmp/100M_raid1_back"
phobos get 100M_raid1_object /tmp/100M_raid1_back
sleep $READER
echo
echo "# Simulate a failure of 128M medium"
echo "phobos dir lock /tmp/128M_dir"
phobos dir lock /tmp/128M_dir
echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
phobos dir list -o name,adm_status,stats.phys_spc_free
sleep $READER
echo
echo "# Get back object under failure"