external/phobos: Update Phobos to dc4ddb6cb705e007186aab43aacc75f2179a2f2d

external/phobos/.pylintrc

@@ -38,7 +38,7 @@ load-plugins=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 #disable=
-disable=I, W, R, import-error, no-name-in-module
+#disable=I, W, R, import-error, no-name-in-module
 
 [REPORTS]
 
@@ -113,9 +113,6 @@ additional-builtins=
 
 [BASIC]
 
-# Required attributes for module, separated by a comma
-required-attributes=
-
 # List of builtins function names that should not be used, separated by a comma
 bad-functions=map,filter,apply,input,file
 
@@ -238,7 +235,7 @@ ignore-mixin-members=yes
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=
+ignored-modules=phobos.core.const,phobos.core.glue
 
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set).
@@ -256,10 +253,6 @@ generated-members=REQUEST,acl_users,aq_parent
 
 [CLASSES]
 
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
 # List of method names used to declare (i.e. assign) instance attributes.
 defining-attr-methods=__init__,__new__,setUp
 

external/phobos/autotools/m4/ax_valgrind_check.m4

@@ -210,7 +210,7 @@ define valgrind_tool_rule
 check-valgrind-$(1)-am:
 ifeq ($$(VALGRIND_ENABLED)-$$(ENABLE_VALGRIND_$(1)),yes-yes)
 ifneq ($$(TESTS),)
-	$$(valgrind_v_use)$$(MAKE) check-TESTS \
+	$$(valgrind_v_use)$$(MAKE) check \
 		TESTS_ENVIRONMENT="$$(VALGRIND_TESTS_ENVIRONMENT)" \
 		LOG_COMPILER="$$(VALGRIND_LOG_COMPILER)" \
 		LOG_FLAGS="$$(valgrind_$(1)_flags)" \

external/phobos/configure.ac

@@ -4,7 +4,7 @@
 
 AC_PREREQ(2.59)
 
-AC_INIT([phobos], [1.90.0], [st-hpc@cea.fr])
+AC_INIT([phobos], [1.92], [st-hpc@cea.fr])
 RELEASE="1"
 
 AC_DEFINE_UNQUOTED(RELEASE, "$RELEASE", [release info])
@@ -67,11 +67,10 @@ CFLAGS="$CFLAGS -I\$(top_srcdir)/src/include"
 AC_ARG_ENABLE( [cli], AS_HELP_STRING([--disable-cli],
                [Do not generate python cli and modules]),
                [enable_cli="$enableval"],[enable_cli="yes"] )
-# force using python2
-PYTHON=$(which python2)
+PYTHON=$(which python3)
 AS_IF( [test "x$enable_cli" != "xno"], [
     # Enable python bindings
-    AX_PYTHON_DEVEL([>='2.6'])
+    AX_PYTHON_DEVEL([>='3.6'])
 ])
 
 # Checks for header files.
@@ -85,6 +84,9 @@ AC_CHECK_HEADERS([scsi/sg_io_linux.h], [], [AC_MSG_ERROR(missing sg3_utils heade
 AC_CHECK_HEADERS([protobuf-c/protobuf-c.h], [], [AC_MSG_ERROR(missing protobuf-c header files)])
 PKG_CHECK_MODULES(JANSSON, [jansson >= 2.5])
 PKG_CHECK_MODULES(LIBPQ, [libpq >= 9.4])
+PKG_CHECK_MODULES(CMOCKA, [cmocka >= 1.1], ,
+    [AC_MSG_WARN([missing package libcmocka-devel, tests will fail to compile])]
+)
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
@@ -111,6 +113,8 @@ AC_CONFIG_FILES([Makefile
          src/layout-modules/Makefile
          src/admin/Makefile
          src/tests/Makefile
+         src/tests/unit/Makefile
+         src/tests/integration/Makefile
          src/cli/Makefile
          phobos.spec
 ])

external/phobos/doc/cfg/template.conf

@@ -13,6 +13,8 @@ families      = tape,dir
 lib_device    = /dev/changer
 # path of the LRS-server socket
 server_socket = /run/phobosd/lrs
+# path of the daemon lock file
+lock_file     = /run/phobosd/phobosd.lock
 
 [scsi]
 # retry count for SCSI requests
@@ -42,6 +44,16 @@ default_layout = simple
 # default resource family for put operations
 default_family = tape
 
+[io]
+# block size used for writing data to multiple media (negative values or 0 will
+# be interpreted as an absence of parameter, and the use of default value
+# sysconf(_SC_PAGESIZE))
+io_block_size = -1
+
+[layout_raid1]
+# number of data replicates for raid1 layout (default is 2)
+repl_count = 2
+
 ######### Tape/drive support and compatibility rules ########
 # You should not modify the following configuration unless:
 #  * You want to add support for a new device or tape model

external/phobos/doc/demo/demo_get.sh

+#!/bin/sh
+
+function noisy_cmd
+{
+    echo "$*" >&2
+    sh -c "$*"
+}
+
+echo "# Starting the daemon"
+noisy_cmd systemctl start phobosd
+
+phobos_db drop_tables
+echo "# Setup database"
+noisy_cmd phobos_db setup_tables
+echo
+
+rm -rf /tmp/test_get
+echo "# Create testing directory"
+noisy_cmd mkdir -p /tmp/test_get
+echo
+
+echo "# Add directory to phobos"
+noisy_cmd phobos dir add /tmp/test_get
+noisy_cmd phobos dir format --unlock --fs posix /tmp/test_get
+echo
+
+echo "# Create an object to get"
+noisy_cmd phobos put --family dir --metadata "a=b" /etc/hosts oid1
+echo
+
+echo "# Get the uuid of the current generation"
+uuid1=$(noisy_cmd phobos object list -o uuid oid1)
+echo "-> $uuid1"
+echo
+
+echo "# Get an alive object"
+noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
+noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
+noisy_cmd phobos get --uuid "$uuid1" oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Overwrite the current object to create a new version"
+noisy_cmd phobos put --family dir --overwrite --metadata "b=c" /etc/hosts oid1
+echo
+
+echo "# Get the alive version"
+noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Get the deprecated version"
+noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Get the most recent version"
+noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Delete the object"
+noisy_cmd phobos delete oid1
+echo
+
+echo "# The get with just oid fails as it only targets alive objects"
+noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# But the deprecated objects can still be accessed"
+noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
+noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Create a new generation by reusing the last oid"
+noisy_cmd phobos put --family dir --metadata "c=d" /etc/hosts oid1
+echo
+
+echo "# Get the uuid of the current generation"
+uuid2=$(noisy_cmd phobos object list -o uuid oid1)
+echo "-> $uuid2"
+echo
+
+echo "# Now we can target the old generation with its uuid"
+noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Create a new version of the current object"
+noisy_cmd phobos put --overwrite --family dir /etc/hosts oid1
+echo
+
+echo "# Target the current generation's old version"
+noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
+echo
+
+echo "# Target the old generation"
+noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
+echo

external/phobos/doc/demo/demo_list.sh

+#!/bin/sh
+
+# This script must be run on a phobos admin node.
+# This script shows some of the list features offered by phobos by executing
+# the following steps:
+# - starts the phobos daemon,
+# - creates and add 3 directories to phobos,
+# - executes various list commands,
+# - stops the phobos daemon and deletes the 3 directories.
+# (TO BE DONE: delete the 3 added directories from phobos)
+
+echo "# START DEMO"
+echo "# Start Phobos daemon"
+echo "systemctl start phobosd"
+systemctl start phobosd
+sleep 5
+echo
+
+echo "# Add directories, format some of them, and unlock one"
+mkdir /tmp/aries /tmp/taurus /tmp/gemini
+echo "phobos dir add /tmp/aries /tmp/taurus /tmp/gemini"
+phobos dir add /tmp/aries /tmp/taurus /tmp/gemini
+sleep 5
+echo "phobos dir format --fs posix /tmp/aries /tmp/gemini"
+phobos dir format --fs posix /tmp/aries /tmp/gemini
+sleep 5
+echo "phobos dir unlock /tmp/aries"
+phobos dir unlock /tmp/aries
+sleep 10
+echo
+
+echo "# List directories (default behavior)"
+echo "phobos dir list"
+phobos dir list
+sleep 10
+echo
+
+echo "# Detailed list"
+echo "phobos dir list --output all"
+phobos dir list --output all
+sleep 10
+echo
+
+echo "# Customized list"
+echo "phobos dir list --output name,path,adm_status,fs.status"
+phobos dir list --output name,adm_status,fs.status
+sleep 10
+echo
+
+echo "# Other list formats"
+echo "phobos dir list --output name,path,adm_status,fs.status --format csv"
+phobos dir list --output name,adm_status,fs.status --format csv
+sleep 5
+echo "phobos dir list --output name,path,adm_status,fs.status --format json"
+phobos dir list --output name,adm_status,fs.status --format json
+sleep 5
+echo "phobos dir list --output name,path,adm_status,fs.status --format yaml"
+phobos dir list --output name,adm_status,fs.status --format yaml
+sleep 5
+echo "phobos dir list --output name,path,adm_status,fs.status --format xml"
+phobos dir list --output name,adm_status,fs.status --format xml
+sleep 10
+echo
+
+# TO BE DONE: delete 3 added directories from phobos.
+
+systemctl stop phobosd
+rm -r /tmp/aries /tmp/taurus /tmp/gemini
+echo "# END DEMO"

external/phobos/doc/demo/demo_phobos_parallel.bash

+#!/bin/bash
+
+# This scripts shows how phobos could be used in parallel on two servers by
+# executing the following steps:
+# - create a file on each server,
+# - put the two files into phobos in parallel by asking separately each server
+# to put its own file into phobos,
+# - getting back the two file in parallel by asking separately each server to
+# to get back its own file,
+# - focus on the need of a future "relocate" call by asking to server1 to get
+# back the file put on server0,
+# - deletes files on each server.
+# (TO BE DONE: delete example files from phobos)
+
+# This scripts needs a running phobos on two servers sharing the same phobos
+# database. The two phobos servers need dir devices ready to store a 1k file.
+# This script must be run on a node able to connect by ssh to the both servers.
+
+SERVER0="${SERVER0}"
+SERVER1="${SERVER1}"
+
+READER=5
+
+echo "# 2 IO servers in parallel"
+echo "# both connected to the same phobos system"
+echo "# metadata are shared through a common db"
+sleep $READER
+sleep $READER
+sleep $READER
+echo
+echo "# First phobos IO server : ${SERVER0}"
+sleep $READER
+echo
+echo "ssh ${SERVER0} sudo head -3 /etc/phobos.conf"
+ssh ${SERVER0} sudo head -3 /etc/phobos.conf
+sleep $READER
+echo
+echo "ssh ${SERVER0} sudo phobos dir list"
+ssh ${SERVER0} sudo phobos dir list
+sleep $READER
+echo
+echo "# Second phobos IO server : ${SERVER1}"
+sleep $READER
+echo
+echo "ssh ${SERVER1} sudo head -3 /etc/phobos.conf"
+ssh ${SERVER1} sudo head -3 /etc/phobos.conf
+sleep $READER
+echo
+echo "ssh ${SERVER1} phobos dir list"
+ssh ${SERVER1} sudo phobos dir list
+sleep $READER
+echo
+echo "# Prepare on both an input object"
+sleep $READER
+sleep $READER
+echo
+echo "ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1"
+ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1
+echo "ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1"
+ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1
+sleep $READER
+echo
+echo "# Put simultaneously the two objects"
+echo "ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &"
+echo "ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object"
+ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &
+ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object
+sleep $READER
+sleep $READER
+echo
+echo "# List object from both IO servers"
+echo "ssh ${SERVER0} sudo phobos extent list -o oid,media_name"
+ssh ${SERVER0} sudo phobos extent list -o oid,media_name
+echo
+echo "ssh ${SERVER1} sudo phobos extent list -o oid,media_name"
+ssh ${SERVER1} sudo phobos extent list -o oid,media_name
+sleep $READER
+sleep $READER
+echo
+echo "# Get back simultaneously the two objects"
+echo "ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &"
+echo "ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back"
+ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &
+ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back
+sleep $READER
+sleep $READER
+echo
+echo "# Next phasis : implement relocate call between IO servers"
+echo "# (currently : ${SERVER1} has no access to ${SERVER0} data)"
+sleep $READER
+sleep $READER
+echo "ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back"
+ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back
+sleep $READER
+sleep $READER
+
+
+echo
+echo "# Clean"
+set -x
+ssh ${SERVER0} sudo rm /tmp/${SERVER0}_file
+ssh ${SERVER0} sudo rm /tmp/${SERVER0}_object_back
+ssh ${SERVER1} sudo rm /tmp/${SERVER1}_file
+ssh ${SERVER1} sudo rm /tmp/${SERVER1}_object_back
+# TO BE DONE: delete files from phobos

external/phobos/doc/demo/demo_phobos_raid1.bash

+#!/bin/bash
+
+# This script shows raid1 layout features offered by phobos as robustness facing
+# media failures and flexibility facing nearly full media.
+# It must be run on a node ready to execute a phobos server.
+# WARNING : the postgres phobos database is dropped.
+
+READER=5
+
+echo "################################"
+echo "# Phobos RAID1 layout quick demo"
+echo "################################"
+sleep $READER
+echo
+echo "# Clean DB"
+sleep $READER
+echo "sudo -u postgres phobos_db drop_db || true"
+sudo -u postgres phobos_db drop_db || true
+sleep $READER
+echo "sudo -u postgres phobos_db setup_db -s -p phobos"
+sudo -u postgres phobos_db setup_db -s -p phobos
+sleep $READER
+echo
+echo "# Start phobos daemon"
+echo "systemctl start phobosd"
+systemctl start phobosd
+sleep $READER
+echo
+echo "################################"
+echo "# First test : raid1 resilience"
+echo "# - store 2 copies of an object on two different media"
+echo "# - retrieve this object even if one medium is unavailable"
+echo "################################"
+sleep $READER
+echo
+echo "# Create a file as input object"
+echo "echo \"Input object example\" > /tmp/input_object"
+echo "Input object example" > /tmp/input_object
+sleep $READER
+echo
+echo "# Add a first directory to phobos ressource"
+echo "mkdir /tmp/dir1"
+mkdir /tmp/dir1
+sleep $READER
+echo "phobos dir add /tmp/dir1"
+phobos dir add /tmp/dir1
+sleep $READER
+echo "phobos dir format --fs posix --unlock /tmp/dir1"
+phobos dir format --fs posix --unlock /tmp/dir1
+sleep $READER
+echo "phobos dir list -o name,adm_status"
+phobos dir list -o name,adm_status
+sleep $READER
+echo
+echo "# Store input_object using simple layout"
+echo "phobos put -f dir -l simple /tmp/input_object simple_layout_object"
+phobos put -f dir -l simple /tmp/input_object simple_layout_object
+sleep $READER
+echo
+echo "# Add a second directory to phobos ressource"
+echo "mkdir /tmp/dir2"
+mkdir /tmp/dir2
+sleep $READER
+echo "phobos dir add /tmp/dir2"
+phobos dir add /tmp/dir2
+sleep $READER
+echo "phobos dir format --fs posix --unlock /tmp/dir2"
+phobos dir format --fs posix --unlock /tmp/dir2
+sleep $READER
+echo "phobos dir list -o name,adm_status"
+phobos dir list -o name,adm_status
+sleep $READER
+echo
+echo "# Set raid1 layout replicat count to 2"
+echo "export PHOBOS_LAYOUT_RAID1_repl_count=2"
+export PHOBOS_LAYOUT_RAID1_repl_count=2
+sleep $READER
+echo
+echo "# Store input_object using raid1 layout"
+echo "phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object"
+phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object
+sleep $READER
+echo
+echo "# List new objects extents"
+echo "phobos extent list -o oid,media_name,layout"
+phobos extent list -o oid,media_name,layout
+sleep $READER
+echo
+echo "# Get back simple and raid1 object"
+echo "phobos get simple_layout_object /tmp/simple_back"
+phobos get simple_layout_object /tmp/simple_object_back
+echo "phobos get raid1_2replica_object /tmp/raid1_back"
+phobos get raid1_2replica_object /tmp/raid1_back
+sleep $READER
+echo
+echo "# Simulate a failure on first medium"
+echo "phobos dir lock /tmp/dir1"
+phobos dir lock /tmp/dir1
+echo "phobos dir list -o name,adm_status"
+phobos dir list -o name,adm_status
+sleep $READER
+echo
+echo "# Due to first medium failure, simple layout object is unavailable."
+echo "phobos get simple_layout_object /tmp/simple_back_with_failure"
+phobos get simple_layout_object /tmp/simple_object_back_with_failure
+sleep $READER
+echo
+echo "# Get raid1 layout object even if first medium is failed."
+echo "phobos get raid1_2replica_object /tmp/raid1_back_failure"
+phobos get raid1_2replica_object /tmp/raid1_back_failure
+sleep $READER
+echo
+echo "# Lock media"
+echo
+echo "phobos dir lock /tmp/dir1 /tmp/dir2"
+phobos dir lock /tmp/dir1 /tmp/dir2
+sleep $READER
+
+
+echo
+echo "################################"
+echo "# Second test : raid1 management of media allocation"
+echo "# - store 2 copies of an object on three different media"
+echo "# - first medium is wide enough to store one copy of the object"
+echo "# - second and third media are smaller and can't store the object"
+echo "# - the second copy is splitted between second and third media"
+echo "# - retrieve this object even if one medium is unavailable"
+sleep $READER
+sleep $READER
+sleep $READER
+echo
+echo "# Create first medium of 128M"
+sleep $READER
+echo
+echo "dd if=/dev/zero of=/tmp/128M_file bs=1M count=128"
+dd if=/dev/zero of=/tmp/128M_device_file bs=1M count=128
+echo
+echo "losetup -f /tmp/128M_device_file"
+losetup -f /tmp/128M_device_file
+echo
+echo "mkfs.ext4 /dev/loop0"
+mkfs.ext4 /dev/loop0
+echo
+echo "mkdir /tmp/128M_dir"
+mkdir /tmp/128M_dir
+echo
+echo "mount /dev/loop0 /tmp/128M_dir"
+mount /dev/loop0 /tmp/128M_dir
+sleep $READER
+echo
+echo "phobos dir add /tmp/128M_dir"
+phobos dir add /tmp/128M_dir
+echo
+echo "phobos dir format --fs posix --unlock /tmp/128M_dir"
+phobos dir format --fs posix --unlock /tmp/128M_dir
+sleep $READER
+echo
+echo "# Create second and third medium of 64M each"
+sleep $READER
+echo
+echo "dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64"
+dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64
+echo "dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64"
+dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64
+echo
+echo "losetup -f /tmp/64M_file_1"
+losetup -f /tmp/64M_file_1
+echo "losetup -f /tmp/64M_file_2"
+losetup -f /tmp/64M_file_2
+echo
+echo "mkfs.ext4 /dev/loop1"
+mkfs.ext4 /dev/loop1
+echo "mkfs.ext4 /dev/loop2"
+mkfs.ext4 /dev/loop2
+echo
+echo "mkdir /tmp/64M_dir_1"
+mkdir /tmp/64M_dir_1
+echo "mkdir /tmp/64M_dir_2"
+mkdir /tmp/64M_dir_2
+echo
+echo "mount /dev/loop1 /tmp/64M_dir_1"
+mount /dev/loop1 /tmp/64M_dir_1
+echo "mount /dev/loop2 /tmp/64M_dir_2"
+mount /dev/loop2 /tmp/64M_dir_2
+sleep $READER
+echo
+echo "phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2"
+phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2
+echo
+echo "phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2"
+phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2
+echo
+echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
+phobos dir list -o name,adm_status,stats.phys_spc_free
+sleep $READER
+echo
+echo "# Create a 100M file as input object"
+sleep $READER
+echo
+echo "dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100"
+dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100
+sleep $READER
+echo
+echo "# Store the 100M file using raid1 layout"
+echo
+echo "phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object"
+phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object
+sleep $READER
+echo
+echo "phobos object list -o all"
+phobos object list -o all
+echo
+echo "phobos extent list -o oid,layout,media_name,size"
+phobos extent list -o oid,layout,media_name,size
+sleep $READER
+echo
+echo "# Get back object"
+echo "phobos get 100M_raid1_object /tmp/100M_raid1_back"
+phobos get 100M_raid1_object /tmp/100M_raid1_back
+sleep $READER
+echo
+echo "# Simulate a failure of 128M medium"
+echo "phobos dir lock /tmp/128M_dir"
+phobos dir lock /tmp/128M_dir
+echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
+phobos dir list -o name,adm_status,stats.phys_spc_free
+sleep $READER
+echo
+echo "# Get back object under failure"