Commit 977c7ecf authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Squashed 'external/phobos/' changes from e07d0af48..dc4ddb6cb

dc4ddb6cb pkg: update to 1.92
59b4ef9e7 bugfix: add missing quote to phobos locate output
59309cdbd lrs: fix medium in place of device into comments
8610b6812 ldm: remove unused lds_loaded field from ldm_dev_state
a0954f74c design: add design for the lock table refactoring
97c615771 bugfix: update dss_lock design for the multiple calls
4e1bbbe8b tests: mutualize the use of test_setup across tests
ee79d5a94 tests: lock all media in test_locate.c instead of specific ones
270d48130 cli: add 'phobos get --best_host'
fb4cd9e47 store: add 'PHO_XFER_OBJ_BEST_HOST' flag to 'phobos_get'
eaf999e10 bugfix: use dir_or_tape instead of always tape in test_locate.sh
df2686730 store: break out of for loop after allocation failure
66273d6d8 test: add unit test for object_md_get
e2b1e08b6 cli: add support for uuid and version in get command
19e16b5cc store: add support for version and uuid in decoder_build()
7f79a42a0 dss: use dss_lazy_find_object() for get and locate
5850b9a82 demo: get
b2b34244d autotools: fix valgrind behavior
2c1b64b1f tests: fix initializations in test_store_object_md
3d78300bc store: fix branch upon uninitialized value
c23ce90ba cli: add tape/dir locate
028031fee phobos_admin: phobos_admin_medium_locate
fe2e7437b optim: mutualize calls to error functions in integration tests
2fe9b6d8e optim: mutualize calls to error functions in tests
c91801647 test: integrate test_scsi_scan.c core function in test_scsi.c
0d3e0b303 bugfix: fix memory leak in test_dss_lock.c
ada400d21 bugfix: fix memory leak in test_dss_medium_locate.c
76feaf0b0 bugfix: fix memory leak in object_md functions
d73cbdac2 cli: phobos locate oid
964ed006e doc: fix dir/tape locate cli example
58d01b7be raid1: take multi-split in locate into account
face68cb8 raid1: layout_repl_count on repl_count without encoder
30dc2c0cb doc: convert to md documentation files (phobos_cmd)
c9e3aecac doc: convert to md documentation files (object_store)
23539d07d doc: convert to md documentation files (lrs)
4c6300766 doc: convert to md documentation files (ldm)
db867b0a5 doc: convert to md documentation files (layout)
d1c610d14 doc: convert to md documentation files (dss_filters)
18e361490 doc: convert to md documentation files (cli)
1e0904212 doc: convert to md documentation files (adapters)
301b1787a design, store: fix typos in comments and design document
915f1d879 layout: avoid using 'pho_attrs_to_json' multiple times
404ee1556 dss: dss_medium_locate
66d7f30c3 store: phobos_locate
8dc71f0a4 store: dss_lazy_find_object
13338fc3e test: cmocka global DSS handle setup and teardown
4a9016658 dss: dss_host_from_lock_owner
fbbb6697f tests: merge 'add_drive' and 'add_dir' in test_store_retry.c
90ec787d8 tests: unit test for object_md_del()
8ee3656d9 fix: bad log in object_md_del
7528bf20a fix: bad return code in object_md_del()
da2513b36 tests: unit test for object_md_save()
e0f363567 extent: remove addr_type from extent structure
adeaa1161 lrs: trim blanks of model string when comparing
c40214e35 rfc: remove id field from pho_lock data structure
f9102d096 optim: remove branch condition that evaluates to garbage value
efb71f6aa pkg: update to 1.91.2
58173fda1 optim: mutualize dss_lock.c functions
30af73093 io: remove ioa_put() call from the io adapter
8ed96415c layout: remove ioa_put() call from raid1
fbfafc3dd dss: decrease log severity when status on an invalid lock
b67126a30 test: add testing for multiple dss_xxx functions
144dbd851 dss: add wrapper for refresh and status
b19e55295 fix: improper use of dss_{un}lock in object_{un}delete
8380c7e56 raid1, cfg: add configurable block size for writing
349c93b99 fix: remove useless pho_lock variable in dss_{device,media}_from_pg_row
3c589e6e1 test: add integration test for overwrite feature
10560bb3b cli, store: put overwrite
25747ac1e store, dss: manage layouts using uuid and version
fd526bed5 fix: improper initialization of xfer in test_undelete.c
a86ce0fee cleanup dss: rename dss lock functions, remove wrappers
28b7e23fb dss: new sql schema to remove old locks
6448886bf dss: remove old lock functions, move new ones to dss_lock.c
d58195de7 dss, lrs and cli: consolidate get using new lock API
19bdf98f7 dss: use new lock API for lock/unlock on device/media
3ad5073fc dss: unit test of dss_object_move
5e649ffac fix: remove duplicate call to pho_cfg_init_local
ff1bfb64b dss: fix invalid call to dss_res_free in dss_generic_get
bbfc0c7cf lrs: fix memory leak in daemon when ping response is successful
0a216bc46 tests: remove DSS client usage from PhobosCLITest.py
b39889dad cli: fix logger cleaning
9bf094074 dss: remove dss_object_delete
b74211472 store and dss: use new lock API into object undelete
cbfccfb18 store and dss: use new lock API into object delete
b63cb96e8 cleaning: compilation warning/error detected by clang
108afa3a1 dss: cleaning UNLOCK simple query
8123042ea doc: add locate.md design file
ab0256b0d cli: implementation of the ping feature
2ec39f91c dss: implement dss_lock_refresh()
8d809a540 dss: implement dss_lock_status()
f78edaa41 dss: implement dss_unlock()
6a8bc04a9 dss: implement dss_lock()
579004bbc optimization: Adds aliases to del/undel, changed delete API
5f913e76a documentation: update the configuration doc
b3492bc46 fix: invalid filter_free after getting layout_list
54ee9efd3 cli: change fs default value for 'dir format' to 'posix'
fa22ac815 cli: add '--pattern' to 'phobos extent list'
9860c4f49 cli: add '--pattern' to 'phobos object list'
979bffa01 db: add schema 1.92
afbe8c297 fix: cli logger error if errno does not exist
cd7b946e9 db: close file after retrieving the sql schema
091981ea5 doc: add dss_api.md
5453b4075 doc: add database.md
97d3db253 cli: fix mput file parser
09942d53b pkg: update to 1.91.1
6f465d59d store: get object after pushing its metadata in database
bbe40ca8d cli: add uuid and version fields to object displayer
3fe39acb3 cli: fix string encoding with 'phobos object/extent list'
0a9a165de cli: fix xfer destruction
305d14e97 doc: fix deletion design
932b1986d doc: extent_key and extent_description into adapters design
78cb6f0a2 io: change io extent path generation
238f3766f store: add uuid and version fields to the XFer structure
2d3f0a043 io: raise an error if getting/deleting an extent without address
a24c3ec8f doc: undelete demo
c480e2bff cli: from "phobos object delete" to "phobos delete"
ae834600a cli: phobos undelete uuid/oid
c81761989 dss: undelete from oid
1b560102e store: init xfer.xd_rc to 0 before trying to execute the xfer
f0d7d6748 fix: memory leak in lrs.c
e02ea384a fix: non-null terminated string
6d96d2ca8 fix: uninitialized rc in lrs.c
0630cb17f fix: resource leaks
be8a860b3 fix: out-of-bounds write after using readlink()
cdbe17c6b fix: avoid dereferencing a null pointer
46d2f1f83 fix: add a 'fallthrough' comment on a no-break switch case
a61c0c304 fix: replace size_t return value types by ssize_t
56bc987e9 fix: remove unnecessary checks
3d244b956 refacto: move up response memory release
065e36abb fix: memory leak in prepare_requests when an error occurs
01c46dae2 cli: add phobos object undelete
29a7de02b undelete: add phobos_undelete() into C API
b5ec00dfa test: test_dss.c, filter from pointer to struct
521f2f077 dss: move a uuid from deprecated object to object table
8e9a5d12f fix: remove 'disk' family from CLI
8656628fa fix: test error on acceptance.sh
a9d23f855 cli: add the option '--deprecated' to 'phobos object list'
99c87664d cli: add 'phobos object delete' command
de8f243b4 store: add phobos_object_delete()
4bb7e0e94 dss: move an oid from object to deprecated_object table
c8071547a fix: free forgotten field of struct dev_info
98f9743a3 dss: add deprecated_object get/set methods
5d139d14a fix: memory leak on dev_info_dup when an error occurs
e65f0a78a doc: replace '-t' option of 'phobos drive list' by '--model'
1faea8817 tests: replace short by long options
bf42879af doc: replace short by long options
526ac8624 fix: memory leak using dss_char4sql()
b24b30513 fix: typo in PhobosCLITest.py
21c23f544 fix: change pending_alloc variable name
db74d8bee fix: misalignment in dss.c
3a08b36f0 fix: remove unnecessary allocations in dss.c
c50c89658 pkg: update to 1.91 (Japet)
6fb786f59 pylint: fix pylint warning issue
608e57758 tests: limit the number of files considered in acceptance.sh
d2c59a87a Add alias functionality
1c056162b Remove loading of the default family from cli
e04f9b8b9 tests: add a mput test to acceptance.sh
0025e2d7f LRS: take into account get operation flag
caf16e1e3 test: add a tag update test
2fd9d101f LRS: take into account put operation flag
1abb0f216 CLI: adding tape access command to set operation flags
3e24babe7 media: add operation flags to struct media_info
802ff1c1c pylint: correct setup.py
59ce4dbb0 pylint: correct tests.PhobosLDMTest.py
e060eb6a4 pylint: correct tests.PhobosCLITest.py
ce0067a25 pylint: correct tests.PhobosDbTest.py
ea89685ae pylint: correct tests.FileDescTest.py
9dab361fa pylint: correct phobos.output.py
2ff44a0f8 pylint: correct db.__main__.py
c78abd4c6 pylint: correct db.__init__.py
df59a3dbf pylint: correct phobos.db.db_config.py
87cab39d4 pylint: correct phobos.core.utils.py
b61140fac pylint: correct phobos.core.store.py
95e1545a2 pylint: correct phobos.core.log.py
7a8543b69 pylint: correct phobos.core.ldm.py
128b4ca14 pylint: correct phobos.core.cfg.py
0f789d1be pylint: correct phobos.core.admin.py
33ebfe60c pylint: correct phobos.core.dss.py
6324f6c35 pylint: correct tests.PhobosDSSTest.py
8ec6d48d4 pylint: correct phobos.core.ffi.py
40c78dfc9 pylint: correct phobos.cli.py
5e4ea31ec pylint: update pylintrc
b223eba5e DB: towards media operation flags
7505f0b35 design: media operations flags
2eaa6eff7 fix: missing 1.91 database schema files for build
7a178375c fix: test_daemon.sh failure due to patch merge
7dcd40dfd python: port to python3
516589670 DB: 1.91, towards deletion and versionning
aa72331a4 phobosd: release locked media on initialization
55b6678cd phobosd: release locked devices on initialization
9aabf4b3f pho_types.h: lock format description
05fb9cc9c lrs: socket default file in same directory as lock file
d7ea98d32 phobosd: check daemon lock file existence before initialization
a18242474 design: deletion_and_versioning.md update
aab689681 fix: fix typo in phobos.db python script
3e3f31b1d demo: add review scripts (tags, list, raid1, pre-parallel)
46b40a66d design: add deletion_and_versioning.md
9bdc862b7 fix: add missing fields to device info duplication
1af227580 cli: remove unused function attrs_from_dict in store.py
808d829ca fix: configuration file loading in PUT requests
349af422d cli: remove unused 'izip' import
31bb81727 python3: change the octal prefix to be python3 compliant
5cef21a40 python3: change division operator to floor division
32a44ef77 tests: shorten python version getter in test shell scripts
bb801cef0 python3: change 'print xxx' to 'print(xxx)'

git-subtree-dir: external/phobos
git-subtree-split: dc4ddb6cb705e007186aab43aacc75f2179a2f2d
parent 2aaebb1b
......@@ -38,7 +38,7 @@ load-plugins=
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
#disable=
disable=I, W, R, import-error, no-name-in-module
#disable=I, W, R, import-error, no-name-in-module
[REPORTS]
......@@ -113,9 +113,6 @@ additional-builtins=
[BASIC]
# Required attributes for module, separated by a comma
required-attributes=
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input,file
......@@ -238,7 +235,7 @@ ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis
ignored-modules=
ignored-modules=phobos.core.const,phobos.core.glue
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
......@@ -256,10 +253,6 @@ generated-members=REQUEST,acl_users,aq_parent
[CLASSES]
# List of interface methods to ignore, separated by a comma. This is used for
# instance to not check methods defines in Zope's Interface base class.
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
......
......@@ -210,7 +210,7 @@ define valgrind_tool_rule
check-valgrind-$(1)-am:
ifeq ($$(VALGRIND_ENABLED)-$$(ENABLE_VALGRIND_$(1)),yes-yes)
ifneq ($$(TESTS),)
$$(valgrind_v_use)$$(MAKE) check-TESTS \
$$(valgrind_v_use)$$(MAKE) check \
TESTS_ENVIRONMENT="$$(VALGRIND_TESTS_ENVIRONMENT)" \
LOG_COMPILER="$$(VALGRIND_LOG_COMPILER)" \
LOG_FLAGS="$$(valgrind_$(1)_flags)" \
......
......@@ -4,7 +4,7 @@
AC_PREREQ(2.59)
AC_INIT([phobos], [1.90.0], [st-hpc@cea.fr])
AC_INIT([phobos], [1.92], [st-hpc@cea.fr])
RELEASE="1"
AC_DEFINE_UNQUOTED(RELEASE, "$RELEASE", [release info])
......@@ -67,11 +67,10 @@ CFLAGS="$CFLAGS -I\$(top_srcdir)/src/include"
AC_ARG_ENABLE( [cli], AS_HELP_STRING([--disable-cli],
[Do not generate python cli and modules]),
[enable_cli="$enableval"],[enable_cli="yes"] )
# force using python2
PYTHON=$(which python2)
PYTHON=$(which python3)
AS_IF( [test "x$enable_cli" != "xno"], [
# Enable python bindings
AX_PYTHON_DEVEL([>='2.6'])
AX_PYTHON_DEVEL([>='3.6'])
])
# Checks for header files.
......@@ -85,6 +84,9 @@ AC_CHECK_HEADERS([scsi/sg_io_linux.h], [], [AC_MSG_ERROR(missing sg3_utils heade
AC_CHECK_HEADERS([protobuf-c/protobuf-c.h], [], [AC_MSG_ERROR(missing protobuf-c header files)])
PKG_CHECK_MODULES(JANSSON, [jansson >= 2.5])
PKG_CHECK_MODULES(LIBPQ, [libpq >= 9.4])
PKG_CHECK_MODULES(CMOCKA, [cmocka >= 1.1], ,
[AC_MSG_WARN([missing package libcmocka-devel, tests will fail to compile])]
)
# Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
......@@ -111,6 +113,8 @@ AC_CONFIG_FILES([Makefile
src/layout-modules/Makefile
src/admin/Makefile
src/tests/Makefile
src/tests/unit/Makefile
src/tests/integration/Makefile
src/cli/Makefile
phobos.spec
])
......
......@@ -13,6 +13,8 @@ families = tape,dir
lib_device = /dev/changer
# path of the LRS-server socket
server_socket = /run/phobosd/lrs
# path of the daemon lock file
lock_file = /run/phobosd/phobosd.lock
[scsi]
# retry count for SCSI requests
......@@ -42,6 +44,16 @@ default_layout = simple
# default resource family for put operations
default_family = tape
[io]
# block size used for writing data to multiple media (negative values or 0 will
# be interpreted as an absence of parameter, and the use of default value
# sysconf(_SC_PAGESIZE))
io_block_size = -1
[layout_raid1]
# number of data replicates for raid1 layout (default is 2)
repl_count = 2
######### Tape/drive support and compatibility rules ########
# You should not modify the following configuration unless:
# * You want to add support for a new device or tape model
......
#!/bin/sh
function noisy_cmd
{
echo "$*" >&2
sh -c "$*"
}
echo "# Starting the daemon"
noisy_cmd systemctl start phobosd
phobos_db drop_tables
echo "# Setup database"
noisy_cmd phobos_db setup_tables
echo
rm -rf /tmp/test_get
echo "# Create testing directory"
noisy_cmd mkdir -p /tmp/test_get
echo
echo "# Add directory to phobos"
noisy_cmd phobos dir add /tmp/test_get
noisy_cmd phobos dir format --unlock --fs posix /tmp/test_get
echo
echo "# Create an object to get"
noisy_cmd phobos put --family dir --metadata "a=b" /etc/hosts oid1
echo
echo "# Get the uuid of the current generation"
uuid1=$(noisy_cmd phobos object list -o uuid oid1)
echo "-> $uuid1"
echo
echo "# Get an alive object"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --uuid "$uuid1" oid1 /tmp/out && rm /tmp/out
echo
echo "# Overwrite the current object to create a new version"
noisy_cmd phobos put --family dir --overwrite --metadata "b=c" /etc/hosts oid1
echo
echo "# Get the alive version"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
echo
echo "# Get the deprecated version"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Get the most recent version"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Delete the object"
noisy_cmd phobos delete oid1
echo
echo "# The get with just oid fails as it only targets alive objects"
noisy_cmd phobos get oid1 /tmp/out && rm /tmp/out
echo
echo "# But the deprecated objects can still be accessed"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Create a new generation by reusing the last oid"
noisy_cmd phobos put --family dir --metadata "c=d" /etc/hosts oid1
echo
echo "# Get the uuid of the current generation"
uuid2=$(noisy_cmd phobos object list -o uuid oid1)
echo "-> $uuid2"
echo
echo "# Now we can target the old generation with its uuid"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Create a new version of the current object"
noisy_cmd phobos put --overwrite --family dir /etc/hosts oid1
echo
echo "# Target the current generation's old version"
noisy_cmd phobos get --version 1 oid1 /tmp/out && rm /tmp/out
echo
echo "# Target the old generation"
noisy_cmd phobos get --uuid $uuid1 oid1 /tmp/out && rm /tmp/out
echo
#!/bin/sh
# This script must be run on a phobos admin node.
# This script shows some of the list features offered by phobos by executing
# the following steps:
# - starts the phobos daemon,
# - creates and add 3 directories to phobos,
# - executes various list commands,
# - stops the phobos daemon and deletes the 3 directories.
# (TO BE DONE: delete the 3 added directories from phobos)
echo "# START DEMO"
echo "# Start Phobos daemon"
echo "systemctl start phobosd"
systemctl start phobosd
sleep 5
echo
echo "# Add directories, format some of them, and unlock one"
mkdir /tmp/aries /tmp/taurus /tmp/gemini
echo "phobos dir add /tmp/aries /tmp/taurus /tmp/gemini"
phobos dir add /tmp/aries /tmp/taurus /tmp/gemini
sleep 5
echo "phobos dir format --fs posix /tmp/aries /tmp/gemini"
phobos dir format --fs posix /tmp/aries /tmp/gemini
sleep 5
echo "phobos dir unlock /tmp/aries"
phobos dir unlock /tmp/aries
sleep 10
echo
echo "# List directories (default behavior)"
echo "phobos dir list"
phobos dir list
sleep 10
echo
echo "# Detailed list"
echo "phobos dir list --output all"
phobos dir list --output all
sleep 10
echo
echo "# Customized list"
echo "phobos dir list --output name,path,adm_status,fs.status"
phobos dir list --output name,adm_status,fs.status
sleep 10
echo
echo "# Other list formats"
echo "phobos dir list --output name,path,adm_status,fs.status --format csv"
phobos dir list --output name,adm_status,fs.status --format csv
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format json"
phobos dir list --output name,adm_status,fs.status --format json
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format yaml"
phobos dir list --output name,adm_status,fs.status --format yaml
sleep 5
echo "phobos dir list --output name,path,adm_status,fs.status --format xml"
phobos dir list --output name,adm_status,fs.status --format xml
sleep 10
echo
# TO BE DONE: delete 3 added directories from phobos.
systemctl stop phobosd
rm -r /tmp/aries /tmp/taurus /tmp/gemini
echo "# END DEMO"
#!/bin/bash
# This scripts shows how phobos could be used in parallel on two servers by
# executing the following steps:
# - create a file on each server,
# - put the two files into phobos in parallel by asking separately each server
# to put its own file into phobos,
# - getting back the two file in parallel by asking separately each server to
# to get back its own file,
# - focus on the need of a future "relocate" call by asking to server1 to get
# back the file put on server0,
# - deletes files on each server.
# (TO BE DONE: delete example files from phobos)
# This scripts needs a running phobos on two servers sharing the same phobos
# database. The two phobos servers need dir devices ready to store a 1k file.
# This script must be run on a node able to connect by ssh to the both servers.
SERVER0="${SERVER0}"
SERVER1="${SERVER1}"
READER=5
echo "# 2 IO servers in parallel"
echo "# both connected to the same phobos system"
echo "# metadata are shared through a common db"
sleep $READER
sleep $READER
sleep $READER
echo
echo "# First phobos IO server : ${SERVER0}"
sleep $READER
echo
echo "ssh ${SERVER0} sudo head -3 /etc/phobos.conf"
ssh ${SERVER0} sudo head -3 /etc/phobos.conf
sleep $READER
echo
echo "ssh ${SERVER0} sudo phobos dir list"
ssh ${SERVER0} sudo phobos dir list
sleep $READER
echo
echo "# Second phobos IO server : ${SERVER1}"
sleep $READER
echo
echo "ssh ${SERVER1} sudo head -3 /etc/phobos.conf"
ssh ${SERVER1} sudo head -3 /etc/phobos.conf
sleep $READER
echo
echo "ssh ${SERVER1} phobos dir list"
ssh ${SERVER1} sudo phobos dir list
sleep $READER
echo
echo "# Prepare on both an input object"
sleep $READER
sleep $READER
echo
echo "ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1"
ssh ${SERVER0} sudo dd if=/dev/urandom of=/tmp/${SERVER0}_file bs=1k count=1
echo "ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1"
ssh ${SERVER1} sudo dd if=/dev/urandom of=/tmp/${SERVER1}_file bs=1k count=1
sleep $READER
echo
echo "# Put simultaneously the two objects"
echo "ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &"
echo "ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object"
ssh ${SERVER0} sudo phobos put -f dir /tmp/${SERVER0}_file ${SERVER0}_object &
ssh ${SERVER1} sudo phobos put -f dir /tmp/${SERVER1}_file ${SERVER1}_object
sleep $READER
sleep $READER
echo
echo "# List object from both IO servers"
echo "ssh ${SERVER0} sudo phobos extent list -o oid,media_name"
ssh ${SERVER0} sudo phobos extent list -o oid,media_name
echo
echo "ssh ${SERVER1} sudo phobos extent list -o oid,media_name"
ssh ${SERVER1} sudo phobos extent list -o oid,media_name
sleep $READER
sleep $READER
echo
echo "# Get back simultaneously the two objects"
echo "ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &"
echo "ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back"
ssh ${SERVER0} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back &
ssh ${SERVER1} sudo phobos get ${SERVER1}_object /tmp/${SERVER1}_object_back
sleep $READER
sleep $READER
echo
echo "# Next phasis : implement relocate call between IO servers"
echo "# (currently : ${SERVER1} has no access to ${SERVER0} data)"
sleep $READER
sleep $READER
echo "ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back"
ssh ${SERVER1} sudo phobos get ${SERVER0}_object /tmp/${SERVER0}_object_back
sleep $READER
sleep $READER
echo
echo "# Clean"
set -x
ssh ${SERVER0} sudo rm /tmp/${SERVER0}_file
ssh ${SERVER0} sudo rm /tmp/${SERVER0}_object_back
ssh ${SERVER1} sudo rm /tmp/${SERVER1}_file
ssh ${SERVER1} sudo rm /tmp/${SERVER1}_object_back
# TO BE DONE: delete files from phobos
#!/bin/bash
# This script shows raid1 layout features offered by phobos as robustness facing
# media failures and flexibility facing nearly full media.
# It must be run on a node ready to execute a phobos server.
# WARNING : the postgres phobos database is dropped.
READER=5
echo "################################"
echo "# Phobos RAID1 layout quick demo"
echo "################################"
sleep $READER
echo
echo "# Clean DB"
sleep $READER
echo "sudo -u postgres phobos_db drop_db || true"
sudo -u postgres phobos_db drop_db || true
sleep $READER
echo "sudo -u postgres phobos_db setup_db -s -p phobos"
sudo -u postgres phobos_db setup_db -s -p phobos
sleep $READER
echo
echo "# Start phobos daemon"
echo "systemctl start phobosd"
systemctl start phobosd
sleep $READER
echo
echo "################################"
echo "# First test : raid1 resilience"
echo "# - store 2 copies of an object on two different media"
echo "# - retrieve this object even if one medium is unavailable"
echo "################################"
sleep $READER
echo
echo "# Create a file as input object"
echo "echo \"Input object example\" > /tmp/input_object"
echo "Input object example" > /tmp/input_object
sleep $READER
echo
echo "# Add a first directory to phobos ressource"
echo "mkdir /tmp/dir1"
mkdir /tmp/dir1
sleep $READER
echo "phobos dir add /tmp/dir1"
phobos dir add /tmp/dir1
sleep $READER
echo "phobos dir format --fs posix --unlock /tmp/dir1"
phobos dir format --fs posix --unlock /tmp/dir1
sleep $READER
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Store input_object using simple layout"
echo "phobos put -f dir -l simple /tmp/input_object simple_layout_object"
phobos put -f dir -l simple /tmp/input_object simple_layout_object
sleep $READER
echo
echo "# Add a second directory to phobos ressource"
echo "mkdir /tmp/dir2"
mkdir /tmp/dir2
sleep $READER
echo "phobos dir add /tmp/dir2"
phobos dir add /tmp/dir2
sleep $READER
echo "phobos dir format --fs posix --unlock /tmp/dir2"
phobos dir format --fs posix --unlock /tmp/dir2
sleep $READER
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Set raid1 layout replicat count to 2"
echo "export PHOBOS_LAYOUT_RAID1_repl_count=2"
export PHOBOS_LAYOUT_RAID1_repl_count=2
sleep $READER
echo
echo "# Store input_object using raid1 layout"
echo "phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object"
phobos put -f dir -l raid1 /tmp/input_object raid1_2replica_object
sleep $READER
echo
echo "# List new objects extents"
echo "phobos extent list -o oid,media_name,layout"
phobos extent list -o oid,media_name,layout
sleep $READER
echo
echo "# Get back simple and raid1 object"
echo "phobos get simple_layout_object /tmp/simple_back"
phobos get simple_layout_object /tmp/simple_object_back
echo "phobos get raid1_2replica_object /tmp/raid1_back"
phobos get raid1_2replica_object /tmp/raid1_back
sleep $READER
echo
echo "# Simulate a failure on first medium"
echo "phobos dir lock /tmp/dir1"
phobos dir lock /tmp/dir1
echo "phobos dir list -o name,adm_status"
phobos dir list -o name,adm_status
sleep $READER
echo
echo "# Due to first medium failure, simple layout object is unavailable."
echo "phobos get simple_layout_object /tmp/simple_back_with_failure"
phobos get simple_layout_object /tmp/simple_object_back_with_failure
sleep $READER
echo
echo "# Get raid1 layout object even if first medium is failed."
echo "phobos get raid1_2replica_object /tmp/raid1_back_failure"
phobos get raid1_2replica_object /tmp/raid1_back_failure
sleep $READER
echo
echo "# Lock media"
echo
echo "phobos dir lock /tmp/dir1 /tmp/dir2"
phobos dir lock /tmp/dir1 /tmp/dir2
sleep $READER
echo
echo "################################"
echo "# Second test : raid1 management of media allocation"
echo "# - store 2 copies of an object on three different media"
echo "# - first medium is wide enough to store one copy of the object"
echo "# - second and third media are smaller and can't store the object"
echo "# - the second copy is splitted between second and third media"
echo "# - retrieve this object even if one medium is unavailable"
sleep $READER
sleep $READER
sleep $READER
echo
echo "# Create first medium of 128M"
sleep $READER
echo
echo "dd if=/dev/zero of=/tmp/128M_file bs=1M count=128"
dd if=/dev/zero of=/tmp/128M_device_file bs=1M count=128
echo
echo "losetup -f /tmp/128M_device_file"
losetup -f /tmp/128M_device_file
echo
echo "mkfs.ext4 /dev/loop0"
mkfs.ext4 /dev/loop0
echo
echo "mkdir /tmp/128M_dir"
mkdir /tmp/128M_dir
echo
echo "mount /dev/loop0 /tmp/128M_dir"
mount /dev/loop0 /tmp/128M_dir
sleep $READER
echo
echo "phobos dir add /tmp/128M_dir"
phobos dir add /tmp/128M_dir
echo
echo "phobos dir format --fs posix --unlock /tmp/128M_dir"
phobos dir format --fs posix --unlock /tmp/128M_dir
sleep $READER
echo
echo "# Create second and third medium of 64M each"
sleep $READER
echo
echo "dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64"
dd if=/dev/zero of=/tmp/64M_file_1 bs=1M count=64
echo "dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64"
dd if=/dev/zero of=/tmp/64M_file_2 bs=1M count=64
echo
echo "losetup -f /tmp/64M_file_1"
losetup -f /tmp/64M_file_1
echo "losetup -f /tmp/64M_file_2"
losetup -f /tmp/64M_file_2
echo
echo "mkfs.ext4 /dev/loop1"
mkfs.ext4 /dev/loop1
echo "mkfs.ext4 /dev/loop2"
mkfs.ext4 /dev/loop2
echo
echo "mkdir /tmp/64M_dir_1"
mkdir /tmp/64M_dir_1
echo "mkdir /tmp/64M_dir_2"
mkdir /tmp/64M_dir_2
echo
echo "mount /dev/loop1 /tmp/64M_dir_1"
mount /dev/loop1 /tmp/64M_dir_1
echo "mount /dev/loop2 /tmp/64M_dir_2"
mount /dev/loop2 /tmp/64M_dir_2
sleep $READER
echo
echo "phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2"
phobos dir add /tmp/64M_dir_1 /tmp/64M_dir_2
echo
echo "phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2"
phobos dir format --fs posix --unlock /tmp/64M_dir_1 /tmp/64M_dir_2
echo
echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
phobos dir list -o name,adm_status,stats.phys_spc_free
sleep $READER
echo
echo "# Create a 100M file as input object"
sleep $READER
echo
echo "dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100"
dd if=/dev/urandom of=/tmp/100M_input_object bs=1M count=100
sleep $READER
echo
echo "# Store the 100M file using raid1 layout"
echo
echo "phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object"
phobos put -f dir -l raid1 /tmp/100M_input_object 100M_raid1_object
sleep $READER
echo
echo "phobos object list -o all"
phobos object list -o all
echo
echo "phobos extent list -o oid,layout,media_name,size"
phobos extent list -o oid,layout,media_name,size
sleep $READER
echo
echo "# Get back object"
echo "phobos get 100M_raid1_object /tmp/100M_raid1_back"
phobos get 100M_raid1_object /tmp/100M_raid1_back
sleep $READER
echo
echo "# Simulate a failure of 128M medium"
echo "phobos dir lock /tmp/128M_dir"
phobos dir lock /tmp/128M_dir
echo "phobos dir list -o name,adm_status,stats.phys_spc_free"
phobos dir list -o name,adm_status,stats.phys_spc_free
sleep $READER
echo
echo "# Get back object under failure"
echo "phobos get 100M_raid1_object /tmp/100M_raid1_back_failure"
phobos get 100M_raid1_object /tmp/100M_raid1_back_failure
sleep $READER
echo
echo "# One more failure"