Commit b5723a4e authored by Sebastien Gougeaud's avatar Sebastien Gougeaud Committed by Thomas Leibovici
Browse files

cli: add a '-f/--family' option for put/mput commands



Instead of using the PHOBOS_STORE_default_family env variable, users
can now indicate to which family they send their data using
the '-f' option of put/mput commands.

Change-Id: Ib7a1278b485bdfc17eab9633316def61c0661c91
Signed-off-by: default avatarSebastien Gougeaud <sebastien.gougeaud@cea.fr>
Reviewed-on: https://cws-fleury.labs.ocre.cea.fr/gerrit/6660


Reviewed-by: default avatarQuentin Bouget <quentin.bouget@cea.fr>
Reviewed-by: Linter
Tested-by: default avatarJenkins s8open_nr <s8open_nr@ccc.ocre.cea.fr>
Reviewed-by: default avatarThomas Leibovici <thomas.leibovici@cea.fr>
parent 12cd08e1
...@@ -267,16 +267,15 @@ phobos configuration file: ...@@ -267,16 +267,15 @@ phobos configuration file:
``` ```
[store] [store]
# indicate 'tape' or 'dir' family # indicate 'tape' or 'dir' family
family = tape default_family = tape
``` ```
Alternatively, you can override this value in the environment (like any other Alternatively, you can override this value by using the '-f' option of your
configuration parameter) by setting the `PHOBOS_STORE_default_family` variable. put commands:
Example:
``` ```
# put data to directory storage # put data to directory storage
PHOBOS_STORE_default_family=dir phobos put file.in obj123 phobos put -f dir file.in obj123
``` ```
### Configuring device and media types ### Configuring device and media types
......
...@@ -45,7 +45,7 @@ from ClusterShell.NodeSet import NodeSet ...@@ -45,7 +45,7 @@ from ClusterShell.NodeSet import NodeSet
from phobos.core.admin import Client as AdminClient from phobos.core.admin import Client as AdminClient
from phobos.core.cfg import load_file as cfg_load_file from phobos.core.cfg import load_file as cfg_load_file
from phobos.core.const import (PHO_LIB_SCSI, from phobos.core.const import (PHO_LIB_SCSI, rsc_family2str,
PHO_RSC_ADM_ST_LOCKED, PHO_RSC_ADM_ST_UNLOCKED) PHO_RSC_ADM_ST_LOCKED, PHO_RSC_ADM_ST_UNLOCKED)
from phobos.core.dss import Client as DSSClient from phobos.core.dss import Client as DSSClient
from phobos.core.ffi import DevInfo, MediaInfo, ResourceFamily from phobos.core.ffi import DevInfo, MediaInfo, ResourceFamily
...@@ -287,6 +287,10 @@ class StoreGenericPutHandler(XferOptHandler): ...@@ -287,6 +287,10 @@ class StoreGenericPutHandler(XferOptHandler):
parser.add_argument('-T', '--tags', type=lambda t: t.split(','), parser.add_argument('-T', '--tags', type=lambda t: t.split(','),
help='Only use media that contain all these tags ' help='Only use media that contain all these tags '
'(comma-separated: foo,bar)') '(comma-separated: foo,bar)')
parser.add_argument('-f', '--family',
choices=map(rsc_family2str,
ResourceFamily.__members__.values()),
help='Targeted storage family')
class StorePutHandler(StoreGenericPutHandler): class StorePutHandler(StoreGenericPutHandler):
...@@ -314,9 +318,12 @@ class StorePutHandler(StoreGenericPutHandler): ...@@ -314,9 +318,12 @@ class StorePutHandler(StoreGenericPutHandler):
self.logger.debug("Loaded attributes set %r", attrs) self.logger.debug("Loaded attributes set %r", attrs)
tags = self.params.get('tags', []) tags = self.params.get('tags', [])
family = self.params.get('family')
self.logger.debug("Inserting object '%s' to 'objid:%s'", src, oid) self.logger.debug("Inserting object '%s' to 'objid:%s'", src, oid)
self.client.put_register(oid, src, attrs=attrs, tags=tags) self.client.put_register(oid, src, family=family, attrs=attrs,
tags=tags)
try: try:
self.client.run() self.client.run()
except IOError as err: except IOError as err:
...@@ -347,6 +354,8 @@ class StoreMPutHandler(StoreGenericPutHandler): ...@@ -347,6 +354,8 @@ class StoreMPutHandler(StoreGenericPutHandler):
fin = open(path) fin = open(path)
tags = self.params.get('tags', []) tags = self.params.get('tags', [])
family = self.params.get('family', [])
for i, line in enumerate(fin): for i, line in enumerate(fin):
# Skip empty lines and comments # Skip empty lines and comments
line = line.strip() line = line.strip()
...@@ -369,7 +378,8 @@ class StoreMPutHandler(StoreGenericPutHandler): ...@@ -369,7 +378,8 @@ class StoreMPutHandler(StoreGenericPutHandler):
self.logger.debug("Loaded attributes set %r", attrs) self.logger.debug("Loaded attributes set %r", attrs)
self.logger.debug("Inserting object '%s' to 'objid:%s'", src, oid) self.logger.debug("Inserting object '%s' to 'objid:%s'", src, oid)
self.client.put_register(oid, src, attrs=attrs, tags=tags) self.client.put_register(oid, src, family=family, attrs=attrs,
tags=tags)
if fin is not sys.stdin: if fin is not sys.stdin:
fin.close() fin.close()
...@@ -885,7 +895,7 @@ class DirOptHandler(MediaOptHandler, DeviceOptHandler): ...@@ -885,7 +895,7 @@ class DirOptHandler(MediaOptHandler, DeviceOptHandler):
"""Directory-related options and actions.""" """Directory-related options and actions."""
label = 'dir' label = 'dir'
descr = 'handle directories' descr = 'handle directories'
family = ResourceFamily(ResourceFamily.DIR) family = ResourceFamily(ResourceFamily.RSC_DIR)
verbs = MediaOptHandler.verbs verbs = MediaOptHandler.verbs
def exec_add(self): def exec_add(self):
...@@ -920,7 +930,7 @@ class DriveOptHandler(DeviceOptHandler): ...@@ -920,7 +930,7 @@ class DriveOptHandler(DeviceOptHandler):
"""Tape Drive options and actions.""" """Tape Drive options and actions."""
label = 'drive' label = 'drive'
descr = 'handle tape drives (use ID or device path to identify resource)' descr = 'handle tape drives (use ID or device path to identify resource)'
family = ResourceFamily(ResourceFamily.TAPE) family = ResourceFamily(ResourceFamily.RSC_TAPE)
verbs = [ verbs = [
AddOptHandler, AddOptHandler,
FormatOptHandler, FormatOptHandler,
...@@ -962,7 +972,7 @@ class TapeOptHandler(MediaOptHandler): ...@@ -962,7 +972,7 @@ class TapeOptHandler(MediaOptHandler):
"""Magnetic tape options and actions.""" """Magnetic tape options and actions."""
label = 'tape' label = 'tape'
descr = 'handle magnetic tape (use tape label to identify resource)' descr = 'handle magnetic tape (use tape label to identify resource)'
family = ResourceFamily(ResourceFamily.TAPE) family = ResourceFamily(ResourceFamily.RSC_TAPE)
verbs = [ verbs = [
TapeAddOptHandler, TapeAddOptHandler,
MediaUpdateOptHandler, MediaUpdateOptHandler,
...@@ -978,7 +988,7 @@ class LibOptHandler(BaseResourceOptHandler): ...@@ -978,7 +988,7 @@ class LibOptHandler(BaseResourceOptHandler):
descr = 'handle tape libraries' descr = 'handle tape libraries'
# for now, this class in only used for tape library actions, but in case # for now, this class in only used for tape library actions, but in case
# it is used for other families, set family as None # it is used for other families, set family as None
family = ResourceFamily(ResourceFamily.TAPE) family = ResourceFamily(ResourceFamily.RSC_TAPE)
verbs = [ verbs = [
ScanOptHandler, ScanOptHandler,
] ]
......
...@@ -76,6 +76,23 @@ static PyObject *py_rsc_family2str(PyObject *self, PyObject *args) ...@@ -76,6 +76,23 @@ static PyObject *py_rsc_family2str(PyObject *self, PyObject *args)
return Py_BuildValue("s", str_repr); return Py_BuildValue("s", str_repr);
} }
static PyObject *ValueError;
static PyObject *py_str2rsc_family(PyObject *self, PyObject *args)
{
enum rsc_family family;
const char *str_repr;
if (!PyArg_ParseTuple(args, "s", &str_repr)) {
PyErr_SetString(ValueError, "Unrecognized family");
return Py_BuildValue("i", PHO_RSC_INVAL);
}
family = str2rsc_family(str_repr);
return Py_BuildValue("i", family);
}
static PyObject *py_rsc_adm_status2str(PyObject *self, PyObject *args) static PyObject *py_rsc_adm_status2str(PyObject *self, PyObject *args)
{ {
enum rsc_adm_status status; enum rsc_adm_status status;
...@@ -136,6 +153,8 @@ static PyMethodDef ConstMethods[] = { ...@@ -136,6 +153,8 @@ static PyMethodDef ConstMethods[] = {
"printable extent state name."}, "printable extent state name."},
{"rsc_family2str", py_rsc_family2str, METH_VARARGS, {"rsc_family2str", py_rsc_family2str, METH_VARARGS,
"printable dev family name."}, "printable dev family name."},
{"str2rsc_family", py_str2rsc_family, METH_VARARGS,
"family enum value from name."},
{"rsc_adm_status2str", py_rsc_adm_status2str, METH_VARARGS, {"rsc_adm_status2str", py_rsc_adm_status2str, METH_VARARGS,
"printable fs status."}, "printable fs status."},
{"fs_status2str", py_fs_status2str, METH_VARARGS, {"fs_status2str", py_fs_status2str, METH_VARARGS,
...@@ -143,7 +162,7 @@ static PyMethodDef ConstMethods[] = { ...@@ -143,7 +162,7 @@ static PyMethodDef ConstMethods[] = {
{"fs_type2str", py_fs_type2str, METH_VARARGS, {"fs_type2str", py_fs_type2str, METH_VARARGS,
"printable fs type."}, "printable fs type."},
{"str2fs_type", py_str2fs_type, METH_VARARGS, {"str2fs_type", py_str2fs_type, METH_VARARGS,
"printable dev family name."}, "fs type enum value from name."},
{NULL, NULL, 0, NULL} {NULL, NULL, 0, NULL}
}; };
......
...@@ -115,9 +115,15 @@ class Resource(Structure): ...@@ -115,9 +115,15 @@ class Resource(Structure):
class ResourceFamily(int): class ResourceFamily(int):
"""Resource family enumeration.""" """Resource family enumeration."""
# XXX: this class will become an IntEnum once we migrate to python3 # XXX: this class will become an IntEnum once we migrate to python3
DISK = PHO_RSC_DISK __members__ = {
TAPE = PHO_RSC_TAPE 'RSC_DISK': PHO_RSC_DISK,
DIR = PHO_RSC_DIR 'RSC_TAPE': PHO_RSC_TAPE,
'RSC_DIR': PHO_RSC_DIR,
}
RSC_DISK = PHO_RSC_DISK
RSC_TAPE = PHO_RSC_TAPE
RSC_DIR = PHO_RSC_DIR
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
......
...@@ -30,8 +30,10 @@ import os ...@@ -30,8 +30,10 @@ import os
from ctypes import * from ctypes import *
from phobos.core.ffi import LIBPHOBOS, Tags from phobos.core.ffi import LIBPHOBOS, Tags
from phobos.core.cfg import get_val as cfg_get_val
from phobos.core.const import (PHO_XFER_OBJ_REPLACE, PHO_XFER_OP_GET, from phobos.core.const import (PHO_XFER_OBJ_REPLACE, PHO_XFER_OP_GET,
PHO_XFER_OP_GETMD, PHO_XFER_OP_PUT) PHO_XFER_OP_GETMD, PHO_XFER_OP_PUT,
PHO_RSC_INVAL, str2rsc_family)
AttrsForeachCBType = CFUNCTYPE(c_int, c_char_p, c_char_p, c_void_p) AttrsForeachCBType = CFUNCTYPE(c_int, c_char_p, c_char_p, c_void_p)
...@@ -130,15 +132,15 @@ class XferDescriptor(Structure): ...@@ -130,15 +132,15 @@ class XferDescriptor(Structure):
the opened file. the opened file.
""" """
self.xd_objid = desc[0] self.xd_objid = desc[0]
self.xd_op = desc[5] self.xd_op = desc[6]
self.xd_layout_name = 0 self.xd_layout_name = 0
self.xd_family = -1 self.xd_family = desc[2]
self.xd_flags = desc[3] self.xd_flags = desc[4]
self.xd_tags = Tags(desc[4]) self.xd_tags = Tags(desc[5])
self.xd_rc = 0 self.xd_rc = 0
if desc[2]: if desc[3]:
for k, v in desc[2].iteritems(): for k, v in desc[3].iteritems():
rc = LIBPHOBOS.pho_attr_set(byref(self.xd_attrs), rc = LIBPHOBOS.pho_attr_set(byref(self.xd_attrs),
str(k), str(v)) str(k), str(v))
if rc: if rc:
...@@ -217,18 +219,21 @@ class Client(object): ...@@ -217,18 +219,21 @@ class Client(object):
def getmd_register(self, oid, data_path, attrs=None): def getmd_register(self, oid, data_path, attrs=None):
"""Enqueue a GETMD transfer.""" """Enqueue a GETMD transfer."""
self.getmd_session.append((oid, data_path, attrs, 0, None, self.getmd_session.append((oid, data_path, -1, attrs, 0, None,
PHO_XFER_OP_GETMD)) PHO_XFER_OP_GETMD))
def get_register(self, oid, data_path, attrs=None): def get_register(self, oid, data_path, attrs=None):
"""Enqueue a GET transfer.""" """Enqueue a GET transfer."""
self.get_session.append((oid, data_path, attrs, 0, None, self.get_session.append((oid, data_path, -1, attrs, 0, None,
PHO_XFER_OP_GET)) PHO_XFER_OP_GET))
def put_register(self, oid, data_path, attrs=None, tags=None): def put_register(self, oid, data_path, family=None, attrs=None, tags=None):
"""Enqueue a PUT transfert.""" """Enqueue a PUT transfert."""
self.put_session.append((oid, data_path, attrs, 0, tags, if family is None:
PHO_XFER_OP_PUT)) family = cfg_get_val("store", "default_family")
self.put_session.append((oid, data_path, str2rsc_family(family), attrs,
0, tags, PHO_XFER_OP_PUT))
def clear(self): def clear(self):
"""Release resources associated to the current queues.""" """Release resources associated to the current queues."""
......
...@@ -194,8 +194,8 @@ err_nores: ...@@ -194,8 +194,8 @@ err_nores:
return rc; return rc;
} }
/** return the resource family to write data */ /** Return the (configured) default resource family. */
static enum rsc_family put_family(void) static enum rsc_family default_family_from_cfg(void)
{ {
const char *fam_str; const char *fam_str;
...@@ -786,11 +786,9 @@ static int phobos_xfer(struct pho_xfer_desc *xfers, size_t n, ...@@ -786,11 +786,9 @@ static int phobos_xfer(struct pho_xfer_desc *xfers, size_t n,
int phobos_put(struct pho_xfer_desc *xfers, size_t n, int phobos_put(struct pho_xfer_desc *xfers, size_t n,
pho_completion_cb_t cb, void *udata) pho_completion_cb_t cb, void *udata)
{ {
enum rsc_family default_family;
const char *default_layout; const char *default_layout;
size_t i; size_t i;
default_family = put_family();
default_layout = PHO_CFG_GET(cfg_store, PHO_CFG_STORE, default_layout); default_layout = PHO_CFG_GET(cfg_store, PHO_CFG_STORE, default_layout);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
...@@ -799,7 +797,7 @@ int phobos_put(struct pho_xfer_desc *xfers, size_t n, ...@@ -799,7 +797,7 @@ int phobos_put(struct pho_xfer_desc *xfers, size_t n,
xfers[i].xd_layout_name = default_layout; xfers[i].xd_layout_name = default_layout;
if (xfers[i].xd_family == PHO_RSC_INVAL) if (xfers[i].xd_family == PHO_RSC_INVAL)
xfers[i].xd_family = default_family; xfers[i].xd_family = default_family_from_cfg();
} }
return phobos_xfer(xfers, n, cb, udata); return phobos_xfer(xfers, n, cb, udata);
......
...@@ -230,6 +230,24 @@ function put_get_test ...@@ -230,6 +230,24 @@ function put_get_test
error "Object attributes do not match expectations" error "Object attributes do not match expectations"
} }
# Test family based media selection
function put_family
{
local PSQL="psql phobos phobos"
local id=test/hosts-fam.$$
local request="SELECT extents FROM extent WHERE oid='$id';"
# phobos put
PHOBOS_STORE_default_family="disk" $LOG_VALG $phobos put \
-f $1 /etc/hosts $id
# PSQL command to get the the extent metadata, with the family of
# the medium it was written to
# XXX: will be replaced by a 'phobos object list' when implemented
$PSQL -t -c "$request" | grep -q "\"fam\": \"$1\"" ||
error "Put with family should have written object on a family medium"
}
# Test tag based media selection # Test tag based media selection
function put_tags function put_tags
{ {
...@@ -474,6 +492,7 @@ PHO_TMP_DIR="$(mktemp -d)" ...@@ -474,6 +492,7 @@ PHO_TMP_DIR="$(mktemp -d)"
trap cleanup EXIT trap cleanup EXIT
dir_setup dir_setup
put_get_test put_get_test
put_family dir
put_tags put_tags
concurrent_put concurrent_put
check_status dir "$dirs" check_status dir "$dirs"
...@@ -489,6 +508,7 @@ if [[ -w /dev/changer ]]; then ...@@ -489,6 +508,7 @@ if [[ -w /dev/changer ]]; then
export PHOBOS_STORE_default_family="tape" export PHOBOS_STORE_default_family="tape"
tape_setup tape_setup
put_get_test put_get_test
put_family tape
put_tags put_tags
concurrent_put concurrent_put
check_status tape "$tapes" check_status tape "$tapes"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment