Commit 989076cc authored by Sebastien Gougeaud's avatar Sebastien Gougeaud Committed by Thomas Leibovici
Browse files

cli: merge list and show commands



Commands list and show were designed to nearly do the same thing, with
list printing every device's or medium's label and show outputting the
whole set of attributes of given targets.

Now they are merged into a single list command which can:
- target a single resource, a set of resources or every resource;
- output any set of attributes of the targeted resources (from
  label to the whole parameters);
- output results in various formats (human, csv, json, yaml, xml);
- select targets depending on their model (for devices) or their tags
  (for media).

Change-Id: I0a7edcec87b9f9d05fe66e605caedf6126a755c5
Signed-off-by: default avatarSebastien Gougeaud <sebastien.gougeaud@cea.fr>
Reviewed-on: https://cws-fleury.labs.ocre.cea.fr/gerrit/6492


Reviewed-by: Linter
Tested-by: default avatarJenkins s8open_nr <s8open_nr@ccc.ocre.cea.fr>
Reviewed-by: default avatarQuentin Bouget <quentin.bouget@cea.fr>
Reviewed-by: default avatarThomas Leibovici <thomas.leibovici@cea.fr>
parent a327ed60
......@@ -171,6 +171,26 @@ cksum=md5:7c28aec5441644094064fcf651ab5e3e,user=foo
```
## Device and media management
### Listing resources
Any device or media can be listed using the 'list' operation. For instance,
the following will list all the existing tapes:
```
phobos tape list
```
The 'list' output can be formatted using the -o option, following a list of
comma-separated attributes:
```
phobos tape list -o stats.phys_spc_used,tags,adm_status
```
If the attribute 'all' is used, then the list is printed with full details:
```
phobos tape list -o all
```
Other options are available using the -h option.
### Locking resources
A device or media can be locked. In this case it cannot be used for
subsequent 'put' or 'get' operations:
......
......@@ -33,6 +33,7 @@ Requires: python-argparse
Requires: python-yaml
Requires: clustershell
Requires: python-psycopg2
Requires: python2-tabulate
Requires: protobuf
Requires: protobuf-c
......
......@@ -16,6 +16,7 @@ EXTRA_DIST=phobos/__init__.py \
phobos/core/ldm.py \
phobos/core/log.py \
phobos/core/store.py \
phobos/core/utils.py \
phobos/db/__init__.py \
phobos/db/__main__.py \
phobos/db/db_config.py \
......
#!/usr/bin/python
#
# All rights reserved (c) 2014-2017 CEA/DAM.
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
......@@ -45,15 +45,16 @@ from phobos.core.const import dev_family2str
from phobos.core.const import PHO_DEV_DIR, PHO_DEV_TAPE, PHO_LIB_SCSI
from phobos.core.const import (PHO_DEV_ADM_ST_LOCKED, PHO_DEV_ADM_ST_UNLOCKED,
PHO_MDA_ADM_ST_LOCKED, PHO_MDA_ADM_ST_UNLOCKED)
from phobos.core.ffi import DevInfo, MediaInfo
from phobos.core.log import LogControl
from phobos.core.log import DISABLED, WARNING, INFO, VERBOSE, DEBUG
from phobos.core.admin import Client as AdminClient
from phobos.core.cfg import load_file as cfg_load_file
from phobos.core.dss import Client as DSSClient
from phobos.core.store import Client as XferClient, attrs_as_dict
from phobos.core.ldm import LibAdapter
from phobos.core.admin import Client as AdminClient
from phobos.core.store import Client as XferClient, attrs_as_dict
from phobos.output import dump_object_list
from ClusterShell.NodeSet import NodeSet
......@@ -429,6 +430,15 @@ class ListOptHandler(DSSInteractHandler):
label = 'list'
descr = 'list all entries of the kind'
@classmethod
def add_options(cls, parser):
"""Add resource-specific options."""
super(ListOptHandler, cls).add_options(parser)
parser.add_argument('res', nargs='*', help='resource(s) to list')
parser.add_argument('-f', '--format', default='human',
help="output format human/xml/json/csv/yaml " \
"(default: human)")
class LockOptHandler(DSSInteractHandler):
"""Lock resource."""
......@@ -456,22 +466,6 @@ class UnlockOptHandler(DSSInteractHandler):
parser.add_argument('--force', action='store_true',
help='Do not check the current lock state')
class ShowOptHandler(DSSInteractHandler):
"""Show resource details."""
label = 'show'
descr = 'show resource details'
@classmethod
def add_options(cls, parser):
"""Add resource-specific options."""
super(ShowOptHandler, cls).add_options(parser)
parser.add_argument('res', nargs='+', help='Resource(s) to show')
parser.add_argument('--numeric', action='store_true', default=False,
help='Output numeric values')
parser.add_argument('--format', default='human',
help="Output format human/xml/json/csv/yaml " \
"(default: human)")
class ScanOptHandler(BaseOptHandler):
"""Scan a physical resource and display retrieved information."""
label = 'scan'
......@@ -489,24 +483,42 @@ class DriveListOptHandler(ListOptHandler):
Specific version of the 'list' command for tape drives, with a couple
extra-options.
"""
@classmethod
def add_options(cls, parser):
"""Add resource-specific options."""
super(DriveListOptHandler, cls).add_options(parser)
parser.add_argument('-m', '--model', help='filter on model')
attr = [x for x in DevInfo().get_display_dict().keys()]
attr.sort()
parser.add_argument('-o', '--output', type=lambda t: t.split(','),
default='label',
help="attributes to output, comma-separated, "
"choose from {" + " ".join(attr) + "} "
"(default: %(default)s)")
class MediaListOptHandler(ListOptHandler):
"""
Specific version of the 'list' command for media, with a couple
extra-options.
"""
@classmethod
def add_options(cls, parser):
"""Add resource-specific options."""
super(MediaListOptHandler, cls).add_options(parser)
parser.add_argument('-T', '--tags', type=lambda t: t.split(','), \
parser.add_argument('-T', '--tags', type=lambda t: t.split(','),
help='filter on tags (comma-separated: foo,bar)')
attr = [x for x in MediaInfo().get_display_dict().keys()]
attr.sort()
parser.add_argument('-o', '--output', type=lambda t: t.split(','),
default='label',
help="attributes to output, comma-separated, "
"choose from {" + " ".join(attr) + "} "
"(default: %(default)s)")
class TapeAddOptHandler(MediaAddOptHandler):
"""Specific version of the 'add' command for tapes, with extra-options."""
@classmethod
......@@ -571,21 +583,22 @@ class DeviceOptHandler(BaseResourceOptHandler):
AddOptHandler,
FormatOptHandler,
ListOptHandler,
ShowOptHandler,
LockOptHandler,
UnlockOptHandler
]
def filter(self, ident):
def filter(self, ident, **kwargs):
"""
Return a list of devices that match the identifier for either serial or
path. You may call it a bug but this is a feature intended to let admins
transparently address devices using one or the other scheme.
"""
dev = self.client.devices.get(family=self.family, serial=ident)
dev = self.client.devices.get(family=self.family, serial=ident,
**kwargs)
if not dev:
# 2nd attempt, by path...
dev = self.client.devices.get(family=self.family, path=ident)
dev = self.client.devices.get(family=self.family, path=ident,
**kwargs)
return dev
def exec_add(self):
......@@ -613,25 +626,6 @@ class DeviceOptHandler(BaseResourceOptHandler):
self.logger.info("Added %d device(s) successfully", len(resources))
def exec_list(self):
"""List devices and display results."""
for obj in self.client.devices.get(family=self.family):
print obj.serial
def exec_show(self):
"""Show device details."""
devs = []
for serial in self.params.get('res'):
curr = self.filter(serial)
if not curr:
self.logger.error("'%s' not found", serial)
sys.exit(os.EX_DATAERR)
assert len(curr) == 1
devs.append(curr[0])
if len(devs) > 0:
dump_object_list(devs, self.params.get('format'),
self.params.get('numeric'))
def exec_lock(self):
"""Device lock"""
devices = []
......@@ -706,7 +700,6 @@ class MediaOptHandler(BaseResourceOptHandler):
MediaAddOptHandler,
MediaUpdateOptHandler,
FormatOptHandler,
ShowOptHandler,
MediaListOptHandler,
LockOptHandler,
UnlockOptHandler
......@@ -802,27 +795,37 @@ class MediaOptHandler(BaseResourceOptHandler):
finally:
adm.fini()
def exec_show(self):
"""Show media details."""
results = []
uids = NodeSet.fromlist(self.params.get('res'))
for uid in uids:
media = self.client.media.get(family=self.family, id=uid)
if not media:
self.logger.warning("Media id %s not found", uid)
continue
assert len(media) == 1
results.append(media[0])
dump_object_list(results, self.params.get('format'),
self.params.get('numeric'))
def exec_list(self):
"""List all media."""
"""List media and display results."""
attrs = [x for x in MediaInfo().get_display_dict().keys()]
attrs.extend(['*', 'all'])
out_attrs = self.params.get('output')
bad_attrs = set(out_attrs).difference(set(attrs))
if bad_attrs:
self.logger.error("Bad output attributes: %s", " ".join(bad_attrs))
sys.exit(os.EX_USAGE)
kwargs = {}
if self.params.get('tags'):
kwargs["tags"] = self.params.get('tags')
for media in self.client.media.get(family=self.family, **kwargs):
print media.ident
objs = []
if self.params.get('res'):
uids = NodeSet.fromlist(self.params.get('res'))
for uid in uids:
curr = self.client.media.get(family=self.family, id=uid,
**kwargs)
if not curr:
continue
assert len(curr) == 1
objs.append(curr[0])
else:
objs = self.client.media.get(family=self.family, **kwargs)
if len(objs) > 0:
dump_object_list(objs, 'media', attr=self.params.get('output'),
fmt=self.params.get('format'))
def exec_lock(self):
"""Lock media"""
......@@ -931,21 +934,38 @@ class DriveOptHandler(DeviceOptHandler):
AddOptHandler,
FormatOptHandler,
DriveListOptHandler,
ShowOptHandler,
LockOptHandler,
UnlockOptHandler
]
def exec_list(self):
"""List devices and display results."""
model = self.params.get('model')
if model:
for obj in self.client.devices.get(family=self.family, model=model):
print obj.serial
attrs = [x for x in DevInfo().get_display_dict().keys()]
attrs.extend(['*', 'all'])
out_attrs = self.params.get('output')
bad_attrs = set(out_attrs).difference(set(attrs))
if bad_attrs:
self.logger.error("Bad output attributes: %s", " ".join(bad_attrs))
sys.exit(os.EX_USAGE)
kwargs = {}
if self.params.get('model'):
kwargs['model'] = self.params.get('model')
objs = []
if self.params.get('res'):
for serial in self.params.get('res'):
curr = self.filter(serial, **kwargs)
if not curr:
continue
assert len(curr) == 1
objs.append(curr[0])
else:
for obj in self.client.devices.get(family=self.family):
print obj.serial
objs = self.client.devices.get(family=self.family, **kwargs)
if len(objs) > 0:
dump_object_list(objs, 'device', attr=self.params.get('output'),
fmt=self.params.get('format'))
class TapeOptHandler(MediaOptHandler):
"""Magnetic tape options and actions."""
......@@ -956,7 +976,6 @@ class TapeOptHandler(MediaOptHandler):
TapeAddOptHandler,
MediaUpdateOptHandler,
FormatOptHandler,
ShowOptHandler,
MediaListOptHandler,
LockOptHandler,
UnlockOptHandler
......
#!/usr/bin/python
#
# All rights reserved (c) 2014-2017 CEA/DAM.
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
......@@ -86,6 +86,8 @@ def dss_filter(obj_type, **kwargs):
filt = JSONFilter()
criteria = []
for key, val in kwargs.iteritems():
if val is None:
continue
key, comp = key_convert(obj_type, key)
if not isinstance(val, list):
val = [val]
......
......@@ -116,19 +116,24 @@ class DevInfo(Structure, CLIManagedResourceMixin):
'host': None,
'model': None,
'path': None,
'serial': None,
'label': None,
'lock_status': None,
'lock_ts': None
}
@property
def label(self):
"""Wrapper to get label"""
return self.serial
@property
def lock_status(self):
""" Wrapper to get lock status"""
"""Wrapper to get lock status"""
return self.lock.lock
@property
def lock_ts(self):
""" Wrapper to get lock timestamp"""
"""Wrapper to get lock timestamp"""
return self.lock.lock_ts
class Tags(Structure):
......@@ -201,7 +206,7 @@ class MediaInfo(Structure, CLIManagedResourceMixin):
'adm_status': adm_status2str,
'addr_type': None,
'model': None,
'ident': None,
'label': None,
'tags': None,
'lock_status': None,
'lock_ts': None
......@@ -229,7 +234,7 @@ class MediaInfo(Structure, CLIManagedResourceMixin):
return self.lock.lock_ts
@property
def ident(self):
def label(self):
return self.id.id
@property
......
#!/usr/bin/python
#
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
# Phobos is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# Phobos is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Phobos. If not, see <http://www.gnu.org/licenses/>.
#
"""
Utilities functions.
"""
from collections import OrderedDict
from itertools import izip
from phobos.core.ffi import DevInfo, MediaInfo
UNIT_PREFIXES = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
def num2human(n, unit='', base=1000, decimals=1):
"""Convert a number to a human readable string"""
base = float(base)
for prefix in UNIT_PREFIXES:
if n < base:
break
n /= base
return '{{:.{:d}f}}{:s}{:s}'.format(decimals, prefix, unit).format(n)
def bytes2human(n, *args, **kwargs):
"""Convert a size in bytes to a human readable string"""
return num2human(n, 'B', 1024, *args, **kwargs)
#!/usr/bin/python
#
# All rights reserved (c) 2014-2017 CEA/DAM.
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
......@@ -23,18 +23,19 @@
Output and formatting utilities.
"""
import logging
import json
import csv
import StringIO
from tabulate import tabulate
import xml.dom.minidom
import xml.etree.ElementTree
from phobos.core.utils import bytes2human
import yaml
def csv_dump(data):
"""Convert a list of dictionaries to a csv string"""
outbuf = StringIO.StringIO()
writer = csv.DictWriter(outbuf, sorted(data[0].keys()))
if hasattr(writer, 'writeheader'):
......@@ -49,7 +50,6 @@ def csv_dump(data):
def xml_dump(data, item_type='item'):
"""Convert a list of dictionaries to xml"""
top = xml.etree.ElementTree.Element('phobos')
for item in data:
# xml only supports strings
......@@ -60,21 +60,44 @@ def xml_dump(data, item_type='item'):
reparsed = xml.dom.minidom.parseString(rough_string)
return reparsed.toprettyxml(indent=" ")
def human_dump(data, item_type='item'):
def human_dump(data):
"""Convert a list of dictionaries to an identifier list text"""
out = "\n".join(str(item['label']) for item in data)
return out
def human_pretty_dump(data):
"""Convert a list of dictionaries to human readable text"""
title = " %s " % (item_type)
out = " {0:_^50}\n".format(str(title))
for item in data:
vals = []
for key in sorted(item.keys()):
vals.append(" |{0:<20}|{1:<27}|".format(key, item[key]))
out = out+ "\n".join(vals) + "\n {0:_^50}\n".format("")
# Convert space sizes to human readable sizes
space_size_attr = [
"stats.phys_spc_free",
"stats.logc_spc_used",
"stats.phys_spc_used"
]
for obj in data:
for key, attr in obj.items():
if key in space_size_attr:
obj[key] = bytes2human(attr)
# Generate formatted printing
out = tabulate(data, headers="keys", tablefmt="github")
return out
def dump_object_list(objs, fmt="human", numeric=False):
"""
Helper for user friendly object display.
"""
def filter_display_dict(objs, attrs):
info = [x.get_display_dict() for x in objs]
# If all/* is an attribute, we fetch them all
obj_list = []
if 'all' in attrs or '*' in attrs:
obj_list = info
else:
for attr_dict in info:
obj_list.append({k: attr_dict[k] for k in attr_dict if k in attrs})
return obj_list
def dump_object_list(objs, item_type, attr=None, fmt="human"):
"""Helper for user friendly object display."""
if not objs:
return
......@@ -86,8 +109,10 @@ def dump_object_list(objs, fmt="human", numeric=False):
'human': human_dump,
}
# Build a list of dict with attributs to export/output
objlist = [x.get_display_dict(numeric) for x in objs]
if attr != ['label']:
formats['human'] = human_pretty_dump
objlist = filter_display_dict(objs, attr)
# Print formatted objects
print formats[fmt](objlist)
# Remove the endstring newline generated by csv, yaml and xml formatters
print formats[fmt](objlist).rstrip()
#!/usr/bin/python
#
# All rights reserved (c) 2014-2017 CEA/DAM.
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
......@@ -90,12 +90,17 @@ class CLIParametersTest(unittest.TestCase):
self.check_cmdline_valid(['dir', 'list'])
self.check_cmdline_valid(['dir', 'add', '--unlock', 'toto'])
self.check_cmdline_valid(['dir', 'add', 'A', 'B', 'C'])
self.check_cmdline_valid(['dir', 'show', 'A,B,C'])
self.check_cmdline_valid(['dir', 'list', 'A,B,C', '-o', 'all'])
self.check_cmdline_valid(['dir', 'list', 'A,B,C', '-o', '*'])
self.check_cmdline_valid(['dir', 'list', 'A,B,C', '-o', 'label,family'])
self.check_cmdline_valid(['tape', 'add', '-t', 'LTO5', 'I,J,K'])
self.check_cmdline_valid(['tape', 'show', 'I,J,K'])
self.check_cmdline_valid(['tape', 'list', 'I,J,K', '-o', 'all'])
self.check_cmdline_valid(['tape', 'list', 'I,J,K', '-o', '*'])
self.check_cmdline_valid(['tape', 'list', 'I,J,K', '-o',
'label,family'])
# Test invalid object and invalid verb
self.check_cmdline_exit(['voynichauthor', 'show'], code=2)
self.check_cmdline_exit(['voynichauthor', 'list'], code=2)
self.check_cmdline_exit(['dir', 'teleport'], code=2)
......@@ -294,14 +299,15 @@ class DeviceAddTest(BasicExecutionTest):
for file in flist:
path = "%s:%s" % (gethostname_short(), file.name)
self.pho_execute(['-v', 'dir', 'show', path])
self.pho_execute(['-v', 'dir', 'list', '-o', 'all', path])
def test_dir_tags(self):
"""Test adding a directory with tags."""
tmp_f = tempfile.NamedTemporaryFile()
tmp_path = tmp_f.name
self.pho_execute(['dir', 'add', tmp_path, '--tags', 'tag-foo,tag-bar'])
output, _ = self.pho_execute_capture(['dir', 'show', tmp_path])
output, _ = self.pho_execute_capture(['dir', 'list', '-o', 'all',
tmp_path])
self.assertIn("['tag-foo', 'tag-bar']", output)
def test_dir_update(self):
......
......@@ -68,8 +68,8 @@ class DSSClientTest(unittest.TestCase):
# Check negative indexation
medias = client.media.get()
self.assertEqual(medias[0].ident, medias[-len(medias)].ident)
self.assertEqual(medias[len(medias) - 1].ident, medias[-1].ident)
self.assertEqual(medias[0].label, medias[-len(medias)].label)
self.assertEqual(medias[len(medias) - 1].label, medias[-1].label)
def test_list_media_by_tags(self):
with Client() as client:
......
......@@ -3,7 +3,7 @@
# vim:expandtab:shiftwidth=4:tabstop=4:
#
# All rights reserved (c) 2014-2017 CEA/DAM.
# All rights reserved (c) 2014-2020 CEA/DAM.
#
# This file is part of Phobos.
#
......@@ -117,7 +117,7 @@ function tape_setup
# show a tape info
local tp1=$(echo $tapes | nodeset -e | awk '{print $1}')
$LOG_VALG $phobos tape show $tp1