cli: pattern-matching for 'phobos object list'

Replace the positional argument of the 'phobos object list' to retrieve
the oids following the given pattern instead of giving an oid list.

Change-Id: Iae19d1bbd90c95f934e5304af58192cfed7f3174
Signed-off-by: Sebastien Gougeaud <sebastien.gougeaud@cea.fr>
Reviewed-on: https://gerrit.ccc.ocre.cea.fr/gerrit/c/phobos/+/6875

Tested-by: Jenkins s8open_nr <s8open_nr@ccc.ocre.cea.fr>
Reviewed-by: Patrice LUCAS <patrice.lucas@cea.fr>
Reviewed-by: Thomas Leibovici <thomas.leibovici@cea.fr>

README.md

@@ -225,6 +225,16 @@ $ phobos object list -o oid,user_md
 | obj02 | {"user": "foo"} |
 ```
 
+You can add a pattern to match object oids:
+```
+phobos object list "obj.*"
+```
+
+The accepted patterns are Basic Regular Expressions (BRE) and Extended
+Regular Expressions (ERE). As defined in [PostgreSQL manual](https://www.postgresql.org/docs/9.3/functions-matching.html#POSIX-SYNTAX-DETAILS),
+PSQL also accepts Advanced Regular Expressions (ARE), but we will not maintain
+this feature as ARE is not a POSIX standard.
+
 ## Device and media management
 ### Listing resources
 Any device or media can be listed using the 'list' operation. For instance,

doc/design/dss_filters.txt

@@ -33,6 +33,7 @@ Other operators are available. They are used as such:
 - $LT: less than (<)
 - $LTE: less or equal than (<=)
 - $LIKE: pattern matching, wildcard is '%'
+- $REGEXP: posix regexp matching
 - $INJSON: json inclusion test
 - $XJSON: json existence test
 

src/cli/phobos/cli.py

@@ -448,6 +448,22 @@ class ListOptHandler(DSSInteractHandler):
                             help="output format human/xml/json/csv/yaml " \
                                  "(default: human)")
 
+class PatternListOptHandler(DSSInteractHandler):
+    """List items of a specific type, with pattern-matching."""
+    label = 'list'
+    descr = 'list all entries of the kind'
+
+    @classmethod
+    def add_options(cls, parser):
+        """Add resource-specific options."""
+        super(PatternListOptHandler, cls).add_options(parser)
+        parser.add_argument('pattern', nargs='?', default='.*',
+                            help="POSIX regexp to match, leave empty to "
+                                 "match all")
+        parser.add_argument('-f', '--format', default='human',
+                            help="output format human/xml/json/csv/yaml "
+                                 "(default: human)")
+
 
 class LockOptHandler(DSSInteractHandler):
     """Lock resource."""
@@ -528,7 +544,7 @@ class MediaListOptHandler(ListOptHandler):
                                  "choose from {" + " ".join(attr) + "} "
                                  "(default: %(default)s)")
 
-class ObjectListOptHandler(ListOptHandler):
+class ObjectListOptHandler(PatternListOptHandler):
     """
     Specific version of the 'list' command for object, with a couple
     extra-options.
@@ -618,16 +634,11 @@ class ObjectOptHandler(BaseResourceOptHandler):
             self.logger.error("Bad output attributes: %s", " ".join(bad_attrs))
             sys.exit(os.EX_USAGE)
 
-        objs = []
-        if self.params.get('res'):
-            for obj in self.params.get('res'):
-                curr = self.client.objects.get(oid=obj)
-                if not curr:
-                    continue
-                assert len(curr) == 1
-                objs.append(curr[0])
-        else:
-            objs = self.client.objects.get()
+        kwargs = {}
+        if self.params.get('pattern'):
+            kwargs["pattern"] = self.params.get('pattern')
+
+        objs = self.client.objects.get(**kwargs)
 
         if len(objs) > 0:
             dump_object_list(objs, 'object', attr=out_attrs,

src/cli/phobos/core/dss.py

@@ -51,6 +51,7 @@ FILTER_OPERATORS = (
     ('__lt', '$LT'),
     ('__le', '$LTE'),
     ('__like', '$LIKE'),
+    ('__regexp', '$REGEXP'),
     ('__jcontain', '$INJSON'),
     ('__jexist', '$XJSON')
 )
@@ -170,6 +171,12 @@ class BaseEntityManager(object):
         """Return the short name of the clas which this proxy wraps."""
         pass
 
+    def convert_kwargs(self, name, keyop, **kwargs):
+        if kwargs.get(name, None):
+            arg = kwargs.pop(name)
+            kwargs[keyop] = arg
+        return kwargs
+
     def get(self, **kwargs):
         """Retrieve objects from DSS."""
         res = POINTER(self.wrapped_class)()
@@ -181,10 +188,10 @@ class BaseEntityManager(object):
         except KeyError:
             pass
 
-        # rename the tags key to be correctly parsed by the dss filter
-        if kwargs.get('tags', None):
-            tags = kwargs.pop('tags')
-            kwargs['tags__jexist'] = tags
+        # rename keys that need a specific operation to be correctly parsed
+        # by the dss filter
+        kwargs = self.convert_kwargs('tags', 'tags__jexist', **kwargs)
+        kwargs = self.convert_kwargs('pattern', 'oid__regexp', **kwargs)
 
         filt = dss_filter(self.wrapped_ident, **kwargs)
         if filt is not None:

src/cli/tests/PhobosCLITest.py

@@ -100,12 +100,14 @@ class CLIParametersTest(unittest.TestCase):
         self.check_cmdline_valid(['tape', 'list', 'I,J,K', '-o',
                                   'name,family'])
         self.check_cmdline_valid(['object', 'list'])
-        self.check_cmdline_valid(['object', 'list', 'oid'])
-        self.check_cmdline_valid(['object', 'list', 'oid1', 'oid2', 'oid3'])
+        self.check_cmdline_valid(['object', 'list', '"obj.*"'])
+        self.check_cmdline_valid(['object', 'list', '"obj.?2"'])
+        self.check_cmdline_valid(['object', 'list', '"obj[0-9]*"'])
+        self.check_cmdline_valid(['object', 'list', '"obj[[:digit:]]*"'])
         self.check_cmdline_valid(['object', 'list', '-o', 'all'])
         self.check_cmdline_valid(['object', 'list', '-o', '*'])
         self.check_cmdline_valid(['object', 'list', '-o', 'oid'])
-        self.check_cmdline_valid(['object', 'list', '-o', 'all', 'oid'])
+        self.check_cmdline_valid(['object', 'list', '-o', 'all', '"obj.*"'])
 
         # Test invalid object and invalid verb
         self.check_cmdline_exit(['voynichauthor', 'list'], code=2)

src/dss/dss.c

@@ -1445,6 +1445,8 @@ static int json2sql_object_begin(struct saj_parser *parser, const char *key,
         g_string_append(str, " <= ");
     } else if (!g_ascii_strcasecmp(current_key, "$LIKE")) {
         g_string_append(str, " LIKE ");
+    } else if (!g_ascii_strcasecmp(current_key, "$REGEXP")) {
+        g_string_append(str, " ~ ");
     } else if (!g_ascii_strcasecmp(current_key, "$INJSON")) {
         g_string_append(str, " @> ");
         str_index = true;

src/tests/test_dss.sh

@@ -165,10 +165,10 @@ test_check_set "media" "delete"
 test_check_get "media" '{"$LIKE": {"DSS::MDA::id": "%COPY%"}}'
 echo "**** TEST: DSS_SET OBJECT  ****"
 test_check_set "object" "insert"
-test_check_get "object" '{"$LIKE": {"DSS::OBJ::oid": "%COPY%"}}'
+test_check_get "object" '{"$REGEXP": {"DSS::OBJ::oid": ".*COPY.*"}}'
 #test_check_set "object" "update"
 test_check_set "object" "delete"
-test_check_get "object" '{"$LIKE": {"DSS::OBJ::oid": "%COPY%"}}'
+test_check_get "object" '{"$REGEXP": {"DSS::OBJ::oid": ".*COPY.*"}}'
 echo "**** TEST: DSS_SET LAYOUT  ****"
 test_check_set "layout" "insert"
 test_check_get "layout" '{"$LIKE": {"DSS::EXT::oid": "%COPY%"}}'