First commit

Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/devtest/os/test/ListCmd.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+import re
+import os
+import asyncio
+from operator import itemgetter
+from jwutils.log import *
+from jwutils import Options
+from devtest.os import *
+from devtest.os.test import *
+from .. import TestCase
+
+class ListCmd(TestCase): # export
+
+    # ------------------------------------- class Row
+
+    class Row:
+
+        def field(self, key, default=None):
+            if key in self.__fields.keys():
+                return self.__fields[key]
+            if default is not None:
+                return default
+            raise KeyError('No field "{}" in row "{}"'.format(key, self))
+
+        def attrib(self, key, default=None):
+            if self.__attribs is not None:
+                return self.__attribs.get(key, default)
+            if default is not None:
+                return default
+            raise KeyError('No attrib "{}" in row "{}"'.format(key, self))
+
+        # "needed": [ "dummyd", "v3.23" ]
+        # "key_":   [ feature ]
+        def check_first_match(self, key_, features):
+            if self.__attribs is None:
+                return False
+            for feature in features:
+                for key, val in self.__attribs.items():
+                    if type(val) == bool:
+                        if key_ == key:
+                            return val
+                        continue
+                    if type(val) != list:
+                        raise Exception('Found attribute {}="{}" of unexpected value type {}'.format(key, val, type(val)))
+                    if feature in val: # first match
+                        return True
+            return False
+
+        def check_attrib(self, key, features):
+            val  = self.attrib(key, False)
+            if type(val) == bool:
+                return val
+            if type(val) != list:
+                raise Exception('Found attribute {}="{}" of unexpected value type {}'.format(key, val, type(val)))
+            for feature in features:
+                if feature in val:
+                    return True
+            return False
+
+        def cmp(self, other):
+            decisive = self.parent.decisive
+            for field in decisive:
+                ret = self.field(field) < other.field(field)
+                if ret:
+                    return ret
+            return 0
+
+        def to_str(self, only_values=False, quotes=None, fields=['fields']):
+            use_fields = None
+            for f_set_name in fields:
+                use_fields = self.parent.row_info(f_set_name, default=False)
+                if use_fields != False:
+                    break
+            if use_fields == False:
+                raise Exception("None of the fields wanted for formatting are available: {}".format(fields))
+            q = '"' if quotes == True else ('' if quotes is None and only_values else '')
+            if only_values:
+                r = ', '.join(['{}{}{}'.format(q, self.field(f), q) for f in use_fields])
+            else:
+                r = ', '.join(['{}={}{}{}'.format(f, q, self.field(f), q) for f in use_fields])
+            if self.__attribs is not None and len(self.__attribs) > 0:
+                r += " | " + str(self.__attribs)
+            return r
+
+        @property
+        def name(self):
+            return '(' + self.to_str(fields=['name-fields', 'cmp-fields'], only_values=True) + ')'
+
+        def __lt__(self, other):
+            return self.cmp(other) < 0
+        def __le__(self, other):
+            return self.cmp(other) <= 0
+        def __eq__(self, other):
+            return self.cmp(other) == 0
+        def __ne__(self, other):
+            return self.cmp(other) != 0
+        def __gt__(self, other):
+            return self.cmp(other) > 0
+        def __ge__(self, other):
+            return self.cmp(other) >= 0
+
+        def __str__(self):
+            return self.to_str()
+
+        def _repr__(self):
+            return self.to_str()
+
+        def __format__(self, fmt):
+            return self.to_str()
+
+        def __init__(self, parent, line, fields, attribs):
+            self.parent = parent
+            self.__line = line
+            self.__fields = fields
+            self.__attribs = attribs
+
+        def __getitem__(self, key):
+            return self.field(key)
+
+        def __hash__(self):
+            decisive = self.parent.decisive
+            return hash(tuple([self.field(field, '') for field in decisive]))
+
+    # ------------------------------------- class ListCmd methods
+
+    def __init__(self, refpath, act_timeout=2, total_timeout=None, write_response=None):
+        self.refpath = refpath
+        self.act_timeout = act_timeout
+        self.total_timeout = total_timeout
+        self.__decisive = None
+        self.__row_info = None
+        if write_response is not None:
+            self.__write_response = write_response
+        else:
+            key = "JW_DEVTEST_WRITE_LIST_CMD_RESPONSE_DEFAULT"
+            val = os.getenv(key)
+            if val is not None:
+                val = val.lower()
+                if val in ['true', '1', 'y', 'yes' ]:
+                    self.__write_response = True
+                elif val in ['false', '0', 'n', 'no' ]:
+                    self.__write_response = False
+                else:
+                    raise Exception('Invalid value "{}" of environment variable "{}"'.format(val, key))
+
+    # override this
+    def _row_info(self):
+        return {
+            'cmd':         "/bin/ps",
+                           # PID USER       VSZ STAT COMMAND
+                           #  38 root         0 SW   [kjournald]
+                           #  40 root      2300 S    {rc.start} /bin/sh /etc/rc.start
+                           #    40        root                 2300       S       {rc.start} /bin/sh /etc/rc.start
+            'regex':       "^ *([0-9]+) +([a-z_][a-z0-9_-]*) +([0-9]+) +([A-Z]+) +(.+)",
+            'fields':      [   'pid',    'user',              'size',   'stat',   'cmd'],
+            'cmp-fields':  [             'user',                                  'cmd'],
+            'name-fields': [                                                      'cmd'],
+         }
+
+    def _filter(self, output):
+        return output
+
+    def row_info(self, key, default=None):
+        if self.__row_info == None:
+            info = self._row_info()
+            if type(info) == dict:
+                self.__row_info = info
+            else: # be backwards compatible and swallow any iterable container
+                self.__row_info = dict()
+                keys = ['cmd', 'regex', 'fields', 'cmp-fields', 'name-fields']
+                for i in range(0, len(info)):
+                    self.__row_info[keys[i]] = info[i]
+        if not key in self.__row_info.keys():
+            if default is not None:
+                return default
+            raise Exception('Required row info "{}" missing'.format(key))
+        return self.__row_info[key]
+
+    @property
+    def decisive(self):
+        if self.__decisive == None:
+            self.__decisive = self.row_info('cmp-fields', True)
+        return self.__decisive
+
+    def parse(self, lines):
+        def parse_line(line, fields):
+            slog(DEBUG, "parsing line >%s<" % line)
+            parsed = line.split('# ')
+            line = parsed[0].strip()
+            match = regex.search(line)
+            if not match:
+                slog(INFO, "Ignoring unparseable line >%s<" % line)
+                return None
+            attribs = None if len(parsed) <= 1 else Options(parsed[1])
+            for k in range(0, len(fields)):
+                slog(DEBUG, "  match {} = >{}<".format(k+1, match.group(k+1)))
+            fields = {fields[k]: match.group(k+1) for k in range(0, len(fields)) }
+            return self.Row(self, line=line, fields=fields, attribs=attribs)
+        fields = self.row_info('fields')
+        re_str = self.row_info('regex')
+        try:
+            regex = re.compile(re_str)
+        except Exception as e:
+            raise Exception('Failed to compile regex ({}): >{}<'.format(e, str(re_str)))
+        return list(filter(None, [parse_line(l, fields) for l in lines]))
+
+    def _eval(self, output, features, header=None):
+
+        def format_rows(rows, quotes=False):
+            def cmp(r1, r2):
+                for k in sort_keys:
+                    if r1[k] < r2[k]:
+                        return True
+                return False
+
+            sort_keys = []
+            key_sets = ['name-fields', 'cmp-fields']
+            for s in key_sets:
+                for k in self.row_info(s, []):
+                    if k not in sort_keys:
+                        sort_keys.append(k)
+
+            #return sorted([ row.to_str(fields=['cmp-fields', 'fields'], only_values=True, quotes=quotes) for row in rows], key=cmp)
+            return [ row.to_str(fields=['cmp-fields', 'fields'], only_values=True, quotes=quotes) for row in sorted(rows, key=itemgetter(*sort_keys))]
+
+        if self.__write_response and not os.path.exists(self.refpath):
+            ref_lines = []
+        else:
+            with open(self.refpath, "r") as f:
+                ref_lines = f.readlines()
+
+        output = self._filter(output)
+        if self.__write_response:
+            response_path = self.refpath + '.last'
+            with open(response_path, "w") as f:
+                slog(INFO, 'Writing response to "{}"'.format(response_path))
+                if header:
+                    f.write(header)
+                f.write('\n'.join(output))
+
+        reference = set(self.parse(ref_lines))
+        actual    = set(self.parse(output))
+
+        # -- Attributes:
+        # default:  error-if-not-present (no matching feature, same as "needed', see below
+        # ignore:   no-error-if-present and no-error-if-not-present
+        # bad:      error-if-present
+
+        ignore    = set([row for row in reference if row.check_first_match('ignore', features)])
+        bad       = set([row for row in reference if row.check_first_match('bad', features)])
+
+        slog_m(DEBUG, "--- reference:\n", format_rows(reference))
+        slog_m(DEBUG, "--- ignore:\n", format_rows(ignore))
+        slog_m(DEBUG, "--- bad:\n", format_rows(bad))
+
+        # the "needed" attribute is actually just as good as any string: it
+        # gets ignored, leading to the line being seen as mandatory
+        needed    = reference - ignore - bad
+        missing   = needed - actual
+        too_many  = actual - needed - ignore
+        ignored   = ignore - (ignore - actual)
+
+        slog_m(NOTICE, "--- needed:\n", format_rows(needed))
+        slog_m(NOTICE, "--- actual:\n", format_rows(actual))
+        slog_m(NOTICE, "--- ignored:\n", format_rows(ignored))
+
+        r = []
+        if len(missing):
+            slog_m(ERR,    "--- should be there but are not:\n", format_rows(missing, quotes=True))
+            r.append("missing:" + ', '.join([row.name for row in missing]))
+        if len(too_many):
+            slog_m(ERR,    "--- too many:\n", format_rows(too_many, quotes=True))
+            r.append("too many: " + ', '.join([row.name for row in too_many]))
+        if not len(r):
+            return None
+
+        return ' and '.join(r)
+
+    async def _run(self, env, machine, phase):
+        console   = get_console(env)
+        cmd       = self.row_info('cmd')
+        output    = await cmd_exec(console, cmd, act_timeout=self.act_timeout,
+                total_timeout=self.total_timeout, echo_cmd=False)
+        if output is None:
+            return "Failed to run command: " + cmd
+        header = '# ' + cmd + '\n' if self.__write_response else None
+        return self._eval(output, env.features, header=header)
+
+    def dump(self, prio, *args, **kwargs):
+        caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
+        with open(self.refpath, "r") as f:
+            reference = self.parse(f.readlines())
+        for l in reference:
+            slog(NOTICE, "{}".format(l), caller=caller)
+
+    def test(self, output, features):
+        self._eval(output, features)