lib.Distro.verify(): Add method

jw-base verify pipes the output of rpm -qV for all janware packages
into a grep for md5 mismatches and missing files. The same capability,
distribution-independent, belongs in the Distro abstraction so that tools
like jw-nettools can check the integrity of the packages they install on a
fleet of hosts.

- Add Distro.verify(), which verifies the given package names, or all
  packages selected by the package filter when none are given, and returns
  the flagged files
- Add Distro._verify() as the distribution hook
- Add FlaggedFile, the record for a flagged file: package, file, and the
  package manager's verify status line - not distribution independent as of
  today, but it's a start
- pm/rpm.py: parse_verify_output() parses the output of `rpm -qV` and
  reports md5 mismatches and missing files, the latter reported with the
  literal word "missing" in the flag field; size, mode, mtime, user, and
  group mismatches are noise on dev machines and are ignored.
  verify_packages() runs `rpm -qV` per package and collects the findings
- pm/dpkg.py: parse_verify_output() parses the output of `dpkg -V` and
  reports md5sum (P) mismatches and missing files (M); verify_packages()
  runs `dpkg -V` per package
- The SUSE, Red Hat, and Debian Distro classes implement _verify() with the
  respective pm helpers; the Arch Distro raises NotImplementedError, like
  its other package manager hooks

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0

test/unit/python/jw/pkg/lib/Distro/test.py

@@ -1,12 +1,19 @@
 from __future__ import annotations
 
+import asyncio
+
 from typing import TYPE_CHECKING, Iterable, cast, override
 
 from jw.pkg.lib.Distro import Distro
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.Package import Package
+from jw.pkg.lib.PackageFilter import PackageFilterString
+from jw.pkg.lib.Result import Result
 
 if TYPE_CHECKING:
     from jw.pkg.lib.ExecContext import ExecContext
-    from jw.pkg.lib.Package import Package
+    from jw.pkg.lib.FlaggedFile import FlaggedFile
 
 OS_RELEASE_UBUNTU = """\
 NAME="Ubuntu"
@@ -59,6 +66,76 @@ class DummyDistro(Distro):
     async def _pkg_files(self, names: list[str] | None) -> Iterable[str]:
         return []
 
+    @override
+    async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]:
+        return []
+
+# -- verify() --
+
+class VerifyDistro(DummyDistro):
+
+    def __init__(
+        self, ec: ExecContext, id: str, os_release_str: str | None = None
+    ) -> None:
+        super().__init__(ec, id, os_release_str)
+        self.verified: list[str] | None = None
+
+    @override
+    async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]:
+        self.verified = list(names)
+        return []
+
+vd = VerifyDistro(ec = cast('ExecContext', object()), id = 'suse')
+
+# -- Given names are passed to the distro backend untouched
+assert asyncio.run(vd.verify(names = ['a', 'b'])) == []
+assert vd.verified == ['a', 'b']
+
+# -- Without names, the (empty) selection is verified, so the
+# backend is not invoked
+vd.verified = None
+assert asyncio.run(vd.verify()) == []
+assert vd.verified is None
+
+# -- pkg_files --
+
+# -- Without a filter, the selection is passed through as the
+#    whole system; the paths come back unfiltered
+
+class TmplDistro(DummyDistro):
+
+    def __init__(self) -> None:
+        super().__init__(ec = cast('ExecContext', object()), id = 'suse')
+        self.files: dict[str, list[str]] = {}
+
+    @override
+    async def _select_by_name(self, names: Iterable[str]) -> list[Package]:
+        return [Package(n) for n in self.files]
+
+    @override
+    async def _pkg_files(self, names: list[str] | None) -> list[str]:
+        if names is None:
+            names = list(self.files)
+        ret: list[str] = []
+        for n in names:
+            ret.extend(self.files[n])
+        return ret
+
+t = TmplDistro()
+t.files = {
+    'a': ['/usr/share/a/a.conf.jw-tmpl', '/usr/bin/a'],
+    'b': ['/etc/b/b.conf.jw-tmpl'],
+}
+assert sorted(asyncio.run(t.pkg_files())) == [
+    '/etc/b/b.conf.jw-tmpl', '/usr/bin/a', '/usr/share/a/a.conf.jw-tmpl'
+]
+
+# -- A single package name as a str is accepted for compatibility
+assert sorted(asyncio.run(t.pkg_files('a'))) == [
+    '/usr/bin/a',
+    '/usr/share/a/a.conf.jw-tmpl',
+]
+
 def distro(os_release_str: str) -> Distro:
     # -- The execution context is stored but not used by the properties
     # under test, so a dummy object is sufficient
@@ -150,4 +227,63 @@ for macro in Distro.macros():
     # argparse help escaping has been undone
     d.expand_macros(macro.replace('%%', '%'))
 
+# -- The rpm distro's batched enumeration --
+
+# -- Without a package filter, the selection is the whole system,
+#    which is listed without package names; with one, the selected
+#    names are listed in one command, and a selection without
+#    matches issues no listing at all
+class CannedRpmLocal(Local):
+
+    def __init__(self) -> None:
+        super().__init__('local')
+        self.cmds: list[list[str]] = []
+
+    @override
+    async def _run(
+        self,
+        cmd: list[str],
+        wd: str | None,
+        verbose: bool,
+        cmd_input: bytes | None,
+        mod_env: dict[str, str] | None,
+        interactive: bool,
+        log_prefix: str,
+    ) -> Result:
+        self.cmds.append(cmd)
+        if '-qal' in cmd:
+            out = '/usr/bin/x\n/etc/a/a.conf.jw-tmpl\n'
+        elif '-ql' in cmd:
+            out = '/etc/a/a.conf.jw-tmpl\n/etc/b/b.conf.jw-tmpl\n'
+        else:
+            out = 'a|v|p|https://jw.dev/a|m\nb|v|p|https://jw.dev/b|m\n'
+        return Result(stdout = out.encode(), status = 0)
+
+cl = CannedRpmLocal()
+sd = SuseDistro(cl, id = 'suse', os_release_str = OS_RELEASE_SUSE)
+assert asyncio.run(sd.pkg_files()) == [
+    '/usr/bin/x',
+    '/etc/a/a.conf.jw-tmpl',
+]
+assert cl.cmds[-1] == ['/usr/bin/rpm', '-qal']
+
+sdf = SuseDistro(
+    cl,
+    id = 'suse',
+    os_release_str = OS_RELEASE_SUSE,
+    default_pkg_filter = PackageFilterString('url=~jw.dev')
+)
+assert asyncio.run(sdf.pkg_files()) == \
+    ['/etc/a/a.conf.jw-tmpl', '/etc/b/b.conf.jw-tmpl']
+assert cl.cmds[-1] == ['/usr/bin/rpm', '-ql', 'a', 'b']
+
+sde = SuseDistro(
+    cl,
+    id = 'suse',
+    os_release_str = OS_RELEASE_SUSE,
+    default_pkg_filter = PackageFilterString('url=~zzz-nomatch')
+)
+assert asyncio.run(sde.pkg_files()) == []
+assert '--queryformat' in cl.cmds[-1]
+
 print('All Distro tests passed')

test/unit/python/jw/pkg/lib/pm/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/dirs.mk

test/unit/python/jw/pkg/lib/pm/rpm/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/pm/rpm/test.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from jw.pkg.lib.FlaggedFile import FlaggedFile
+from jw.pkg.lib.pm.rpm import _parse_verify_output
+
+# -- _parse_verify_output --
+
+# -- A clean line is not reported
+assert _parse_verify_output('pkg', '.....UG..    /usr/bin/foo\n') == []
+
+# -- Mismatches in size, mode, mtime, user, and group are noise
+assert _parse_verify_output('pkg', 'SM...UGT.  c /etc/foo\n') == []
+
+# -- A content (md5) mismatch is reported, with the flags preserved
+ret = _parse_verify_output('pkg', 'SM5....T.  c /etc/foo\n')
+assert len(ret) == 1
+assert ret[0].pkg == 'pkg'
+assert ret[0].file == '/etc/foo'
+assert ret[0].flags == 'SM5....T.'
+
+# -- The 8-character flag layout of older rpm versions is understood
+ret = _parse_verify_output('pkg', 'SM5....T /etc/foo\n')
+assert len(ret) == 1
+assert ret[0].flags == 'SM5....T'
+
+# -- A missing file is reported, config files included
+ret = _parse_verify_output('pkg', 'missing     /etc/opt/jw-ws/pi\n')
+assert len(ret) == 1
+assert ret[0].file == '/etc/opt/jw-ws/pi'
+assert ret[0].flags == 'missing'
+ret = _parse_verify_output('pkg', 'missing   c /etc/opt/jw-ws/pi/models.json\n')
+assert len(ret) == 1
+assert ret[0].file == '/etc/opt/jw-ws/pi/models.json'
+
+# -- A trailing parenthesized annotation is not part of the file
+# name
+ret = _parse_verify_output(
+    'pkg', 'missing   c /etc/phpMyAdmin/config.inc.php.jw-tmpl (Permission denied)\n'
+)
+assert len(ret) == 1
+assert ret[0].file == '/etc/phpMyAdmin/config.inc.php.jw-tmpl'
+
+# -- Lines without a file name and foreign output are ignored
+assert _parse_verify_output('pkg', '.....UG..\n') == []
+assert _parse_verify_output('pkg', 'package foo is not installed\n') == []
+assert _parse_verify_output('pkg', 'error: rpm verify failed\n') == []
+
+# -- The output of several files is accumulated
+ret = _parse_verify_output(
+    'pkg', 'SM5....T.  c /etc/foo\nmissing   c /etc/bar\n.....UG..  c /etc/baz\n'
+)
+assert [f.file for f in ret] == ['/etc/foo', '/etc/bar']
+
+# -- FlaggedFile renders as pkg file flags
+f = FlaggedFile('pkg', '/etc/foo', 'SM5....T.')
+assert repr(f) == 'pkg /etc/foo SM5....T.'
+
+print('All rpm pm tests passed')