From cae983479c9520dd309b139835e15704db08d1de Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 15 Oct 2026 15:38:06 +0200 Subject: [PATCH] lib.Distro.verify(): Add method jw-base verify pipes the output of rpm -qV for all janware packages into a grep for md5 mismatches and missing files. The same capability, distribution-independent, belongs in the Distro abstraction so that tools like jw-nettools can check the integrity of the packages they install on a fleet of hosts. - Add Distro.verify(), which verifies the given package names, or all packages selected by the package filter when none are given, and returns the flagged files - Add Distro._verify() as the distribution hook - Add FlaggedFile, the record for a flagged file: package, file, and the package manager's verify status line - not distribution independent as of today, but it's a start - pm/rpm.py: parse_verify_output() parses the output of `rpm -qV` and reports md5 mismatches and missing files, the latter reported with the literal word "missing" in the flag field; size, mode, mtime, user, and group mismatches are noise on dev machines and are ignored. verify_packages() runs `rpm -qV` per package and collects the findings - pm/dpkg.py: parse_verify_output() parses the output of `dpkg -V` and reports md5sum (P) mismatches and missing files (M); verify_packages() runs `dpkg -V` per package - The SUSE, Red Hat, and Debian Distro classes implement _verify() with the respective pm helpers; the Arch Distro raises NotImplementedError, like its other package manager hooks Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0 --- src/python/jw/pkg/lib/Distro.py | 25 ++++ src/python/jw/pkg/lib/FlaggedFile.py | 21 +++ src/python/jw/pkg/lib/distros/arch/Distro.py | 7 + .../jw/pkg/lib/distros/debian/Distro.py | 7 +- .../jw/pkg/lib/distros/redhat/Distro.py | 7 +- src/python/jw/pkg/lib/distros/suse/Distro.py | 7 +- src/python/jw/pkg/lib/pm/dpkg.py | 36 +++++ src/python/jw/pkg/lib/pm/rpm.py | 58 ++++++++ test/unit/python/jw/pkg/lib/Distro/test.py | 138 +++++++++++++++++- test/unit/python/jw/pkg/lib/pm/Makefile | 4 + test/unit/python/jw/pkg/lib/pm/rpm/Makefile | 7 + test/unit/python/jw/pkg/lib/pm/rpm/test.py | 58 ++++++++ 12 files changed, 371 insertions(+), 4 deletions(-) create mode 100644 src/python/jw/pkg/lib/FlaggedFile.py create mode 100644 test/unit/python/jw/pkg/lib/pm/Makefile create mode 100644 test/unit/python/jw/pkg/lib/pm/rpm/Makefile create mode 100644 test/unit/python/jw/pkg/lib/pm/rpm/test.py diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index d5ee106e..b4dbd5bf 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from .base import Result from .ExecContext import ExecContext + from .FlaggedFile import FlaggedFile from .Package import Package from .PackageFilter import PackageFilter @@ -486,3 +487,27 @@ class Distro(abc.ABC): if names is None and self.default_pkg_filter is not None: names = [p.name for p in await self.select()] return await self._pkg_files(names) + + # -- verify + + @abc.abstractmethod + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + pass + + async def verify( + self, + names: Collection[str] = [], + filter: PackageFilter | None = None, + ) -> list[FlaggedFile]: + """Verify the integrity of installed packages: report files that + were modified or deleted outside of the package manager. + + Without names, all packages selected by filter (or the default + package filter, if any) are verified. + """ + if not names: + names = [p.name for p in await self.select([], filter = filter)] + if not names: + log(WARNING, 'No packages to verify') + return [] + return list(await self._verify(names)) diff --git a/src/python/jw/pkg/lib/FlaggedFile.py b/src/python/jw/pkg/lib/FlaggedFile.py new file mode 100644 index 00000000..a45b234c --- /dev/null +++ b/src/python/jw/pkg/lib/FlaggedFile.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import override + +class FlaggedFile: # export + """A file of an installed package that failed the package manager's + integrity verification (Distro.verify()). + + flags is the verify status line of the package manager for this + file, or the string "missing" for a file the package manager + reports as missing. + """ + + def __init__(self, pkg: str, file: str, flags: str) -> None: + self.pkg = pkg + self.file = file + self.flags = flags + + @override + def __repr__(self) -> str: + return f'{self.pkg} {self.file} {self.flags}' diff --git a/src/python/jw/pkg/lib/distros/arch/Distro.py b/src/python/jw/pkg/lib/distros/arch/Distro.py index b86555be..cc31b9c4 100644 --- a/src/python/jw/pkg/lib/distros/arch/Distro.py +++ b/src/python/jw/pkg/lib/distros/arch/Distro.py @@ -8,6 +8,7 @@ if TYPE_CHECKING: from typing import Iterable from ...base import Result + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -70,3 +71,9 @@ class Distro(Base): raise NotImplementedError( 'distro pkg ls yet implemented for Arch-like distributions' ) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + raise NotImplementedError( + 'distro verify is not yet implemented for Arch-like distributions' + ) diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index adc50246..33a0e203 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -6,12 +6,13 @@ from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base from ...log import NOTICE, log -from ...pm.dpkg import list_files, query_packages, run_dpkg +from ...pm.dpkg import list_files, query_packages, run_dpkg, verify_packages if TYPE_CHECKING: from typing import Iterable from ...base import Result + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -87,3 +88,7 @@ class Distro(Base): @override async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/distros/redhat/Distro.py b/src/python/jw/pkg/lib/distros/redhat/Distro.py index e9f5f195..fbf1f37d 100644 --- a/src/python/jw/pkg/lib/distros/redhat/Distro.py +++ b/src/python/jw/pkg/lib/distros/redhat/Distro.py @@ -3,13 +3,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm +from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages if TYPE_CHECKING: from typing import Any, Collection, Iterable from ...base import Result from ...ExecContext import ExecContext + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -73,6 +74,10 @@ class Distro(Base): async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) + async def _download(self, name: str, tmp_dir: str) -> str: raise NotImplementedError( 'distro pkg download is not yet implemented for RedHat distributions' diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index c1d1ce2b..b23bb278 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm +from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages if TYPE_CHECKING: from collections.abc import Collection @@ -11,6 +11,7 @@ if TYPE_CHECKING: from ...base import Result from ...ExecContext import ExecContext + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -97,3 +98,7 @@ class Distro(Base): @override async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index d1193a68..339ddb48 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from ..ExecContext import ExecContext from ..base import InputMode +from ..FlaggedFile import FlaggedFile from ..Package import Package from ..util import run_cmd, run_sudo @@ -88,3 +89,38 @@ async def list_files( return [] file_list_str = await run_dpkg(['-L', *names], sudo = False, ec = ec) return file_list_str.splitlines() + +def _parse_verify_output( + pkg: str, + output: str, +) -> list[FlaggedFile]: + """Parse the output of `dpkg -V ` into a list of flagged + files. + + Reports md5sum (P) mismatches and missing files (M), the cases + that indicate a package was modified outside of the package + manager. The remaining dpkg verify flags (access permissions, + configuration file rdev, user, group, size, mtime, and + executable bit) are noise on dev machines and are ignored. + """ + ret: list[FlaggedFile] = [] + for line in output.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2: + continue + flags, file = parts + if 'P' in flags or 'M' in flags: + ret.append(FlaggedFile(pkg, file, flags)) + return ret + +async def verify_packages( + names: Iterable[str], + ec: ExecContext | None = None, +) -> list[FlaggedFile]: + ret: list[FlaggedFile] = [] + for name in names: + # dpkg -V reports its findings on standard output; the exit + # status is not informative, so it is ignored + output = await run_dpkg(['-V', name], sudo = False, throw = False, ec = ec) + ret.extend(_parse_verify_output(name, output)) + return ret diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index 7beebed6..eef7e794 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -1,8 +1,11 @@ from __future__ import annotations +import re + from typing import TYPE_CHECKING, Any, Collection, Iterable, cast from ..base import InputMode +from ..FlaggedFile import FlaggedFile from ..Package import Package from ..util import run_cmd, run_sudo @@ -76,3 +79,58 @@ async def list_files( args, throw = True, sudo = False, mode = InputMode.NonInteractive, ec = ec ) return stdout.splitlines() + +def _parse_verify_output( + pkg: str, + output: str, +) -> list[FlaggedFile]: + """Parse the output of `rpm -qV ` into a list of flagged + files. + + Reports content (md5) mismatches and missing files, the cases + that indicate a package was modified outside of the package + manager. Mismatches in size, mode, mtime, user, and group are + noise on dev machines and are ignored. + """ + ret: list[FlaggedFile] = [] + for line in output.splitlines(): + # rpm appends a parenthesized annotation, for instance + # "(Permission denied)", to a line whose file could not be + # accessed; it is not part of the file name + line = re.sub(r'\s*\([^)]*\)$', '', line) + parts = line.split() + if not parts: + continue + # rpm reports a missing file with the literal word "missing" + # in the flag field, the file name in the usual place + if parts[0] == 'missing': + if len(parts) >= 2: + ret.append(FlaggedFile(pkg, parts[-1], 'missing')) + continue + flags = parts[0] + if len(flags) not in (8, 9) or len(parts) < 2: + continue + if not all(ch in 'SM5DLUGTP.?' for ch in flags): + continue + if flags[2] == '5': + ret.append(FlaggedFile(pkg, parts[-1], flags)) + return ret + +async def verify_packages( + names: Iterable[str], + ec: ExecContext | None = None, +) -> list[FlaggedFile]: + ret: list[FlaggedFile] = [] + for name in names: + # rpm exits with 1 both when differences are found and when + # the package is not installed, so only the output is + # informative + output = await run_rpm( + ['-qV', name], + throw = False, + sudo = False, + mode = InputMode.NonInteractive, + ec = ec + ) + ret.extend(_parse_verify_output(name, output)) + return ret diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index 88630113..f1bd8e4d 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -1,12 +1,19 @@ from __future__ import annotations +import asyncio + from typing import TYPE_CHECKING, Iterable, cast, override from jw.pkg.lib.Distro import Distro +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Package import Package +from jw.pkg.lib.PackageFilter import PackageFilterString +from jw.pkg.lib.Result import Result if TYPE_CHECKING: from jw.pkg.lib.ExecContext import ExecContext - from jw.pkg.lib.Package import Package + from jw.pkg.lib.FlaggedFile import FlaggedFile OS_RELEASE_UBUNTU = """\ NAME="Ubuntu" @@ -59,6 +66,76 @@ class DummyDistro(Distro): async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return [] + @override + async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]: + return [] + +# -- verify() -- + +class VerifyDistro(DummyDistro): + + def __init__( + self, ec: ExecContext, id: str, os_release_str: str | None = None + ) -> None: + super().__init__(ec, id, os_release_str) + self.verified: list[str] | None = None + + @override + async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]: + self.verified = list(names) + return [] + +vd = VerifyDistro(ec = cast('ExecContext', object()), id = 'suse') + +# -- Given names are passed to the distro backend untouched +assert asyncio.run(vd.verify(names = ['a', 'b'])) == [] +assert vd.verified == ['a', 'b'] + +# -- Without names, the (empty) selection is verified, so the +# backend is not invoked +vd.verified = None +assert asyncio.run(vd.verify()) == [] +assert vd.verified is None + +# -- pkg_files -- + +# -- Without a filter, the selection is passed through as the +# whole system; the paths come back unfiltered + +class TmplDistro(DummyDistro): + + def __init__(self) -> None: + super().__init__(ec = cast('ExecContext', object()), id = 'suse') + self.files: dict[str, list[str]] = {} + + @override + async def _select_by_name(self, names: Iterable[str]) -> list[Package]: + return [Package(n) for n in self.files] + + @override + async def _pkg_files(self, names: list[str] | None) -> list[str]: + if names is None: + names = list(self.files) + ret: list[str] = [] + for n in names: + ret.extend(self.files[n]) + return ret + +t = TmplDistro() +t.files = { + 'a': ['/usr/share/a/a.conf.jw-tmpl', '/usr/bin/a'], + 'b': ['/etc/b/b.conf.jw-tmpl'], +} +assert sorted(asyncio.run(t.pkg_files())) == [ + '/etc/b/b.conf.jw-tmpl', '/usr/bin/a', '/usr/share/a/a.conf.jw-tmpl' +] + +# -- A single package name as a str is accepted for compatibility +assert sorted(asyncio.run(t.pkg_files('a'))) == [ + '/usr/bin/a', + '/usr/share/a/a.conf.jw-tmpl', +] + def distro(os_release_str: str) -> Distro: # -- The execution context is stored but not used by the properties # under test, so a dummy object is sufficient @@ -150,4 +227,63 @@ for macro in Distro.macros(): # argparse help escaping has been undone d.expand_macros(macro.replace('%%', '%')) +# -- The rpm distro's batched enumeration -- + +# -- Without a package filter, the selection is the whole system, +# which is listed without package names; with one, the selected +# names are listed in one command, and a selection without +# matches issues no listing at all +class CannedRpmLocal(Local): + + def __init__(self) -> None: + super().__init__('local') + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qal' in cmd: + out = '/usr/bin/x\n/etc/a/a.conf.jw-tmpl\n' + elif '-ql' in cmd: + out = '/etc/a/a.conf.jw-tmpl\n/etc/b/b.conf.jw-tmpl\n' + else: + out = 'a|v|p|https://jw.dev/a|m\nb|v|p|https://jw.dev/b|m\n' + return Result(stdout = out.encode(), status = 0) + +cl = CannedRpmLocal() +sd = SuseDistro(cl, id = 'suse', os_release_str = OS_RELEASE_SUSE) +assert asyncio.run(sd.pkg_files()) == [ + '/usr/bin/x', + '/etc/a/a.conf.jw-tmpl', +] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-qal'] + +sdf = SuseDistro( + cl, + id = 'suse', + os_release_str = OS_RELEASE_SUSE, + default_pkg_filter = PackageFilterString('url=~jw.dev') +) +assert asyncio.run(sdf.pkg_files()) == \ + ['/etc/a/a.conf.jw-tmpl', '/etc/b/b.conf.jw-tmpl'] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-ql', 'a', 'b'] + +sde = SuseDistro( + cl, + id = 'suse', + os_release_str = OS_RELEASE_SUSE, + default_pkg_filter = PackageFilterString('url=~zzz-nomatch') +) +assert asyncio.run(sde.pkg_files()) == [] +assert '--queryformat' in cl.cmds[-1] + print('All Distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/pm/Makefile b/test/unit/python/jw/pkg/lib/pm/Makefile new file mode 100644 index 00000000..c87020cb --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/Makefile b/test/unit/python/jw/pkg/lib/pm/rpm/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/rpm/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/test.py b/test/unit/python/jw/pkg/lib/pm/rpm/test.py new file mode 100644 index 00000000..c743d13b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/rpm/test.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from jw.pkg.lib.FlaggedFile import FlaggedFile +from jw.pkg.lib.pm.rpm import _parse_verify_output + +# -- _parse_verify_output -- + +# -- A clean line is not reported +assert _parse_verify_output('pkg', '.....UG.. /usr/bin/foo\n') == [] + +# -- Mismatches in size, mode, mtime, user, and group are noise +assert _parse_verify_output('pkg', 'SM...UGT. c /etc/foo\n') == [] + +# -- A content (md5) mismatch is reported, with the flags preserved +ret = _parse_verify_output('pkg', 'SM5....T. c /etc/foo\n') +assert len(ret) == 1 +assert ret[0].pkg == 'pkg' +assert ret[0].file == '/etc/foo' +assert ret[0].flags == 'SM5....T.' + +# -- The 8-character flag layout of older rpm versions is understood +ret = _parse_verify_output('pkg', 'SM5....T /etc/foo\n') +assert len(ret) == 1 +assert ret[0].flags == 'SM5....T' + +# -- A missing file is reported, config files included +ret = _parse_verify_output('pkg', 'missing /etc/opt/jw-ws/pi\n') +assert len(ret) == 1 +assert ret[0].file == '/etc/opt/jw-ws/pi' +assert ret[0].flags == 'missing' +ret = _parse_verify_output('pkg', 'missing c /etc/opt/jw-ws/pi/models.json\n') +assert len(ret) == 1 +assert ret[0].file == '/etc/opt/jw-ws/pi/models.json' + +# -- A trailing parenthesized annotation is not part of the file +# name +ret = _parse_verify_output( + 'pkg', 'missing c /etc/phpMyAdmin/config.inc.php.jw-tmpl (Permission denied)\n' +) +assert len(ret) == 1 +assert ret[0].file == '/etc/phpMyAdmin/config.inc.php.jw-tmpl' + +# -- Lines without a file name and foreign output are ignored +assert _parse_verify_output('pkg', '.....UG..\n') == [] +assert _parse_verify_output('pkg', 'package foo is not installed\n') == [] +assert _parse_verify_output('pkg', 'error: rpm verify failed\n') == [] + +# -- The output of several files is accumulated +ret = _parse_verify_output( + 'pkg', 'SM5....T. c /etc/foo\nmissing c /etc/bar\n.....UG.. c /etc/baz\n' +) +assert [f.file for f in ret] == ['/etc/foo', '/etc/bar'] + +# -- FlaggedFile renders as pkg file flags +f = FlaggedFile('pkg', '/etc/foo', 'SM5....T.') +assert repr(f) == 'pkg /etc/foo SM5....T.' + +print('All rpm pm tests passed')