From 2fad5d189c97e1bb77a1998fb7101d98f3c0b288 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 15 Oct 2026 14:29:17 +0200 Subject: [PATCH 1/3] lib.Distro.pkg_files(): Take list[str] | None Distro.pkg_files() takes a package name and returns a list of files in that package. This commit generalizes that - pass None to it and get the list of all packaged files in the system, and pass a list of strings to retrieve all files belonging to the named packages. The same arguments are propagated to _pkg_files(), which in turn propagates them to list_files(). This allows faster generation of file lists than before. Having to retrieve a list of matching packages and query each package for its contained files can be collapsed into one or two backend calls. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/Distro.py | 19 ++++++--- src/python/jw/pkg/lib/distros/arch/Distro.py | 2 +- .../jw/pkg/lib/distros/debian/Distro.py | 4 +- .../jw/pkg/lib/distros/redhat/Distro.py | 4 +- src/python/jw/pkg/lib/distros/suse/Distro.py | 4 +- src/python/jw/pkg/lib/pm/dpkg.py | 40 +++++++++++++++---- src/python/jw/pkg/lib/pm/rpm.py | 19 ++++++--- test/integration/jw_pkg/secrets/test.py | 9 ++++- test/unit/python/jw/pkg/lib/Distro/test.py | 2 +- 9 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index 147583ec..d5ee106e 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -471,11 +471,18 @@ class Distro(abc.ABC): # -- pkg_files @abc.abstractmethod - async def _pkg_files(self, name: str) -> Iterable[str]: + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: pass - async def pkg_files(self, name: str) -> Iterable[str]: - if not name: - log(WARNING, 'No package specified for inspection') - return [] - return await self._pkg_files(name) + async def pkg_files(self, names: str | list[str] | None = None) -> Iterable[str]: + # -- The file paths of the packages selected by the + # argument: None, the default selection, filtered to the + # app's packages when a filter is set, all installed + # packages otherwise; [], no packages; [names], the + # listed packages; a str names one package, accepted for + # compatibility with the single-package call + if isinstance(names, str): + names = [names] + if names is None and self.default_pkg_filter is not None: + names = [p.name for p in await self.select()] + return await self._pkg_files(names) diff --git a/src/python/jw/pkg/lib/distros/arch/Distro.py b/src/python/jw/pkg/lib/distros/arch/Distro.py index f7b10f8a..b86555be 100644 --- a/src/python/jw/pkg/lib/distros/arch/Distro.py +++ b/src/python/jw/pkg/lib/distros/arch/Distro.py @@ -66,7 +66,7 @@ class Distro(Base): ) @override - async def _pkg_files(self, name: str) -> Iterable[str]: + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: raise NotImplementedError( 'distro pkg ls yet implemented for Arch-like distributions' ) diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index d0f0291b..adc50246 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -85,5 +85,5 @@ class Distro(Base): await self.dpkg(['-P', *names], sudo = True) @override - async def _pkg_files(self, name: str) -> Iterable[str]: - return await list_files(name, ec = self.ctx) + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: + return await list_files(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/distros/redhat/Distro.py b/src/python/jw/pkg/lib/distros/redhat/Distro.py index 0ba5d473..e9f5f195 100644 --- a/src/python/jw/pkg/lib/distros/redhat/Distro.py +++ b/src/python/jw/pkg/lib/distros/redhat/Distro.py @@ -70,8 +70,8 @@ class Distro(Base): await self.rpm(['-e', *names], sudo = True) @override - async def _pkg_files(self, name: str) -> Iterable[str]: - return await list_files(name, ec = self.ctx) + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: + return await list_files(names, ec = self.ctx) async def _download(self, name: str, tmp_dir: str) -> str: raise NotImplementedError( diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index 02967ef6..c1d1ce2b 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -95,5 +95,5 @@ class Distro(Base): await self.rpm(['-e', *names], sudo = True) @override - async def _pkg_files(self, name: str) -> Iterable[str]: - return await list_files(name, ec = self.ctx) + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: + return await list_files(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index a7b30813..d1193a68 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, cast +from typing import TYPE_CHECKING, Any, Iterable, cast if TYPE_CHECKING: from ..ExecContext import ExecContext @@ -26,21 +26,25 @@ def meta_map() -> dict[str, str]: return _meta_map async def _run( - cmd: list[str], sudo: bool = False, ec: ExecContext | None = None + cmd: list[str], + sudo: bool = False, + ec: ExecContext | None = None, + **kwargs: Any, ) -> str: return ( - await run_sudo(cmd) - if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive) + await run_sudo(cmd) if sudo else await + run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive, **kwargs) ).stdout_str async def run_dpkg( # export args: list[str], sudo: bool = False, - ec: ExecContext | None = None + ec: ExecContext | None = None, + **kwargs: Any, ) -> str: cmd = ['/usr/bin/dpkg'] cmd.extend(args) - return await _run(cmd, sudo, ec) + return await _run(cmd, sudo, ec, **kwargs) async def run_dpkg_query( # export args: list[str], @@ -61,6 +65,26 @@ async def query_packages(names: Iterable[str] = [], specs = await run_dpkg_query(['-W', '-f=' + fmt_str, *names], sudo = False, ec = ec) return cast('Iterable[Package]', Package.parse_specs_str(specs)) -async def list_files(pkg: str, ec: ExecContext | None = None) -> list[str]: - file_list_str = await run_dpkg(['-L', pkg], sudo = False, ec = ec) +async def list_files( + names: list[str] | None = None, + ec: ExecContext | None = None, +) -> list[str]: + # -- None: all installed packages. The standard interface, two + # queries: select the installed packages, then list their + # files in one command; the list output carries a + # package-name header and a blank line per package, both + # dropped. []: no packages, no command + if names is None: + names = ( + await run_dpkg_query( + ['-W', '-f=${?installed:binary:Package}\n'], sudo = False, ec = ec + ) + ).split() + if not names: + return [] + stdout = await run_dpkg_query(['-L', *names], sudo = False, ec = ec) + return [line for line in stdout.splitlines() if line and not line.endswith(':')] + if not names: + return [] + file_list_str = await run_dpkg(['-L', *names], sudo = False, ec = ec) return file_list_str.splitlines() diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index ece689ce..7beebed6 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -60,12 +60,19 @@ async def query_packages( # export ) return cast('Iterable[Package]', Package.parse_specs_str(specs)) -async def list_files(pkg: str, ec: ExecContext | None = None) -> list[str]: +async def list_files( + names: list[str] | None = None, + ec: ExecContext | None = None, +) -> list[str]: + # -- None: all installed packages, one query without package + # names. []: no packages, no command + if names is None: + args = ['-qal'] + elif not names: + return [] + else: + args = ['-ql', *names] stdout = await run_rpm( - ['-ql', pkg], - throw = True, - sudo = False, - mode = InputMode.NonInteractive, - ec = ec + args, throw = True, sudo = False, mode = InputMode.NonInteractive, ec = ec ) return stdout.splitlines() diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index faf67a68..e23ae6b6 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -71,8 +71,13 @@ class FakeDistro(SuseDistro): return [Package(name) for name in names if name in self.__files] @override - async def _pkg_files(self, name: str) -> list[str]: - return list(self.__files.get(name, [])) + async def _pkg_files(self, names: list[str] | None) -> list[str]: + if names is None: + names = list(self.__files) + ret: list[str] = [] + for n in names: + ret.extend(self.__files.get(n, [])) + return ret def run_cli(distro: FakeDistro, argv: list[str]) -> tuple[list[str], list[str]]: """Run the real jw-pkg CLI in-process, return stdout and stderr diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index dbde4818..88630113 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -56,7 +56,7 @@ class DummyDistro(Distro): pass @override - async def _pkg_files(self, name: str) -> Iterable[str]: + async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return [] def distro(os_release_str: str) -> Distro: -- 2.56.0 From cae983479c9520dd309b139835e15704db08d1de Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 15 Oct 2026 15:38:06 +0200 Subject: [PATCH 2/3] lib.Distro.verify(): Add method jw-base verify pipes the output of rpm -qV for all janware packages into a grep for md5 mismatches and missing files. The same capability, distribution-independent, belongs in the Distro abstraction so that tools like jw-nettools can check the integrity of the packages they install on a fleet of hosts. - Add Distro.verify(), which verifies the given package names, or all packages selected by the package filter when none are given, and returns the flagged files - Add Distro._verify() as the distribution hook - Add FlaggedFile, the record for a flagged file: package, file, and the package manager's verify status line - not distribution independent as of today, but it's a start - pm/rpm.py: parse_verify_output() parses the output of `rpm -qV` and reports md5 mismatches and missing files, the latter reported with the literal word "missing" in the flag field; size, mode, mtime, user, and group mismatches are noise on dev machines and are ignored. verify_packages() runs `rpm -qV` per package and collects the findings - pm/dpkg.py: parse_verify_output() parses the output of `dpkg -V` and reports md5sum (P) mismatches and missing files (M); verify_packages() runs `dpkg -V` per package - The SUSE, Red Hat, and Debian Distro classes implement _verify() with the respective pm helpers; the Arch Distro raises NotImplementedError, like its other package manager hooks Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0 --- src/python/jw/pkg/lib/Distro.py | 25 ++++ src/python/jw/pkg/lib/FlaggedFile.py | 21 +++ src/python/jw/pkg/lib/distros/arch/Distro.py | 7 + .../jw/pkg/lib/distros/debian/Distro.py | 7 +- .../jw/pkg/lib/distros/redhat/Distro.py | 7 +- src/python/jw/pkg/lib/distros/suse/Distro.py | 7 +- src/python/jw/pkg/lib/pm/dpkg.py | 36 +++++ src/python/jw/pkg/lib/pm/rpm.py | 58 ++++++++ test/unit/python/jw/pkg/lib/Distro/test.py | 138 +++++++++++++++++- test/unit/python/jw/pkg/lib/pm/Makefile | 4 + test/unit/python/jw/pkg/lib/pm/rpm/Makefile | 7 + test/unit/python/jw/pkg/lib/pm/rpm/test.py | 58 ++++++++ 12 files changed, 371 insertions(+), 4 deletions(-) create mode 100644 src/python/jw/pkg/lib/FlaggedFile.py create mode 100644 test/unit/python/jw/pkg/lib/pm/Makefile create mode 100644 test/unit/python/jw/pkg/lib/pm/rpm/Makefile create mode 100644 test/unit/python/jw/pkg/lib/pm/rpm/test.py diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index d5ee106e..b4dbd5bf 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from .base import Result from .ExecContext import ExecContext + from .FlaggedFile import FlaggedFile from .Package import Package from .PackageFilter import PackageFilter @@ -486,3 +487,27 @@ class Distro(abc.ABC): if names is None and self.default_pkg_filter is not None: names = [p.name for p in await self.select()] return await self._pkg_files(names) + + # -- verify + + @abc.abstractmethod + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + pass + + async def verify( + self, + names: Collection[str] = [], + filter: PackageFilter | None = None, + ) -> list[FlaggedFile]: + """Verify the integrity of installed packages: report files that + were modified or deleted outside of the package manager. + + Without names, all packages selected by filter (or the default + package filter, if any) are verified. + """ + if not names: + names = [p.name for p in await self.select([], filter = filter)] + if not names: + log(WARNING, 'No packages to verify') + return [] + return list(await self._verify(names)) diff --git a/src/python/jw/pkg/lib/FlaggedFile.py b/src/python/jw/pkg/lib/FlaggedFile.py new file mode 100644 index 00000000..a45b234c --- /dev/null +++ b/src/python/jw/pkg/lib/FlaggedFile.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import override + +class FlaggedFile: # export + """A file of an installed package that failed the package manager's + integrity verification (Distro.verify()). + + flags is the verify status line of the package manager for this + file, or the string "missing" for a file the package manager + reports as missing. + """ + + def __init__(self, pkg: str, file: str, flags: str) -> None: + self.pkg = pkg + self.file = file + self.flags = flags + + @override + def __repr__(self) -> str: + return f'{self.pkg} {self.file} {self.flags}' diff --git a/src/python/jw/pkg/lib/distros/arch/Distro.py b/src/python/jw/pkg/lib/distros/arch/Distro.py index b86555be..cc31b9c4 100644 --- a/src/python/jw/pkg/lib/distros/arch/Distro.py +++ b/src/python/jw/pkg/lib/distros/arch/Distro.py @@ -8,6 +8,7 @@ if TYPE_CHECKING: from typing import Iterable from ...base import Result + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -70,3 +71,9 @@ class Distro(Base): raise NotImplementedError( 'distro pkg ls yet implemented for Arch-like distributions' ) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + raise NotImplementedError( + 'distro verify is not yet implemented for Arch-like distributions' + ) diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index adc50246..33a0e203 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -6,12 +6,13 @@ from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base from ...log import NOTICE, log -from ...pm.dpkg import list_files, query_packages, run_dpkg +from ...pm.dpkg import list_files, query_packages, run_dpkg, verify_packages if TYPE_CHECKING: from typing import Iterable from ...base import Result + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -87,3 +88,7 @@ class Distro(Base): @override async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/distros/redhat/Distro.py b/src/python/jw/pkg/lib/distros/redhat/Distro.py index e9f5f195..fbf1f37d 100644 --- a/src/python/jw/pkg/lib/distros/redhat/Distro.py +++ b/src/python/jw/pkg/lib/distros/redhat/Distro.py @@ -3,13 +3,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm +from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages if TYPE_CHECKING: from typing import Any, Collection, Iterable from ...base import Result from ...ExecContext import ExecContext + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -73,6 +74,10 @@ class Distro(Base): async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) + async def _download(self, name: str, tmp_dir: str) -> str: raise NotImplementedError( 'distro pkg download is not yet implemented for RedHat distributions' diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index c1d1ce2b..b23bb278 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm +from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages if TYPE_CHECKING: from collections.abc import Collection @@ -11,6 +11,7 @@ if TYPE_CHECKING: from ...base import Result from ...ExecContext import ExecContext + from ...FlaggedFile import FlaggedFile from ...Package import Package class Distro(Base): @@ -97,3 +98,7 @@ class Distro(Base): @override async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return await list_files(names, ec = self.ctx) + + @override + async def _verify(self, names: Iterable[str]) -> Iterable[FlaggedFile]: + return await verify_packages(names, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index d1193a68..339ddb48 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from ..ExecContext import ExecContext from ..base import InputMode +from ..FlaggedFile import FlaggedFile from ..Package import Package from ..util import run_cmd, run_sudo @@ -88,3 +89,38 @@ async def list_files( return [] file_list_str = await run_dpkg(['-L', *names], sudo = False, ec = ec) return file_list_str.splitlines() + +def _parse_verify_output( + pkg: str, + output: str, +) -> list[FlaggedFile]: + """Parse the output of `dpkg -V ` into a list of flagged + files. + + Reports md5sum (P) mismatches and missing files (M), the cases + that indicate a package was modified outside of the package + manager. The remaining dpkg verify flags (access permissions, + configuration file rdev, user, group, size, mtime, and + executable bit) are noise on dev machines and are ignored. + """ + ret: list[FlaggedFile] = [] + for line in output.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2: + continue + flags, file = parts + if 'P' in flags or 'M' in flags: + ret.append(FlaggedFile(pkg, file, flags)) + return ret + +async def verify_packages( + names: Iterable[str], + ec: ExecContext | None = None, +) -> list[FlaggedFile]: + ret: list[FlaggedFile] = [] + for name in names: + # dpkg -V reports its findings on standard output; the exit + # status is not informative, so it is ignored + output = await run_dpkg(['-V', name], sudo = False, throw = False, ec = ec) + ret.extend(_parse_verify_output(name, output)) + return ret diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index 7beebed6..eef7e794 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -1,8 +1,11 @@ from __future__ import annotations +import re + from typing import TYPE_CHECKING, Any, Collection, Iterable, cast from ..base import InputMode +from ..FlaggedFile import FlaggedFile from ..Package import Package from ..util import run_cmd, run_sudo @@ -76,3 +79,58 @@ async def list_files( args, throw = True, sudo = False, mode = InputMode.NonInteractive, ec = ec ) return stdout.splitlines() + +def _parse_verify_output( + pkg: str, + output: str, +) -> list[FlaggedFile]: + """Parse the output of `rpm -qV ` into a list of flagged + files. + + Reports content (md5) mismatches and missing files, the cases + that indicate a package was modified outside of the package + manager. Mismatches in size, mode, mtime, user, and group are + noise on dev machines and are ignored. + """ + ret: list[FlaggedFile] = [] + for line in output.splitlines(): + # rpm appends a parenthesized annotation, for instance + # "(Permission denied)", to a line whose file could not be + # accessed; it is not part of the file name + line = re.sub(r'\s*\([^)]*\)$', '', line) + parts = line.split() + if not parts: + continue + # rpm reports a missing file with the literal word "missing" + # in the flag field, the file name in the usual place + if parts[0] == 'missing': + if len(parts) >= 2: + ret.append(FlaggedFile(pkg, parts[-1], 'missing')) + continue + flags = parts[0] + if len(flags) not in (8, 9) or len(parts) < 2: + continue + if not all(ch in 'SM5DLUGTP.?' for ch in flags): + continue + if flags[2] == '5': + ret.append(FlaggedFile(pkg, parts[-1], flags)) + return ret + +async def verify_packages( + names: Iterable[str], + ec: ExecContext | None = None, +) -> list[FlaggedFile]: + ret: list[FlaggedFile] = [] + for name in names: + # rpm exits with 1 both when differences are found and when + # the package is not installed, so only the output is + # informative + output = await run_rpm( + ['-qV', name], + throw = False, + sudo = False, + mode = InputMode.NonInteractive, + ec = ec + ) + ret.extend(_parse_verify_output(name, output)) + return ret diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index 88630113..f1bd8e4d 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -1,12 +1,19 @@ from __future__ import annotations +import asyncio + from typing import TYPE_CHECKING, Iterable, cast, override from jw.pkg.lib.Distro import Distro +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Package import Package +from jw.pkg.lib.PackageFilter import PackageFilterString +from jw.pkg.lib.Result import Result if TYPE_CHECKING: from jw.pkg.lib.ExecContext import ExecContext - from jw.pkg.lib.Package import Package + from jw.pkg.lib.FlaggedFile import FlaggedFile OS_RELEASE_UBUNTU = """\ NAME="Ubuntu" @@ -59,6 +66,76 @@ class DummyDistro(Distro): async def _pkg_files(self, names: list[str] | None) -> Iterable[str]: return [] + @override + async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]: + return [] + +# -- verify() -- + +class VerifyDistro(DummyDistro): + + def __init__( + self, ec: ExecContext, id: str, os_release_str: str | None = None + ) -> None: + super().__init__(ec, id, os_release_str) + self.verified: list[str] | None = None + + @override + async def _verify(self, names: Iterable[str]) -> list[FlaggedFile]: + self.verified = list(names) + return [] + +vd = VerifyDistro(ec = cast('ExecContext', object()), id = 'suse') + +# -- Given names are passed to the distro backend untouched +assert asyncio.run(vd.verify(names = ['a', 'b'])) == [] +assert vd.verified == ['a', 'b'] + +# -- Without names, the (empty) selection is verified, so the +# backend is not invoked +vd.verified = None +assert asyncio.run(vd.verify()) == [] +assert vd.verified is None + +# -- pkg_files -- + +# -- Without a filter, the selection is passed through as the +# whole system; the paths come back unfiltered + +class TmplDistro(DummyDistro): + + def __init__(self) -> None: + super().__init__(ec = cast('ExecContext', object()), id = 'suse') + self.files: dict[str, list[str]] = {} + + @override + async def _select_by_name(self, names: Iterable[str]) -> list[Package]: + return [Package(n) for n in self.files] + + @override + async def _pkg_files(self, names: list[str] | None) -> list[str]: + if names is None: + names = list(self.files) + ret: list[str] = [] + for n in names: + ret.extend(self.files[n]) + return ret + +t = TmplDistro() +t.files = { + 'a': ['/usr/share/a/a.conf.jw-tmpl', '/usr/bin/a'], + 'b': ['/etc/b/b.conf.jw-tmpl'], +} +assert sorted(asyncio.run(t.pkg_files())) == [ + '/etc/b/b.conf.jw-tmpl', '/usr/bin/a', '/usr/share/a/a.conf.jw-tmpl' +] + +# -- A single package name as a str is accepted for compatibility +assert sorted(asyncio.run(t.pkg_files('a'))) == [ + '/usr/bin/a', + '/usr/share/a/a.conf.jw-tmpl', +] + def distro(os_release_str: str) -> Distro: # -- The execution context is stored but not used by the properties # under test, so a dummy object is sufficient @@ -150,4 +227,63 @@ for macro in Distro.macros(): # argparse help escaping has been undone d.expand_macros(macro.replace('%%', '%')) +# -- The rpm distro's batched enumeration -- + +# -- Without a package filter, the selection is the whole system, +# which is listed without package names; with one, the selected +# names are listed in one command, and a selection without +# matches issues no listing at all +class CannedRpmLocal(Local): + + def __init__(self) -> None: + super().__init__('local') + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qal' in cmd: + out = '/usr/bin/x\n/etc/a/a.conf.jw-tmpl\n' + elif '-ql' in cmd: + out = '/etc/a/a.conf.jw-tmpl\n/etc/b/b.conf.jw-tmpl\n' + else: + out = 'a|v|p|https://jw.dev/a|m\nb|v|p|https://jw.dev/b|m\n' + return Result(stdout = out.encode(), status = 0) + +cl = CannedRpmLocal() +sd = SuseDistro(cl, id = 'suse', os_release_str = OS_RELEASE_SUSE) +assert asyncio.run(sd.pkg_files()) == [ + '/usr/bin/x', + '/etc/a/a.conf.jw-tmpl', +] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-qal'] + +sdf = SuseDistro( + cl, + id = 'suse', + os_release_str = OS_RELEASE_SUSE, + default_pkg_filter = PackageFilterString('url=~jw.dev') +) +assert asyncio.run(sdf.pkg_files()) == \ + ['/etc/a/a.conf.jw-tmpl', '/etc/b/b.conf.jw-tmpl'] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-ql', 'a', 'b'] + +sde = SuseDistro( + cl, + id = 'suse', + os_release_str = OS_RELEASE_SUSE, + default_pkg_filter = PackageFilterString('url=~zzz-nomatch') +) +assert asyncio.run(sde.pkg_files()) == [] +assert '--queryformat' in cl.cmds[-1] + print('All Distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/pm/Makefile b/test/unit/python/jw/pkg/lib/pm/Makefile new file mode 100644 index 00000000..c87020cb --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/Makefile b/test/unit/python/jw/pkg/lib/pm/rpm/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/rpm/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/test.py b/test/unit/python/jw/pkg/lib/pm/rpm/test.py new file mode 100644 index 00000000..c743d13b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/rpm/test.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from jw.pkg.lib.FlaggedFile import FlaggedFile +from jw.pkg.lib.pm.rpm import _parse_verify_output + +# -- _parse_verify_output -- + +# -- A clean line is not reported +assert _parse_verify_output('pkg', '.....UG.. /usr/bin/foo\n') == [] + +# -- Mismatches in size, mode, mtime, user, and group are noise +assert _parse_verify_output('pkg', 'SM...UGT. c /etc/foo\n') == [] + +# -- A content (md5) mismatch is reported, with the flags preserved +ret = _parse_verify_output('pkg', 'SM5....T. c /etc/foo\n') +assert len(ret) == 1 +assert ret[0].pkg == 'pkg' +assert ret[0].file == '/etc/foo' +assert ret[0].flags == 'SM5....T.' + +# -- The 8-character flag layout of older rpm versions is understood +ret = _parse_verify_output('pkg', 'SM5....T /etc/foo\n') +assert len(ret) == 1 +assert ret[0].flags == 'SM5....T' + +# -- A missing file is reported, config files included +ret = _parse_verify_output('pkg', 'missing /etc/opt/jw-ws/pi\n') +assert len(ret) == 1 +assert ret[0].file == '/etc/opt/jw-ws/pi' +assert ret[0].flags == 'missing' +ret = _parse_verify_output('pkg', 'missing c /etc/opt/jw-ws/pi/models.json\n') +assert len(ret) == 1 +assert ret[0].file == '/etc/opt/jw-ws/pi/models.json' + +# -- A trailing parenthesized annotation is not part of the file +# name +ret = _parse_verify_output( + 'pkg', 'missing c /etc/phpMyAdmin/config.inc.php.jw-tmpl (Permission denied)\n' +) +assert len(ret) == 1 +assert ret[0].file == '/etc/phpMyAdmin/config.inc.php.jw-tmpl' + +# -- Lines without a file name and foreign output are ignored +assert _parse_verify_output('pkg', '.....UG..\n') == [] +assert _parse_verify_output('pkg', 'package foo is not installed\n') == [] +assert _parse_verify_output('pkg', 'error: rpm verify failed\n') == [] + +# -- The output of several files is accumulated +ret = _parse_verify_output( + 'pkg', 'SM5....T. c /etc/foo\nmissing c /etc/bar\n.....UG.. c /etc/baz\n' +) +assert [f.file for f in ret] == ['/etc/foo', '/etc/bar'] + +# -- FlaggedFile renders as pkg file flags +f = FlaggedFile('pkg', '/etc/foo', 'SM5....T.') +assert repr(f) == 'pkg /etc/foo SM5....T.' + +print('All rpm pm tests passed') -- 2.56.0 From 8106d4a7b0604afa9ebbc53094cf00b475ac2d65 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 6 Oct 2026 18:12:38 +0200 Subject: [PATCH 3/3] cmds.pkg.CmdVerify: Add command Add the "packages verify" command: it runs Distro.verify() with the app's package filter and prints one line per flagged file. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0 --- src/python/jw/pkg/cmds/pkg/CmdVerify.py | 29 +++++++ .../integration/jw_pkg/help/test-expected.txt | 10 +++ test/unit/python/jw/pkg/lib/pm/dpkg/Makefile | 7 ++ test/unit/python/jw/pkg/lib/pm/dpkg/test.py | 81 +++++++++++++++++++ test/unit/python/jw/pkg/lib/pm/rpm/test.py | 47 ++++++++++- 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 src/python/jw/pkg/cmds/pkg/CmdVerify.py create mode 100644 test/unit/python/jw/pkg/lib/pm/dpkg/Makefile create mode 100644 test/unit/python/jw/pkg/lib/pm/dpkg/test.py diff --git a/src/python/jw/pkg/cmds/pkg/CmdVerify.py b/src/python/jw/pkg/cmds/pkg/CmdVerify.py new file mode 100644 index 00000000..9de6afaa --- /dev/null +++ b/src/python/jw/pkg/cmds/pkg/CmdVerify.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, override + +from .Cmd import Cmd, Parent + +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + +class CmdVerify(Cmd): # export + + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'verify', + help = ( + 'Verify the integrity of installed packages: report files' + ' modified or deleted outside of the package manager' + ) + ) + + @override + def add_arguments(self, parser: ArgumentParser) -> None: + super().add_arguments(parser) + + @override + async def _run(self, args: Namespace) -> None: + for file in await self.distro.verify(): + print(file) diff --git a/test/integration/jw_pkg/help/test-expected.txt b/test/integration/jw_pkg/help/test-expected.txt index b0363da8..0b0c0b39 100644 --- a/test/integration/jw_pkg/help/test-expected.txt +++ b/test/integration/jw_pkg/help/test-expected.txt @@ -66,6 +66,8 @@ Available subcommands of packages: reboot-required Check whether the machine needs rebooting refresh Refresh the distribution's notion of available packages select Select packages by filter + verify Verify the integrity of installed packages: report files + modified or deleted outside of the package manager ============= Running: jw-pkg.py -t ../../../.. --log-level info packages delete --help usage: jw-pkg.py packages delete [-h] [names ...] @@ -140,6 +142,14 @@ Select packages by filter positional arguments: filter Package filter string +options: + -h, --help show this help message and exit +============= Running: jw-pkg.py -t ../../../.. --log-level info packages verify --help +usage: jw-pkg.py packages verify [-h] + +Verify the integrity of installed packages: report files modified or deleted +outside of the package manager + options: -h, --help show this help message and exit ============= Running: jw-pkg.py -t ../../../.. --log-level info platform --help diff --git a/test/unit/python/jw/pkg/lib/pm/dpkg/Makefile b/test/unit/python/jw/pkg/lib/pm/dpkg/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/dpkg/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/pm/dpkg/test.py b/test/unit/python/jw/pkg/lib/pm/dpkg/test.py new file mode 100644 index 00000000..66f06df6 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/pm/dpkg/test.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.pm.dpkg import list_files +from jw.pkg.lib.Result import Result + +class CannedLocal(Local): + + def __init__(self, stdout: str) -> None: + super().__init__('local') + self.stdout = stdout + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + return Result(stdout = self.stdout.encode(), status = 0) + +# -- One listing over several packages +cl = CannedLocal('/usr/bin/b\n/etc/b/b.conf.jw-tmpl\n') +assert asyncio.run(list_files(['a', 'b'], ec = cl)) == \ + ['/usr/bin/b', '/etc/b/b.conf.jw-tmpl'] +assert cl.cmds[-1] == [ + '/usr/bin/dpkg', + '-L', + 'a', + 'b', +] + +# -- The unfiltered selection is the standard interface, two +# queries: select the installed packages, then list their files +# in one command; the list output carries a package-name header +# and a blank line per package, both dropped +class CannedTwoStage(Local): + + def __init__(self, query: str, listing: str) -> None: + super().__init__('local') + self.query = query + self.listing = listing + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + out = self.query if '-W' in cmd else self.listing + return Result(stdout = out.encode(), status = 0) + +cl2 = CannedTwoStage('a\nb\n', 'a:\n/usr/bin/a\n\nb:\n/etc/b/b.conf.jw-tmpl\n') +assert asyncio.run(list_files(None, ec = cl2)) == \ + ['/usr/bin/a', '/etc/b/b.conf.jw-tmpl'] +assert cl2.cmds == [ + ['/usr/bin/dpkg-query', '-W', '-f=${?installed:binary:Package}\n'], + ['/usr/bin/dpkg-query', '-L', 'a', 'b'], +] + +# -- An empty selection lists nothing and runs no query +cl3 = CannedTwoStage('', '') +assert asyncio.run(list_files([], ec = cl3)) == [] +assert cl3.cmds == [] diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/test.py b/test/unit/python/jw/pkg/lib/pm/rpm/test.py index c743d13b..e42ace0a 100644 --- a/test/unit/python/jw/pkg/lib/pm/rpm/test.py +++ b/test/unit/python/jw/pkg/lib/pm/rpm/test.py @@ -1,7 +1,13 @@ from __future__ import annotations +import asyncio + +from typing import override + +from jw.pkg.lib.ec.Local import Local from jw.pkg.lib.FlaggedFile import FlaggedFile -from jw.pkg.lib.pm.rpm import _parse_verify_output +from jw.pkg.lib.pm.rpm import _parse_verify_output, list_files +from jw.pkg.lib.Result import Result # -- _parse_verify_output -- @@ -56,3 +62,42 @@ f = FlaggedFile('pkg', '/etc/foo', 'SM5....T.') assert repr(f) == 'pkg /etc/foo SM5....T.' print('All rpm pm tests passed') + +# -- list_files -- + +class CannedLocal(Local): + + def __init__(self, stdout: str) -> None: + super().__init__('local') + self.stdout = stdout + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + return Result(stdout = self.stdout.encode(), status = 0) + +# -- One query over several packages, the output split into lines +cl = CannedLocal('/usr/bin/a\n/etc/a/a.conf.jw-tmpl\n') +assert asyncio.run(list_files(['a', 'b'], ec = cl)) == \ + ['/usr/bin/a', '/etc/a/a.conf.jw-tmpl'] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-ql', 'a', 'b'] + +# -- The unfiltered selection is one query without package names +cl = CannedLocal('/usr/bin/a\n/etc/a/a.conf.jw-tmpl\n') +assert asyncio.run(list_files(None, ec = cl)) == ['/usr/bin/a', '/etc/a/a.conf.jw-tmpl'] +assert cl.cmds[-1] == ['/usr/bin/rpm', '-qal'] + +# -- An empty selection lists nothing and runs no query +cl = CannedLocal('') +assert asyncio.run(list_files([], ec = cl)) == [] +assert cl.cmds == [] -- 2.56.0