From afbdc2e4b07fe361c875698d37d45969a7e6a8bc Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 27 Oct 2026 14:59:14 +0100 Subject: [PATCH 1/4] lib.Package: Add and fill version field Add a version field to the Package class and make query_packages() fill it. It's very much distribution and package dependent what shape that version has, so from Package's perspective, it's just a str-typed member. Further improvements and fixes: - Make query_packages() accept throw, defaulting to the previous behaviour, so a caller that wants to be prepared for a missing package / an empty result can pass throw = False. - Make dpkg's _run() and run_dpkg_query() forward ec and the remaining keyword arguments on the sudo path, which dropped them before. Add unit tests for the file type tree and its StatResult predicates. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/Package.py | 11 ++++- src/python/jw/pkg/lib/pm/dpkg.py | 30 ++++++------ src/python/jw/pkg/lib/pm/rpm.py | 18 ++++---- test/unit/python/jw/pkg/lib/Distro/test.py | 2 +- test/unit/python/jw/pkg/lib/Package/test.py | 21 +++++---- .../python/jw/pkg/lib/file_types/Makefile | 8 ++++ .../unit/python/jw/pkg/lib/file_types/test.py | 46 +++++++++++++++++++ 7 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 test/unit/python/jw/pkg/lib/file_types/Makefile create mode 100644 test/unit/python/jw/pkg/lib/file_types/test.py diff --git a/src/python/jw/pkg/lib/Package.py b/src/python/jw/pkg/lib/Package.py index 6fd19e24..7d883a8c 100644 --- a/src/python/jw/pkg/lib/Package.py +++ b/src/python/jw/pkg/lib/Package.py @@ -8,19 +8,25 @@ meta_tags = [ 'packager', 'url', 'maintainer', + 'version', ] class Package: + """A package as reported by a package manager. The version is the + full version string as the PM reports it, for instance + version-release for rpm and the single version for dpkg.""" + name: str vendor: str | None = None packager: str | None = None url: str | None = None maintainer: str | None = None + version: str | None = None @classmethod def parse_spec_str(cls, spec: str, delimiter: str = '|') -> 'Package': tags = spec.split(delimiter) - if len(tags) != 5: + if len(tags) != len(meta_tags): raise ValueError(f'Invalid package spec string "{spec}"') return cls( name = tags[0], @@ -28,6 +34,7 @@ class Package: packager = tags[2], url = tags[3], maintainer = tags[4], + version = tags[5], ) @classmethod @@ -51,12 +58,14 @@ class Package: packager: str | None = None, url: str | None = None, maintainer: str | None = None, + version: str | None = None, ): self.name = name self.vendor = vendor self.packager = packager self.url = url self.maintainer = maintainer + self.version = version @override def __repr__(self) -> str: diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index 339ddb48..b2ad671e 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -17,11 +17,12 @@ def meta_map() -> dict[str, str]: if _meta_map is None: _meta_map = Package.order_tags( { - 'name': 'binary:Package', + 'name': '${binary:Package}', 'vendor': None, # deb doesn't have vendor field 'packager': None, # -- packager -- - 'url': 'Homepage', - 'maintainer': 'Maintainer', + 'url': '${Homepage}', + 'maintainer': '${Maintainer}', + 'version': '${Version}', } ) return _meta_map @@ -33,7 +34,7 @@ async def _run( **kwargs: Any, ) -> str: return ( - await run_sudo(cmd) if sudo else await + await run_sudo(cmd, ec = ec, **kwargs) if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive, **kwargs) ).stdout_str @@ -50,20 +51,23 @@ async def run_dpkg( # export async def run_dpkg_query( # export args: list[str], sudo: bool = False, - ec: ExecContext | None = None + ec: ExecContext | None = None, + **kwargs: Any, ) -> str: cmd = ['/usr/bin/dpkg-query'] cmd.extend(args) - return await _run(cmd, sudo, ec) + return await _run(cmd, sudo, ec, **kwargs) -async def query_packages(names: Iterable[str] = [], - ec: ExecContext | None = None) -> Iterable[Package]: - fmt_str = ( - '|'.join([(f'${{{tag}}}' if tag else '') - for tag in meta_map().values()]) + r'\n' - ) +async def query_packages( + names: Iterable[str] = [], + ec: ExecContext | None = None, + throw: bool = True +) -> Iterable[Package]: + fmt_str = '|'.join(tag or '' for tag in meta_map().values()) + r'\n' # dpkg-query -W -f='${binary:Package}|${Maintainer}| ... \n' - specs = await run_dpkg_query(['-W', '-f=' + fmt_str, *names], sudo = False, ec = ec) + specs = await run_dpkg_query( + ['-W', '-f=' + fmt_str, *names], sudo = False, throw = throw, ec = ec + ) return cast('Iterable[Package]', Package.parse_specs_str(specs)) async def list_files( diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index eef7e794..cb63e26a 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -19,11 +19,13 @@ def meta_map() -> dict[str, str]: if _meta_map is None: _meta_map = Package.order_tags( { - 'name': 'Name', - 'vendor': 'Vendor', - 'packager': 'Packager', - 'url': 'URL', + 'name': '%{Name}', + 'vendor': '%{Vendor}', + 'packager': '%{Packager}', + 'url': '%{URL}', 'maintainer': None, # RPM doesn't have a maintainer field + # -- the full version identity as one column + 'version': '%{VERSION}-%{RELEASE}', } ) return _meta_map @@ -46,17 +48,15 @@ async def run_rpm( # export async def query_packages( # export names: Collection[str] = [], ec: ExecContext | None = None, + throw: bool = True, ) -> Iterable[Package]: - fmt_str = ( - '|'.join([(f'%{{{tag}}}' if tag else '') - for tag in meta_map().values()]) + r'\n' - ) + fmt_str = '|'.join(tag or '' for tag in meta_map().values()) + r'\n' opts = ['-q', '--queryformat', fmt_str] if not names: opts.append('-a') specs = await run_rpm( [*opts, *names], - throw = True, + throw = throw, sudo = False, mode = InputMode.NonInteractive, ec = ec diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index f1bd8e4d..37902e22 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -256,7 +256,7 @@ class CannedRpmLocal(Local): elif '-ql' in cmd: out = '/etc/a/a.conf.jw-tmpl\n/etc/b/b.conf.jw-tmpl\n' else: - out = 'a|v|p|https://jw.dev/a|m\nb|v|p|https://jw.dev/b|m\n' + out = 'a|v|p|https://jw.dev/a|m|1.0.0-1\nb|v|p|https://jw.dev/b|m|1.0.0-1\n' return Result(stdout = out.encode(), status = 0) cl = CannedRpmLocal() diff --git a/test/unit/python/jw/pkg/lib/Package/test.py b/test/unit/python/jw/pkg/lib/Package/test.py index 09269a91..52979cfb 100644 --- a/test/unit/python/jw/pkg/lib/Package/test.py +++ b/test/unit/python/jw/pkg/lib/Package/test.py @@ -3,29 +3,32 @@ from jw.pkg.lib.Package import Package, meta_tags # -- parse_spec_str -- p = Package.parse_spec_str( - 'jw-core|jw|Jan|https://example.com/jw-core|Jan ' + 'jw-core|jw|Jan|https://example.com/jw-core|Jan |1.0.0-1' ) assert p.name == 'jw-core' assert p.vendor == 'jw' assert p.packager == 'Jan' assert p.url == 'https://example.com/jw-core' assert p.maintainer == 'Jan ' +assert p.version == '1.0.0-1' # An empty field is preserved as an empty string -p = Package.parse_spec_str('name||||') +p = Package.parse_spec_str('name|||||') assert p.name == 'name' assert p.vendor == '' assert p.packager == '' assert p.url == '' assert p.maintainer == '' +assert p.version == '' # A custom delimiter works -p = Package.parse_spec_str('a,b,c,d,e', delimiter = ',') +p = Package.parse_spec_str('a,b,c,d,e,f', delimiter = ',') assert p.name == 'a' assert p.maintainer == 'e' +assert p.version == 'f' # Wrong field counts raise -for spec in ('a', 'a|b', 'a|b|c|d', 'a|b|c|d|e|f'): +for spec in ('a', 'a|b', 'a|b|c', 'a|b|c|d|e', 'a|b|c|d|e|f|g'): try: Package.parse_spec_str(spec) assert False, f'Should have raised for "{spec}"' @@ -35,8 +38,8 @@ for spec in ('a', 'a|b', 'a|b|c|d', 'a|b|c|d|e|f'): # -- parse_specs_str -- specs = ( - 'jw-core|jw|Jan|https://example.com/jw-core|Jan\n' - 'jw-base|jw|Jan|https://example.com/jw-base|Jan' + 'jw-core|jw|Jan|https://example.com/jw-core|Jan|1.0.0-1\n' + 'jw-base|jw|Jan|https://example.com/jw-base|Jan|2.0.0-1' ) packages = Package.parse_specs_str(specs) assert [p.name for p in packages] == ['jw-core', 'jw-base'] @@ -64,17 +67,19 @@ assert ordered == { 'packager': '', 'url': 'https://example.com/jw-core', 'maintainer': 'Jan', + 'version': '', } # -- __repr__ -- -p = Package.parse_spec_str('jw-core|jw|Jan|url|Jan') +p = Package.parse_spec_str('jw-core|jw|Jan|url|Jan|1.0.0-1') assert repr(p) == ( 'name : jw-core\n' 'vendor : jw\n' 'packager : Jan\n' 'url : url\n' - 'maintainer : Jan' + 'maintainer : Jan\n' + 'version : 1.0.0-1' ) print('All Package tests passed') diff --git a/test/unit/python/jw/pkg/lib/file_types/Makefile b/test/unit/python/jw/pkg/lib/file_types/Makefile new file mode 100644 index 00000000..3d3138c5 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/file_types/Makefile @@ -0,0 +1,8 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: + +test: run diff --git a/test/unit/python/jw/pkg/lib/file_types/test.py b/test/unit/python/jw/pkg/lib/file_types/test.py new file mode 100644 index 00000000..7b77d0bc --- /dev/null +++ b/test/unit/python/jw/pkg/lib/file_types/test.py @@ -0,0 +1,46 @@ +import stat + +from jw.pkg.lib.base import StatResult +from jw.pkg.lib.file_types import ( + FT_COMPOUND, + FT_DIR, + FT_REG, + FT_TOP, + FileType, + file_types, +) + +# -- The compound hangs under the regular files: it is a file whose +# content is gettable, and only its position in the tree makes +# it listable, too +assert FT_COMPOUND.parent is FT_REG +assert FT_COMPOUND.is_subtype(FT_REG) +assert not FT_COMPOUND.is_subtype(FT_DIR) +assert FT_DIR.is_subtype(FT_TOP) +assert FT_REG.is_subtype(FT_TOP) + +def __stat(ftype: FileType) -> StatResult: + return StatResult(0o400, 'u', 'g', 0, 0.0, 0.0, 0.0, ftype = ftype) + +# -- The predicates answer from the type: a directory lists, a file +# is gettable, a compound is both +assert __stat(FT_DIR).is_dir and not __stat(FT_DIR).is_gettable +assert not __stat(FT_REG).is_dir and __stat(FT_REG).is_gettable +st = __stat(FT_COMPOUND) +assert st.is_dir and st.is_gettable + +# -- A mode-resolved stat answers the same way: the host's file +# types are the tree's file types +st = StatResult(stat.S_IFREG | 0o400, 'u', 'g', 0, 0.0, 0.0, 0.0) +assert st.ftype is FT_REG and st.is_gettable and not st.is_dir +st = StatResult(stat.S_IFDIR | 0o755, 'u', 'g', 0, 0.0, 0.0, 0.0) +assert st.ftype is FT_DIR and st.is_dir and not st.is_gettable + +# -- A mode the tree cannot resolve is an error +try: + file_types.from_mode(stat.S_IFLNK | 0o777) + raise AssertionError('expected NotImplementedError') +except NotImplementedError: + pass + +print('All file_types tests passed') -- 2.56.0 From 36978c55e2d8c32c625040709b5522520b158831 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 27 Oct 2026 14:51:56 +0100 Subject: [PATCH 2/4] lib.pm.*.install_packages(): Add functions Currently, jw-pkg install is sometimes left to the high-level package manager, which is allowed to install dependencies along, and that's sometimes unwanted, but there's no backend in the low-level package manager module lib.pm to allow specifying that. This commit adds it: lib.pm.*.install_packages(). The functions install a list of local package files using the low-level package managers rpm / dpkg, bypassing the high-level package managers yum / apt / zypper. pm.rpm runs rpm -U and pm.dpkg runs dpkg -i. Every file is classified against the installed state of its package, queried by name: a file for a package that is not installed is installed fresh, a file with another version installed is upgraded, and a file whose version is already installed is reinstalled when force is set and skipped otherwise. On rpm, plain -U refuses a package that is already installed, and --reinstall aborts the transaction when a file of the incoming package differs from the installed one, so the upgrades run before the reinstalls and the same-version files need force. With only_update, a file for a package that is not installed raises before anything is installed. The dpkg route is gated on an apt simulation, because dpkg -i leaves a half-configured package when a dependency is missing: the simulation runs first and the install is refused when it would install or remove anything but the given files' own packages. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/pm/dpkg.py | 105 ++++++++ src/python/jw/pkg/lib/pm/rpm.py | 63 +++++ test/unit/python/jw/pkg/lib/pm/dpkg/test.py | 260 +++++++++++++++++++- test/unit/python/jw/pkg/lib/pm/rpm/test.py | 233 +++++++++++++++++- 4 files changed, 659 insertions(+), 2 deletions(-) diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index b2ad671e..9255976f 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -94,6 +94,111 @@ async def list_files( file_list_str = await run_dpkg(['-L', *names], sudo = False, ec = ec) return file_list_str.splitlines() +async def _file_name_version( + path: str, + ec: ExecContext | None = None, +) -> tuple[str, str]: + """Return the (name, version) pair of a local deb file.""" + out = await run_dpkg(['-I', path], sudo = False, throw = True, ec = ec) + name: str | None = None + version: str | None = None + for line in out.splitlines(): + line = line.strip() + if line.startswith('Package: '): + name = line[len('Package: '):].strip() + elif line.startswith('Version: '): + version = line[len('Version: '):].strip() + if name is None or version is None: + raise Exception(f'cannot read the package name and version of {path}') + return name, version + +async def _simulate_install( + files: list[str], + expected: set[str], + ec: ExecContext | None = None, +) -> None: + """Gate a direct dpkg install on an apt simulation. + + dpkg -i leaves a half-configured package when a dependency is + missing, instead of aborting the transaction cleanly, so the + install is simulated first and refused when the simulation + would install or remove anything but the expected packages. + """ + result = await run_sudo( + ['/usr/bin/apt-get', '-s', '--no-install-recommends', 'install', *files], + ec = ec, + cmd_input = InputMode.NonInteractive, + throw = False, + ) + out = result.stdout_str + if result.status != 0: + raise Exception(f'apt simulation of the direct install failed:\n{out}') + inst: list[str] = [] + remv: list[str] = [] + for line in out.splitlines(): + parts = line.split() + if parts and parts[0] == 'Inst': + inst.append(parts[1]) + elif parts and parts[0] == 'Remv': + remv.append(parts[1]) + pulled = [pkg for pkg in inst if pkg not in expected] + if pulled or remv: + raise Exception(f'apt simulation would install {pulled} or remove {remv}') + +async def install_packages( + paths: Iterable[str], + ec: ExecContext | None = None, + force: bool = False, + only_update: bool = False, +) -> None: + """Install local deb files. + + Every install is gated on an apt simulation, which refuses the + transaction when it would install or remove anything but the + given files' own packages; see _simulate_install(). + + Files are classified against the installed state of their + packages: files for packages that are not installed are + installed fresh, files with another version installed are + upgraded, and files whose version is already installed are + reinstalled with -i --force-reinstall when force is set and + skipped otherwise. The upgrades run before the reinstalls. + + With only_update, a file for a package that is not installed + raises before anything is installed. + """ + paths = list(paths) + if not paths: + return + update: list[str] = [] + reinstall: list[str] = [] + not_installed: list[str] = [] + names: dict[str, str] = {} + for path in paths: + name, version = await _file_name_version(path, ec = ec) + names[path] = name + pkgs = list(await query_packages([name], ec = ec, throw = False)) + if not pkgs: + (not_installed if only_update else update).append(path) + continue + if version in {p.version for p in pkgs if p.version}: + reinstall.append(path) + continue + update.append(path) + if not_installed: + raise Exception( + 'only_update: package not installed for ' + + '\n'.join(f' {p}' for p in not_installed) + ) + selected = [*update, *reinstall] if force else update + if not selected: + return + await _simulate_install(selected, {names[path] for path in selected}, ec = ec) + if update: + await run_dpkg(['-i', *update], sudo = True, ec = ec) + if force and reinstall: + await run_dpkg(['-i', '--force-reinstall', *reinstall], sudo = True, ec = ec) + def _parse_verify_output( pkg: str, output: str, diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index cb63e26a..29070878 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -80,6 +80,69 @@ async def list_files( ) return stdout.splitlines() +async def _file_name_version( + path: str, + ec: ExecContext | None = None, +) -> tuple[str, str]: + """Return the (name, version-release) pair of a local rpm file.""" + out = await run_rpm( + ['-qp', '--queryformat', '%{NAME}\t%{VERSION}-%{RELEASE}', path], + throw = True, + sudo = False, + mode = InputMode.NonInteractive, + ec = ec + ) + name, version = out.split('\t', 1) + return name, version + +async def install_packages( + paths: Iterable[str], + ec: ExecContext | None = None, + force: bool = False, + only_update: bool = False, +) -> None: + """Install local rpm files. + + Files are classified against the installed state of their + packages: files for packages that are not installed are + installed fresh, files with another version installed are + upgraded, and files whose version is already installed are + reinstalled with -U --reinstall when force is set and + skipped otherwise, because plain -U refuses a package that + is already installed. The upgrades run before the + reinstalls: --reinstall aborts the whole transaction when a + file of the incoming package differs from the installed + one, which is exactly what an upgrade changes. + + With only_update, a file for a package that is not installed + raises before anything is installed. + """ + paths = list(paths) + if not paths: + return + update: list[str] = [] + reinstall: list[str] = [] + not_installed: list[str] = [] + for path in paths: + name, version = await _file_name_version(path, ec = ec) + pkgs = list(await query_packages([name], ec = ec, throw = False)) + if not pkgs: + (not_installed if only_update else update).append(path) + continue + if version in {p.version for p in pkgs if p.version}: + reinstall.append(path) + continue + update.append(path) + if not_installed: + raise Exception( + 'only_update: package not installed for ' + + '\n'.join(f' {p}' for p in not_installed) + ) + if update: + await run_rpm(['-U', *update], sudo = True, ec = ec) + if force and reinstall: + await run_rpm(['-U', '--reinstall', *reinstall], sudo = True, ec = ec) + def _parse_verify_output( pkg: str, output: str, diff --git a/test/unit/python/jw/pkg/lib/pm/dpkg/test.py b/test/unit/python/jw/pkg/lib/pm/dpkg/test.py index 66f06df6..bb8a9dba 100644 --- a/test/unit/python/jw/pkg/lib/pm/dpkg/test.py +++ b/test/unit/python/jw/pkg/lib/pm/dpkg/test.py @@ -5,7 +5,7 @@ import asyncio from typing import override from jw.pkg.lib.ec.Local import Local -from jw.pkg.lib.pm.dpkg import list_files +from jw.pkg.lib.pm.dpkg import install_packages, list_files from jw.pkg.lib.Result import Result class CannedLocal(Local): @@ -79,3 +79,261 @@ assert cl2.cmds == [ cl3 = CannedTwoStage('', '') assert asyncio.run(list_files([], ec = cl3)) == [] assert cl3.cmds == [] + +# -- install_packages -- + +class InstallLocal(Local): + """A canned execution context that answers dpkg queries and + the apt simulation""" + + def __init__( + self, + files: dict[str, str], + installed: dict[str, str], + simulate: str, + simulate_status: int = 0, + ) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.simulate = simulate + self.simulate_status = simulate_status + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-I' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if cmd[0].endswith('dpkg-query'): + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + if '/usr/bin/apt-get' in cmd: + return Result( + stdout = self.simulate.encode(), status = self.simulate_status + ) + return Result(stdout = b'', status = 0) + +def dpkg_args(cmd: list[str]) -> list[str]: + """The dpkg command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/dpkg'):] + +# -- Without force or only_update, all files go through a single +# plain -i, gated on the apt simulation +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-41'}, + simulate = 'Inst jw-base [1.0.0-42]\n', +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il)) +sim = [cmd for cmd in il.cmds if '/usr/bin/apt-get' in cmd] +assert len(sim) == 1 +args = sim[0][sim[0].index('/usr/bin/apt-get'):] +assert '-s' in args +assert '--no-install-recommends' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.deb' +installs = [dpkg_args(cmd) for cmd in il.cmds if '-i' in cmd] +assert installs == [['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-42.deb']] + +# -- Without force, a file whose version is already installed is +# skipped +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il)) +assert not [cmd for cmd in il.cmds if '-i' in cmd] +assert not [cmd for cmd in il.cmds if '/usr/bin/apt-get' in cmd] + +# -- An empty selection runs no commands +il = InstallLocal(files = {}, installed = {}, simulate = '') +assert asyncio.run(install_packages([], ec = il)) is None +assert il.cmds == [] + +# -- force reinstalls a file whose version is already installed +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True)) +assert dpkg_args(il.cmds[-1]) == [ + '/usr/bin/dpkg', + '-i', + '--force-reinstall', + '/a/jw-base-run-1.0.0-42.deb', +] + +# -- force upgrades a file whose package is installed in another +# version with plain -i +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-41'}, + simulate = 'Inst jw-base [1.0.0-42]\n', +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True)) +assert dpkg_args(il.cmds[-1]) == ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-42.deb'] + +# -- force installs a file whose package is not installed with +# plain -i +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {}, + simulate = 'Inst jw-base [1.0.0-42]\n', +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True)) +assert dpkg_args(il.cmds[-1]) == ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-42.deb'] + +# -- A file that cannot be queried raises before the simulation +# or the install +il = InstallLocal( + files = {}, + installed = {}, + simulate = '', +) +try: + asyncio.run(install_packages(['/a/broken.deb'], ec = il, force = True)) +except Exception as e: + assert 'package name and version' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in il.cmds if '-i' in cmd] +assert not [cmd for cmd in il.cmds if '/usr/bin/apt-get' in cmd] + +# -- only_update raises on files whose package is not installed, +# before simulating or installing anything +il = InstallLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-foo-run-1.0.0-42.deb': 'Package: jw-foo\nVersion: 1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +try: + asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-foo-run-1.0.0-42.deb'], + ec = il, + force = True, + only_update = True, + ) + ) +except Exception as e: + assert 'only_update' in str(e) + assert '/a/jw-foo-run-1.0.0-42.deb' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in il.cmds if '-i' in cmd] +assert not [cmd for cmd in il.cmds if '/usr/bin/apt-get' in cmd] + +# -- only_update installs an update of an installed package +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-41'}, + simulate = 'Inst jw-base [1.0.0-42]\n', +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True, only_update = True + ) +) +assert dpkg_args(il.cmds[-1]) == ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-42.deb'] + +# -- force runs two installs for a mixed selection, the upgrades +# first +il = InstallLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-base-run-1.0.0-41.deb': 'Package: jw-base\nVersion: 1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = 'Inst jw-base [1.0.0-41]\n', +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-base-run-1.0.0-41.deb'], + ec = il, + force = True, + ) +) +installs = [dpkg_args(cmd) for cmd in il.cmds if '-i' in cmd] +assert installs == [ + ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-41.deb'], + [ + '/usr/bin/dpkg', + '-i', + '--force-reinstall', + '/a/jw-base-run-1.0.0-42.deb', + ], +] + +# -- The simulation refuses the install when it would pull a +# dependency +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {}, + simulate = 'Inst jw-base [1.0.0-42]\nInst jw-foo [2.0]\n', +) +try: + asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True) + ) +except Exception as e: + assert 'would install' in str(e) + assert 'jw-foo' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in il.cmds if '-i' in cmd] + +# -- The simulation refuses the install when it would remove a +# package +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {}, + simulate = 'Inst jw-base [1.0.0-42]\nRemv jw-old [1.0]\n', +) +try: + asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True) + ) +except Exception as e: + assert 'remove' in str(e) + assert 'jw-old' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in il.cmds if '-i' in cmd] + +# -- A failed simulation, for instance unmet dependencies, aborts +# with its output +il = InstallLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {}, + simulate = ( + 'The following packages have unmet dependencies:\n' + ' jw-base : Depends: jw-foo but it is not going to be installed\n' + ), + simulate_status = 100, +) +try: + asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.deb'], ec = il, force = True) + ) +except Exception as e: + assert 'unmet dependencies' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in il.cmds if '-i' in cmd] diff --git a/test/unit/python/jw/pkg/lib/pm/rpm/test.py b/test/unit/python/jw/pkg/lib/pm/rpm/test.py index e42ace0a..deed1875 100644 --- a/test/unit/python/jw/pkg/lib/pm/rpm/test.py +++ b/test/unit/python/jw/pkg/lib/pm/rpm/test.py @@ -6,7 +6,7 @@ from typing import override from jw.pkg.lib.ec.Local import Local from jw.pkg.lib.FlaggedFile import FlaggedFile -from jw.pkg.lib.pm.rpm import _parse_verify_output, list_files +from jw.pkg.lib.pm.rpm import _parse_verify_output, install_packages, list_files from jw.pkg.lib.Result import Result # -- _parse_verify_output -- @@ -101,3 +101,234 @@ assert cl.cmds[-1] == ['/usr/bin/rpm', '-qal'] cl = CannedLocal('') assert asyncio.run(list_files([], ec = cl)) == [] assert cl.cmds == [] + +# -- install_packages -- + +def rpm_args(cmd: list[str]) -> list[str]: + """The rpm command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/rpm'):] + +class DispatchLocal(Local): + """A canned execution context that answers rpm queries""" + + def __init__(self, files: dict[str, str], installed: dict[str, str]) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qp' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if '-q' in cmd: + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + return Result(stdout = b'', status = 0) + +# -- Without force, a file whose version is already installed is +# skipped, because plain -U refuses it +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +asyncio.run(install_packages(['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl)) +assert not [cmd for cmd in dl.cmds if '-U' in cmd] + +# -- Without force, fresh and update files go through a single +# plain -U +dl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/b/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-41\n'}, +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/b/jw-foo-run-1.0.0-42.x86_64.rpm'], + ec = dl, + ) +) +installs = [cmd for cmd in dl.cmds if '-U' in cmd] +assert len(installs) == 1 +assert installs[0][0] == '/usr/bin/sudo' +assert rpm_args(installs[0]) == [ + '/usr/bin/rpm', + '-U', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + '/b/jw-foo-run-1.0.0-42.x86_64.rpm', +] + +# -- An empty selection installs nothing and runs no command +dl = DispatchLocal(files = {}, installed = {}) +asyncio.run(install_packages([], ec = dl)) +assert dl.cmds == [] + +# -- With force, a file whose version is already installed is reinstalled +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl, force = True) +) +assert rpm_args(dl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- With force, a file with another version installed is a plain upgrade +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-41\n'}, +) +asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl, force = True) +) +assert rpm_args(dl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- With force, a file for a package that is not installed is a plain +# install +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {}, +) +asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl, force = True) +) +assert rpm_args(dl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- An unqueryable file raises before anything is installed +dl = DispatchLocal(files = {}, installed = {}) +try: + asyncio.run( + install_packages(['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl, force = True) + ) +except Exception: + pass +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in dl.cmds if '-U' in cmd] + +# -- With force, a mixed transaction runs two installs, the upgrades first +dl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-base-run-1.0.0-41.x86_64.rpm': 'jw-base\t1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + ec = dl, + force = True, + ) +) +installs = [rpm_args(cmd) for cmd in dl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + ], +] + +# -- With only_update, a file whose package is not installed raises +# before anything is installed +dl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +try: + asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-foo-run-1.0.0-42.x86_64.rpm'], + ec = dl, + force = True, + only_update = True, + ) + ) +except Exception as e: + assert 'only_update' in str(e) + assert '/a/jw-foo-run-1.0.0-42.x86_64.rpm' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in dl.cmds if '-U' in cmd] + +# -- With only_update, an update of an installed package is installed +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-43.x86_64.rpm': 'jw-base\t1.0.0-43'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-43.x86_64.rpm'], + ec = dl, + force = True, + only_update = True, + ) +) +installs = [rpm_args(cmd) for cmd in dl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-43.x86_64.rpm'], +] + +# -- With only_update and force off, a file whose version is +# already installed is skipped +dl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +asyncio.run( + install_packages( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], ec = dl, only_update = True + ) +) +assert not [cmd for cmd in dl.cmds if '-U' in cmd] + +# -- With only_update, all files absent raises without installing +dl = DispatchLocal( + files = {'/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42'}, + installed = {}, +) +try: + asyncio.run( + install_packages( + ['/a/jw-foo-run-1.0.0-42.x86_64.rpm'], ec = dl, only_update = True + ) + ) +except Exception as e: + assert 'only_update' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in dl.cmds if '-U' in cmd] -- 2.56.0 From c5ad946d6ef1d9b4bbc8efd4bffd301fedb28525 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 27 Oct 2026 14:54:28 +0100 Subject: [PATCH 3/4] lib.distros.Distro.install_*(): Add install_deps The install methods of Distro inconsistently resolve dependencies or don't depending on the descendant's implementation. Pin that more unambigously by adding a new parameter install_deps, defaulting to True, to the following methods of lib.Distro: _install_local_files() _install_urls() _install_urls_and_names() install() The implementations in lib.distros.* make use of the new lib.pm.*.install_packages() method introduced by the previous commit. Add a unit test directory per Distro descendant. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/Distro.py | 42 ++++-- .../jw/pkg/lib/distros/debian/Distro.py | 19 ++- .../jw/pkg/lib/distros/redhat/Distro.py | 19 ++- src/python/jw/pkg/lib/distros/suse/Distro.py | 17 ++- test/unit/python/jw/pkg/lib/Distro/test.py | 19 +++ test/unit/python/jw/pkg/lib/distros/Makefile | 4 + .../python/jw/pkg/lib/distros/debian/Makefile | 7 + .../python/jw/pkg/lib/distros/debian/test.py | 127 +++++++++++++++++ .../python/jw/pkg/lib/distros/redhat/Makefile | 7 + .../python/jw/pkg/lib/distros/redhat/test.py | 134 ++++++++++++++++++ .../python/jw/pkg/lib/distros/suse/Makefile | 7 + .../python/jw/pkg/lib/distros/suse/test.py | 134 ++++++++++++++++++ 12 files changed, 522 insertions(+), 14 deletions(-) create mode 100644 test/unit/python/jw/pkg/lib/distros/Makefile create mode 100644 test/unit/python/jw/pkg/lib/distros/debian/Makefile create mode 100644 test/unit/python/jw/pkg/lib/distros/debian/test.py create mode 100644 test/unit/python/jw/pkg/lib/distros/redhat/Makefile create mode 100644 test/unit/python/jw/pkg/lib/distros/redhat/test.py create mode 100644 test/unit/python/jw/pkg/lib/distros/suse/Makefile create mode 100644 test/unit/python/jw/pkg/lib/distros/suse/test.py diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index b4dbd5bf..a34f4246 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -401,17 +401,27 @@ class Distro(abc.ABC): async def _install(self, names: Iterable[str], only_update: bool) -> None: pass - # Default implementation assumes package manager can handle local files. - # Not true for all distros. Override if Distro knows better. + # The default route passes local files to the package manager, which + # resolves their dependencies. The direct route (install_deps = False) + # is not available here; override in descendants that offer it. async def _install_local_files( - self, paths: Iterable[str], only_update: bool + self, paths: Iterable[str], only_update: bool, install_deps: bool ) -> None: + assert not (only_update and install_deps) + if not install_deps: + raise NotImplementedError( + f'Installing local files without dependency resolution ' + f'is not supported on {self.os}' + ) await self._install(paths, only_update = only_update) # Download first and then install. Override if Distro knows better. - async def _install_urls(self, urls: Iterable[str], only_update: bool) -> None: + async def _install_urls( + self, urls: Iterable[str], only_update: bool, install_deps: bool + ) -> None: from .util import copy + assert not (only_update and install_deps) tmp: str | None = None try: tmp = await self.__exec_context.mktemp( @@ -424,7 +434,9 @@ class Distro(abc.ABC): raise paths if isinstance(paths, str): paths = [paths] - await self._install_local_files(paths, only_update = only_update) + await self._install_local_files( + paths, only_update = only_update, install_deps = install_deps + ) finally: if tmp is not None: await self.__exec_context.erase(tmp) @@ -434,8 +446,10 @@ class Distro(abc.ABC): # - Pass names to package manager # Override if Distro knows better. async def _install_urls_and_names( - self, packages: Iterable[str], only_update: bool + self, packages: Iterable[str], only_update: bool, install_deps: bool ) -> None: + + assert not (only_update and install_deps) urls: list[str] = [] names: list[str] = [] for package in packages: @@ -447,15 +461,25 @@ class Distro(abc.ABC): continue names.append(package) if urls: - await self._install_urls(urls, only_update = only_update) + await self._install_urls( + urls, only_update = only_update, install_deps = install_deps + ) if names: await self._install(names, only_update = only_update) - async def install(self, names: Collection[str], only_update: bool = False) -> None: + async def install( + self, + names: Collection[str], + only_update: bool, + install_deps: bool = True, + ) -> None: + assert not (only_update and install_deps) if not names: log(WARNING, 'No packages specified for installation') return - await self._install_urls_and_names(names, only_update = only_update) + await self._install_urls_and_names( + names, only_update = only_update, install_deps = install_deps + ) # -- delete diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index 33a0e203..ac719267 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -6,7 +6,13 @@ from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base from ...log import NOTICE, log -from ...pm.dpkg import list_files, query_packages, run_dpkg, verify_packages +from ...pm.dpkg import ( + install_packages, + list_files, + query_packages, + run_dpkg, + verify_packages, +) if TYPE_CHECKING: from typing import Iterable @@ -81,6 +87,17 @@ class Distro(Base): args.extend(names) await self.apt_get(args) + @override + async def _install_local_files( + self, paths: Iterable[str], only_update: bool, install_deps: bool + ) -> None: + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) + @override async def _delete(self, names: Iterable[str]) -> None: await self.dpkg(['-P', *names], sudo = True) diff --git a/src/python/jw/pkg/lib/distros/redhat/Distro.py b/src/python/jw/pkg/lib/distros/redhat/Distro.py index fbf1f37d..b365c3ad 100644 --- a/src/python/jw/pkg/lib/distros/redhat/Distro.py +++ b/src/python/jw/pkg/lib/distros/redhat/Distro.py @@ -3,7 +3,13 @@ from __future__ import annotations from typing import TYPE_CHECKING, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages +from ...pm.rpm import ( + install_packages, + list_files, + query_packages, + run_rpm, + verify_packages, +) if TYPE_CHECKING: from typing import Any, Collection, Iterable @@ -66,6 +72,17 @@ class Distro(Base): args.extend(names) await self.yum(args) + @override + async def _install_local_files( + self, paths: Iterable[str], only_update: bool, install_deps: bool + ) -> None: + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) + @override async def _delete(self, names: Iterable[str]) -> None: await self.rpm(['-e', *names], sudo = True) diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index b23bb278..b143d839 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -3,7 +3,13 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages +from ...pm.rpm import ( + install_packages, + list_files, + query_packages, + run_rpm, + verify_packages, +) if TYPE_CHECKING: from collections.abc import Collection @@ -87,9 +93,14 @@ class Distro(Base): @override async def _install_local_files( - self, paths: Iterable[str], only_update: bool + self, paths: Iterable[str], only_update: bool, install_deps: bool ) -> None: - await self.rpm(['-U', '--reinstall', *paths]) + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) @override async def _delete(self, names: Iterable[str]) -> None: diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index 37902e22..1725d5de 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -286,4 +286,23 @@ sde = SuseDistro( assert asyncio.run(sde.pkg_files()) == [] assert '--queryformat' in cl.cmds[-1] +# -- _install_local_files -- + +# -- With dependency resolution, the files go to the package manager; +# DummyDistro._install() is a no-op +asyncio.run( + d._install_local_files(['/a/jw-base-run-1.0.0-42.rpm'], False, install_deps = True) +) + +# -- Without, the base implementation raises: there is no direct route +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.rpm'], False, install_deps = False + ) + ) + assert False, 'Should have raised' +except NotImplementedError: + pass + print('All Distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/Makefile b/test/unit/python/jw/pkg/lib/distros/Makefile new file mode 100644 index 00000000..7b64e78c --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/jw/pkg/lib/distros/debian/Makefile b/test/unit/python/jw/pkg/lib/distros/debian/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/debian/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/debian/test.py b/test/unit/python/jw/pkg/lib/distros/debian/test.py new file mode 100644 index 00000000..f5370e8a --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/debian/test.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.debian.Distro import Distro as DebianDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers dpkg queries and + the apt simulation""" + + def __init__( + self, + files: dict[str, str], + installed: dict[str, str], + simulate: str, + ) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.simulate = simulate + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-I' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if cmd[0].endswith('dpkg-query'): + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + if '/usr/bin/apt-get' in cmd and '-s' in cmd: + return Result(stdout = self.simulate.encode(), status = 0) + return Result(stdout = b'', status = 0) + +def dpkg_args(cmd: list[str]) -> list[str]: + """The dpkg command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/dpkg'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-base-run-1.0.0-41.deb': 'Package: jw-base\nVersion: 1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = 'Inst jw-base [1.0.0-41]\n', +) +d = DebianDistro(cl, id = 'debian') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-base-run-1.0.0-41.deb'], + False, + install_deps = False, + ) +) +installs = [dpkg_args(cmd) for cmd in cl.cmds if '-i' in cmd] +assert installs == [ + ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-41.deb'], + [ + '/usr/bin/dpkg', + '-i', + '--force-reinstall', + '/a/jw-base-run-1.0.0-42.deb', + ], +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-foo-run-1.0.0-42.deb': 'Package: jw-foo\nVersion: 1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +d = DebianDistro(cl, id = 'debian') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-foo-run-1.0.0-42.deb'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.deb' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-i' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +d = DebianDistro(cl, id = 'debian') +asyncio.run( + d._install_local_files(['/a/jw-base-run-1.0.0-42.deb'], False, install_deps = True) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/apt-get' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/apt-get'):] +assert 'install' in args +assert '--no-install-recommends' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.deb' +assert not [cmd for cmd in cl.cmds if '-i' in cmd] + +print('All debian distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/redhat/Makefile b/test/unit/python/jw/pkg/lib/distros/redhat/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/redhat/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/redhat/test.py b/test/unit/python/jw/pkg/lib/distros/redhat/test.py new file mode 100644 index 00000000..121fcb96 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/redhat/test.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.redhat.Distro import Distro as RedhatDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers rpm queries""" + + def __init__(self, files: dict[str, str], installed: dict[str, str]) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qp' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if '-q' in cmd: + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + return Result(stdout = b'', status = 0) + +def rpm_args(cmd: list[str]) -> list[str]: + """The rpm command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/rpm'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-base-run-1.0.0-41.x86_64.rpm': 'jw-base\t1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + False, + install_deps = False, + ) +) +installs = [rpm_args(cmd) for cmd in cl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + ], +] + +# -- A single file whose version is already installed is reinstalled +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = False + ) +) +assert rpm_args(cl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-foo-run-1.0.0-42.x86_64.rpm'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.x86_64.rpm' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = True + ) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/yum' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/yum'):] +assert 'install' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.x86_64.rpm' +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +print('All redhat distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/suse/Makefile b/test/unit/python/jw/pkg/lib/distros/suse/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/suse/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/suse/test.py b/test/unit/python/jw/pkg/lib/distros/suse/test.py new file mode 100644 index 00000000..ca95cc5f --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/suse/test.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers rpm queries""" + + def __init__(self, files: dict[str, str], installed: dict[str, str]) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qp' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if '-q' in cmd: + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + return Result(stdout = b'', status = 0) + +def rpm_args(cmd: list[str]) -> list[str]: + """The rpm command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/rpm'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-base-run-1.0.0-41.x86_64.rpm': 'jw-base\t1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + False, + install_deps = False, + ) +) +installs = [rpm_args(cmd) for cmd in cl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + ], +] + +# -- A single file whose version is already installed is reinstalled +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = False + ) +) +assert rpm_args(cl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-foo-run-1.0.0-42.x86_64.rpm'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.x86_64.rpm' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = True + ) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/zypper' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/zypper'):] +assert 'install' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.x86_64.rpm' +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +print('All suse distro tests passed') -- 2.56.0 From afd3a8c40b3cac359c5bc69af0225b7aa61a4f21 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 27 Oct 2026 14:56:40 +0100 Subject: [PATCH 4/4] cmds.pkg: Add --exact to jw-pkg packages install Add an additional option -x, which inhibits dependency resolution during installation of packages. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 --- src/python/jw/pkg/cmds/pkg/CmdInstall.py | 14 +++++++++++++- test/integration/jw_pkg/help/test-expected.txt | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/python/jw/pkg/cmds/pkg/CmdInstall.py b/src/python/jw/pkg/cmds/pkg/CmdInstall.py index 066b8b8e..a041652b 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdInstall.py +++ b/src/python/jw/pkg/cmds/pkg/CmdInstall.py @@ -26,6 +26,14 @@ class CmdInstall(Cmd): # export action = 'store_true', help = "Only update the listed packages, don't install them", ) + parser.add_argument( + '-x', + '--exact', + default = False, + action = 'store_true', + help = "Install local files directly, without resolving " + "dependencies", + ) parser.add_argument( '-F', '--fixed-strings', @@ -38,4 +46,8 @@ class CmdInstall(Cmd): # export names = ( args.names if args.fixed_strings else self.distro.expand_macros(args.names) ) - return await self.distro.install(names, only_update = args.only_update) + return await self.distro.install( + names, + only_update = args.only_update, + install_deps = not (args.only_update or args.exact), + ) diff --git a/test/integration/jw_pkg/help/test-expected.txt b/test/integration/jw_pkg/help/test-expected.txt index 9c2e5f12..fe76fc8d 100644 --- a/test/integration/jw_pkg/help/test-expected.txt +++ b/test/integration/jw_pkg/help/test-expected.txt @@ -88,7 +88,7 @@ options: --download-only Only download packages from the repos, don't install them, yet (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info packages install --help -usage: jw-pkg.py packages install [-h] [--only-update] [-F] [names ...] +usage: jw-pkg.py packages install [-h] [--only-update] [-x] [-F] [names ...] Install the distribution's notion of available packages @@ -99,6 +99,8 @@ options: -h, --help show this help message and exit --only-update Only update the listed packages, don't install them (default: False) + -x, --exact Install local files directly, without resolving + dependencies (default: False) -F, --fixed-strings Don't expand macros in (default: False) ============= Running: jw-pkg.py -t ../../../.. --log-level info packages ls --help usage: jw-pkg.py packages ls [-h] [names ...] -- 2.56.0