diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index b4dbd5bf..a34f4246 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -401,17 +401,27 @@ class Distro(abc.ABC): async def _install(self, names: Iterable[str], only_update: bool) -> None: pass - # Default implementation assumes package manager can handle local files. - # Not true for all distros. Override if Distro knows better. + # The default route passes local files to the package manager, which + # resolves their dependencies. The direct route (install_deps = False) + # is not available here; override in descendants that offer it. async def _install_local_files( - self, paths: Iterable[str], only_update: bool + self, paths: Iterable[str], only_update: bool, install_deps: bool ) -> None: + assert not (only_update and install_deps) + if not install_deps: + raise NotImplementedError( + f'Installing local files without dependency resolution ' + f'is not supported on {self.os}' + ) await self._install(paths, only_update = only_update) # Download first and then install. Override if Distro knows better. - async def _install_urls(self, urls: Iterable[str], only_update: bool) -> None: + async def _install_urls( + self, urls: Iterable[str], only_update: bool, install_deps: bool + ) -> None: from .util import copy + assert not (only_update and install_deps) tmp: str | None = None try: tmp = await self.__exec_context.mktemp( @@ -424,7 +434,9 @@ class Distro(abc.ABC): raise paths if isinstance(paths, str): paths = [paths] - await self._install_local_files(paths, only_update = only_update) + await self._install_local_files( + paths, only_update = only_update, install_deps = install_deps + ) finally: if tmp is not None: await self.__exec_context.erase(tmp) @@ -434,8 +446,10 @@ class Distro(abc.ABC): # - Pass names to package manager # Override if Distro knows better. async def _install_urls_and_names( - self, packages: Iterable[str], only_update: bool + self, packages: Iterable[str], only_update: bool, install_deps: bool ) -> None: + + assert not (only_update and install_deps) urls: list[str] = [] names: list[str] = [] for package in packages: @@ -447,15 +461,25 @@ class Distro(abc.ABC): continue names.append(package) if urls: - await self._install_urls(urls, only_update = only_update) + await self._install_urls( + urls, only_update = only_update, install_deps = install_deps + ) if names: await self._install(names, only_update = only_update) - async def install(self, names: Collection[str], only_update: bool = False) -> None: + async def install( + self, + names: Collection[str], + only_update: bool, + install_deps: bool = True, + ) -> None: + assert not (only_update and install_deps) if not names: log(WARNING, 'No packages specified for installation') return - await self._install_urls_and_names(names, only_update = only_update) + await self._install_urls_and_names( + names, only_update = only_update, install_deps = install_deps + ) # -- delete diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index 33a0e203..ac719267 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -6,7 +6,13 @@ from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base from ...log import NOTICE, log -from ...pm.dpkg import list_files, query_packages, run_dpkg, verify_packages +from ...pm.dpkg import ( + install_packages, + list_files, + query_packages, + run_dpkg, + verify_packages, +) if TYPE_CHECKING: from typing import Iterable @@ -81,6 +87,17 @@ class Distro(Base): args.extend(names) await self.apt_get(args) + @override + async def _install_local_files( + self, paths: Iterable[str], only_update: bool, install_deps: bool + ) -> None: + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) + @override async def _delete(self, names: Iterable[str]) -> None: await self.dpkg(['-P', *names], sudo = True) diff --git a/src/python/jw/pkg/lib/distros/redhat/Distro.py b/src/python/jw/pkg/lib/distros/redhat/Distro.py index fbf1f37d..b365c3ad 100644 --- a/src/python/jw/pkg/lib/distros/redhat/Distro.py +++ b/src/python/jw/pkg/lib/distros/redhat/Distro.py @@ -3,7 +3,13 @@ from __future__ import annotations from typing import TYPE_CHECKING, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages +from ...pm.rpm import ( + install_packages, + list_files, + query_packages, + run_rpm, + verify_packages, +) if TYPE_CHECKING: from typing import Any, Collection, Iterable @@ -66,6 +72,17 @@ class Distro(Base): args.extend(names) await self.yum(args) + @override + async def _install_local_files( + self, paths: Iterable[str], only_update: bool, install_deps: bool + ) -> None: + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) + @override async def _delete(self, names: Iterable[str]) -> None: await self.rpm(['-e', *names], sudo = True) diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index b23bb278..b143d839 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -3,7 +3,13 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, override from ...Distro import Distro as Base -from ...pm.rpm import list_files, query_packages, run_rpm, verify_packages +from ...pm.rpm import ( + install_packages, + list_files, + query_packages, + run_rpm, + verify_packages, +) if TYPE_CHECKING: from collections.abc import Collection @@ -87,9 +93,14 @@ class Distro(Base): @override async def _install_local_files( - self, paths: Iterable[str], only_update: bool + self, paths: Iterable[str], only_update: bool, install_deps: bool ) -> None: - await self.rpm(['-U', '--reinstall', *paths]) + if install_deps: + await self._install(paths, only_update = False) + return + await install_packages( + paths, ec = self.ctx, force = True, only_update = only_update + ) @override async def _delete(self, names: Iterable[str]) -> None: diff --git a/test/unit/python/jw/pkg/lib/Distro/test.py b/test/unit/python/jw/pkg/lib/Distro/test.py index 37902e22..1725d5de 100644 --- a/test/unit/python/jw/pkg/lib/Distro/test.py +++ b/test/unit/python/jw/pkg/lib/Distro/test.py @@ -286,4 +286,23 @@ sde = SuseDistro( assert asyncio.run(sde.pkg_files()) == [] assert '--queryformat' in cl.cmds[-1] +# -- _install_local_files -- + +# -- With dependency resolution, the files go to the package manager; +# DummyDistro._install() is a no-op +asyncio.run( + d._install_local_files(['/a/jw-base-run-1.0.0-42.rpm'], False, install_deps = True) +) + +# -- Without, the base implementation raises: there is no direct route +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.rpm'], False, install_deps = False + ) + ) + assert False, 'Should have raised' +except NotImplementedError: + pass + print('All Distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/Makefile b/test/unit/python/jw/pkg/lib/distros/Makefile new file mode 100644 index 00000000..7b64e78c --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/jw/pkg/lib/distros/debian/Makefile b/test/unit/python/jw/pkg/lib/distros/debian/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/debian/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/debian/test.py b/test/unit/python/jw/pkg/lib/distros/debian/test.py new file mode 100644 index 00000000..f5370e8a --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/debian/test.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.debian.Distro import Distro as DebianDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers dpkg queries and + the apt simulation""" + + def __init__( + self, + files: dict[str, str], + installed: dict[str, str], + simulate: str, + ) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.simulate = simulate + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-I' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if cmd[0].endswith('dpkg-query'): + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + if '/usr/bin/apt-get' in cmd and '-s' in cmd: + return Result(stdout = self.simulate.encode(), status = 0) + return Result(stdout = b'', status = 0) + +def dpkg_args(cmd: list[str]) -> list[str]: + """The dpkg command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/dpkg'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-base-run-1.0.0-41.deb': 'Package: jw-base\nVersion: 1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = 'Inst jw-base [1.0.0-41]\n', +) +d = DebianDistro(cl, id = 'debian') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-base-run-1.0.0-41.deb'], + False, + install_deps = False, + ) +) +installs = [dpkg_args(cmd) for cmd in cl.cmds if '-i' in cmd] +assert installs == [ + ['/usr/bin/dpkg', '-i', '/a/jw-base-run-1.0.0-41.deb'], + [ + '/usr/bin/dpkg', + '-i', + '--force-reinstall', + '/a/jw-base-run-1.0.0-42.deb', + ], +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42', + '/a/jw-foo-run-1.0.0-42.deb': 'Package: jw-foo\nVersion: 1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +d = DebianDistro(cl, id = 'debian') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.deb', '/a/jw-foo-run-1.0.0-42.deb'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.deb' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-i' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.deb': 'Package: jw-base\nVersion: 1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42'}, + simulate = '', +) +d = DebianDistro(cl, id = 'debian') +asyncio.run( + d._install_local_files(['/a/jw-base-run-1.0.0-42.deb'], False, install_deps = True) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/apt-get' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/apt-get'):] +assert 'install' in args +assert '--no-install-recommends' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.deb' +assert not [cmd for cmd in cl.cmds if '-i' in cmd] + +print('All debian distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/redhat/Makefile b/test/unit/python/jw/pkg/lib/distros/redhat/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/redhat/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/redhat/test.py b/test/unit/python/jw/pkg/lib/distros/redhat/test.py new file mode 100644 index 00000000..121fcb96 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/redhat/test.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.redhat.Distro import Distro as RedhatDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers rpm queries""" + + def __init__(self, files: dict[str, str], installed: dict[str, str]) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qp' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if '-q' in cmd: + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + return Result(stdout = b'', status = 0) + +def rpm_args(cmd: list[str]) -> list[str]: + """The rpm command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/rpm'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-base-run-1.0.0-41.x86_64.rpm': 'jw-base\t1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + False, + install_deps = False, + ) +) +installs = [rpm_args(cmd) for cmd in cl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + ], +] + +# -- A single file whose version is already installed is reinstalled +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = False + ) +) +assert rpm_args(cl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-foo-run-1.0.0-42.x86_64.rpm'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.x86_64.rpm' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = RedhatDistro(cl, id = 'redhat') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = True + ) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/yum' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/yum'):] +assert 'install' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.x86_64.rpm' +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +print('All redhat distro tests passed') diff --git a/test/unit/python/jw/pkg/lib/distros/suse/Makefile b/test/unit/python/jw/pkg/lib/distros/suse/Makefile new file mode 100644 index 00000000..4801122b --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/suse/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/distros/suse/test.py b/test/unit/python/jw/pkg/lib/distros/suse/test.py new file mode 100644 index 00000000..ca95cc5f --- /dev/null +++ b/test/unit/python/jw/pkg/lib/distros/suse/test.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import asyncio + +from typing import override + +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.Result import Result + +class DispatchLocal(Local): + """A canned execution context that answers rpm queries""" + + def __init__(self, files: dict[str, str], installed: dict[str, str]) -> None: + super().__init__('local') + self.files = files + self.installed = installed + self.cmds: list[list[str]] = [] + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.cmds.append(cmd) + if '-qp' in cmd: + return Result(stdout = self.files.get(cmd[-1], '').encode(), status = 0) + if '-q' in cmd: + name = cmd[-1] + return Result( + stdout = self.installed.get(name, '').encode(), + status = 0 if name in self.installed else 1, + ) + return Result(stdout = b'', status = 0) + +def rpm_args(cmd: list[str]) -> list[str]: + """The rpm command without the sudo prefix and its options""" + return cmd[cmd.index('/usr/bin/rpm'):] + +# -- A mixed transaction runs two installs, the upgrades first: the +# distro installs local files with force, which keeps the reinstall +# semantics for already-installed versions +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-base-run-1.0.0-41.x86_64.rpm': 'jw-base\t1.0.0-41', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + False, + install_deps = False, + ) +) +installs = [rpm_args(cmd) for cmd in cl.cmds if '-U' in cmd] +assert installs == [ + ['/usr/bin/rpm', '-U', '/a/jw-base-run-1.0.0-41.x86_64.rpm'], + [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', + ], +] + +# -- A single file whose version is already installed is reinstalled +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = False + ) +) +assert rpm_args(cl.cmds[-1]) == [ + '/usr/bin/rpm', + '-U', + '--reinstall', + '/a/jw-base-run-1.0.0-42.x86_64.rpm', +] + +# -- only_update raises on files whose package is not installed +cl = DispatchLocal( + files = { + '/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42', + '/a/jw-foo-run-1.0.0-42.x86_64.rpm': 'jw-foo\t1.0.0-42', + }, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +try: + asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm', '/a/jw-foo-run-1.0.0-42.x86_64.rpm'], + True, + install_deps = False, + ) + ) +except Exception as e: + assert '/a/jw-foo-run-1.0.0-42.x86_64.rpm' in str(e) +else: + raise AssertionError('expected an exception') +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +# -- deps routes local files through the package manager +cl = DispatchLocal( + files = {'/a/jw-base-run-1.0.0-42.x86_64.rpm': 'jw-base\t1.0.0-42'}, + installed = {'jw-base': 'jw-base|||||1.0.0-42\n'}, +) +d = SuseDistro(cl, id = 'suse') +asyncio.run( + d._install_local_files( + ['/a/jw-base-run-1.0.0-42.x86_64.rpm'], False, install_deps = True + ) +) +pm_cmds = [cmd for cmd in cl.cmds if '/usr/bin/zypper' in cmd] +assert len(pm_cmds) == 1 +args = pm_cmds[0][pm_cmds[0].index('/usr/bin/zypper'):] +assert 'install' in args +assert args[-1] == '/a/jw-base-run-1.0.0-42.x86_64.rpm' +assert not [cmd for cmd in cl.cmds if '-U' in cmd] + +print('All suse distro tests passed')