From aa7275f4261856697d869dfc292f32a45a59a786 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 18 Apr 2026 12:08:00 +0200 Subject: [PATCH] App.distro_xxx: Move properties to Distro.xxx Commit a19679fec reverted the first attempt to make AsyncSSH reuse one connection during an instance lifetime. That failed because a lot of distribution-specific properties were filled in a new event loop thread started by AsyncRunner, and AsyncSSH didn't like that. This commit is the first part of the solution: Move those properties from the App class to the Distro class, and load the Distro class in an async loader. As soon as it's instantiated, it can provide all its properties without cluttering the code with async keywords. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/App.py | 217 +--------------- src/python/jw/pkg/cmds/distro/CmdInfo.py | 9 +- src/python/jw/pkg/cmds/distro/CmdInstall.py | 4 +- src/python/jw/pkg/cmds/posix/CmdCopy.py | 4 +- .../pkg/cmds/projects/BaseCmdPkgRelations.py | 2 +- .../jw/pkg/cmds/projects/CmdRequiredOsPkg.py | 2 +- src/python/jw/pkg/lib/Distro.py | 238 ++++++++++++++++-- 7 files changed, 243 insertions(+), 233 deletions(-) diff --git a/src/python/jw/pkg/App.py b/src/python/jw/pkg/App.py index 6daeaa74..e10c0681 100644 --- a/src/python/jw/pkg/App.py +++ b/src/python/jw/pkg/App.py @@ -211,21 +211,15 @@ class App(Base): return True return False - def __init__(self) -> None: + def __init__(self, distro: Distro|None=None) -> None: super().__init__("jw-pkg swiss army knife", modules=["jw.pkg.cmds"]) # -- Members without default values - self.__opt_os: str|None = None self.__opt_interactive: bool|None = None self.__opt_verbose: bool|None = None self.__top_name: str|None = None - self.__os_release: str|None = None - self.__distro_id: str|None = None - self.__distro_name: str|None = None - self.__distro_codename: str|None = None - self.__distro: Distro|None = None - self.__os_cascade: list[str]|None = None + self.__distro = distro self.__res_cache = ResultCache() self.__topdir: str|None = None self.__pretty_topdir: str|None = None @@ -236,6 +230,10 @@ class App(Base): self.__projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj" self.__pretty_projs_root = None + async def __init_async(self) -> None: + if self.__distro is None: + self.__distro = await Distro.instantiate(ec=self.exec_context, id=self.args.distro_id) + async def __aexit__(self, exc_type, exc, tb) -> None: if self.__exec_context is not None: await self.__exec_context.close() @@ -249,14 +247,12 @@ class App(Base): + 'one of "make:", "unaltered", "absolute". Absolute topdir by default') parser.add_argument('-p', '--prefix', default = None, help='Parent directory of project source directories') - parser.add_argument('-O', '--os', default = None, help='Target operating system') parser.add_argument('--distro-id', default=None, help='Distribution ID (default is taken from /etc/os-release)') parser.add_argument('--interactive', choices=['true', 'false', 'auto'], default='true', help="Wait for user input or try to proceed unattended") parser.add_argument('--verbose', action='store_true', default=False, help="Be verbose on stderr about what's being done on the distro level") parser.add_argument('--uri', default='local', help="Run commands on this host") async def _run(self, args: argparse.Namespace) -> None: - self.__opt_os = args.os self.__topdir = args.topdir self.__pretty_topdir = self.__format_topdir(self.__topdir, args.topdir_format) self.__topdir_fmt = args.topdir_format @@ -267,6 +263,7 @@ class App(Base): if args.prefix is not None: self.__projs_root = args.prefix self.__pretty_projs_root = args.prefix + await self.__init_async() return await super()._run(args) @property @@ -303,158 +300,15 @@ class App(Base): def projs_root(self): return self.__projs_root - @property - def os_release(self) -> str: - if self.__os_release is None: - release_file = '/etc/os-release' - try: - result = self.call_async(self.exec_context.get(release_file, throw=True)) - self.__os_release = result.stdout.decode().strip() - except Exception as e: - log(INFO, f'Failed to read {release_file} ({str(e)}), falling back to uname') - result = self.call_async( - self.exec_context.run( - ['uname', '-s'], - throw=False, - cmd_input=InputMode.NonInteractive - ) - ) - if result.status != 0: - log(ERR, f'/etc/os-release and uname both failed, the latter with exit status {result.status}') - raise - uname = result.decode().stdout.strip().lower() - self.__os_release = f'ID={uname}\nVERSION_CODENAME=unknown' - - return self.__os_release - - def os_release_field(self, key: str, throw: bool=False) -> str: - m = re.search(r'^\s*' + key + r'\s*=\s*("?)([^"\n]+)\1\s*$', self.os_release, re.MULTILINE) - if m is None: - if throw: - raise Exception(f'Could not read "{key}=" from /etc/os-release') - return None - return m.group(2) - - @property - def distro_name(self) -> str: - if self.__distro_name is None: - self.__distro_name = self.os_release_field('NAME', throw=True) - return self.__distro_name - - @property - def distro_id(self) -> str: - if self.__distro_id is None: - ret = self.args.distro_id # The distribution ID requested by the command line - if ret is None: - # The ID of the distribution we run on - ret = self.os_release_field('ID', throw=True) - match ret: - case 'opensuse-tumbleweed': - ret = 'suse' - case 'kali': - ret = 'kali' - self.__distro_id = ret - return self.__distro_id - - @property - def distro_codename(self) -> str: - match self.distro_id: - case 'suse': - self.__distro_codename = \ - self.os_release_field('ID', throw=True).split('-')[1] - case 'kali': - self.__distro_codename = \ - self.os_release_field('VERSION_CODENAME', throw=True).split('-')[1] - case _: - self.__distro_codename = \ - self.os_release_field('VERSION_CODENAME', throw=True) - return self.__distro_codename - - @property - def distro_cascade(self) -> str: - return ' '.join(self.os_cascade()) - - @cached_property - def distro_pkg_ext(self) -> str: - for entry in self.os_cascade(): - ret = entry.replace('pkg-', '') - if ret != entry: - return ret - raise RuntimeError(f'No package extension in found in {self.distro_cascade}') - - @property - def distro_gnu_triplet(self) -> str: - - import sysconfig - import shutil - import subprocess - - # Best: GNU host triplet Python was built for - for key in ("HOST_GNU_TYPE", "BUILD_GNU_TYPE"): # BUILD_GNU_TYPE can exist too - ret = sysconfig.get_config_var(key) - if isinstance(ret, str) and ret: - return ret - - # Common on Debian/Ubuntu: multiarch component (often looks like a triplet) - ret = sysconfig.get_config_var("MULTIARCH") - if isinstance(ret, str) and ret: - return ret - - # Sometimes exposed (privately) by CPython - ret = getattr(sys.implementation, "_multiarch", None) - if isinstance(ret, str) and ret: - return ret - - # Last resort: ask the system compiler - for cc in ("gcc", "cc", "clang"): - path = shutil.which(cc) - if not path: - continue - try: - ret = subprocess.check_output([path, "-dumpmachine"], text=True, stderr=subprocess.DEVNULL).strip() - if ret: - return ret - except Exception: - pass - - raise RuntimeError('Failed to get GNU triplet from running machine') - @property def distro(self) -> Distro: if self.__distro is None: - ret = Distro.instantiate(self.distro_id, ec=self.exec_context) - self.__distro = ret + raise Exception('No distro object') return self.__distro - def distro_info(self, fmt: str|Iterable) -> str|Iterable: - if not isinstance(fmt, str): - ret: list[str] = [] - for entry in fmt: - ret.append(self.distro_info(entry)) - return ret - ret = fmt - for macro in re.findall("%{([A-Za-z_-]+)}", fmt): - try: - name = 'distro_' + macro.replace('-', '_') - if name == 'distro_info': - continue - val = getattr(self, name) - patt = r'%{' + macro + r'}' - if ret.find(patt) == -1: - continue - ret = ret.replace(patt, val) - except Exception as e: - log(ERR, f'Failed to expand macro "{macro}" inside "{fmt}": {str(e)}') - raise - return ret - def find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True): return self.__find_dir(name, search_subdirs, search_absdirs, pretty) - @lru_cache(maxsize=None) - def get_os(self) -> str: - return self.distro_id + '-' + self.distro_codename - # TODO: add support for customizing this in project.conf def htdocs_dir(self, project: str) -> str: return self.find_dir(project, ["/src/html/htdocs", "/tools/html/htdocs", "/htdocs"], @@ -464,59 +318,6 @@ class App(Base): def tmpl_dir(self, name: str) -> str: return self.find_dir(name, ["/tmpl"], ["/opt/" + name + "/share/tmpl"]) - def os_cascade(self) -> list[str]: - def __append(entry: str): - if not entry in ret: - ret.append(entry) - if self.__os_cascade is None: - ret = [ 'os' ] - match self.distro_id: - case 'centos': - __append('linux') - __append('pkg-rpm') - __append('pm-yum') - __append('redhat') - __append('rhel') - case 'fedora' | 'rhel': - __append('linux') - __append('pkg-rpm') - __append('pm-yum') - __append('redhat') - case 'suse': - __append('linux') - __append('pkg-rpm') - __append('pm-zypper') - case 'kali' | 'raspbian': - __append('linux') - __append('pkg-debian') - __append('pm-apt') - __append('debian') - case 'ubuntu': - __append('linux') - __append('pkg-debian') - __append('pm-apt') - case 'archlinux': - __append('linux') - __append('pkg-pm') - __append('pm-pacman') - - os = self.__opt_os if self.__opt_os is not None else self.get_os() - name = re.sub(r'-.*', '', os) - series = os - rx = re.compile(r'\.[0-9]+$') - while True: - n = re.sub(rx, '', series) - if n == series: - break - ret.append(n) - series = n - __append(name) - __append(os) - __append(self.distro_id) - # e.g. os, linux, suse, suse-tumbleweed - self.__os_cascade = ret - return self.__os_cascade - def strip_module_from_spec(self, mod): return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip()) @@ -660,7 +461,7 @@ class App(Base): log(DEBUG, "checking if project " + project + " is excluded from build") exclude = self.get_project_refs([ project ], ['build'], 'exclude', scope = Scope.One, add_self=False, names_only=True) - cascade = self.os_cascade() + [ 'all' ] + cascade = self.distro.os_cascade + [ 'all' ] for p1 in exclude: for p2 in cascade: if p1 == p2: diff --git a/src/python/jw/pkg/cmds/distro/CmdInfo.py b/src/python/jw/pkg/cmds/distro/CmdInfo.py index 821812ac..d8c1a1ba 100644 --- a/src/python/jw/pkg/cmds/distro/CmdInfo.py +++ b/src/python/jw/pkg/cmds/distro/CmdInfo.py @@ -3,20 +3,19 @@ from argparse import Namespace, ArgumentParser from ...lib.log import * +from ...lib.Distro import Distro from ..Cmd import Cmd from ..CmdProjects import CmdProjects class CmdInfo(Cmd): # export def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'info', help='Print information about target platform') + super().__init__(parent, 'info', help='Retrieve information about target platform') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - macro_names = ', '.join(['%%{' + name.replace('distro_', '').replace('_', '-') + '}' - for name in dir(self.app) if name.startswith('distro_')]) parser.add_argument('--format', default='%{cascade}', - help=f'Format string, expanding macros {macro_names}') + help=f'Format string, expanding macros {", ".join(Distro.macros())}') async def _run(self, args: Namespace) -> None: - print(self.app.distro_info(args.format)) + print(self.app.distro.expand_macros(args.format)) diff --git a/src/python/jw/pkg/cmds/distro/CmdInstall.py b/src/python/jw/pkg/cmds/distro/CmdInstall.py index 0f7441ff..9ebcf9f3 100644 --- a/src/python/jw/pkg/cmds/distro/CmdInstall.py +++ b/src/python/jw/pkg/cmds/distro/CmdInstall.py @@ -15,8 +15,8 @@ class CmdInstall(Cmd): # export parser.add_argument("names", nargs="*", help="Packages to be installed") parser.add_argument('--only-update', default=False, action='store_true', help='Only update the listed packages, don\'t install them') parser.add_argument('-F', '--fixed-strings', action='store_true', - help='Don\'t expand platform info macros in ') + help='Don\'t expand platform.expand_macros macros in ') async def _run(self, args: Namespace) -> None: - names = names if args.fixed_strings else self.app.distro_info(args.names) + names = names if args.fixed_strings else self.app.distro.expand_macros(args.names) return await self.distro.install(names, only_update=args.only_update) diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index cf666d05..6eb5bfb5 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -23,12 +23,12 @@ class CmdCopy(Cmd): # export parser.add_argument('-g', '--group', default=None, help='Destination file group') parser.add_argument('-m', '--mode', default=None, help='Destination file mode') parser.add_argument('-F', '--fixed-strings', action='store_true', - help='Don\'t expand platform info macros in and ') + help='Don\'t expand platform.expand_macros macros in and ') async def _run(self, args: Namespace) -> None: def __expand(url: str) -> str: if args.fixed_strings: return url - return self.app.distro_info(url) + return self.app.distro.expand_macros(url) await copy(__expand(args.src), __expand(args.dst), owner=args.owner, group=args.group, mode=int(args.mode, 0)) diff --git a/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py b/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py index 0683e748..f7ecdaef 100644 --- a/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py +++ b/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py @@ -37,7 +37,7 @@ class BaseCmdPkgRelations(Cmd): ) -> list[str]: if subsections is None: - subsections = self.app.os_cascade() + subsections = self.app.distro.os_cascade subsections.append('jw') expand_semver_revision_range = expand_semver_revision_range diff --git a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py index 8d282c2c..9721f207 100644 --- a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py +++ b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py @@ -38,7 +38,7 @@ class CmdRequiredOsPkg(Cmd): # export for d in deps: if self.app.is_excluded_from_build(d) is not None: deps.remove(d) - subsecs = self.app.os_cascade() + subsecs = self.app.distro.os_cascade log(DEBUG, "subsecs = ", subsecs) requires: set[str] = set() for sec in subsecs: diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index c0f331b9..cbde46bf 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -3,32 +3,86 @@ from __future__ import annotations from typing import TYPE_CHECKING +from functools import cached_property if TYPE_CHECKING: import Iterable -import abc, importlib +import abc, importlib, re from .ExecContext import ExecContext -from .base import Result +from .base import Result, InputMode from .Package import Package from .log import * class Distro(abc.ABC): - def __init__(self, ec: ExecContext): + def __init__(self, ec: ExecContext, id: str|None=None, os_release_str: str|None=None): assert ec is not None + assert id is not None self.__exec_context = ec + self.__os_release_str: str|None = os_release_str self.__id: str|None = None - def __set_id(self, id: str) -> None: - self.__id = id + # Names that can be used by code outside this class to retrieve + # distribution properties by + # getattr(instance, name.replace('-', '_')) + macro_names = [ + 'os', + 'id', + 'name', + 'codename', + 'gnu-triplet', + 'os-cascade', + 'os-release', + 'pkg-ext', + ] # == Load @classmethod - def instantiate(cls, distro_id: str, *args, **kwargs): - backend_id = distro_id.lower().replace('-', '_') + async def read_os_release_str(cls, ec: ExecContext) -> None: + release_file = '/etc/os-release' + try: + result = await ec.get(release_file, throw=True) + ret = result.stdout.decode().strip() + except Exception as e: + log(INFO, f'Failed to read {release_file} ({str(e)}), falling back to uname') + result = await ec.run( + ['uname', '-s'], + throw=False, + cmd_input=InputMode.NonInteractive + ) + if result.status != 0: + log(ERR, f'/etc/os-release and uname both failed, the latter with exit status {result.status}') + raise + uname = result.decode().stdout.strip().lower() + ret = f'ID={uname}\nVERSION_CODENAME=unknown' + return ret + + @classmethod + def parse_os_release_field(self, key: str, os_release_str: str, throw: bool=False) -> str: + m = re.search(r'^\s*' + key + r'\s*=\s*("?)([^"\n]+)\1\s*$', os_release_str, re.MULTILINE) + if m is None: + if throw: + raise Exception(f'Could not read "{key}=" from /etc/os-release') + return None + return m.group(2) + + @classmethod + def parse_os_release_field_id(cls, os_release_str: str, throw: bool=False) -> str: + ret = cls.parse_os_release_field('ID', os_release_str, throw=throw) + match ret: + case 'opensuse-tumbleweed': + return 'suse' + return ret + + @classmethod + async def instantiate(cls, ec: ExecContext, *args, id: str|None=None, os_release_str: str|None=None, **kwargs): + if id is None: + os_release_str = await cls.read_os_release_str(ec) + id = cls.parse_os_release_field_id(os_release_str, throw=True) + backend_id = id.lower().replace('-', '_') match backend_id: case 'ubuntu' | 'raspbian' | 'kali': backend_id = 'debian' @@ -43,16 +97,172 @@ class Distro(abc.ABC): log(ERR, f'Failed to import Distro module {module_path} ({str(e)})') raise cls = getattr(module, 'Distro') - ret = cls(*args, **kwargs) - ret.__set_id(backend_id) + ret = cls(ec, *args, id=id, os_release_str=os_release_str, **kwargs) + return ret + + def os_release_field(self, key: str, throw: bool=False) -> str: + return self.parse_os_release_field(key, self.os_release_str, throw) + + async def cache(self) -> None: + if self.__os_release_str is None: + self.__os_release_str = await self.read_os_release_str(self.__exec_context) + + @cached_property + def os_cascade(self) -> list[str]: + def __append(entry: str): + if not entry in ret: + ret.append(entry) + ret = [ 'os' ] + match self.id: + case 'centos': + __append('linux') + __append('pkg-rpm') + __append('pm-yum') + __append('redhat') + __append('rhel') + case 'fedora' | 'rhel': + __append('linux') + __append('pkg-rpm') + __append('pm-yum') + __append('redhat') + case 'suse': + __append('linux') + __append('pkg-rpm') + __append('pm-zypper') + case 'kali' | 'raspbian': + __append('linux') + __append('pkg-debian') + __append('pm-apt') + __append('debian') + case 'ubuntu': + __append('linux') + __append('pkg-debian') + __append('pm-apt') + case 'archlinux': + __append('linux') + __append('pkg-pm') + __append('pm-pacman') + + os = self.os + name = re.sub(r'-.*', '', os) + series = os + rx = re.compile(r'\.[0-9]+$') + while True: + n = re.sub(rx, '', series) + if n == series: + break + ret.append(n) + series = n + __append(name) + __append(os) + __append(self.id) + # e.g. os, linux, suse, suse-tumbleweed + return ret + + @cached_property + def cascade(self) -> str: + return ' '.join(self.os_cascade) + + @property + def os_release_str(self) -> str: + if self.__os_release_str is None: + raise Exception(f'Tried to access OS release from an incompletely loaded Distro instance. Call reacache() before') + return self.__os_release_str + + @cached_property + def name(self) -> str: + return self.os_release_field('NAME', throw=True) + + @cached_property + def id(self) -> str: + return self.parse_os_release_field_id(self.__os_release_str, throw=True) + + @cached_property + def codename(self) -> str: + match self.id: + case 'suse': + return self.os_release_field('ID', throw=True).split('-')[1] + case 'kali': + return self.os_release_field('VERSION_CODENAME', throw=True).split('-')[1] + case _: + return self.os_release_field('VERSION_CODENAME', throw=True) + raise NotImplementedError(f'Can\'t determine code name from distribution ID {self.id}') + + @cached_property + def os(self) -> str: + return self.id + '-' + self.codename + + @cached_property + def pkg_ext(self) -> str: + for entry in self.os_cascade(): + ret = entry.replace('pkg-', '') + if ret != entry: + return ret + raise RuntimeError(f'No package extension in found in {self.os_cascade}') + + @cached_property + def gnu_triplet(self) -> str: + + import sysconfig + import shutil + import subprocess + + # Best: GNU host triplet Python was built for + for key in ("HOST_GNU_TYPE", "BUILD_GNU_TYPE"): # BUILD_GNU_TYPE can exist too + ret = sysconfig.get_config_var(key) + if isinstance(ret, str) and ret: + return ret + + # Common on Debian/Ubuntu: multiarch component (often looks like a triplet) + ret = sysconfig.get_config_var("MULTIARCH") + if isinstance(ret, str) and ret: + return ret + + # Sometimes exposed (privately) by CPython + ret = getattr(sys.implementation, "_multiarch", None) + if isinstance(ret, str) and ret: + return ret + + # Last resort: ask the system compiler + for cc in ("gcc", "cc", "clang"): + path = shutil.which(cc) + if not path: + continue + try: + ret = subprocess.check_output([path, "-dumpmachine"], text=True, stderr=subprocess.DEVNULL).strip() + if ret: + return ret + except Exception: + pass + + raise RuntimeError('Failed to get GNU triplet from running machine') + + @classmethod + def macros(cls) -> list[str]: + return ['%%{' + name + '}' for name in cls.macro_names] + + def expand_macros(self, fmt: str|Iterable) -> str|Iterable: + if not isinstance(fmt, str): + ret: list[str] = [] + for entry in fmt: + ret.append(self.expand_macros(entry)) + return ret + ret = fmt + for macro in re.findall("%{([A-Za-z_-]+)}", fmt): + try: + name = macro.replace('-', '_') + val = getattr(self, name) + patt = r'%{' + macro + r'}' + if ret.find(patt) == -1: + continue + ret = ret.replace(patt, val) + except Exception as e: + log(ERR, f'Failed to expand macro "{macro}" inside "{fmt}": {str(e)}') + raise return ret # == Convenience methods - @property - def id(self) -> str: - return self.__id - @property def ctx(self) -> ExecContext: return self.__exec_context @@ -67,7 +277,7 @@ class Distro(abc.ABC): def interactive(self) -> bool: return self.__exec_context.interactive - # == Distribution specific methods + # == Distribution abstraction methods # -- ref