janware/jw-pkg
1
Code Pull requests Activity Actions

App.distro_xxx: Move properties to Distro.xxx

Browse source 
Commit a19679fec reverted the first attempt to make AsyncSSH reuse one connection during an instance lifetime. That failed because a lot of distribution-specific properties were filled in a new event loop thread started by AsyncRunner, and AsyncSSH didn't like that.
This commit is the first part of the solution: Move those properties from the App class to the Distro class, and load the Distro class in an async loader. As soon as it's instantiated, it can provide all its properties without cluttering the code with async keywords.
Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-04-18 12:08:00 +02:00
commit aa7275f426
7 changed files with 241 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

217
src/python/jw/pkg/App.py
View file

@ -211,21 +211,15 @@ class App(Base):
return True
return False
def __init__(self) -> None:
def __init__(self, distro: Distro|None=None) -> None:
super().__init__("jw-pkg swiss army knife", modules=["jw.pkg.cmds"])
# -- Members without default values
self.__opt_os: str|None = None
self.__opt_interactive: bool|None = None
self.__opt_verbose: bool|None = None
self.__top_name: str|None = None
self.__os_release: str|None = None
self.__distro_id: str|None = None
self.__distro_name: str|None = None
self.__distro_codename: str|None = None
self.__distro: Distro|None = None
self.__os_cascade: list[str]|None = None
self.__distro = distro
self.__res_cache = ResultCache()
self.__topdir: str|None = None
self.__pretty_topdir: str|None = None
@ -236,6 +230,10 @@ class App(Base):
self.__projs_root = pwd.getpwuid(os.getuid()).pw_dir + "/local/src/jw.dev/proj"
self.__pretty_projs_root = None
async def __init_async(self) -> None:
if self.__distro is None:
self.__distro = await Distro.instantiate(ec=self.exec_context, id=self.args.distro_id)
async def __aexit__(self, exc_type, exc, tb) -> None:
if self.__exec_context is not None:
await self.__exec_context.close()
@ -249,14 +247,12 @@ class App(Base):
+ 'one of "make:<var-name>", "unaltered", "absolute". Absolute topdir by default')
parser.add_argument('-p', '--prefix', default = None,
help='Parent directory of project source directories')
parser.add_argument('-O', '--os', default = None, help='Target operating system')
parser.add_argument('--distro-id', default=None, help='Distribution ID (default is taken from /etc/os-release)')
parser.add_argument('--interactive', choices=['true', 'false', 'auto'], default='true', help="Wait for user input or try to proceed unattended")
parser.add_argument('--verbose', action='store_true', default=False, help="Be verbose on stderr about what's being done on the distro level")
parser.add_argument('--uri', default='local', help="Run commands on this host")
async def _run(self, args: argparse.Namespace) -> None:
self.__opt_os = args.os
self.__topdir = args.topdir
self.__pretty_topdir = self.__format_topdir(self.__topdir, args.topdir_format)
self.__topdir_fmt = args.topdir_format
@ -267,6 +263,7 @@ class App(Base):
if args.prefix is not None:
self.__projs_root = args.prefix
self.__pretty_projs_root = args.prefix
await self.__init_async()
return await super()._run(args)
@property
@ -303,158 +300,15 @@ class App(Base):
def projs_root(self):
return self.__projs_root
@property
def os_release(self) -> str:
if self.__os_release is None:
release_file = '/etc/os-release'
try:
result = self.call_async(self.exec_context.get(release_file, throw=True))
self.__os_release = result.stdout.decode().strip()
except Exception as e:
log(INFO, f'Failed to read {release_file} ({str(e)}), falling back to uname')
result = self.call_async(
self.exec_context.run(
['uname', '-s'],
throw=False,
cmd_input=InputMode.NonInteractive
)
)
if result.status != 0:
log(ERR, f'/etc/os-release and uname both failed, the latter with exit status {result.status}')
raise
uname = result.decode().stdout.strip().lower()
self.__os_release = f'ID={uname}\nVERSION_CODENAME=unknown'
return self.__os_release
def os_release_field(self, key: str, throw: bool=False) -> str:
m = re.search(r'^\s*' + key + r'\s*=\s*("?)([^"\n]+)\1\s*$', self.os_release, re.MULTILINE)
if m is None:
if throw:
raise Exception(f'Could not read "{key}=" from /etc/os-release')
return None
return m.group(2)
@property
def distro_name(self) -> str:
if self.__distro_name is None:
self.__distro_name = self.os_release_field('NAME', throw=True)
return self.__distro_name
@property
def distro_id(self) -> str:
if self.__distro_id is None:
ret = self.args.distro_id # The distribution ID requested by the command line
if ret is None:
# The ID of the distribution we run on
ret = self.os_release_field('ID', throw=True)
match ret:
case 'opensuse-tumbleweed':
ret = 'suse'
case 'kali':
ret = 'kali'
self.__distro_id = ret
return self.__distro_id
@property
def distro_codename(self) -> str:
match self.distro_id:
case 'suse':
self.__distro_codename = \
self.os_release_field('ID', throw=True).split('-')[1]
case 'kali':
self.__distro_codename = \
self.os_release_field('VERSION_CODENAME', throw=True).split('-')[1]
case _:
self.__distro_codename = \
self.os_release_field('VERSION_CODENAME', throw=True)
return self.__distro_codename
@property
def distro_cascade(self) -> str:
return ' '.join(self.os_cascade())
@cached_property
def distro_pkg_ext(self) -> str:
for entry in self.os_cascade():
ret = entry.replace('pkg-', '')
if ret != entry:
return ret
raise RuntimeError(f'No package extension in found in {self.distro_cascade}')
@property
def distro_gnu_triplet(self) -> str:
import sysconfig
import shutil
import subprocess
# Best: GNU host triplet Python was built for
for key in ("HOST_GNU_TYPE", "BUILD_GNU_TYPE"): # BUILD_GNU_TYPE can exist too
ret = sysconfig.get_config_var(key)
if isinstance(ret, str) and ret:
return ret
# Common on Debian/Ubuntu: multiarch component (often looks like a triplet)
ret = sysconfig.get_config_var("MULTIARCH")
if isinstance(ret, str) and ret:
return ret
# Sometimes exposed (privately) by CPython
ret = getattr(sys.implementation, "_multiarch", None)
if isinstance(ret, str) and ret:
return ret
# Last resort: ask the system compiler
for cc in ("gcc", "cc", "clang"):
path = shutil.which(cc)
if not path:
continue
try:
ret = subprocess.check_output([path, "-dumpmachine"], text=True, stderr=subprocess.DEVNULL).strip()
if ret:
return ret
except Exception:
pass
raise RuntimeError('Failed to get GNU triplet from running machine')
@property
def distro(self) -> Distro:
if self.__distro is None:
ret = Distro.instantiate(self.distro_id, ec=self.exec_context)
self.__distro = ret
raise Exception('No distro object')
return self.__distro
def distro_info(self, fmt: str|Iterable) -> str|Iterable:
if not isinstance(fmt, str):
ret: list[str] = []
for entry in fmt:
ret.append(self.distro_info(entry))
return ret
ret = fmt
for macro in re.findall("%{([A-Za-z_-]+)}", fmt):
try:
name = 'distro_' + macro.replace('-', '_')
if name == 'distro_info':
continue
val = getattr(self, name)
patt = r'%{' + macro + r'}'
if ret.find(patt) == -1:
continue
ret = ret.replace(patt, val)
except Exception as e:
log(ERR, f'Failed to expand macro "{macro}" inside "{fmt}": {str(e)}')
raise
return ret
def find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True):
return self.__find_dir(name, search_subdirs, search_absdirs, pretty)
@lru_cache(maxsize=None)
def get_os(self) -> str:
return self.distro_id + '-' + self.distro_codename
# TODO: add support for customizing this in project.conf
def htdocs_dir(self, project: str) -> str:
return self.find_dir(project, ["/src/html/htdocs", "/tools/html/htdocs", "/htdocs"],
@ -464,59 +318,6 @@ class App(Base):
def tmpl_dir(self, name: str) -> str:
return self.find_dir(name, ["/tmpl"], ["/opt/" + name + "/share/tmpl"])
def os_cascade(self) -> list[str]:
def __append(entry: str):
if not entry in ret:
ret.append(entry)
if self.__os_cascade is None:
ret = [ 'os' ]
match self.distro_id:
case 'centos':
__append('linux')
__append('pkg-rpm')
__append('pm-yum')
__append('redhat')
__append('rhel')
case 'fedora' | 'rhel':
__append('linux')
__append('pkg-rpm')
__append('pm-yum')
__append('redhat')
case 'suse':
__append('linux')
__append('pkg-rpm')
__append('pm-zypper')
case 'kali' | 'raspbian':
__append('linux')
__append('pkg-debian')
__append('pm-apt')
__append('debian')
case 'ubuntu':
__append('linux')
__append('pkg-debian')
__append('pm-apt')
case 'archlinux':
__append('linux')
__append('pkg-pm')
__append('pm-pacman')
os = self.__opt_os if self.__opt_os is not None else self.get_os()
name = re.sub(r'-.*', '', os)
series = os
rx = re.compile(r'\.[0-9]+$')
while True:
n = re.sub(rx, '', series)
if n == series:
break
ret.append(n)
series = n
__append(name)
__append(os)
__append(self.distro_id)
# e.g. os, linux, suse, suse-tumbleweed
self.__os_cascade = ret
return self.__os_cascade
def strip_module_from_spec(self, mod):
return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip())
@ -660,7 +461,7 @@ class App(Base):
log(DEBUG, "checking if project " + project + " is excluded from build")
exclude = self.get_project_refs([ project ], ['build'], 'exclude',
scope = Scope.One, add_self=False, names_only=True)
cascade = self.os_cascade() + [ 'all' ]
cascade = self.distro.os_cascade + [ 'all' ]
for p1 in exclude:
for p2 in cascade:
if p1 == p2: