diff --git a/scripts/jw-pkg.py b/scripts/jw-pkg.py index 3c78f0d8..91407a82 100644 --- a/scripts/jw-pkg.py +++ b/scripts/jw-pkg.py @@ -1,5 +1,4 @@ #!/usr/bin/python3 -# -*- coding: utf-8 -*- # PYTHON_ARGCOMPLETE_OK from jw.pkg.App import App diff --git a/src/python/jw/__init__.py b/src/python/jw/__init__.py index 3ad9513f..b36383a6 100644 --- a/src/python/jw/__init__.py +++ b/src/python/jw/__init__.py @@ -1,2 +1,3 @@ from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/src/python/jw/pkg/App.py b/src/python/jw/pkg/App.py index 3835b7eb..70a5689c 100644 --- a/src/python/jw/pkg/App.py +++ b/src/python/jw/pkg/App.py @@ -1,29 +1,40 @@ -# -*- coding: utf-8 -*- # # This source code file is a merge of various build tools and a horrible mess. # from __future__ import annotations -from typing import TYPE_CHECKING, Iterable +import argparse +import os +import pwd +import re +import sys -if TYPE_CHECKING: - from typing import TypeAlias - import os, sys, pwd, re - -import os, sys, argparse, pwd, re -from functools import lru_cache, cached_property from enum import Enum, auto +from functools import lru_cache +from typing import TYPE_CHECKING from .lib.App import App as Base -from .lib.log import * from .lib.Distro import Distro -from .lib.base import InputMode +from .lib.log import DEBUG, ERR, log + +if TYPE_CHECKING: + import os + import pwd + import re + import sys + + from typing import TypeAlias + + from .lib.ExecContext import ExecContext + from .lib.PackageFilter import PackageFilter # Meaning of pkg.requires.xxx variables # build: needs to be built and installed before this can be built -# devel: needs to be installed before this-devel can be installed, i.e. before _other_ packages can be built against this -# run: needs to be installed before this-run can be installed, i.e. before this and other packages can run with this +# devel: needs to be installed before this-devel can be installed, +# i.e. before _other_ packages can be built against this +# run: needs to be installed before this-run can be installed, +# i.e. before this and other packages can run with this # --------------------------------------------------------------------- Helpers @@ -35,32 +46,34 @@ class ResultCache(object): def run(self, func, args): d = self.__cache depth = 0 - keys = [ func.__name__ ] + args - l = len(keys) + keys = [func.__name__] + args + sz = len(keys) for k in keys: if k is None: k = 'None' else: k = str(k) depth += 1 - #log(DEBUG, 'depth = ', depth, 'key = ', k, 'd = ', str(d)) + # log(DEBUG, 'depth = ', depth, 'key = ', k, 'd = ', str(d)) if k in d: - if l == depth: + if sz == depth: return d[k] d = d[k] continue - if l == depth: + if sz == depth: r = func(*args) d[k] = r return r d = d[k] = {} - #d = d[k] - raise Exception('cache algorithm failed for function', func.__name__, 'in depth', depth) + # d = d[k] + raise Exception( + 'cache algorithm failed for function', func.__name__, 'in depth', depth + ) class Scope(Enum): - Self = auto() - One = auto() - Subtree = auto() + Self = auto() + One = auto() + Subtree = auto() Graph: TypeAlias = dict[str, set[str]] @@ -68,27 +81,41 @@ Graph: TypeAlias = dict[str, set[str]] class App(Base): - def __format_topdir(self, topdir: None|str, fmt: str) -> str: - if topdir is None: + def __format_topdir(self, path: None | str, fmt: str) -> str | None: + if path is None: return None match fmt: case 'unaltered': - return topdir + return path case None | 'absolute': - return os.path.abspath(self.__topdir) + return os.path.abspath(path) case _: m = re.search(r'^make:(\S+)$', fmt) if m is None: - raise Exception(f'Can\'t interpret "{fmt}" as valid topdir ' + - 'reference, expecting "unaltered", "absolute", or "make:"') + raise Exception( + f'Can\'t interpret "{fmt}" as valid topdir reference, ' + 'expecting "unaltered", "absolute", or "make:"' + ) return '$(' + m.group(1) + ')' - def __proj_dir(self, name: str, pretty) -> str: + @property + def __topdir(self) -> str: + if self.___topdir is None: + raise Exception('Tried to access undefined top directory') + return self.___topdir + + @property + def __pretty_topdir(self) -> str: + if self.___pretty_topdir is None: + raise Exception('Tried to access undefined pretty top directory') + return self.___pretty_topdir + + def __proj_dir(self, name: str, pretty: bool) -> str | None: if name == self.__top_name: if pretty: return self.__pretty_topdir return self.__topdir - for d in [ self.__projs_root, '/opt' ]: + for d in [self.__projs_root, '/opt']: ret = d + '/' + name if os.path.exists(ret): return ret @@ -97,7 +124,14 @@ class App(Base): return None raise Exception('No project path found for module "{}"'.format(name)) - def __find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True) -> str|None: + def __find_dir( + self, + name: str, + search_subdirs: list[str] = [], + search_absdirs: list[str] = [], + pretty: bool = True, + ) -> str | None: + def format_pd(name: str, pd: str, pretty: bool): if not pretty: return pd @@ -107,7 +141,10 @@ class App(Base): return pd if name == self.__top_name: return self.__pretty_topdir - raise NotImplementedError(f'Tried to pretty-format directory {pd}, not implemented') + raise NotImplementedError( + f'Tried to pretty-format directory {pd}, not implemented' + ) + pd = self.__proj_dir(name, False) if pd is None: return None @@ -126,11 +163,25 @@ class App(Base): return ret return None - def __get_project_refs_cached(self, buf, visited, spec, section, key, add_self, scope, names_only): - return self.__res_cache.run(self.__get_project_refs, [buf, visited, spec, section, key, add_self, scope, names_only]) + def __get_project_refs_cached( + self, buf, visited, spec, section, key, add_self, scope, names_only + ): + return self.__res_cache.run( + self.__get_project_refs, + [buf, visited, spec, section, key, add_self, scope, names_only], + ) - def __get_project_refs(self, buf: list[str], visited: set[str], spec: str, - section: str, key: str, add_self: bool, scope: Scope, names_only: bool) -> None: + def __get_project_refs( + self, + buf: list[str], + visited: set[str], + spec: str, + section: str, + key: str, + add_self: bool, + scope: Scope, + names_only: bool, + ) -> None: name = self.strip_module_from_spec(spec) if names_only: spec = name @@ -142,59 +193,95 @@ class App(Base): return visited.add(spec) deps = self.get_value(name, section, key) - log(DEBUG, 'name = ', name, 'section = ', section, 'key = ', key, 'deps = ', deps, 'scope = ', scope.name, 'visited = ', visited) + log( + DEBUG, + ( + f'name={name}, section={section}, key={key}, deps={deps}, ' + f'scope={scope.name}, visited={visited}' + ), + ) if deps and scope != Scope.Self: if scope == Scope.One: subscope = Scope.Self else: subscope = Scope.Subtree - deps = deps.split(',') - for dep in deps: + for dep in deps.split(','): dep = dep.strip() - if not(len(dep)): + if not (len(dep)): continue - self.__get_project_refs_cached(buf, visited, dep, - section, key, add_self=True, scope=subscope, - names_only=names_only) + self.__get_project_refs_cached( + buf, + visited, + dep, + section, + key, + add_self = True, + scope = subscope, + names_only = names_only, + ) if add_self: buf.append(spec) - def __read_dep_graph(self, projects: list[str], section: str, graph: Graph) -> None: + def __read_dep_graph( + self, + projects: list[str], + sections: str | list[str], + graph: Graph, + ) -> None: + if isinstance(sections, str): + sections = [sections] for project in projects: if project in graph: continue - deps = self.get_project_refs([ project ], ['pkg.requires.jw'], section, - scope = Scope.One, add_self=False, names_only=True) - if not deps is None: + for section in sections: + deps = self.get_project_refs( + [project], + ['pkg.requires.jw'], + sections, + scope = Scope.One, + add_self = False, + names_only = True, + ) + if deps is None: + continue graph[project] = set(deps) for dep in deps: - self.__read_dep_graph([ dep ], section, graph) + self.__read_dep_graph([dep], sections, graph) def __flip_dep_graph(self, graph: Graph): ret: Graph = {} for project, deps in graph.items(): for d in deps: - if not d in ret: + if d not in ret: ret[d] = set() ret[d].add(project) return ret - def __find_circular_deps_recursive(self, project: str, graph: Graph, unvisited: list[str], - temp: set[str], path: str) -> str|None: + def __find_circular_deps_recursive( + self, + project: str, + graph: Graph, + unvisited: list[str], + temp: set[str], + path: list[str], + ) -> str | None: if project in temp: log(DEBUG, 'found circular dependency at project', project) return project - if not project in unvisited: + if project not in unvisited: return None temp.add(project) if project in graph: for dep in graph[project]: - last = self.__find_circular_deps_recursive(dep, graph, unvisited, temp, path) + last = self.__find_circular_deps_recursive( + dep, graph, unvisited, temp, path + ) if last is not None: path.insert(0, dep) return last unvisited.remove(project) temp.remove(project) + return None def __find_circular_deps(self, projects: list[str], flavours: list[str]) -> bool: graph: Graph = {} @@ -205,25 +292,27 @@ class App(Base): while unvisited: project = unvisited[0] log(DEBUG, 'Checking circular dependency of', project) - last = self.__find_circular_deps_recursive(project, self.__flip_dep_graph(graph), unvisited, temp, ret) + last = self.__find_circular_deps_recursive( + project, self.__flip_dep_graph(graph), unvisited, temp, ret + ) if last is not None: log(DEBUG, f'Found circular dependency below {project}, last is {last}') return True return False - def __init__(self, distro: Distro|None=None) -> None: + def __init__(self, distro: Distro | None = None) -> None: - super().__init__('jw-pkg swiss army knife', modules=['jw.pkg.cmds']) + super().__init__('jw-pkg swiss army knife', modules = ['jw.pkg.cmds']) # -- Members without default values - self.__opt_interactive: bool|None = None - self.__opt_verbose: bool|None = None - self.__top_name: str|None = None + self.__opt_interactive: bool | None = None + self.__opt_verbose: bool | None = None + self.__top_name: str | None = None self.__distro = distro self.__res_cache = ResultCache() - self.__topdir: str|None = None - self.__pretty_topdir: str|None = None - self.__exec_context: ExecContext|None = None + self.___topdir: str | None = None + self.___pretty_topdir: str | None = None + self.__exec_context: ExecContext | None = None # -- Members with default values self.__topdir_fmt = 'absolute' @@ -235,48 +324,83 @@ class App(Base): pkg_filter_str = self.args.pkg_filter if pkg_filter_str is None: pkg_filter_str = os.getenv('JW_DEFAULT_PKG_FILTER') - pkg_filter: PackageFilter|None = None + pkg_filter: PackageFilter | None = None if pkg_filter_str is not None: from .lib.PackageFilter import PackageFilterString + pkg_filter = PackageFilterString(pkg_filter_str) self.__distro = await Distro.instantiate( - ec = self.exec_context, - id = self.args.distro_id, - default_pkg_filter = pkg_filter, - ) + ec = self.exec_context, + id = self.args.distro_id, + default_pkg_filter = pkg_filter, + ) async def __aexit__(self, exc_type, exc, tb) -> None: if self.__exec_context is not None: await self.__exec_context.close() self.__exec_context = None - return super().__aexit__(exc_type, exc, tb) def _add_arguments(self, parser) -> None: super()._add_arguments(parser) - parser.add_argument('-t', '--topdir', default = None, help='Project Path') - parser.add_argument('--topdir-format', default = 'absolute', help='Output references to topdir as ' - + 'one of "make:", "unaltered", "absolute". Absolute topdir by default') - parser.add_argument('-p', '--prefix', default = None, - help='Parent directory of project source directories') - parser.add_argument('--distro-id', default=None, help='Distribution ID (default is taken from /etc/os-release)') - parser.add_argument('--interactive', choices=['true', 'false', 'auto'], default='true', help='Wait for user input or try to proceed unattended') - parser.add_argument('--verbose', action='store_true', default=False, help='Be verbose on stderr about what\'s being done on the distro level') - parser.add_argument('--target', default='local', help='Run commands on this host') - parser.add_argument('--pkg-filter', help='Default filter for all distribution package-related operations') + parser.add_argument('-t', '--topdir', default = None, help = 'Project Path') + parser.add_argument( + '--topdir-format', + default = 'absolute', + help = ( + 'Output references to topdir as one of "make:", ' + '"unaltered", "absolute". Absolute topdir by default' + ), + ) + parser.add_argument( + '-p', + '--prefix', + default = None, + help = 'Parent directory of project source directories', + ) + parser.add_argument( + '--distro-id', + default = None, + help = 'Distribution ID (default is taken from /etc/os-release)', + ) + parser.add_argument( + '--interactive', + choices = ['true', 'false', 'auto'], + default = 'true', + help = 'Wait for user input or try to proceed unattended', + ) + parser.add_argument( + '--verbose', + action = 'store_true', + default = False, + help = "Be verbose on stderr about what's being done on the distro level", + ) + parser.add_argument( + '--target', default = 'local', help = 'Run commands on this host' + ) + parser.add_argument( + '--pkg-filter', + help = 'Default filter for all distribution package-related operations', + ) async def _run(self, args: argparse.Namespace) -> None: - self.__topdir = args.topdir - self.__pretty_topdir = self.__format_topdir(self.__topdir, args.topdir_format) + self.___topdir = args.topdir + self.___pretty_topdir = self.__format_topdir(self.___topdir, args.topdir_format) self.__topdir_fmt = args.topdir_format - if self.__topdir is not None: - self.__top_name = self.read_value(self.__topdir + '/make/project.conf', 'build', 'name') + if self.___topdir is not None: + self.__top_name = self.read_value( + self.___topdir + '/make/project.conf', 'build', 'name' + ) if not self.__top_name: - self.__top_name = re.sub('-[0-9.-]*$', '', os.path.basename(os.path.realpath(self.__topdir))) + self.__top_name = re.sub( + '-[0-9.-]*$', + '', + os.path.basename(os.path.realpath(self.___topdir)) + ) if args.prefix is not None: self.__projs_root = args.prefix self.__pretty_projs_root = args.prefix await self.__init_async() - return await super()._run(args) + await super()._run(args) @property def interactive(self) -> bool: @@ -288,20 +412,32 @@ class App(Base): self.__opt_interactive = False case 'auto': self.__opt_interactive = sys.stdin.isatty() + case _: + raise ValueError( + f'Unknown --interactive value: {self.args.interactive}' + ) + # Not logically possible to fail, but this keeps pyright happy + assert self.__opt_interactive is not None return self.__opt_interactive @property def verbose(self) -> bool: if self.__opt_verbose is None: self.__opt_verbose = self.args.verbose + # Not logically possible to fail, but this keeps pyright happy + assert self.__opt_verbose is not None return self.__opt_verbose @property - def exec_context(self) -> str: + def exec_context(self) -> ExecContext: if self.__exec_context is None: from .lib.ExecContext import ExecContext - self.__exec_context = ExecContext.create(self.args.target, interactive=self.interactive, - verbose_default=self.verbose) + + self.__exec_context = ExecContext.create( + self.args.target, + interactive = self.interactive, + verbose_default = self.verbose, + ) return self.__exec_context @property @@ -318,29 +454,51 @@ class App(Base): raise Exception('No distro object') return self.__distro - def find_dir(self, name: str, search_subdirs: list[str]=[], search_absdirs: list[str]=[], pretty: bool=True): - return self.__find_dir(name, search_subdirs, search_absdirs, pretty) + def find_dir( + self, + name: str, + search_subdirs: list[str] = [], + search_absdirs: list[str] = [], + pretty: bool = True, + ) -> str: + ret = self.__find_dir(name, search_subdirs, search_absdirs, pretty) + if ret is None: + msg = f'Failed to find directory for "{name}"' + log(ERR, msg) + for search_name, search in [ + ('subdirs', search_subdirs), + ('absdirs', search_absdirs), + ]: + if search: + log(ERR, f' Searched {search_name} in:') + for d in search: + log(ERR, f' - {d}') + raise FileNotFoundError(msg) + return ret # TODO: add support for customizing this in project.conf def htdocs_dir(self, project: str) -> str: - return self.find_dir(project, ['/src/html/htdocs', '/tools/html/htdocs', '/htdocs'], - ['/srv/www/proj/' + project]) + return self.find_dir( + project, + ['/src/html/htdocs', '/tools/html/htdocs', '/htdocs'], + ['/srv/www/proj/' + project], + ) # TODO: add support for customizing this in project.conf def tmpl_dir(self, name: str) -> str: - return self.find_dir(name, ['/tmpl'], ['/opt/' + name + '/share/tmpl']) + return self.find_dir(name, ['/tmpl'], ['/opt/' + name + '/share/tmpl']) def strip_module_from_spec(self, mod): return re.sub(r'-dev$|-devel$|-run$', '', re.split('([=><]+)', mod)[0].strip()) - @lru_cache(maxsize=None) + @lru_cache(maxsize = None) def get_section(self, path: str, section: str) -> str: ret = '' pat = '[' + section + ']' in_section = False file = open(path) for line in file: - if (line.rstrip() == pat): + if line.rstrip() == pat: in_section = True continue if in_section: @@ -350,10 +508,10 @@ class App(Base): file.close() return ret.rstrip() - @lru_cache(maxsize=None) - def read_value(self, path: str, section: str, key: str) -> str|None: + @lru_cache(maxsize = None) + def read_value(self, path: str, section: str, key: str) -> str | None: - def scan_section(f, key: str) -> str|None: + def scan_section(f, key: str) -> str | None: if key is None: ret = '' for line in f: @@ -374,43 +532,44 @@ class App(Base): cont_line = '' rx = re.compile(r'^\s*' + key + r'\s*=\s*(.*)\s*$') for line in lines: - #log(DEBUG, ' looking for "%s" in line="%s"' % (key, line)) + # log(DEBUG, ' looking for "%s" in line="%s"' % (key, line)) m = re.search(rx, line) if m is not None: return m.group(1) return None - def scan_section_debug(f, key: str) -> str|None: + def scan_section_debug(f, key: str) -> str | None: ret = scan_section(f, key) - #log(DEBUG, ' returning', rr) + # log(DEBUG, ' returning', rr) return ret try: - #log(DEBUG, 'looking for {}::[{}].{}'.format(path, section, key)) + # log(DEBUG, 'looking for {}::[{}].{}'.format(path, section, key)) + # TODO: Parse with open(path, 'r') as f: if not len(section): - rr = scan_section(f, key) + return scan_section(f, key) pat = '[' + section + ']' for line in f: if line.rstrip() == pat: return scan_section(f, key) return None - except: - log(DEBUG, path, 'not found') + except Exception: + log(DEBUG, f'Not found: {path}') # TODO: handle this special case cleaner somewhere up the stack if section == 'build' and key == 'libname': return 'none' return None - @lru_cache(maxsize=None) - def get_value(self, project: str, section: str, key: str) -> str: - if self.__top_name and project == self.__top_name: - proj_root = self.__topdir - else: - proj_root = self.__projs_root + '/' + project + @lru_cache(maxsize = None) + def get_value(self, project: str, section: str, key: str) -> str | None: + ret: str | None + proj_dir = self.__proj_dir(project, pretty = False) + if proj_dir is None: + raise Exception(f"Can't get project directory for {project}") if section == 'version': - proj_version_dirs = [ proj_root ] - if proj_root != self.__topdir: + proj_version_dirs = [proj_dir] + if proj_dir != self.___topdir: proj_version_dirs.append('/usr/share/doc/packages/' + project) for d in proj_version_dirs: version_path = d + '/VERSION' @@ -423,13 +582,24 @@ class App(Base): log(DEBUG, f'Ignoring unreadable file "{version_path}"') continue raise Exception(f'No version file found for project "{project}"') - path = proj_root + '/make/project.conf' + path = proj_dir + '/make/project.conf' ret = self.read_value(path, section, key) - log(DEBUG, 'Lookup %s -> %s / [%s%s] -> "%s"' % - (self.__top_name, project, section, '.' + key if key else '', ret)) + log( + DEBUG, + 'Lookup %s -> %s / [%s%s] -> "%s"' % + (self.__top_name, project, section, '.' + key if key else '', ret), + ) return ret - def get_values(self, projects: list[str], sections: list[str], keys: list[str]) -> list[str]: + @lru_cache(maxsize = None) + def get_version(self, project) -> str: + ret = self.get_value(project, 'version', '') + if ret is None: + raise Exception(f"Can't get version of project {project}") + return ret + + def get_values(self, projects: list[str], sections: list[str], + keys: list[str]) -> list[str]: """ Collect a list of values from a list of given projects, sections and keys, maintaining order @@ -441,39 +611,60 @@ class App(Base): vals = self.get_value(p, section, key) if vals: ret += [val.strip() for val in vals.split(',')] - return list(dict.fromkeys(ret)) # Remove duplicates, keep ordering + return list(dict.fromkeys(ret)) # Remove duplicates, keep ordering - def get_project_refs(self, projects: list[str], sections: list[str], - keys: str|list[str], add_self: bool, scope: Scope, names_only=True) -> list[str]: + def get_project_refs( + self, + projects: list[str], + sections: list[str], + keys: str | list[str], + add_self: bool, + scope: Scope, + names_only = True, + ) -> list[str]: if isinstance(keys, str): - keys = [ keys ] + keys = [keys] ret: list[str] = [] for section in sections: for key in keys: - visited = set() + visited: set[str] = set() for name in projects: rr: list[str] = [] - self.__get_project_refs_cached(rr, visited, name, section, key, add_self, scope, names_only) + self.__get_project_refs_cached( + rr, visited, name, section, key, add_self, scope, names_only + ) # TODO: this looks like a performance hogger for m in rr: - if not m in ret: - ret.append(m) + if m not in ret: + ret.append(m) return ret def get_libname(self, projects) -> str: - vals = self.get_project_refs(projects, ['build'], 'libname', - scope = Scope.One, add_self=False, names_only=True) + vals = self.get_project_refs( + projects, + ['build'], + 'libname', + scope = Scope.One, + add_self = False, + names_only = True, + ) if not vals: return ' '.join(projects) if 'none' in vals: vals.remove('none') return ' '.join(reversed(vals)) - def is_excluded_from_build(self, project: str) -> str|None: + def is_excluded_from_build(self, project: str) -> str | None: log(DEBUG, 'checking if project ' + project + ' is excluded from build') - exclude = self.get_project_refs([ project ], ['build'], 'exclude', - scope = Scope.One, add_self=False, names_only=True) - cascade = self.distro.os_cascade + [ 'all' ] + exclude = self.get_project_refs( + [project], + ['build'], + 'exclude', + scope = Scope.One, + add_self = False, + names_only = True, + ) + cascade = self.distro.os_cascade + ['all'] for p1 in exclude: for p2 in cascade: if p1 == p2: diff --git a/src/python/jw/pkg/cmds/Cmd.py b/src/python/jw/pkg/cmds/Cmd.py index 30afddb0..4f333dc3 100644 --- a/src/python/jw/pkg/cmds/Cmd.py +++ b/src/python/jw/pkg/cmds/Cmd.py @@ -1,15 +1,16 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from argparse import ArgumentParser +from typing import TYPE_CHECKING -from ..App import App -from ..lib.Cmd import Cmd as Base +from ..CmdBase import CmdBase -class Cmd(Base): # export +if TYPE_CHECKING: + from ..App import App + from ..lib.Distro import Distro - def __init__(self, parent: App|Base, name: str, help: str) -> None: +class Cmd(CmdBase): # export + + def __init__(self, parent: App | CmdBase, name: str, help: str) -> None: super().__init__(parent, name, help) async def _run(self, args): diff --git a/src/python/jw/pkg/cmds/CmdPkg.py b/src/python/jw/pkg/cmds/CmdPkg.py index 836eb3a3..0bfe5b53 100644 --- a/src/python/jw/pkg/cmds/CmdPkg.py +++ b/src/python/jw/pkg/cmds/CmdPkg.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- - from argparse import ArgumentParser from ..App import App from .Cmd import Cmd as CmdBase -class CmdPkg(CmdBase): # export +class CmdPkg(CmdBase): # export def __init__(self, parent: App) -> None: - super().__init__(parent, 'pkg', help="System package manager wrapper") + super().__init__(parent, 'pkg', help = 'System package manager wrapper') self.load_subcommands() def add_arguments(self, p: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/CmdPlatform.py b/src/python/jw/pkg/cmds/CmdPlatform.py index 0db24096..c497d614 100644 --- a/src/python/jw/pkg/cmds/CmdPlatform.py +++ b/src/python/jw/pkg/cmds/CmdPlatform.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- - from argparse import ArgumentParser from ..App import App from .Cmd import Cmd as CmdBase -class CmdPlatform(CmdBase): # export +class CmdPlatform(CmdBase): # export def __init__(self, parent: App) -> None: - super().__init__(parent, 'platform', help="Miscellaneous platform-related comamnds") + super().__init__( + parent, 'platform', help = 'Miscellaneous platform-related comamnds' + ) self.load_subcommands() def add_arguments(self, p: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/CmdPosix.py b/src/python/jw/pkg/cmds/CmdPosix.py index c98d67ab..afcd75d5 100644 --- a/src/python/jw/pkg/cmds/CmdPosix.py +++ b/src/python/jw/pkg/cmds/CmdPosix.py @@ -1,14 +1,19 @@ -# -*- coding: utf-8 -*- - from argparse import ArgumentParser from ..App import App from .Cmd import Cmd as CmdBase -class CmdPosix(CmdBase): # export +class CmdPosix(CmdBase): # export def __init__(self, parent: App) -> None: - super().__init__(parent, 'posix', help='Perform various operations on a distro through its POSIX utility interface') + super().__init__( + parent, + 'posix', + help = ( + 'Perform various operations on a distro through its ' + 'POSIX utility interface' + ), + ) self.load_subcommands() def add_arguments(self, p: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/CmdProjects.py b/src/python/jw/pkg/cmds/CmdProjects.py index f266c417..321cfa39 100644 --- a/src/python/jw/pkg/cmds/CmdProjects.py +++ b/src/python/jw/pkg/cmds/CmdProjects.py @@ -1,15 +1,18 @@ -# -*- coding: utf-8 -*- - import sys + from argparse import ArgumentParser from ..App import App from .Cmd import Cmd as CmdBase -class CmdProjects(CmdBase): # export +class CmdProjects(CmdBase): # export def __init__(self, parent: App) -> None: - super().__init__(parent, 'projects', help='Project metadata evaluation for building packages') + super().__init__( + parent, + 'projects', + help = 'Project metadata evaluation for building packages' + ) self.load_subcommands() def add_arguments(self, p: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/CmdSecrets.py b/src/python/jw/pkg/cmds/CmdSecrets.py index 7d3f5ce8..506b14bb 100644 --- a/src/python/jw/pkg/cmds/CmdSecrets.py +++ b/src/python/jw/pkg/cmds/CmdSecrets.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- - from argparse import ArgumentParser from ..App import App from .Cmd import Cmd as CmdBase -class CmdSecrets(CmdBase): # export +class CmdSecrets(CmdBase): # export def __init__(self, parent: App) -> None: - super().__init__(parent, 'secrets', help="Manage package secrets") + super().__init__(parent, 'secrets', help = 'Manage package secrets') self.load_subcommands() def add_arguments(self, p: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/__init__.py b/src/python/jw/pkg/cmds/__init__.py index 1fed7a71..d4f3a76e 100644 --- a/src/python/jw/pkg/cmds/__init__.py +++ b/src/python/jw/pkg/cmds/__init__.py @@ -4,6 +4,6 @@ __all__ = detect_modules( package_name = __name__, package_path = __path__, namespace = globals(), - prefix = "Cmd", - skip = {"Cmd"}, -) # pyright: ignore[reportUnsupportedDunderAll] + prefix = 'Cmd', + skip = {'Cmd'}, +) # pyright: ignore[reportUnsupportedDunderAll] diff --git a/src/python/jw/pkg/cmds/pkg/Cmd.py b/src/python/jw/pkg/cmds/pkg/Cmd.py index ffc6cefd..3da8c7d3 100644 --- a/src/python/jw/pkg/cmds/pkg/Cmd.py +++ b/src/python/jw/pkg/cmds/pkg/Cmd.py @@ -1,16 +1,13 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: - from ...lib.Distro import Distro from ..CmdPkg import CmdPkg from ..Cmd import Cmd as Base -class Cmd(Base): # export +class Cmd(Base): # export def __init__(self, parent: CmdPkg, name: str, help: str) -> None: super().__init__(parent, name, help) diff --git a/src/python/jw/pkg/cmds/pkg/CmdDelete.py b/src/python/jw/pkg/cmds/pkg/CmdDelete.py index 41650031..e4bced49 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdDelete.py +++ b/src/python/jw/pkg/cmds/pkg/CmdDelete.py @@ -1,18 +1,18 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd from ..CmdPkg import CmdPkg +from .Cmd import Cmd -class CmdDelete(Cmd): # export +class CmdDelete(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'delete', help="Delete packages by name") + super().__init__(parent, 'delete', help = 'Delete packages by name') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("names", nargs="*", help="Names of packages to be deleted") + parser.add_argument( + 'names', nargs = '*', help = 'Names of packages to be deleted' + ) async def _run(self, args: Namespace) -> None: return await self.distro.delete(args.names) diff --git a/src/python/jw/pkg/cmds/pkg/CmdDup.py b/src/python/jw/pkg/cmds/pkg/CmdDup.py index 4aed7e7f..7d442098 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdDup.py +++ b/src/python/jw/pkg/cmds/pkg/CmdDup.py @@ -1,18 +1,21 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd from ..CmdPkg import CmdPkg +from .Cmd import Cmd -class CmdDup(Cmd): # export +class CmdDup(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'dup', help="Upgrade distribution") + super().__init__(parent, 'dup', help = 'Upgrade distribution') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--download-only', default=False, action='store_true', - help='Only download packages from the repos, don\'t install them, yet') + parser.add_argument( + '--download-only', + default = False, + action = 'store_true', + help = "Only download packages from the repos, don't install them, yet", + ) + async def _run(self, args: Namespace) -> None: - return await self.distro.dup(download_only=args.download_only) + return await self.distro.dup(download_only = args.download_only) diff --git a/src/python/jw/pkg/cmds/pkg/CmdInstall.py b/src/python/jw/pkg/cmds/pkg/CmdInstall.py index f3f55201..ad77aa11 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdInstall.py +++ b/src/python/jw/pkg/cmds/pkg/CmdInstall.py @@ -1,22 +1,35 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd from ..CmdPkg import CmdPkg +from .Cmd import Cmd -class CmdInstall(Cmd): # export +class CmdInstall(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'install', help="Install the distribution's notion of available packages") + super().__init__( + parent, + 'install', + help = "Install the distribution's notion of available packages", + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("names", nargs="*", help="Packages to be installed") - parser.add_argument('--only-update', default=False, action='store_true', help='Only update the listed packages, don\'t install them') - parser.add_argument('-F', '--fixed-strings', action='store_true', - help='Don\'t expand platform.expand_macros macros in ') + parser.add_argument('names', nargs = '*', help = 'Packages to be installed') + parser.add_argument( + '--only-update', + default = False, + action = 'store_true', + help = "Only update the listed packages, don't install them", + ) + parser.add_argument( + '-F', + '--fixed-strings', + action = 'store_true', + help = "Don't expand platform.expand_macros macros in ", + ) async def _run(self, args: Namespace) -> None: - names = names if args.fixed_strings else self.app.distro.expand_macros(args.names) - return await self.distro.install(names, only_update=args.only_update) + names = ( + args.names if args.fixed_strings else self.distro.expand_macros(args.names) + ) + return await self.distro.install(names, only_update = args.only_update) diff --git a/src/python/jw/pkg/cmds/pkg/CmdLs.py b/src/python/jw/pkg/cmds/pkg/CmdLs.py index c301a5cd..806ac324 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdLs.py +++ b/src/python/jw/pkg/cmds/pkg/CmdLs.py @@ -1,18 +1,16 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .NamedPkgsCmd import NamedPkgsCmd as Base from ..CmdPkg import CmdPkg +from .NamedPkgsCmd import NamedPkgsCmd as Base -class CmdLs(Base): # export +class CmdLs(Base): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'ls', help="List package contents") + super().__init__(parent, 'ls', help = 'List package contents') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) async def _run(self, args: Namespace) -> None: for name in args.names: - print('\n'.join(await self.parent.distro.pkg_files(name))) + print('\n'.join(await self.distro.pkg_files(name))) diff --git a/src/python/jw/pkg/cmds/pkg/CmdMeta.py b/src/python/jw/pkg/cmds/pkg/CmdMeta.py index 5209eb56..e44a77de 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdMeta.py +++ b/src/python/jw/pkg/cmds/pkg/CmdMeta.py @@ -1,21 +1,19 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .NamedPkgsCmd import NamedPkgsCmd as Base from ..CmdPkg import CmdPkg +from .NamedPkgsCmd import NamedPkgsCmd as Base -class CmdMeta(Base): # export +class CmdMeta(Base): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'meta', help="List package metadata") + super().__init__(parent, 'meta', help = 'List package metadata') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) async def _run(self, args: Namespace) -> None: - packages = await self.distro.select(args.names) - for package in packages: + names = await self.distro.select(args.names) + for name in names: if len(args.names) > 1: print(f'-- {name}') - print(package) + print(name) diff --git a/src/python/jw/pkg/cmds/pkg/CmdRebootRequired.py b/src/python/jw/pkg/cmds/pkg/CmdRebootRequired.py index a74f8405..411ee761 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdRebootRequired.py +++ b/src/python/jw/pkg/cmds/pkg/CmdRebootRequired.py @@ -1,17 +1,19 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd from ..CmdPkg import CmdPkg +from .Cmd import Cmd -class CmdRebootRequired(Cmd): # export +class CmdRebootRequired(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'reboot-required', help="Check whether the machine needs rebooting") + super().__init__( + parent, + 'reboot-required', + help = 'Check whether the machine needs rebooting' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) async def _run(self, args: Namespace) -> None: - return await self.distro.reboot_required() + await self.distro.reboot_required() diff --git a/src/python/jw/pkg/cmds/pkg/CmdRefresh.py b/src/python/jw/pkg/cmds/pkg/CmdRefresh.py index 0e6c65a6..1ee060ee 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdRefresh.py +++ b/src/python/jw/pkg/cmds/pkg/CmdRefresh.py @@ -1,14 +1,16 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd from ..CmdPkg import CmdPkg +from .Cmd import Cmd -class CmdRefresh(Cmd): # export +class CmdRefresh(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'refresh', help="Refresh the distribution's notion of available packages") + super().__init__( + parent, + 'refresh', + help = "Refresh the distribution's notion of available packages", + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) diff --git a/src/python/jw/pkg/cmds/pkg/CmdSelect.py b/src/python/jw/pkg/cmds/pkg/CmdSelect.py index 1dedca9f..387877d1 100644 --- a/src/python/jw/pkg/cmds/pkg/CmdSelect.py +++ b/src/python/jw/pkg/cmds/pkg/CmdSelect.py @@ -1,23 +1,19 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser -import re - -from ...lib.Package import Package from ...lib.PackageFilter import PackageFilterString from ..CmdPkg import CmdPkg from .Cmd import Cmd -class CmdSelect(Cmd): # export +class CmdSelect(Cmd): # export def __init__(self, parent: CmdPkg) -> None: - super().__init__(parent, 'select', help="Select packages by filter") + super().__init__(parent, 'select', help = 'Select packages by filter') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("filter", help="Package filter string") + parser.add_argument('filter', help = 'Package filter string') async def _run(self, args: Namespace) -> None: filter = PackageFilterString(args.filter) if args.filter else None - for p in await self.distro.select(filter=filter): + for p in await self.distro.select(filter = filter): print(p.name) diff --git a/src/python/jw/pkg/cmds/pkg/NamedPkgsCmd.py b/src/python/jw/pkg/cmds/pkg/NamedPkgsCmd.py index 2dfe2c6a..bc5875a6 100644 --- a/src/python/jw/pkg/cmds/pkg/NamedPkgsCmd.py +++ b/src/python/jw/pkg/cmds/pkg/NamedPkgsCmd.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser -from argparse import Namespace, ArgumentParser - -from .Cmd import Cmd as Base from ..CmdPkg import CmdPkg as Parent +from .Cmd import Cmd as Base -class NamedPkgsCmd(Base): # export +class NamedPkgsCmd(Base): # export def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('names', nargs='*', help='Package names') + parser.add_argument('names', nargs = '*', help = 'Package names') diff --git a/src/python/jw/pkg/cmds/pkg/__init__.py b/src/python/jw/pkg/cmds/pkg/__init__.py index 475565ba..b1f52bd0 100644 --- a/src/python/jw/pkg/cmds/pkg/__init__.py +++ b/src/python/jw/pkg/cmds/pkg/__init__.py @@ -4,6 +4,6 @@ __all__ = detect_modules( package_name = __name__, package_path = __path__, namespace = globals(), - prefix = "Cmd", - skip = {"Cmd"}, -) # pyright: ignore[reportUnsupportedDunderAll] + prefix = 'Cmd', + skip = {'Cmd'}, +) # pyright: ignore[reportUnsupportedDunderAll] diff --git a/src/python/jw/pkg/cmds/platform/Cmd.py b/src/python/jw/pkg/cmds/platform/Cmd.py index 71e6326f..35f9031b 100644 --- a/src/python/jw/pkg/cmds/platform/Cmd.py +++ b/src/python/jw/pkg/cmds/platform/Cmd.py @@ -1,16 +1,13 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: - from ...lib.Distro import Distro from ..CmdPlatform import CmdPlatform from ..Cmd import Cmd as Base -class Cmd(Base): # export +class Cmd(Base): # export def __init__(self, parent: CmdPlatform, name: str, help: str) -> None: super().__init__(parent, name, help) diff --git a/src/python/jw/pkg/cmds/platform/CmdInfo.py b/src/python/jw/pkg/cmds/platform/CmdInfo.py index 537d8b63..9d649423 100644 --- a/src/python/jw/pkg/cmds/platform/CmdInfo.py +++ b/src/python/jw/pkg/cmds/platform/CmdInfo.py @@ -1,21 +1,23 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from ...lib.log import * from ...lib.Distro import Distro from ..Cmd import Cmd from ..CmdPlatform import CmdPlatform -class CmdInfo(Cmd): # export +class CmdInfo(Cmd): # export def __init__(self, parent: CmdPlatform) -> None: - super().__init__(parent, 'info', help='Retrieve information about target platform') + super().__init__( + parent, 'info', help = 'Retrieve information about target platform' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--format', default='%{cascade}', - help=f'Format string, expanding macros {", ".join(Distro.macros())}') + parser.add_argument( + '--format', + default = '%{cascade}', + help = f'Format string, expanding macros {", ".join(Distro.macros())}', + ) async def _run(self, args: Namespace) -> None: print(self.app.distro.expand_macros(args.format)) diff --git a/src/python/jw/pkg/cmds/platform/__init__.py b/src/python/jw/pkg/cmds/platform/__init__.py index 475565ba..b1f52bd0 100644 --- a/src/python/jw/pkg/cmds/platform/__init__.py +++ b/src/python/jw/pkg/cmds/platform/__init__.py @@ -4,6 +4,6 @@ __all__ = detect_modules( package_name = __name__, package_path = __path__, namespace = globals(), - prefix = "Cmd", - skip = {"Cmd"}, -) # pyright: ignore[reportUnsupportedDunderAll] + prefix = 'Cmd', + skip = {'Cmd'}, +) # pyright: ignore[reportUnsupportedDunderAll] diff --git a/src/python/jw/pkg/cmds/posix/Cmd.py b/src/python/jw/pkg/cmds/posix/Cmd.py index 9939a072..4f11c3df 100644 --- a/src/python/jw/pkg/cmds/posix/Cmd.py +++ b/src/python/jw/pkg/cmds/posix/Cmd.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations +from argparse import ArgumentParser from typing import TYPE_CHECKING -from ..Cmd import Cmd as Base +from ...CmdBase import CmdBase if TYPE_CHECKING: - from ..CmdPosix import CmdPosix + from ..CmdPosix import CmdPosix as Parent -class Cmd(Base): # export +class Cmd(CmdBase): # export - def __init__(self, parent: CmdPosix, name: str, help: str) -> None: + def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) def add_arguments(self, parser: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/posix/CmdCopy.py b/src/python/jw/pkg/cmds/posix/CmdCopy.py index 6eb5bfb5..89831019 100644 --- a/src/python/jw/pkg/cmds/posix/CmdCopy.py +++ b/src/python/jw/pkg/cmds/posix/CmdCopy.py @@ -1,34 +1,56 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from ...lib.util import copy from .Cmd import Cmd if TYPE_CHECKING: - from ..CmdPosix import CmdPosix - from argparse import Namespace, ArgumentParser + from argparse import ArgumentParser, Namespace -class CmdCopy(Cmd): # export + from ..CmdPosix import CmdPosix + +class CmdCopy(Cmd): # export def __init__(self, parent: CmdPosix) -> None: - super().__init__(parent, 'copy', help="Copy files") + super().__init__(parent, 'copy', help = 'Copy files') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('src', help='Source file URI') - parser.add_argument('dst', help='Destination file URI') - parser.add_argument('-o', '--owner', default=None, help='Destination file owner') - parser.add_argument('-g', '--group', default=None, help='Destination file group') - parser.add_argument('-m', '--mode', default=None, help='Destination file mode') - parser.add_argument('-F', '--fixed-strings', action='store_true', - help='Don\'t expand platform.expand_macros macros in and ') + parser.add_argument('src', help = 'Source file URI') + parser.add_argument('dst', help = 'Destination file URI') + parser.add_argument( + '-o', '--owner', default = None, help = 'Destination file owner' + ) + parser.add_argument( + '-g', '--group', default = None, help = 'Destination file group' + ) + parser.add_argument( + '-m', '--mode', default = None, help = 'Destination file mode' + ) + parser.add_argument( + '-F', + '--fixed-strings', + action = 'store_true', + help = "Don't expand platform.expand_macros macros in and ", + ) async def _run(self, args: Namespace) -> None: + def __expand(url: str) -> str: if args.fixed_strings: return url - return self.app.distro.expand_macros(url) - await copy(__expand(args.src), __expand(args.dst), - owner=args.owner, group=args.group, mode=int(args.mode, 0)) + ret = self.app.distro.expand_macros(url) + if not isinstance(ret, str): + raise Exception( + f'Expanding macros in "{url}" returned unexpected ret "{ret}"' + ) + return ret + + await copy( + __expand(args.src), + __expand(args.dst), + owner = args.owner, + group = args.group, + mode = int(args.mode, 0), + ) diff --git a/src/python/jw/pkg/cmds/posix/CmdTar.py b/src/python/jw/pkg/cmds/posix/CmdTar.py index 87239a27..88236530 100644 --- a/src/python/jw/pkg/cmds/posix/CmdTar.py +++ b/src/python/jw/pkg/cmds/posix/CmdTar.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from ..CmdPosix import CmdPosix as Parent +from .Cmd import Cmd as Base -from .Cmd import Cmd -from ..CmdPosix import CmdPosix +class CmdTar(Base): # export -class CmdTar(Cmd): # export - - def __init__(self, parent: CmdPosix) -> None: - super().__init__(parent, 'tar', help='Handle tar archives') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'tar', help = 'Handle tar archives') self.load_subcommands() def add_arguments(self, parser: ArgumentParser) -> None: diff --git a/src/python/jw/pkg/cmds/posix/tar/Cmd.py b/src/python/jw/pkg/cmds/posix/tar/Cmd.py index 277906ca..2ecbdb98 100644 --- a/src/python/jw/pkg/cmds/posix/tar/Cmd.py +++ b/src/python/jw/pkg/cmds/posix/tar/Cmd.py @@ -1,27 +1,29 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser +from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from ..Cmd import Cmd as Base +from ....CmdBase import CmdBase +from ....lib.FileContext import FileContext +from ....lib.ProcFilterGpg import ProcFilterGpg +from ....lib.TarIo import TarIo from ..CmdTar import CmdTar as Parent -from ....lib.TarIo import TarIo -from ....lib.ProcFilterGpg import ProcFilterGpg -from ....lib.FileContext import FileContext - -class Cmd(Base): # export +class Cmd(CmdBase): # export def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) self.__tar_io: None = None @asynccontextmanager - async def ctx(self, **kwargs) -> TarIo: - async with TarIo.create(src=self.app.args.archive_path, **kwargs) as ret: - ret.src.add_proc_filter(FileContext.Direction.In, ProcFilterGpg(ec=self.app.exec_context)) + async def ctx(self, **kwargs) -> AsyncIterator[TarIo]: + async with TarIo.create(src = self.app.args.archive_path, **kwargs) as ret: + ret.src.add_proc_filter( + FileContext.Direction.In, ProcFilterGpg(ec = self.app.exec_context) + ) yield ret def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('-f', '--archive-path', required=True, help='Archive path') + parser.add_argument( + '-f', '--archive-path', required = True, help = 'Archive path' + ) diff --git a/src/python/jw/pkg/cmds/posix/tar/CmdExtract.py b/src/python/jw/pkg/cmds/posix/tar/CmdExtract.py index c7bf7de0..53f9aa02 100644 --- a/src/python/jw/pkg/cmds/posix/tar/CmdExtract.py +++ b/src/python/jw/pkg/cmds/posix/tar/CmdExtract.py @@ -1,27 +1,20 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from ....lib.log import DEBUG, log +from .Cmd import Cmd, Parent -from .Cmd import Cmd -from ..CmdTar import CmdTar +class CmdExtract(Cmd): # export -from ....lib.FileContext import FileContext -from ....lib.log import * - -class CmdExtract(Cmd): # export - - def __init__(self, parent: CmdTar) -> None: - super().__init__(parent, 'x', help="Extract a tar archive") + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'x', help = 'Extract a tar archive') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('dst', help='Destination root URI') + parser.add_argument('dst', help = 'Destination root URI') async def _run(self, args: Namespace) -> None: - async with self.ctx(dst=args.dst) as ctx: + async with self.ctx(dst = args.dst) as ctx: paths = await ctx.extract(ctx.dst.root) log(DEBUG, f'Extracted {len(paths)} files') diff --git a/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py b/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py index 9f87af7e..fbc4ce50 100644 --- a/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py +++ b/src/python/jw/pkg/cmds/projects/BaseCmdPkgRelations.py @@ -106,8 +106,8 @@ class BaseCmdPkgRelations(Cmd): default = False, help = ( "Don't consider or output modules matching the os cascade in their " - "[build].exclude config" - ) + '[build].exclude config' + ), ) parser.add_argument( '--hide-self', diff --git a/src/python/jw/pkg/cmds/projects/CmdBuild.py b/src/python/jw/pkg/cmds/projects/CmdBuild.py index 886070e9..d1c49da0 100644 --- a/src/python/jw/pkg/cmds/projects/CmdBuild.py +++ b/src/python/jw/pkg/cmds/projects/CmdBuild.py @@ -1,65 +1,120 @@ -# -*- coding: utf-8 -*- +import datetime +import os +import re -import os, re, sys, subprocess, datetime, time -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from functools import lru_cache -from ...lib.util import get_profile_env -from ...lib.log import * -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects from ...App import Scope +from ...lib.log import DEBUG, ERR, NOTICE, log +from ...lib.util import get_profile_env, pretty_cmd +from .Cmd import Cmd, Parent -class CmdBuild(Cmd): # export +class CmdBuild(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'build', help='janware software project build tool') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'build', help = 'janware software project build tool') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build') - parser.add_argument('-n', '--dry-run', action='store_true', - default=False, help='Don\'t build anything, just print what would be done.') - parser.add_argument('-O', '--build-order', action='store_true', - default=False, help='Don\'t build anything, just print the build order.') - parser.add_argument('-I', '--ignore-deps', action='store_true', - default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line') - parser.add_argument('--env-reinit', action='store_true', - default=False, help='Source /etc/profile before each build step. Discard environment unless --env-keep is specified') - parser.add_argument('--env-keep', default='none', help='Comma seperated list of environment variables to keep, ' - + '"all" or "none", only meaningful if --env-reinit is specified') - parser.add_argument('target', default='all', help='Build target') - parser.add_argument('modules', nargs='+', default='', help='Modules to be built') + parser.add_argument( + '--exclude', + default = '', + help = 'Space seperated ist of modules to be excluded from build', + ) + parser.add_argument( + '-n', + '--dry-run', + action = 'store_true', + default = False, + help = "Don't build anything, just print what would be done.", + ) + parser.add_argument( + '-O', + '--build-order', + action = 'store_true', + default = False, + help = "Don't build anything, just print the build order.", + ) + parser.add_argument( + '-I', + '--ignore-deps', + action = 'store_true', + default = False, + help = ( + "Don't build dependencies, i.e. build only modules specified " + 'on the command line' + ), + ) + parser.add_argument( + '--env-reinit', + action = 'store_true', + default = False, + help = ( + 'Source /etc/profile before each build step. Discard environment ' + 'unless --env-keep is specified' + ), + ) + parser.add_argument( + '--env-keep', + default = 'none', + help = ( + 'Comma seperated list of environment variables to keep, ' + '"all" or "none", only meaningful if --env-reinit is specified' + ), + ) + parser.add_argument( + 'target', + default = 'all', + help = 'Build target', + ) + parser.add_argument( + 'modules', + nargs = '+', + default = '', + help = 'Modules to be built', + ) async def _run(self, args: Namespace) -> None: - @lru_cache(maxsize=None) - def read_deps(cur, prereq_type): + @lru_cache(maxsize = None) + def read_deps(cur, prereq_type: str) -> list[str]: # dep cache doesn't make a difference at all if prereq_type in dep_cache: if cur in dep_cache[prereq_type]: return dep_cache[prereq_type][cur] else: - dep_cache[prereq_type]: dict[str, str] = {} + dep_cache[prereq_type] = {} - ret = self.app.get_project_refs([ cur ], ['pkg.requires.jw'], - prereq_type, scope = Scope.Subtree, add_self=False, names_only=True) + ret = self.app.get_project_refs( + [cur], + ['pkg.requires.jw'], + prereq_type, + scope = Scope.Subtree, + add_self = False, + names_only = True, + ) log(DEBUG, 'prerequisites = ' + ' '.join(ret)) if cur in ret: ret.remove(cur) - log(DEBUG, 'inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(ret)) + + log( + DEBUG, + (f'Inserting {prereq_type}, prerequisites of {cur}: {" ".join(ret)}'), + ) + dep_cache[prereq_type][cur] = ret return ret def add_dep_tree(cur, prereq_types, tree, all_deps): - log(DEBUG, "adding prerequisites " + ' '.join(prereq_types) + " of module " + cur) + log(DEBUG, 'Adding deps "{" ".join(prereq_types)}" of module {cur)') if cur in all_deps: - log(DEBUG, 'already handled module ' + cur) + log(DEBUG, 'Already handled module "{cur}"') return 0 deps = set() all_deps.add(cur) for t in prereq_types: - log(DEBUG, "checking prereqisites of type " + t) + log(DEBUG, 'Checking deps of type "{t}"') deps.update(read_deps(cur, t)) for d in deps: add_dep_tree(d, prereq_types, tree, all_deps) @@ -70,17 +125,20 @@ class CmdBuild(Cmd): # export all_deps = set() dep_tree = {} for m in modules: - log(DEBUG, "--- adding dependency tree of module " + m) + log(DEBUG, '--- Adding dependency tree of module "{m}"') add_dep_tree(m, prereq_types, dep_tree, all_deps) while len(all_deps): # Find any leaf for d in all_deps: - if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself - break # found - else: # no Leaf found + # Dependency d doesn't have dependencies itself + if not len(dep_tree[d]): + break # found + else: # no Leaf found print(all_deps) - raise Exception("fatal: the dependencies between these modules are unresolvable") - order.append(d) # do it + raise Exception( + 'Fatal: the dependencies between these modules are unresolvable' + ) + order.append(d) # do it # bookkeep it all_deps.remove(d) for k in dep_tree.keys(): @@ -88,56 +146,68 @@ class CmdBuild(Cmd): # export dep_tree[k].remove(d) return 1 - async def run_make(module, target, cur_project, num_projects): + async def run_make(module, target, cur_project, num_projects) -> None: patt = self.app.is_excluded_from_build(module) if patt is not None: + title = f'---- {module}' log(NOTICE, f',{title} >') log(NOTICE, f'| Configured to skip build on platform >{patt}<') log(NOTICE, f'`{title} <') return - make_cmd = [ "make", target ] - wd = self.app.find_dir(module, pretty=False) - title = '---- [%d/%d]: Running "%s" in %s -' % (cur_project, num_projects, ' '.join(make_cmd), wd) + make_cmd = ['make', target] + wd = self.app.find_dir(module, pretty = False) + title = '---- [%d/%d]: Running "%s" in %s -' % ( + cur_project, + num_projects, + ' '.join(make_cmd), + wd, + ) mod_env = None if args.env_reinit: - keep: bool|list[str] = False + keep: bool | list[str] = False if args.env_keep is not None: match args.env_keep: case 'all': - keep=True + keep = True case 'none': - keep=False + keep = False case _: keep = args.env_keep.split(',') - mod_env = await get_profile_env(keep=keep) + mod_env = await get_profile_env(keep = keep) try: await self.app.exec_context.run( make_cmd, - wd=wd, - throw=True, - verbose=True, - mod_env=mod_env, - title=title + wd = wd, + throw = True, + verbose = True, + mod_env = mod_env, + title = title, ) except Exception as e: - log(ERR, f'Failed to make target "{target}" in module "{module}" below base {self.app.projs_root}: {str(e)}') + log( + ERR, + ( + f'Failed to make target "{target}" in module "{module}" ' + f'below base {self.app.projs_root}: {str(e)}' + ), + ) raise async def run_make_on_modules(modules, order, target): cur_project = 0 num_projects = len(order) - if target in ["clean", "distclean"]: + if target in ['clean', 'distclean']: for m in reversed(order): cur_project += 1 await run_make(m, target, cur_project, num_projects) if m in modules: modules.remove(m) if not len(modules): - log(NOTICE, "All modules cleaned") + log(NOTICE, 'All modules cleaned') return else: for m in order: @@ -146,7 +216,7 @@ class CmdBuild(Cmd): # export async def run(args): - log(DEBUG, "----------------------------------------- running ", ' '.join(sys.argv)) + log(DEBUG, f'-------------------------------------- running {pretty_cmd()}') modules = args.modules exclude = args.exclude.split() @@ -154,19 +224,19 @@ class CmdBuild(Cmd): # export env_exclude = os.getenv('BUILD_EXCLUDE', '') if len(env_exclude): - log(NOTICE, "Exluding modules from environment: " + env_exclude) - exclude += " " + env_exclude + log(NOTICE, 'Exluding modules from environment: ' + env_exclude) + exclude += ' ' + env_exclude # -- build order = [] - glob_prereq_types = [ "build" ] - if re.match("pkg-.*", target) is not None: - glob_prereq_types = [ "build", "run", "release", "devel" ] + glob_prereq_types = ['build'] + if re.match('pkg-.*', target) is not None: + glob_prereq_types = ['build', 'run', 'release', 'devel'] if target != 'order' and not args.build_order: - log(NOTICE, "Using prerequisite types " + ' '.join(glob_prereq_types)) - log(NOTICE, "Calculating order for modules ... ") + log(NOTICE, 'Using prerequisite types ' + ' '.join(glob_prereq_types)) + log(NOTICE, 'Calculating order for modules ... ') calculate_order(order, modules, glob_prereq_types) if args.ignore_deps: @@ -177,18 +247,24 @@ class CmdBuild(Cmd): # export exit(0) cur_project = 0 - log(NOTICE, "Building target %s in %d projects:" % (target, len(order))) + log(NOTICE, 'Building target %s in %d projects:' % (target, len(order))) for m in order: cur_project += 1 - log(NOTICE, " %3d %s" % (cur_project, m)) + log(NOTICE, ' %3d %s' % (cur_project, m)) if args.dry_run: exit(0) await run_make_on_modules(modules, order, target) - log(NOTICE, 'Build done at %s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) + log( + NOTICE, + ( + 'Build done at %s' % + (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + ), + ) - dep_cache: dict[dict[str, str]] = {} + dep_cache: dict[str, dict[str, list[str]]] = {} await run(args) diff --git a/src/python/jw/pkg/cmds/projects/CmdCanonicalizeRemotes.py b/src/python/jw/pkg/cmds/projects/CmdCanonicalizeRemotes.py index 9ffa4560..03ea8ea2 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCanonicalizeRemotes.py +++ b/src/python/jw/pkg/cmds/projects/CmdCanonicalizeRemotes.py @@ -1,54 +1,72 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + +from argparse import ArgumentParser, Namespace from typing import TYPE_CHECKING -from argparse import Namespace, ArgumentParser - -from ...lib.log import * from ...lib.base import InputMode -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects -from ...App import Scope +from ...lib.log import NOTICE, log +from .Cmd import Cmd, Parent if TYPE_CHECKING: from ...lib.base import Result -class CmdCanonicalizeRemotes(Cmd): # export +class CmdCanonicalizeRemotes(Cmd): # export def __rewrite_url(self, url: str) -> str: return url.replace('/srv/git', '') - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'canonicalize-remotes', help='Streamline janware Git remotes') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'canonicalize-remotes', help = 'Streamline janware Git remotes' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('-n', '--dry-run', default=False, action='store_true', help='Only log what would be done') + parser.add_argument( + '-n', + '--dry-run', + default = False, + action = 'store_true', + help = 'Only log what would be done', + ) async def _run(self, args: Namespace) -> None: - async def git(cmd: list[str], ro=False, throw=True) -> Result: + async def git(cmd: list[str], ro = False, throw = True) -> Result: cmd = ['/usr/bin/git', *cmd] log(NOTICE, f'-- {" ".join(cmd)}') if ro or not args.dry_run: - return await self.app.exec_context.run(cmd, cmd_input=InputMode.NonInteractive, throw=throw) - remotes: dict[str, dict[str, str]] = {} - stdout, stderr, status = await git(['remote', '-v'], ro=True) - for line in stdout.decode().splitlines(): + return await self.app.exec_context.run( + cmd, cmd_input = InputMode.NonInteractive, throw = throw + ) + return Result(b'', None, 0) + + remotes: dict[str, dict[str, str | list[str]]] = {} + result = await git(['remote', '-v'], ro = True) + for line in result.stdout_str.splitlines(): name, url, fp = line.split() remote = remotes.setdefault(name, {}) key = 'url' if fp == '(fetch)' else 'pushurl' fpurls = remote.setdefault(key, []) + assert isinstance(fpurls, list) fpurls.append(url) - for remote, config in remotes.items(): + for name, remote in remotes.items(): dirty_keys: set[str] = set() - for key, urls in config.items(): + for key, urls in remote.items(): for url in urls: if url != self.__rewrite_url(url): dirty_keys.add(key) for key in dirty_keys: - urls = config[key] - await git(['config', '--unset-all', f'remote.{remote}.{key}'], throw=False) + urls = remote[key] + await git( + ['config', '--unset-all', f'remote.{name}.{key}'], throw = False + ) for url in urls: - await git(['config', '--add', f'remote.{remote}.{key}', self.__rewrite_url(url)]) + await git( + [ + 'config', + '--add', + f'remote.{name}.{key}', + self.__rewrite_url(url), + ] + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdCflags.py b/src/python/jw/pkg/cmds/projects/CmdCflags.py index 913c899b..9434d922 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCflags.py +++ b/src/python/jw/pkg/cmds/projects/CmdCflags.py @@ -1,23 +1,26 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser - -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects from ...App import Scope +from .Cmd import Cmd, Parent -class CmdCflags(Cmd): # export +class CmdCflags(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'cflags', help='cflags') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'cflags', help = 'cflags') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], 'build', - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + 'build', + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) out = [] for m in reversed(deps): path = self.app.find_dir(m, ['/include']) diff --git a/src/python/jw/pkg/cmds/projects/CmdCheck.py b/src/python/jw/pkg/cmds/projects/CmdCheck.py index 8668c85d..9a8ee871 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCheck.py +++ b/src/python/jw/pkg/cmds/projects/CmdCheck.py @@ -1,24 +1,28 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from ...lib.log import NOTICE, log +from .Cmd import Cmd, Parent -from ...lib.log import * -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdCheck(Cmd): # export -class CmdCheck(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'check', help='Check for circular dependencies between given modules') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'check', + help = 'Check for circular dependencies between given modules', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('-f', '--flavour', nargs='?', default = 'build') + parser.add_argument('module', nargs = '*', help = 'Modules') + parser.add_argument('-f', '--flavour', nargs = '?', default = 'build') async def _run(self, args: Namespace) -> None: - path = self.app.find_circular_deps(args.module, args.flavour) - if path: - log(NOTICE, f'Found circular dependency in flavour {args.flavour}:', ' -> '.join(path)) + if self.app.find_circular_deps(args.module, args.flavour): + log(NOTICE, f'Found circular dependency in flavour {args.flavour}') exit(1) - log(NOTICE, f'No circular dependency found for flavour {args.flavour} in modules:', ' '.join(args.module)) + log( + NOTICE, + f'No circular dependency found for flavour {args.flavour} in modules:', + ' '.join(args.module), + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdCommands.py b/src/python/jw/pkg/cmds/projects/CmdCommands.py index 7e5bf1cd..a0694d46 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCommands.py +++ b/src/python/jw/pkg/cmds/projects/CmdCommands.py @@ -1,23 +1,23 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdCommands(Cmd): # export -class CmdCommands(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'commands', help='List available commands') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'commands', help = 'List available commands') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) async def _run(self, args: Namespace) -> None: - import sys, re, os, glob - this_dir = os.path.dirname(sys.modules[__name__].__file__) + import glob + import os + import re + + this_dir = os.path.dirname(__file__) ret = [] - for file_name in glob.glob('Cmd*.py', root_dir=this_dir): + for file_name in glob.glob('Cmd*.py', root_dir = this_dir): cc_name = re.sub(r'^Cmd|\.py', '', file_name) name = re.sub(r'(? None: - super().__init__(parent, 'exepath', help='exepath') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'exepath', help = 'exepath') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + ['run', 'build', 'devel'], + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) out = [] for m in deps: path = self.app.find_dir(m, ['/bin']) diff --git a/src/python/jw/pkg/cmds/projects/CmdGetAuthInfo.py b/src/python/jw/pkg/cmds/projects/CmdGetAuthInfo.py index 22f55e34..2c0167c4 100644 --- a/src/python/jw/pkg/cmds/projects/CmdGetAuthInfo.py +++ b/src/python/jw/pkg/cmds/projects/CmdGetAuthInfo.py @@ -1,44 +1,72 @@ -# -*- coding: utf-8 -*- +import os +import re -import re, os -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace -from ...lib.log import * +from ...lib.log import DEBUG, log from ...lib.Uri import Uri -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdGetAuthInfo(Cmd): # export +class CmdGetAuthInfo(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'get-auth-info', help='Try to retrieve authentication information from the source tree') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'get-auth-info', + help = 'Try to retrieve authentication information from the source tree', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--only-values', default=False, action='store_true', - help='Don\'t prefix values by "="') - parser.add_argument('--username', default=False, action='store_true', - help='Show user name') - parser.add_argument('--password', default=False, action='store_true', - help='Show password') - parser.add_argument('--remote-owner-base', default=False, action='store_true', - help='Show remote base URL for owner jw-pkg was cloned from') - parser.add_argument('--remote-base', default=False, action='store_true', - help='Show remote base URL') + parser.add_argument( + '--only-values', + default = False, + action = 'store_true', + help = 'Don\'t prefix values by "="', + ) + parser.add_argument( + '--username', + default = False, + action = 'store_true', + help = 'Show user name' + ) + parser.add_argument( + '--password', + default = False, + action = 'store_true', + help = 'Show password' + ) + parser.add_argument( + '--remote-owner-base', + default = False, + action = 'store_true', + help = 'Show remote base URL for owner jw-pkg was cloned from', + ) + parser.add_argument( + '--remote-base', + default = False, + action = 'store_true', + help = 'Show remote base URL', + ) async def _run(self, args: Namespace) -> None: keys = ['username', 'password'] # --- Milk jw-pkg repo - jw_pkg_dir = self.app.find_dir('jw-pkg', pretty=False) + jw_pkg_dir = self.app.find_dir('jw-pkg', pretty = False) if not os.path.isdir(jw_pkg_dir + '/.git'): log(DEBUG, f'jw-pkg directory is not a Git repo: {jw_pkg_dir}') return - remotes, stderr, status = (await self.app.exec_context.run(['git', '-C', jw_pkg_dir, 'remote', '-v'])).decode() + git_result = await self.app.exec_context.run( + ['git', '-C', jw_pkg_dir, 'remote', '-v'] + ) result: dict[str, str] = {} - for line in remotes.splitlines(): + for line in git_result.stdout_str.splitlines(): name, url, typ = re.split(r'\s+', line) - if name == 'origin' and typ in ['(pull)', '(fetch)']: # TODO: Use other remotes, too? + if name == 'origin' and typ in [ + '(pull)', + '(fetch)', + ]: # TODO: Use other remotes, too? parsed = Uri(url) for key in keys: result[key] = getattr(parsed, key) @@ -52,7 +80,7 @@ class CmdGetAuthInfo(Cmd): # export # --- Print results for key, val in result.items(): - if getattr(args, key, None) != True: + if not getattr(args, key, None): continue if val is None: continue diff --git a/src/python/jw/pkg/cmds/projects/CmdGetval.py b/src/python/jw/pkg/cmds/projects/CmdGetval.py index 0e13d200..58cc5d50 100644 --- a/src/python/jw/pkg/cmds/projects/CmdGetval.py +++ b/src/python/jw/pkg/cmds/projects/CmdGetval.py @@ -1,18 +1,19 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdGetval(Cmd): # export -class CmdGetval(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'getval', help='Get value from project config') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'getval', help = 'Get value from project config') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--project', default = None, help = 'Project name, default is name of project\'s topdir') + parser.add_argument( + '--project', + default = None, + help = "Project name, default is name of project's topdir", + ) parser.add_argument('section', default = '', help = 'Config section') parser.add_argument('key', default = '', help = 'Config key') diff --git a/src/python/jw/pkg/cmds/projects/CmdHtdocsDir.py b/src/python/jw/pkg/cmds/projects/CmdHtdocsDir.py index 2d3cb104..abff33e6 100644 --- a/src/python/jw/pkg/cmds/projects/CmdHtdocsDir.py +++ b/src/python/jw/pkg/cmds/projects/CmdHtdocsDir.py @@ -1,21 +1,21 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdHtdocsDir(Cmd): # export -class CmdHtdocsDir(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'htdocs-dir', help='Print source directory containing document root of a given module') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'htdocs-dir', + help = 'Print source directory containing document root of a given module', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: r = [] for m in args.module: r.append(self.app.htdocs_dir(m)) - print(' '.join(r)) diff --git a/src/python/jw/pkg/cmds/projects/CmdLdflags.py b/src/python/jw/pkg/cmds/projects/CmdLdflags.py index 5c2ca2b5..e19dcfb4 100644 --- a/src/python/jw/pkg/cmds/projects/CmdLdflags.py +++ b/src/python/jw/pkg/cmds/projects/CmdLdflags.py @@ -1,27 +1,42 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdLdflags(Cmd): # export +class CmdLdflags(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'ldflags', help='ldflags') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'ldflags', help = 'ldflags') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') - parser.add_argument('--exclude', action='append', help='Exclude Modules', default=[]) - parser.add_argument('-s', '--add-self', action='store_true', - default=False, help='Include libflags of specified modules, too, not only their dependencies') + parser.add_argument('module', nargs = '*', help = 'Modules') + parser.add_argument( + '--exclude', action = 'append', help = 'Exclude Modules', default = [] + ) + parser.add_argument( + '-s', + '--add-self', + action = 'store_true', + default = False, + help = ( + 'Include libflags of specified modules, too, ' + 'not only their dependencies' + ), + ) # -L needs to contain more paths than libs linked with -l would require - def __get_ldpathflags(self, names: list[str], exclude: list[str] = []) -> str: - deps = self.app.get_project_refs(names, ['pkg.requires.jw'], 'build', - scope = Scope.Subtree, add_self=True, names_only=True) + def __get_ldpathflags( + self, names: list[str], exclude: list[str] = [] + ) -> str | None: + deps = self.app.get_project_refs( + names, + ['pkg.requires.jw'], + 'build', + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) ret = [] for m in deps: if m in exclude: @@ -35,11 +50,17 @@ class CmdLdflags(Cmd): # export ret.append('-L' + path) if not ret: return None - return(' '.join(ret)) + return ' '.join(ret) async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], 'build', - scope = Scope.One, add_self=args.add_self, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + 'build', + scope = Scope.One, + add_self = args.add_self, + names_only = True, + ) out = [] for m in reversed(deps): if m in args.exclude: diff --git a/src/python/jw/pkg/cmds/projects/CmdLdlibpath.py b/src/python/jw/pkg/cmds/projects/CmdLdlibpath.py index c5fc3b16..e4fa0028 100644 --- a/src/python/jw/pkg/cmds/projects/CmdLdlibpath.py +++ b/src/python/jw/pkg/cmds/projects/CmdLdlibpath.py @@ -1,23 +1,26 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdLdlibpath(Cmd): # export +class CmdLdlibpath(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'ldlibpath', help='ldlibpath') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'ldlibpath', help = 'ldlibpath') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], [ 'run', 'build', 'devel' ], - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + ['run', 'build', 'devel'], + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) out = [] for m in deps: path = self.app.find_dir(m, ['/lib']) diff --git a/src/python/jw/pkg/cmds/projects/CmdLibname.py b/src/python/jw/pkg/cmds/projects/CmdLibname.py index ad5cc12f..2f76022f 100644 --- a/src/python/jw/pkg/cmds/projects/CmdLibname.py +++ b/src/python/jw/pkg/cmds/projects/CmdLibname.py @@ -1,18 +1,15 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdLibname(Cmd): # export -class CmdLibname(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'libname', help='libname') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'libname', help = 'libname') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: print(self.app.get_libname(args.module)) diff --git a/src/python/jw/pkg/cmds/projects/CmdListRepos.py b/src/python/jw/pkg/cmds/projects/CmdListRepos.py index 9f2bb97a..0aa1cecb 100644 --- a/src/python/jw/pkg/cmds/projects/CmdListRepos.py +++ b/src/python/jw/pkg/cmds/projects/CmdListRepos.py @@ -1,62 +1,105 @@ -# -*- coding: utf-8 -*- +import os +import re -import re, os -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace -from ...lib.util import get_username, get_password, run_curl -from ...lib.log import * +from ...lib.base import Input +from ...lib.log import DEBUG, log from ...lib.Uri import Uri -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from ...lib.util import get_password, get_username, run_curl_into +from .Cmd import Cmd, Parent -class CmdListRepos(Cmd): # export +class CmdListRepos(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'list-repos', help='Query a remote GIT server for repositories') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'list-repos', help = 'Query a remote GIT server for repositories' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('base_url', help='Base URL of all Git repositories without user part') - parser.add_argument('--username', help='Username for SSH or HTTP authentication, don\'t specify for unauthenticated', default=None) - parser.add_argument('--askpass', help='Program to echo password for SSH or HTTP authentication, don\'t specify for unauthenticated', default=None) - parser.add_argument('--from-owner', help='List from-owner\'s projects', default='janware') + parser.add_argument( + 'base_url', help = 'Base URL of all Git repositories without user part' + ) + parser.add_argument( + '--username', + help = ( + "Username for SSH or HTTP authentication, don't " + 'specify for unauthenticated' + ), + default = None, + ) + parser.add_argument( + '--askpass', + help = ( + 'Program to echo password for SSH or HTTP authentication, ' + "don't specify for unauthenticated" + ), + default = None, + ) + parser.add_argument( + '--from-owner', help = "List from-owner's projects", default = 'janware' + ) async def _run(self, args: Namespace) -> None: base_url = Uri(args.base_url) - askpass_env=['GIT_ASKPASS', 'SSH_ASKPASS'] - username = await get_username(args=args, url=args.base_url, askpass_env=askpass_env) + askpass_env = ['GIT_ASKPASS', 'SSH_ASKPASS'] + username = await get_username( + args = args, url = args.base_url, askpass_env = askpass_env + ) password = None if username is not None: - password = await get_password(args=args, url=args.base_url, askpass_env=askpass_env) + password = await get_password( + args = args, url = args.base_url, askpass_env = askpass_env + ) match base_url.scheme: case 'ssh': if re.match(r'ssh://.*devgit\.janware\.com/', args.base_url): - from jw.pkg.lib.ec.SSHClient import SSHClient, ssh_client + from jw.pkg.lib.ec.SSHClient import ssh_client + if username is not None: base_url.set_username(username) if password is not None: base_url.set_password(password) - ssh = ssh_client(base_url, interactive=self.app.interactive, verbose_default=self.app.verbose) - cmd = ['/opt/jw-pkg/bin/git-srv-admin.sh', '-u', args.from_owner, '-j', 'list-personal-projects'] + ssh = ssh_client( + base_url, + interactive = self.app.interactive, + verbose_default = self.app.verbose, + ) + cmd = [ + '/opt/jw-pkg/bin/git-srv-admin.sh', + '-u', + args.from_owner, + '-j', + 'list-personal-projects', + ] result = await ssh.run(cmd) - print('\n'.join(result.stdout.decode().splitlines())) + print('\n'.join(result.stdout_str.splitlines())) return case 'https': from jw.pkg.lib.base import InputMode - cmd_input = InputMode.NonInteractive + + cmd_input: Input = InputMode.NonInteractive if re.match(r'https://github.com', args.base_url): curl_args = [ '-f', - '-H', 'Accept: application/vnd.github+json', - '-H', 'X-GitHub-Api-Version: 2022-11-28', + '-H', + 'Accept: application/vnd.github+json', + '-H', + 'X-GitHub-Api-Version: 2022-11-28', ] if password is not None: - assert username is not None, f'Assertion failed: username is empty but password isn\'t for "{args.base_url}"' + assert username is not None, ( + 'Assertion failed: username is empty but password ' + 'isn\'t for "{args.base_url}"' + ) cmd_input = (f'-u {username}:{password}').encode('utf-8') curl_args.extend(['-K-']) - curl_args.append(f'https://api.github.com/users/{args.from_owner}/repos') - repos, stderr, status = await run_curl(curl_args, cmd_input=cmd_input, parse_json=True) + curl_args.append( + f'https://api.github.com/users/{args.from_owner}/repos' + ) + repos = await run_curl_into(list, curl_args, cmd_input = cmd_input) for repo in repos: print(repo['name']) return @@ -69,33 +112,44 @@ class CmdListRepos(Cmd): # export cmd_input = (f'-u {username}:{password}').encode('utf-8') curl_args.extend(['-K-']) for entities_dir in ['orgs', 'users']: - api_url = f'{args.base_url}/api/v1/{entities_dir}/{args.from_owner}/repos' + api_url = ( + f'{args.base_url}/api/v1/{entities_dir}/' + f'{args.from_owner}/repos' + ) try: tried.append(api_url) - repos, stderr, status = await run_curl(curl_args + [api_url], cmd_input=cmd_input, parse_json=True) + repos = await run_curl_into( + list, + curl_args + [api_url], + cmd_input = cmd_input, + ) for repo in repos: print(repo['name']) break except Exception as e: msg = 'curl {} failed ({}), trying next'.format( - ' '.join(curl_args + [api_url]), - str(e) + ' '.join(curl_args + [api_url]), str(e) ) log(DEBUG, msg) tried[-1] += ': ' + msg raise else: - raise RuntimeError(f'Failed to fetch repository list from assumed Forgejo instance at {args.base_url}, tried {', '.join(tried)}') + raise RuntimeError( + f'Failed to fetch repository list from assumed Forgejo ' + f'instance at {args.base_url}, tried {", ".join(tried)}' + ) return if os.path.isdir(args.base_url): - for subdir in ["." , args.from_owner]: + for subdir in ['.', args.from_owner]: out = [] - for entry in os.scandir(args.base_url + "/" + subdir): + for entry in os.scandir(args.base_url + '/' + subdir): path = entry.path - if os.path.isdir(path + "/.git") or os.path.exists(path + "/HEAD"): + if os.path.isdir(path + '/.git') or os.path.exists(path + '/HEAD'): out.append(path) if out: print('\n'.join(out)) break return - raise Exception(f'Don\'t know how to enumerate Git repos at base url {args.base_url}') + raise Exception( + f"Don't know how to enumerate Git repos at base url {args.base_url}" + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdModules.py b/src/python/jw/pkg/cmds/projects/CmdModules.py index 63f28800..9b3b4c6e 100644 --- a/src/python/jw/pkg/cmds/projects/CmdModules.py +++ b/src/python/jw/pkg/cmds/projects/CmdModules.py @@ -1,44 +1,57 @@ -# -*- coding: utf-8 -*- +import re -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace -from ...lib.log import * -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from ...lib.log import DEBUG, log +from .Cmd import Cmd, Parent -class CmdModules(Cmd): # export +class CmdModules(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'modules', help='Query existing janware packages') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'modules', help = 'Query existing janware packages') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('-F', '--filter', nargs='?', default=None, help='Key-value pairs, seperated by commas, to be searched for in project.conf') + parser.add_argument( + '-F', + '--filter', + nargs = '?', + default = None, + help = + 'Key-value pairs, seperated by commas, to be searched for in project.conf', + ) async def _run(self, args: Namespace) -> None: import pathlib + proj_root = self.app.projs_root - log(DEBUG, "proj_root = " + proj_root) + log(DEBUG, 'proj_root = ' + proj_root) path = pathlib.Path(self.app.projs_root) modules = [p.parents[1].name for p in path.glob('*/make/project.conf')] - log(DEBUG, "modules = ", modules) + log(DEBUG, 'modules = ', modules) out = [] - filters = None if args.filter is None else [re.split("=", f) for f in re.split(",", args.filter)] + filters = ( + None if args.filter is None else + [re.split('=', f) for f in re.split(',', args.filter)] + ) for m in modules: - if filters: - for f in filters: - path = f[0].rsplit('.') - if len(path) > 1: - sec = path[0] - key = path[1] - else: - sec = None - key = path[0] - val = self.app.get_value(m, sec, key) - log(DEBUG, 'Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val)) - if val and val == f[1]: - out.append(m) - break - else: + if not filters: out.append(m) + continue + for f in filters: + path_str = f[0].rsplit('.') + if len(path_str) > 1: + sec = path_str[0] + key = path_str[1] + else: + sec = None + key = path_str[0] + val = self.app.get_value(m, sec, key) + log( + DEBUG, + 'Checking in {} if {}="{}", is "{}"'.format(m, f[0], f[1], val), + ) + if val and val == f[1]: + out.append(m) + break print(' '.join(out)) diff --git a/src/python/jw/pkg/cmds/projects/CmdPath.py b/src/python/jw/pkg/cmds/projects/CmdPath.py index d257f022..95b91f29 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPath.py +++ b/src/python/jw/pkg/cmds/projects/CmdPath.py @@ -1,26 +1,29 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdPath(Cmd): # export +class CmdPath(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'path', help='path') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'path', help = 'path') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], 'run', - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + 'run', + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) out = [] for m in deps: - path = self.app.find_dir(m, '/bin') + path = self.app.find_dir(m, ['/bin']) if path is not None: out.append(path) print(':'.join(out)) diff --git a/src/python/jw/pkg/cmds/projects/CmdPkgConflicts.py b/src/python/jw/pkg/cmds/projects/CmdPkgConflicts.py index b5e18998..232aabc1 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPkgConflicts.py +++ b/src/python/jw/pkg/cmds/projects/CmdPkgConflicts.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser - from .BaseCmdPkgRelations import BaseCmdPkgRelations as Base +from .Cmd import Parent -class CmdPkgConflicts(Base): # export +class CmdPkgConflicts(Base): # export - def __init__(self, parent: Base) -> None: - super().__init__(parent, 'conflicts', help='Print packages conflicting with a given package') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'conflicts', + help = 'Print packages conflicting with a given package' + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdPkgProvides.py b/src/python/jw/pkg/cmds/projects/CmdPkgProvides.py index 091f9fd8..7f77e7da 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPkgProvides.py +++ b/src/python/jw/pkg/cmds/projects/CmdPkgProvides.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser - from .BaseCmdPkgRelations import BaseCmdPkgRelations as Base +from .Cmd import Parent -class CmdPkgProvides(Base): # export +class CmdPkgProvides(Base): # export - def __init__(self, parent: Base) -> None: - super().__init__(parent, 'provides', help='Print packages and capabilities provided by a given package') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'provides', + help = 'Print packages and capabilities provided by a given package', + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdPkgRequires.py b/src/python/jw/pkg/cmds/projects/CmdPkgRequires.py index 2e766286..caae655c 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPkgRequires.py +++ b/src/python/jw/pkg/cmds/projects/CmdPkgRequires.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser - from .BaseCmdPkgRelations import BaseCmdPkgRelations as Base +from .Cmd import Parent -class CmdPkgRequires(Base): # export +class CmdPkgRequires(Base): # export - def __init__(self, parent: Base) -> None: - super().__init__(parent, 'requires', help='Print packages required for a given package') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'requires', help = 'Print packages required for a given package' + ) diff --git a/src/python/jw/pkg/cmds/projects/CmdProjDir.py b/src/python/jw/pkg/cmds/projects/CmdProjDir.py index 99d76c51..2893b282 100644 --- a/src/python/jw/pkg/cmds/projects/CmdProjDir.py +++ b/src/python/jw/pkg/cmds/projects/CmdProjDir.py @@ -1,19 +1,18 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from ...lib.log import WARNING, log +from .Cmd import Cmd, Parent -from ...lib.log import * -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdProjDir(Cmd): # export -class CmdProjDir(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'proj-dir', help='Print directory of a given package') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'proj-dir', help = 'Print directory of a given package' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: out = [] diff --git a/src/python/jw/pkg/cmds/projects/CmdPythonpath.py b/src/python/jw/pkg/cmds/projects/CmdPythonpath.py index 3d95fdc3..cd9f1176 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPythonpath.py +++ b/src/python/jw/pkg/cmds/projects/CmdPythonpath.py @@ -1,23 +1,28 @@ -# -*- coding: utf-8 -*- - -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdPythonpath(Cmd): # export +class CmdPythonpath(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'pythonpath', help='Generate PYTHONPATH for given modules') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'pythonpath', help = 'Generate PYTHONPATH for given modules' + ) def add_arguments(self, p: ArgumentParser) -> None: super().add_arguments(p) - p.add_argument('module', help='Modules', nargs='*') + p.add_argument('module', help = 'Modules', nargs = '*') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], [ 'run', 'build' ], - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + ['run', 'build'], + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) out = [] for m in deps: path = self.app.find_dir(m, ['src/python', 'tools/python']) diff --git a/src/python/jw/pkg/cmds/projects/CmdPythonpathOrig.py b/src/python/jw/pkg/cmds/projects/CmdPythonpathOrig.py index 18a98a40..1c6d6ee9 100644 --- a/src/python/jw/pkg/cmds/projects/CmdPythonpathOrig.py +++ b/src/python/jw/pkg/cmds/projects/CmdPythonpathOrig.py @@ -1,30 +1,35 @@ -# -*- coding: utf-8 -*- +import os -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from .Cmd import Cmd, Parent -class CmdPythonpathOrig(Cmd): # export +class CmdPythonpathOrig(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'pythonpath_orig', help='pythonpath') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'pythonpath_orig', help = 'pythonpath') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: - deps = self.app.get_project_refs(args.module, ['pkg.requires.jw'], [ 'run', 'build' ], - scope = Scope.Subtree, add_self=True, names_only=True) + deps = self.app.get_project_refs( + args.module, + ['pkg.requires.jw'], + ['run', 'build'], + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) r = '' for m in deps: - pd = self.app.find_dir(m, pretty=False) + pd = self.app.find_dir(m, pretty = False) if pd is None: continue - for subdir in [ 'src/python', 'tools/python' ]: - cand = pd + "/" + subdir - if isdir(cand): + for subdir in ['src/python', 'tools/python']: + cand = pd + '/' + subdir + if os.path.isdir(cand): r = r + ':' + cand print(r[1:]) diff --git a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py index 9721f207..5d812b84 100644 --- a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py +++ b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py @@ -1,45 +1,60 @@ -# -*- coding: utf-8 -*- - -from typing import Iterable -from argparse import Namespace, ArgumentParser +from argparse import ArgumentParser, Namespace from ...App import Scope -from ...lib.log import * -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +from ...lib.log import DEBUG, log +from .Cmd import Cmd, Parent # TODO: seems at least partly redundant to CmdPkgRequires / print_pkg_relations -class CmdRequiredOsPkg(Cmd): # export +class CmdRequiredOsPkg(Cmd): # export - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'required-os-pkg', help='List distribution packages required for a package') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'required-os-pkg', + help = 'List distribution packages required for a package', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('flavours', help='Dependency flavours', default='build') - parser.add_argument('modules', nargs='*', help='Modules') - parser.add_argument('--skip-excluded', action='store_true', default=False, - help='Output empty prerequisite list for excluded modules') - parser.add_argument('--quote', action='store_true', default=False, - help='Put double quotes around each listed dependency') + parser.add_argument('flavours', help = 'Dependency flavours', default = 'build') + parser.add_argument('modules', nargs = '*', help = 'Modules') + parser.add_argument( + '--skip-excluded', + action = 'store_true', + default = False, + help = 'Output empty prerequisite list for excluded modules', + ) + parser.add_argument( + '--quote', + action = 'store_true', + default = False, + help = 'Put double quotes around each listed dependency', + ) async def _run(self, args: Namespace) -> None: modules = args.modules flavours = set(args.flavours.split(',')) if 'build' in flavours: - # TODO: This adds too much. Only the run dependencies of the build dependencies would be needed. + # TODO: This adds too much. Only the run dependencies of the build + # dependencies would be needed. flavours.add('run') if 'release' in flavours: flavours |= set(['run', 'devel', 'build']) - log(DEBUG, "flavours = " + args.flavours) - deps = self.app.get_project_refs(modules, ['pkg.requires.jw'], flavours, - scope = Scope.Subtree, add_self=True, names_only=True) + log(DEBUG, 'flavours = ' + args.flavours) + deps = self.app.get_project_refs( + modules, + ['pkg.requires.jw'], + list(flavours), + scope = Scope.Subtree, + add_self = True, + names_only = True, + ) if args.skip_excluded: for d in deps: if self.app.is_excluded_from_build(d) is not None: deps.remove(d) subsecs = self.app.distro.os_cascade - log(DEBUG, "subsecs = ", subsecs) + log(DEBUG, 'subsecs = ', subsecs) requires: set[str] = set() for sec in subsecs: for flavour in flavours: @@ -47,6 +62,6 @@ class CmdRequiredOsPkg(Cmd): # export if vals: requires |= set(vals) if args.quote: - requires = [f'"{dep}"' for dep in requires] + out = [f'"{dep}"' for dep in requires] # TODO: add all not in build tree as -devel - print(' '.join(requires)) + print(' '.join(out)) diff --git a/src/python/jw/pkg/cmds/projects/CmdSummary.py b/src/python/jw/pkg/cmds/projects/CmdSummary.py index d2573fb7..ec340d1e 100644 --- a/src/python/jw/pkg/cmds/projects/CmdSummary.py +++ b/src/python/jw/pkg/cmds/projects/CmdSummary.py @@ -1,23 +1,22 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdSummary(Cmd): # export -class CmdSummary(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'summary', help='Print summary description of given modules') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, 'summary', help = 'Print summary description of given modules' + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: r = [] for m in args.module: - summary = self.app.get_value(m, "summary", None) + summary = self.app.get_value(m, 'summary', None) if summary is not None: r.append(summary) print(' '.join(r)) diff --git a/src/python/jw/pkg/cmds/projects/CmdTest.py b/src/python/jw/pkg/cmds/projects/CmdTest.py index 5eb61088..7c7e03d7 100644 --- a/src/python/jw/pkg/cmds/projects/CmdTest.py +++ b/src/python/jw/pkg/cmds/projects/CmdTest.py @@ -1,18 +1,15 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdTest(Cmd): # export -class CmdTest(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'test', help='Test') + def __init__(self, parent: Parent) -> None: + super().__init__(parent, 'test', help = 'Test') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('blah', default='', help='The blah argument') + parser.add_argument('blah', default = '', help = 'The blah argument') async def _run(self, args: Namespace) -> None: - print("blah = " + args.blah) + print('blah = ' + args.blah) diff --git a/src/python/jw/pkg/cmds/projects/CmdTmplDir.py b/src/python/jw/pkg/cmds/projects/CmdTmplDir.py index ad889049..22def81d 100644 --- a/src/python/jw/pkg/cmds/projects/CmdTmplDir.py +++ b/src/python/jw/pkg/cmds/projects/CmdTmplDir.py @@ -1,18 +1,19 @@ -# -*- coding: utf-8 -*- +from argparse import ArgumentParser, Namespace -from argparse import Namespace, ArgumentParser +from .Cmd import Cmd, Parent -from ..Cmd import Cmd -from ..CmdProjects import CmdProjects +class CmdTmplDir(Cmd): # export -class CmdTmplDir(Cmd): # export - - def __init__(self, parent: CmdProjects) -> None: - super().__init__(parent, 'tmpl-dir', help='Print directory containing templates of a given module') + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'tmpl-dir', + help = 'Print directory containing templates of a given module', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('module', nargs='*', help='Modules') + parser.add_argument('module', nargs = '*', help = 'Modules') async def _run(self, args: Namespace) -> None: r = [] diff --git a/src/python/jw/pkg/cmds/projects/__init__.py b/src/python/jw/pkg/cmds/projects/__init__.py index 475565ba..b1f52bd0 100644 --- a/src/python/jw/pkg/cmds/projects/__init__.py +++ b/src/python/jw/pkg/cmds/projects/__init__.py @@ -4,6 +4,6 @@ __all__ = detect_modules( package_name = __name__, package_path = __path__, namespace = globals(), - prefix = "Cmd", - skip = {"Cmd"}, -) # pyright: ignore[reportUnsupportedDunderAll] + prefix = 'Cmd', + skip = {'Cmd'}, +) # pyright: ignore[reportUnsupportedDunderAll] diff --git a/src/python/jw/pkg/cmds/secrets/Cmd.py b/src/python/jw/pkg/cmds/secrets/Cmd.py index d49f3211..c7d1412e 100644 --- a/src/python/jw/pkg/cmds/secrets/Cmd.py +++ b/src/python/jw/pkg/cmds/secrets/Cmd.py @@ -1,24 +1,22 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING +from argparse import ArgumentParser from functools import cached_property +from typing import TYPE_CHECKING -from ..Cmd import Cmd as Base +from ...CmdBase import CmdBase +from ..CmdSecrets import CmdSecrets as Parent if TYPE_CHECKING: from typing import Iterable - from ...lib.Distro import Distro - from ...lib.ExecContext import ExecContext - from ..CmdSecrets import CmdSecrets + from .lib.base import Attrs from .lib.DistroContext import DistroContext -class Cmd(Base): # export +class Cmd(CmdBase): # export @cached_property - def ctx(self) -> ExecContext: + def ctx(self) -> DistroContext: return DistroContext(self.app.distro) async def _match_files(self, packages: Iterable[str], pattern: str) -> list[str]: @@ -27,21 +25,27 @@ class Cmd(Base): # export async def _list_template_files(self, packages: Iterable[str]) -> list[str]: return await self.ctx.list_template_files(packages) - async def _list_secret_paths(self, packages: Iterable[str], ignore_missing: bool=False) -> list[str]: + async def _list_secret_paths( + self, packages: Iterable[str], ignore_missing: bool = False + ) -> list[str]: return await self.ctx.list_secret_paths(packages, ignore_missing) - async def _list_compilation_targets(self, packages: Iterable[str], ignore_missing: bool=False) -> list[str]: + async def _list_compilation_targets( + self, packages: Iterable[str], ignore_missing: bool = False + ) -> list[str]: return await self.ctx.list_compilation_targets(packages, ignore_missing) async def _remove_compilation_targets(self, packages: Iterable[str]) -> list[str]: return await self.ctx.remove_compilation_targets(packages) - async def _compile_template_files(self, packages: Iterable[str], default_attrs: Attrs) -> list[str]: + async def _compile_template_files( + self, packages: Iterable[str], default_attrs: Attrs + ) -> list[str]: return await self.ctx.compile_template_files(packages, default_attrs) - def __init__(self, parent: CmdSecrets, name: str, help: str) -> None: + def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("packages", nargs='*', help="Package names") + parser.add_argument('packages', nargs = '*', help = 'Package names') diff --git a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py index c5172816..9f821da1 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py +++ b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py @@ -1,20 +1,21 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING +from .Cmd import Cmd from .lib.base import Attrs -from .Cmd import Cmd - if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import ArgumentParser, Namespace -class CmdCompileTemplates(Cmd): # export + from ..CmdSecrets import CmdSecrets + +class CmdCompileTemplates(Cmd): # export def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'compile-templates', help="Compile package template files") + super().__init__( + parent, 'compile-templates', help = 'Compile package template files' + ) async def _run(self, args: Namespace) -> None: attrs = Attrs(args.mode, args.owner, args.group, None) @@ -22,6 +23,12 @@ class CmdCompileTemplates(Cmd): # export def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument('--owner', '-o', default=None, help='Default output file owner') - parser.add_argument('--group', '-g', default=None, help='Default output file group') - parser.add_argument('--mode', '-m', default=None, help='Default output file mode') + parser.add_argument( + '--owner', '-o', default = None, help = 'Default output file owner' + ) + parser.add_argument( + '--group', '-g', default = None, help = 'Default output file group' + ) + parser.add_argument( + '--mode', '-m', default = None, help = 'Default output file mode' + ) diff --git a/src/python/jw/pkg/cmds/secrets/CmdInstall.py b/src/python/jw/pkg/cmds/secrets/CmdInstall.py index 3175391a..c1dbc5ed 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdInstall.py +++ b/src/python/jw/pkg/cmds/secrets/CmdInstall.py @@ -1,22 +1,33 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from .Cmd import Cmd if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import ArgumentParser, Namespace -class CmdInstall(Cmd): # export + from ..CmdSecrets import CmdSecrets + +class CmdInstall(Cmd): # export def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'install', help='Install secrets from various sources as static secrets onto the target') + super().__init__( + parent, + 'install', + help = ( + 'Install secrets from various sources as static secrets onto the target' + ), + ) def add_arguments(self, parser: ArgumentParser) -> None: - parser.add_argument('src', help='URI of secret source') - parser.add_argument('--only-missing', action='store_true', default=False, help='Install only secrets not already on the target') + parser.add_argument('src', help = 'URI of secret source') + parser.add_argument( + '--only-missing', + action = 'store_true', + default = False, + help = 'Install only secrets not already on the target', + ) super().add_arguments(parser) async def _run(self, args: Namespace) -> None: diff --git a/src/python/jw/pkg/cmds/secrets/CmdListCompilationOutput.py b/src/python/jw/pkg/cmds/secrets/CmdListCompilationOutput.py index 7f976bcd..7b4c621a 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdListCompilationOutput.py +++ b/src/python/jw/pkg/cmds/secrets/CmdListCompilationOutput.py @@ -1,22 +1,37 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from .Cmd import Cmd if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import ArgumentParser, Namespace -class CmdListCompilationOutput(Cmd): # export + from ..CmdSecrets import CmdSecrets + +class CmdListCompilationOutput(Cmd): # export def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'list-compilation-output', help="List package compilation output files") + super().__init__( + parent, + 'list-compilation-output', + help = 'List package compilation output files', + ) def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("--all", action='store_true', default=False, help="Show all output targets, including non-existent files") + parser.add_argument( + '--all', + action = 'store_true', + default = False, + help = 'Show all output targets, including non-existent files', + ) async def _run(self, args: Namespace) -> None: - print('\n'.join(await self._list_compilation_targets(args.packages, ignore_missing=(not args.all)))) + print( + '\n'.join( + await self._list_compilation_targets( + args.packages, ignore_missing = (not args.all) + ) + ) + ) diff --git a/src/python/jw/pkg/cmds/secrets/CmdListSecrets.py b/src/python/jw/pkg/cmds/secrets/CmdListSecrets.py index bd4d7672..8ff8a608 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdListSecrets.py +++ b/src/python/jw/pkg/cmds/secrets/CmdListSecrets.py @@ -1,22 +1,32 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from .Cmd import Cmd if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import ArgumentParser, Namespace -class CmdListSecrets(Cmd): # export + from ..CmdSecrets import CmdSecrets + +class CmdListSecrets(Cmd): # export def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'list-secrets', help="List package secret files") + super().__init__(parent, 'list-secrets', help = 'List package secret files') def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) - parser.add_argument("--all", action='store_true', default=False, help="Show all secret paths, including non-existent files") + parser.add_argument( + '--all', + action = 'store_true', + default = False, + help = 'Show all secret paths, including non-existent files', + ) async def _run(self, args: Namespace) -> None: - print('\n'.join(await self._list_secret_paths(args.packages, ignore_missing=(not args.all)))) + print( + '\n'.join( + await + self._list_secret_paths(args.packages, ignore_missing = (not args.all)) + ) + ) diff --git a/src/python/jw/pkg/cmds/secrets/CmdListTemplates.py b/src/python/jw/pkg/cmds/secrets/CmdListTemplates.py index 21933c77..1750f663 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdListTemplates.py +++ b/src/python/jw/pkg/cmds/secrets/CmdListTemplates.py @@ -1,18 +1,18 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from .Cmd import Cmd if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import Namespace -class CmdListTemplates(Cmd): # export + from ..CmdSecrets import CmdSecrets + +class CmdListTemplates(Cmd): # export def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'list-templates', help="List package template files") + super().__init__(parent, 'list-templates', help = 'List package template files') async def _run(self, args: Namespace) -> None: print('\n'.join(await self._list_template_files(args.packages))) diff --git a/src/python/jw/pkg/cmds/secrets/CmdRmCompilationOutput.py b/src/python/jw/pkg/cmds/secrets/CmdRmCompilationOutput.py index efc40f4e..d1631af4 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdRmCompilationOutput.py +++ b/src/python/jw/pkg/cmds/secrets/CmdRmCompilationOutput.py @@ -1,18 +1,20 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING -from .Cmd import Cmd +from .Cmd import Cmd, Parent if TYPE_CHECKING: - from ..CmdSecrets import CmdSecrets - from argparse import Namespace, ArgumentParser + from argparse import Namespace -class CmdRmCompilationOutput(Cmd): # export +class CmdRmCompilationOutput(Cmd): # export - def __init__(self, parent: CmdSecrets) -> None: - super().__init__(parent, 'rm-compilation-output', help="Remove package compilation output files") + def __init__(self, parent: Parent) -> None: + super().__init__( + parent, + 'rm-compilation-output', + help = 'Remove package compilation output files', + ) async def _run(self, args: Namespace) -> None: await self._remove_compilation_targets(args.packages) diff --git a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py index 82136d3c..1ff00fd5 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py @@ -1,24 +1,20 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations import re +import sys from pathlib import Path - from typing import TYPE_CHECKING +from ....lib.log import DEBUG, NOTICE, WARNING, log +from ....lib.ProcFilterGpg import ProcFilterGpg +from .base import Attrs +from .FilesContext import FilesContext + if TYPE_CHECKING: from typing import Iterable - from ....lib.FileContext import FileContext -from ....lib.log import * -from ....lib.util import run_cmd -from ....lib.TarIo import TarIo -from ....lib.ProcFilterGpg import ProcFilterGpg - -from .base import Attrs -from .FilesContext import FilesContext + from ....lib.Distro import Distro class DistroContext(FilesContext): @@ -37,18 +33,22 @@ class DistroContext(FilesContext): async def list_template_files(self, pkg_names: Iterable[str]) -> list[str]: if not pkg_names: pkg_names = [p.name for p in await self.__distro.select()] - return await self.match_files(pkg_names, pattern=r'.*\.jw-tmpl$') + return await self.match_files(pkg_names, pattern = r'.*\.jw-tmpl$') - async def list_secret_paths(self, pkg_names: Iterable[str], ignore_missing: bool=False) -> list[str]: + async def list_secret_paths( + self, pkg_names: Iterable[str], ignore_missing: bool = False + ) -> list[str]: ret = [] for tmpl in await self.list_template_files(pkg_names): - path = str(Path(tmpl).with_suffix(".jw-secret")) + path = str(Path(tmpl).with_suffix('.jw-secret')) if ignore_missing and not await self.ctx.file_exists(path): continue ret.append(path) return ret - async def list_compilation_targets(self, pkg_names: Iterable[str], ignore_missing: bool=False) -> list[str]: + async def list_compilation_targets( + self, pkg_names: Iterable[str], ignore_missing: bool = False + ) -> list[str]: ret = [] for tmpl in await self.list_template_files(pkg_names): path = tmpl.removesuffix('.jw-tmpl') @@ -58,72 +58,119 @@ class DistroContext(FilesContext): return ret async def remove_compilation_targets(self, pkg_names: Iterable[str]) -> list[str]: + ret: list[str] = [] for path in await self.list_compilation_targets(pkg_names): try: - self.ctx.stat(path) + await self.ctx.stat(path) log(NOTICE, f'Removing {path}') await self.ctx.unlink(path) - except FileNotFoundError as e: - log(DEBUG, f'Compilation target {path} doesn\'t exist (ignored)') + ret.append(path) + except FileNotFoundError: + log(DEBUG, f"Compilation target {path} doesn't exist (ignored)") continue + return ret - async def compile_template_files(self, pkg_names: Iterable[str], default_attrs: Attrs) -> list[str]: + async def compile_template_files( + self, pkg_names: Iterable[str], default_attrs: Attrs + ) -> list[str]: + ret: list[str] = [] missing = 0 for target in await self.list_compilation_targets(pkg_names): - if not await self.compile_template_file(target, default_attrs): + if await self.compile_template_file(target, default_attrs): + ret.append(target) + else: missing += 1 if missing > 0: - log(WARNING, f'{missing} missing secrets found. You might want to add them and run sudo {app.cmdline} again') + from ....lib.util import pretty_cmd - async def install(self, src_uri: str, pkg_names: Iterable[str], only_missing: bool=False, verbose: bool=False) -> None: + cmdline = pretty_cmd(sys.argv) + log( + WARNING, + ( + f'{missing} missing secrets found. You might want to add them and ' + f'run sudo {cmdline} again' + ), + ) + return ret - async def _read_secret_tar_blob(src_uri: str): + async def install( + self, + src_uri: str, + pkg_names: Iterable[str], + only_missing: bool = False, + verbose: bool = False, + ) -> None: + + async def _read_secret_tar_blob(src_uri: str) -> bytes: ec = self.ctx from ....lib.ec.Local import Local from ....lib.util import get + if not isinstance(ec, Local): - ec = Local() # Security: Use a local exec context for decrypting and filtering secrets - return (await get(src_uri, content_filter=ProcFilterGpg(ec=ec))).stdout + # Security: Use a local exec context for decrypting and + # filtering secrets + ec = Local() + return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout def _matches_host_prefix(path: str) -> bool: - return re.match(r'^' + root_in_tar, path) + return re.match(r'^' + host_root_in_tar, path) is not None def _crop_host_prefix(path: str) -> bool: - return re.sub(r'^' + root_in_tar, '', path) + return re.sub(r'^' + host_root_in_tar, '', path) is not None def _crop_default_prefix(path: str) -> bool: - return re.sub(r'^' + default, '', path) + return re.sub(default_rx, '', path) is not None def _matches_default_prefix(path: str) -> bool: - return re.match(r'^default', path) + return re.match(r'^default', path) is not None def _is_needed_secret(path: str) -> bool: return path in secret_paths - from .tar import filter as tar_filter, rewrite as tar_rewrite, extract as tar_extract, merge as tar_merge + from .tar import extract as tar_extract + from .tar import filter as tar_filter + from .tar import merge as tar_merge + from .tar import rewrite as tar_rewrite + if only_missing: raise NotImplementedError('--only-missing is not yet implemented') secret_paths = await self.list_secret_paths(pkg_names) - - host_root_in_tar = '/'.join(reversed(self.ctx.hostname.split('.'))) + hostname = self.ctx.uri.hostname + if not hostname: + raise Exception('Have no hostname to find secrets in tar file') + host_root_in_tar = '/'.join(reversed(hostname.split('.'))) host_rx = re.compile(r'^' + host_root_in_tar) default_rx = re.compile(r'^default') filtered_paths: list[str] = [] extracted_paths: list[str] = [] - blob_all = await _read_secret_tar_blob(src_uri) - blob_host_filtered = tar_filter(blob_all, lambda p: re.match(host_rx, p), filtered_paths) - blob_host_transformed = tar_rewrite(blob_host_filtered, lambda p: re.sub(host_rx, '', p)) - blob_default_filtered = tar_filter(blob_all, lambda p: re.match(default_rx, p), filtered_paths) - blob_default_transformed = tar_rewrite(blob_default_filtered, lambda p: re.sub(default_rx, '', p)) - blob_secret_material = tar_merge([blob_host_transformed, blob_default_transformed], overwrite=False) - blob_needed = tar_filter(blob_secret_material, _is_needed_secret, extracted_paths) + blob_all = await _read_secret_tar_blob(src_uri) + if blob_all is None: + raise Exception(f'Tar blob {src_uri} is empty') + blob_host_filtered = tar_filter( + blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths + ) + blob_host_transformed = tar_rewrite( + blob_host_filtered, lambda p: re.sub(host_rx, '', p) + ) + blob_default_filtered = tar_filter( + blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths + ) + blob_default_transformed = tar_rewrite( + blob_default_filtered, lambda p: re.sub(default_rx, '', p) + ) + blob_secret_material = tar_merge( + [blob_host_transformed, blob_default_transformed], overwrite = False + ) + blob_needed = tar_filter( + blob_secret_material, _is_needed_secret, extracted_paths + ) - await tar_extract(self.ctx, blob_needed, root='/', verbose=verbose) + await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose) for path in secret_paths: - if not path in extracted_paths: - log(NOTICE, f'not extracted: {path}') - else: - log(NOTICE, f'extracted: {path}') + if path not in extracted_paths: + log(NOTICE, f'not extracted: {path}') + else: + log(NOTICE, f'extracted: {path}') diff --git a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py index 28bc2cc1..e7fbdd5e 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py @@ -1,21 +1,17 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -import re, stat, copy +import copy +import re +import stat from contextlib import suppress -from pathlib import Path - from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Iterable from ....lib.FileContext import FileContext -from ....lib.log import * -from ....lib.util import run_cmd - +from ....lib.log import DEBUG, NOTICE, WARNING, log from .base import Attrs class FilesContext: @@ -27,17 +23,17 @@ class FilesContext: def ctx(self) -> FileContext: return self.__ctx - async def _read_key_value_file(self, path: str, throw=False) -> dict[str, str]: + async def _read_key_value_file(self, path: str, throw = False) -> dict[str, str]: ret: dict[str, str] = {} try: result = await self.ctx.get(path) - for line in result.stdout.decode().splitlines(): + for line in result.stdout_str.splitlines(): line = line.strip() - if not line or line.startswith("#"): + if not line or line.startswith('#'): continue - if "=" not in line: + if '=' not in line: continue - key, val = line.split("=", 1) + key, val = line.split('=', 1) key = key.strip() val = val.strip() if key: @@ -46,39 +42,39 @@ class FilesContext: log(DEBUG, f'File not found {path}') return ret - def _parse_attributes(self, content: str) -> Attrs: + def _parse_attributes(self, content: str) -> Attrs | None: if not content: return None first_line = content.splitlines()[0] - if not re.match(r"^\s*#\s*conf\s*:", first_line): + if not re.match(r'^\s*#\s*conf\s*:', first_line): return None ret = Attrs() ret.conf = first_line - m = re.match(r"^\s*#\s*conf\s*:\s*(.*?)\s*$", first_line) + m = re.match(r'^\s*#\s*conf\s*:\s*(.*?)\s*$', first_line) if not m: return ret for part in re.split(r'[; ,]', m.group(1)): part = part.strip() - if not part or "=" not in part: + if not part or '=' not in part: continue - key, val = part.split("=", 1) + key, val = part.split('=', 1) key = key.strip() val = val.strip() - if key == "owner": + if key == 'owner': ret.owner = val or None - elif key == "group": + elif key == 'group': ret.group = val or None - elif key == "mode": + elif key == 'mode': if val: try: - if re.fullmatch(r"0[0-7]+", val): + if re.fullmatch(r'0[0-7]+', val): ret.mode = int(val, 8) else: ret.mode = int(val, 0) @@ -87,20 +83,26 @@ class FilesContext: return ret - async def _read_attributes(self, paths: Iterable[str]) -> Attrs|None: + async def _read_attributes(self, paths: Iterable[str]) -> Attrs | None: ret = Attrs() for path in paths: try: result = await self.ctx.get(path) - lines = result.stdout.decode().splitlines() + lines = result.stdout_str.splitlines() if lines: ret.update(self._parse_attributes(lines[0])) except FileNotFoundError: - log(DEBUG, f'Can\'t parse "{path}" for attributes, file doesn\'t exist (ignored)') + log( + DEBUG, + ( + f'Can\'t parse "{path}" for attributes, ' + "file doesn't exist (ignored)" + ), + ) return ret def _format_metadata(self, owner: str, group: str, mode: int) -> str: - return f"{owner}:{group} {mode:o}" + return f'{owner}:{group} {mode:o}' async def _compile_one_template_file( self, @@ -110,11 +112,12 @@ class FilesContext: replace: dict[str, str] = {}, ) -> None: - owner = "root" - group = "root" + owner = 'root' + group = 'root' mode = 0o400 - new_content = (await self.ctx.get(src)).stdout.decode() + result = await self.ctx.get(src) + new_content = result.stdout_str attrs = self._parse_attributes(new_content) if attrs is None: @@ -133,15 +136,21 @@ class FilesContext: for key, val in replace.items(): new_content = new_content.replace(key, val) - tmp_path: str|None = None + tmp_path: str | None = None try: tmp_path = await self.ctx.mktemp(dst + '.jw-pkg.XXXXX') - await self.ctx.put(tmp_path, new_content.encode('utf-8'), owner=owner, group=group, mode=mode) + await self.ctx.put( + tmp_path, + new_content.encode('utf-8'), + owner = owner, + group = group, + mode = mode, + ) content_changed = True metadata_changed = True - old_meta = "" + old_meta = '' try: st = await self.ctx.stat(dst) @@ -150,23 +159,21 @@ class FilesContext: else: old_mode = stat.S_IMODE(st.mode) old_meta = self._format_metadata(st.owner, st.group, old_mode) - old_content = (await self.ctx.get(dst)).stdout.decode() + old_content = (await self.ctx.get(dst)).stdout_str content_changed = old_content != new_content metadata_changed = ( - st.owner != owner - or st.group != group - or old_mode != mode + st.owner != owner or st.group != group or old_mode != mode ) changes = [] if content_changed: - changes.append("@content") + changes.append('@content') if metadata_changed: - changes.append(f"@metadata ({old_meta} -> {new_meta})") + changes.append(f'@metadata ({old_meta} -> {new_meta})') - details = ", ".join(changes) if changes else "no changes" - log(NOTICE, f"Applying macros in {src} to {dst}: {details}") + details = ', '.join(changes) if changes else 'no changes' + log(NOTICE, f'Applying macros in {src} to {dst}: {details}') if not changes: await self.ctx.unlink(tmp_path) @@ -181,23 +188,28 @@ class FilesContext: with suppress(FileNotFoundError): await self.ctx.unlink(tmp_path) - async def compile_template_file(self, target_path: str, default_attrs: Attrs|None=None) -> bool: - path_tmpl = target_path + '.jw-tmpl' + async def compile_template_file( + self, target_path: str, default_attrs: Attrs | None = None + ) -> bool: + path_tmpl = target_path + '.jw-tmpl' path_secret_file = target_path + '.jw-secret-file' - path_secret = target_path + '.jw-secret' + path_secret = target_path + '.jw-secret' attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs()) - attrs.update(await self._read_attributes([ - path_tmpl, - path_secret_file, - path_secret - ])) + attrs.update( + await self._read_attributes([path_tmpl, path_secret_file, path_secret]) + ) replace = await self._read_key_value_file(path_secret) - for src in [ path_secret_file, path_tmpl ]: + for src in [path_secret_file, path_tmpl]: try: - await self._compile_one_template_file(src=src, dst=target_path, default_attrs=attrs, replace=replace) + await self._compile_one_template_file( + src = src, + dst = target_path, + default_attrs = attrs, + replace = replace + ) return True - except FileNotFoundError as e: - log(DEBUG, f'Compilation source {src} doesn\'t exist (ignored)') + except FileNotFoundError: + log(DEBUG, f"Compilation source {src} doesn't exist (ignored)") continue log(WARNING, f'No secret found for target {target_path}, not compiling') diff --git a/src/python/jw/pkg/cmds/secrets/lib/base.py b/src/python/jw/pkg/cmds/secrets/lib/base.py index 408aa885..80d5da08 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/base.py +++ b/src/python/jw/pkg/cmds/secrets/lib/base.py @@ -1,18 +1,15 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations from dataclasses import dataclass @dataclass class Attrs: - mode: int | None = None owner: str | None = None group: str | None = None conf: str | None = None - def update(self, rhs: Args|None) -> Args: + def update(self, rhs: Attrs | None) -> Attrs: if rhs is not None: if rhs.mode: self.mode = rhs.mode @@ -32,4 +29,3 @@ class Attrs: if self.group is not None: return False return True - diff --git a/src/python/jw/pkg/cmds/secrets/lib/tar.py b/src/python/jw/pkg/cmds/secrets/lib/tar.py index cf497614..43e9312d 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/tar.py +++ b/src/python/jw/pkg/cmds/secrets/lib/tar.py @@ -1,21 +1,27 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations +import io +import tarfile + +from tarfile import TarFile from typing import TYPE_CHECKING, Callable -import tarfile, io -from tarfile import TarFile - -from ....lib.log import * +from ....lib.log import DEBUG, log if TYPE_CHECKING: - from ....lib.ExecContext import ExecContext + from typing import Iterable -def filter(blob: bytes, path_filter: Callable[[str], bool]|None, matched: list[str]|None=None) -> bytes: + from ....lib.ExecContext import ExecContext + from ....lib.FileContext import FileContext + +def filter( + blob: bytes, + path_filter: Callable[[str], bool] | None, + matched: list[str] | None = None, +) -> bytes: ret = io.BytesIO() - with tarfile.open(fileobj=ret, mode='w') as tf_out: - tf_in = TarFile(fileobj=io.BytesIO(blob)) + with tarfile.open(fileobj = ret, mode = 'w') as tf_out: + tf_in = TarFile(fileobj = io.BytesIO(blob)) for info in tf_in.getmembers(): if path_filter is not None and not path_filter(info.name): continue @@ -28,8 +34,8 @@ def filter(blob: bytes, path_filter: Callable[[str], bool]|None, matched: list[s def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes: ret = io.BytesIO() - with tarfile.open(fileobj=ret, mode='w') as tf_out: - tf_in = TarFile(fileobj=io.BytesIO(blob)) + with tarfile.open(fileobj = ret, mode = 'w') as tf_out: + tf_in = TarFile(fileobj = io.BytesIO(blob)) for info in tf_in.getmembers(): new_name = rewrite_filter(info.name) log(DEBUG, f'Rewriting {info.name} -> {new_name}') @@ -38,11 +44,11 @@ def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes: tf_out.addfile(info, buf) return ret.getvalue() -def merge(blobs: Iterable[bytes], overwrite: bool=False) -> bytes: +def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes: ret = io.BytesIO() - with tarfile.open(fileobj=ret, mode='w') as tf_out: + with tarfile.open(fileobj = ret, mode = 'w') as tf_out: for blob in blobs: - tf_in = TarFile(fileobj=io.BytesIO(blob)) + tf_in = TarFile(fileobj = io.BytesIO(blob)) existing_names = tf_out.getnames() for info in tf_in.getmembers(): if not overwrite and info.name in existing_names: @@ -51,11 +57,21 @@ def merge(blobs: Iterable[bytes], overwrite: bool=False) -> bytes: tf_out.addfile(info, buf) return ret.getvalue() -async def extract(dst: ExecContext, blob: bytes, root: str|None=None, verbose: bool=False) -> None: +async def extract( + dst: FileContext, + blob: bytes, + root: str | None = None, + verbose: bool = False +) -> None: cmd = ['tar'] if root is not None: cmd += ['-C', root] if verbose: cmd += '-v' cmd += ['-x', '-f', '-'] - await dst.run(cmd, verbose=verbose, cmd_input=blob) + if not isinstance(dst, ExecContext): + raise NotImplementedError( + 'Extracting tar files to a non-executable ' + f'context is not yet implemented: {dst}' + ) + await dst.run(cmd, verbose = verbose, cmd_input = blob) diff --git a/src/python/jw/pkg/cmds/secrets/lib/util.py b/src/python/jw/pkg/cmds/secrets/lib/util.py index 34e16257..5056ceab 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/util.py +++ b/src/python/jw/pkg/cmds/secrets/lib/util.py @@ -1,17 +1,21 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations from typing import TYPE_CHECKING -from ....lib.FileContext import FileContext from ....lib.ec.Local import Local +from ....lib.FileContext import FileContext from .FilesContext import FilesContext if TYPE_CHECKING: from .base import Attrs -async def compile_template_file(target_path: str, default_attrs: Attrs|None=None, ctx: FileContext|None=None) -> bool: +async def compile_template_file( + target_path: str, + default_attrs: Attrs | None = None, + ctx: FileContext | None = None +) -> bool: if ctx is None: ctx = Local() - await FilesContext(ctx).compile_template_file(target_path, default_attrs=default_attrs) + return await FilesContext(ctx).compile_template_file( + target_path, default_attrs = default_attrs + ) diff --git a/src/python/jw/pkg/lib/App.py b/src/python/jw/pkg/lib/App.py index 3755d2f2..4385b431 100644 --- a/src/python/jw/pkg/lib/App.py +++ b/src/python/jw/pkg/lib/App.py @@ -1,76 +1,119 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Any, TYPE_CHECKING +import asyncio +import cProfile +import os +import sys -import os, sys, argparse, re, asyncio, cProfile +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace +from typing import TYPE_CHECKING, Any from .AsyncRunner import AsyncRunner -from .log import * +from .log import DEBUG, ERR, NOTICE, log, set_log_flags, set_log_level from .Types import LoadTypes if TYPE_CHECKING: - from typing import TypeVar from collections.abc import Awaitable - T = TypeVar("T") + from typing import TypeVar -class App: # export + T = TypeVar('T') + +class App: # export def _add_arguments(self, parser): - self.__parser.add_argument('--log-flags', help='Log flags', default=self.__default_log_flags) - self.__parser.add_argument('--log-level', help='Log level', default=self.__default_log_level) - self.__parser.add_argument('--log-file', help='Log file', default=self.__default_log_file) - self.__parser.add_argument('--backtrace', help='Show exception backtraces', action='store_true', default=self.__back_trace) - self.__parser.add_argument('--write-profile', help='Profile code and store output to file', default=None) + self.__parser.add_argument( + '--log-flags', help = 'Log flags', default = self.__default_log_flags + ) + self.__parser.add_argument( + '--log-level', help = 'Log level', default = self.__default_log_level + ) + self.__parser.add_argument( + '--log-file', help = 'Log file', default = self.__default_log_file + ) + self.__parser.add_argument( + '--backtrace', + help = 'Show exception backtraces', + action = 'store_true', + default = self.__back_trace, + ) + self.__parser.add_argument( + '--write-profile', + help = 'Profile code and store output to file', + default = None, + ) - def __init__(self, description: str = '', name_filter: str = '^Cmd.*', modules: None=None, eloop: None=None) -> None: + def __init__( + self, + description: str = '', + name_filter: str = '^Cmd.*', + modules: list[str] | None = None, + eloop: None = None, + ) -> None: def add_cmd_to_parser(cmd, parsers): parser = parsers.add_parser( cmd.name, help = cmd.help, description = cmd.description, - formatter_class = argparse.ArgumentDefaultsHelpFormatter, + formatter_class = ArgumentDefaultsHelpFormatter, ) - parser.set_defaults(func=cmd.run) + parser.set_defaults(func = cmd.run) cmd.add_arguments(parser) cmd.set_parser(parser) return parser - def add_cmds_to_parser(parent, parser, cmds, all=False): + def add_cmds_to_parser( + parent: AbstractCmd | App, + parser: ArgumentParser, + cmds, + all = False + ) -> None: if not cmds: return + class SubCommand: - def __init__(self, cmd: Cmd, parser: Any): + + def __init__(self, cmd: AbstractCmd, parser: Any): self.cmd = cmd self.parser = parser + title = 'Available subcommands' if hasattr(parent, 'name'): title += ' of ' + getattr(parent, 'name') - subparsers = parser.add_subparsers(title=title, metavar='', dest='command') + subparsers = parser.add_subparsers( + title = title, metavar = '', dest = 'command' + ) scs: dict[str, SubCommand] = {} for cmd in cmds: cmd.set_parent(parent) scs[cmd.name] = SubCommand(cmd, add_cmd_to_parser(cmd, subparsers)) if all: for sc in scs.values(): - add_cmds_to_parser(sc.cmd, sc.parser, sc.cmd.children, all=all) + add_cmds_to_parser(sc.cmd, sc.parser, sc.cmd.children, all = all) return args, unknown = self.__parser.parse_known_args() if args.command in scs: sc = scs[args.command] - add_cmds_to_parser(sc.cmd, sc.parser, sc.cmd.children, all=all) + add_cmds_to_parser(sc.cmd, sc.parser, sc.cmd.children, all = all) - from .Cmd import Cmd + from .Cmd import AbstractCmd - self.__args: Namespace|None = None - self.__cmdline: str|None = None - self.__default_log_flags: str = os.getenv('JW_DEFAULT_LOG_FLAGS', default='stderr,position,prio,color') - self.__default_log_level: str|int|None = os.getenv('JW_DEFAULT_LOG_LEVEL', default=NOTICE) - self.__default_log_file: str|None = os.getenv('JW_DEFAULT_LOG_FILE', default=None) - backtrace: str|bool = os.getenv('JW_DEFAULT_SHOW_BACKTRACE', False) - self.__back_trace = True if isinstance(backtrace, str) and backtrace.lower() in ['1', 'true'] else False + self.__args: Namespace | None = None + self.__cmdline: str | None = None + self.__default_log_flags: str = os.getenv( + 'JW_DEFAULT_LOG_FLAGS', default = 'stderr,position,prio,color' + ) + self.__default_log_level: str | int | None = os.getenv( + 'JW_DEFAULT_LOG_LEVEL', default = NOTICE + ) + self.__default_log_file: str | None = os.getenv( + 'JW_DEFAULT_LOG_FILE', default = None + ) + backtrace: str | bool = os.getenv('JW_DEFAULT_SHOW_BACKTRACE', False) + self.__back_trace = isinstance(backtrace, str) and backtrace.lower() in { + '1', + 'true', + } set_log_flags(self.__default_log_flags) set_log_level(self.__default_log_level) @@ -79,10 +122,13 @@ class App: # export if eloop is None: self.__eloop = asyncio.get_event_loop() self.__own_eloop = True - self.__async_runner: AsyncRunner|None = None + self.__async_runner: AsyncRunner | None = None - self.__parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=description, add_help=False) + self.__parser = ArgumentParser( + formatter_class = ArgumentDefaultsHelpFormatter, + description = description, + add_help = False, + ) self._add_arguments(self.__parser) args, unknown = self.__parser.parse_known_args() @@ -91,12 +137,26 @@ class App: # export log(DEBUG, '-------------- Running: >' + ' '.join(sys.argv) + '<') - cmd_classes = LoadTypes(modules if modules else ['__main__'], type_name_filter=name_filter, type_filter=[Cmd]) - add_all_parsers = '-h' in sys.argv or '--help' in sys.argv or '_ARGCOMPLETE' in os.environ - add_cmds_to_parser(self, self.__parser, [cmd_class(self) for cmd_class in cmd_classes], all=add_all_parsers) + cmd_classes = LoadTypes( + modules if modules else ['__main__'], + type_name_filter = name_filter, + type_filter = [AbstractCmd], # type: ignore[type-abstract] + ) + add_all_parsers = ( + '-h' in sys.argv or '--help' in sys.argv or '_ARGCOMPLETE' in os.environ + ) + add_cmds_to_parser( + self, + self.__parser, + [cmd_class(self) for cmd_class in cmd_classes], + all = add_all_parsers, + ) - # -- Add help only now, wouldn't want to have parse_known_args() exit on --help with subcommands missing - self.__parser.add_argument('-h', '--help', action='help', help='Show this help message and exit') + # -- Add help only now, wouldn't want to have parse_known_args() exit + # on --help with subcommands missing + self.__parser.add_argument( + '-h', '--help', action = 'help', help = 'Show this help message and exit' + ) def __del__(self): if self.__own_eloop: @@ -105,24 +165,32 @@ class App: # export self.__eloop = None self.__own_eloop = False - async def __aenter__(self) ->None: - return self + async def __aenter__(self) -> None: + pass async def __aexit__(self, exc_type, exc, tb) -> None: pass - async def __run(self, argv=None) -> None: + async def __run(self, argv = None) -> None: try: - class NoopCompleter: + # Import argcomplete only here to not require it to be compatible + # with minimal environments + from argcomplete.completers import BaseCompleter + + class NoopCompleter(BaseCompleter): + def __call__(self, **kwargs): return () - import argcomplete # Don't require it to be compatible with minimal environments - argcomplete.autocomplete(self.__parser, default_completer=NoopCompleter()) - except: + + import argcomplete + + argcomplete.autocomplete(self.__parser, default_completer = NoopCompleter()) + + except Exception: pass - self.__args = self.__parser.parse_args(args=argv) + self.__args = self.__parser.parse_args(args = argv) set_log_flags(self.__args.log_flags) set_log_level(self.__args.log_level) @@ -152,14 +220,17 @@ class App: # export finally: if pr is not None: pr.disable() - log(NOTICE, f'Writing profile statistics to {self.__args.write_profile}') + log( + NOTICE, + f'Writing profile statistics to {self.__args.write_profile}' + ) pr.dump_stats(self.__args.write_profile) if exit_status: sys.exit(exit_status) # Run sub-command. Overwrite if you want to do anything before or after - async def _run(self, args: argparse.Namespace) -> None: + async def _run(self, args: Namespace) -> None | int: return await self.args.func(args) def call_async(self, awaitable: Awaitable[T], timeout: float | None = None) -> T: @@ -167,6 +238,8 @@ class App: # export @property def eloop(self) -> asyncio.AbstractEventLoop: + if self.__eloop is None: + raise Exception('Tried to get inexistent event loop from application') return self.__eloop @property @@ -178,28 +251,34 @@ class App: # export @property def cmdline(self) -> str: if self.__cmdline is None: + import shlex + with open('/proc/self/cmdline', 'rb') as f: raw = f.read().split(b'\0')[:-1] self.__cmdline = ' '.join(shlex.quote(arg.decode()) for arg in raw) return self.__cmdline @property - def args(self) -> argparse.Namespace: + def args(self) -> Namespace: + if self.__args is None: + raise Exception('Tried to get inexistent argument list from application') return self.__args @property - def parser(self) -> argparse.ArgumentParser: + def parser(self) -> ArgumentParser: return self.__parser - def run(self, argv=None) -> None: + def run(self, argv = None) -> None: try: - ret = self.__eloop.run_until_complete(self.__run(argv)) # type: ignore + ret = self.eloop.run_until_complete(self.__run(argv)) finally: if self.__async_runner: self.__async_runner.close() self.__async_runner = None return ret -def run_sub_commands(description = '', name_filter = '^Cmd.*', modules=None, argv=None): # export - with App(description, name_filter, modules) as app: - return app.run(argv=argv) +def run_sub_commands( + description = '', name_filter = '^Cmd.*', modules = None, argv = None +): # export + app = App(description, name_filter, modules) + return app.run(argv = argv) diff --git a/src/python/jw/pkg/lib/AsyncRunner.py b/src/python/jw/pkg/lib/AsyncRunner.py index c10a845f..5d50309c 100644 --- a/src/python/jw/pkg/lib/AsyncRunner.py +++ b/src/python/jw/pkg/lib/AsyncRunner.py @@ -1,10 +1,13 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations + +import asyncio +import concurrent.futures +import contextlib -import abc, asyncio, contextlib, concurrent.futures from collections.abc import Awaitable, Generator from typing import TypeVar -T = TypeVar("T") +T = TypeVar('T') @contextlib.contextmanager def loop_in_thread() -> Generator[asyncio.AbstractEventLoop, None, None]: @@ -19,8 +22,7 @@ def loop_in_thread() -> Generator[asyncio.AbstractEventLoop, None, None]: loop_fut.set_result(asyncio.get_running_loop()) await stop_event.wait() - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as tpe: - + with concurrent.futures.ThreadPoolExecutor(max_workers = 1) as tpe: complete_fut = tpe.submit(asyncio.run, main()) for fut in concurrent.futures.as_completed((loop_fut, complete_fut)): @@ -40,13 +42,13 @@ class AsyncRunner: self._loop = self._cm.__enter__() def call(self, awaitable: Awaitable[T], timeout: float | None = None) -> T: - fut = asyncio.run_coroutine_threadsafe(awaitable, self._loop) + fut = asyncio.run_coroutine_threadsafe(awaitable, self._loop) # type: ignore return fut.result(timeout) def close(self) -> None: self._cm.__exit__(None, None, None) - def __enter__(self) -> "AsyncRunner": + def __enter__(self) -> AsyncRunner: return self def __exit__(self, exc_type, exc, tb) -> None: diff --git a/src/python/jw/pkg/lib/Cmd.py b/src/python/jw/pkg/lib/Cmd.py index 68a6ff73..b63e7b2f 100644 --- a/src/python/jw/pkg/lib/Cmd.py +++ b/src/python/jw/pkg/lib/Cmd.py @@ -1,38 +1,44 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Any +import abc +import sys -import inspect, sys, re, abc, argparse -from argparse import ArgumentParser, _SubParsersAction +from argparse import ArgumentParser +from typing import TYPE_CHECKING -from .log import * -from .Types import Types, LoadTypes +from .log import ERR +from .Types import LoadTypes, Types -class Cmd(abc.ABC): # export +if TYPE_CHECKING: + from typing import Any - def __init__(self, parent: App|Cmd, name: str, help: str, description: str|None=None) -> None: - from . import App - self.__parent: App|Cmd|None = parent - self.__app: App|None = None - self.__name = name - self.__help = help - self.__description = description if description else help + from .App import App + +class AbstractCmd(abc.ABC): + + def __init__( + self, + parent: App | AbstractCmd, + ) -> None: + self.__parent: App | AbstractCmd | None = parent + self.__app: App | None = None self.__children: list[Cmd] = [] self.__child_classes: list[type[Cmd]] = [] - self.__parser: ArgumentParser|None = None + self.__parser: ArgumentParser | None = None - @abc.abstractmethod - async def _run(self, args) -> None: - if isinstance(self.__parent, Cmd): # Calling App.run() would loop - return await self.__parent._run(args) - - def set_parent(self, parent: Any|Cmd): + def set_parent(self, parent: Any | Cmd): self.__parent = parent @property - def parent(self) -> App|Cmd: + def name(self) -> str: + return self._name() + + @abc.abstractmethod + def _name(self) -> str: + raise NotImplementedError('Called pure virtual base class method') + + @property + def parent(self) -> App | AbstractCmd: if self.__parent is None: raise Exception(f'Tried to access inexistent parent of command {self.name}') return self.__parent @@ -40,55 +46,45 @@ class Cmd(abc.ABC): # export @property def app(self) -> App: from .App import App + if self.__app is None: parent = self.__parent while True: if parent is None: - raise Exception("Can't get application object from command without parent") + raise Exception( + "Can't get application object from command without parent" + ) if isinstance(parent, App): self.__app = parent break - assert parent != parent.__parent, f'Assertion failed: Parent mismatch' + assert parent != parent.__parent, 'Assertion failed: Parent mismatch' parent = parent.__parent return self.__app + @property + def children(self) -> tuple[Cmd, ...]: + return tuple(self.__children) + + @property + def child_classes(self) -> tuple[type[Cmd], ...]: + return tuple(self.__child_classes) + + @property + def parser(self) -> ArgumentParser: + if self.__parser is None: + raise Exception(f'Tried to get a non-existing parser from {self}') + return self.__parser + # Don't use a setter decorator to force using a grepable method def set_parser(self, parser: ArgumentParser): self.__parser = parser - @property - def parser(self) -> str: - return self.__parser - - @property - def name(self) -> str: - return self.__name - - @property - def help(self) -> str: - return self.__help - - @property - def description(self) -> str: - return self.__description - - @property - def children(self) -> list[Cmd]: - return tuple(self.__children) - - @property - def child_classes(self) -> list[type[Cmd]]: - return tuple(self.__child_classes) - - def print_help(self, exit_status: int|None=None) -> None: + def print_help(self, exit_status: int | None = None) -> None: self.parser.print_help() if exit_status is not None: sys.exit(exit_status) - async def run(self, args): - return await self._run(args) - - def add_subcommands(self, cmds: Cmd|list[Cmds]|Types|list[Types]) -> None: + def add_subcommands(self, cmds: Cmd | list[Cmd] | Types | list[Types]) -> None: if isinstance(cmds, Cmd): assert False return @@ -106,21 +102,73 @@ class Cmd(abc.ABC): # export self.__children.append(cmd) assert len(self.__children) == len(self.__child_classes) except Exception as e: - cmds.dump(ERR, f"Failed to add subcommands ({str(e)})") + cmds.dump(ERR, f'Failed to add subcommands ({str(e)})') raise return raise Exception(f'Tried to add sub-commands of unknown type {type(cmds)}') - def load_subcommands(self, modules: str|list[str]|None=None, name_filter: str=r'Cmd[^.]') -> None: + def load_subcommands( + self, + modules: str | list[str] | None = None, + name_filter: str = r'Cmd[^.]' + ) -> None: if modules is None: # Derive module search path for the calling module's subcommands # from the module path of the calling module itself modules = [type(self).__module__.replace('Cmd', '').lower()] elif isinstance(modules, str): modules = [modules] - self.add_subcommands(LoadTypes(modules, type_name_filter=name_filter)) + self.add_subcommands(LoadTypes(modules, type_name_filter = name_filter)) + + # -- Interface to derived classes # To be overridden by derived class in case the command does take arguments. # Will be called from App base class constructor and set up the parser hierarchy def add_arguments(self, parser: ArgumentParser) -> None: pass + + @abc.abstractmethod + async def _run(self, args) -> None: + if isinstance(self.__parent, Cmd): # Calling App.run() would loop + return await self.__parent._run(args) + + async def run(self, args): + return await self._run(args) + + @abc.abstractmethod + def _help(self) -> str: + raise NotImplementedError('Called pure virtual base class method') + + @property + def help(self) -> str: + return self._help() + + def _description(self) -> str: + raise NotImplementedError('Called pure virtual base class method') + + @property + def description(self) -> str: + return self._description() + +class Cmd(AbstractCmd): # export + + def __init__( + self, + parent: App | Cmd, + name: str, + help: str, + description: str | None = None + ) -> None: + super().__init__(parent) + self.__name = name + self.__help = help + self.__description = description if description else help + + def _name(self) -> str: + return self.__name + + def _help(self) -> str: + return self.__help + + def _description(self) -> str: + return self.__description diff --git a/src/python/jw/pkg/lib/CopyContext.py b/src/python/jw/pkg/lib/CopyContext.py index 57fb60a3..cae43bac 100644 --- a/src/python/jw/pkg/lib/CopyContext.py +++ b/src/python/jw/pkg/lib/CopyContext.py @@ -1,32 +1,47 @@ -# -*- coding: utf-8 -*- - from typing import Self from .FileContext import FileContext +from .Uri import Uri class CopyContext: - def __init__(self, src: str|FileContext, dst: str|FileContext, chroot=False) -> None: - if isinstance(src, FileContext): - self.__src = src - self.__src_uri = src.uri - else: - self.__src: FileContext|None = None - self.__src_uri = src - if isinstance(dst, FileContext): - self.__dst = dst - self.__dst_uri = dst.uri - else: - self.__dst: FileContext|None = None - self.__dst_uri = dst + def __init__( + self, + src: Uri | str | FileContext, + dst: Uri | str | FileContext, + chroot = False + ) -> None: + + def __uri(ctx: FileContext | Uri | str) -> Uri | str | None: + if ctx is None: + return None + if isinstance(ctx, Uri): + return ctx + if isinstance(ctx, str): + return ctx + assert isinstance(ctx, FileContext) + return ctx.uri + + def __info( + ctx: FileContext | Uri | str, + ) -> tuple[FileContext | None, str | Uri | None]: + fc: FileContext | None = ctx if isinstance(ctx, FileContext) else None + return fc, __uri(ctx) + + self.__src, self.__src_uri = __info(src) + self.__dst, self.__dst_uri = __info(dst) self.__chroot = chroot async def __aenter__(self) -> Self: if self.__src is None: - self.__src = FileContext.create(self.__src_uri, chroot=self.__chroot) + if self.__src_uri is None: + raise Exception('Tried to create source context without URI') + self.__src = FileContext.create(self.__src_uri, chroot = self.__chroot) await self.__src.open() if self.__dst is None: - self.__dst = FileContext.create(self.__dst_uri, chroot=self.__chroot) + if self.__dst_uri is None: + raise Exception('Tried to create destination context without URI') + self.__dst = FileContext.create(self.__dst_uri, chroot = self.__chroot) await self.__dst.open() return self @@ -40,10 +55,14 @@ class CopyContext: @property def src(self) -> FileContext: + if self.__src is None: + raise Exception('Tried to access inexistent source context') return self.__src @property def dst(self) -> FileContext: + if self.__dst is None: + raise Exception('Tried to access inexistent destination context') return self.__dst async def _run(self) -> None: diff --git a/src/python/jw/pkg/lib/Distro.py b/src/python/jw/pkg/lib/Distro.py index f1019cbe..b27896ea 100644 --- a/src/python/jw/pkg/lib/Distro.py +++ b/src/python/jw/pkg/lib/Distro.py @@ -1,38 +1,41 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING +import abc +import importlib +import re +import sys + from functools import cached_property +from typing import TYPE_CHECKING + +from .log import ERR, INFO, WARNING, log if TYPE_CHECKING: - import Iterable + from typing import Iterable + + from .base import InputMode, Result + from .ExecContext import ExecContext + from .Package import Package from .PackageFilter import PackageFilter -import abc, importlib, re - -from .PackageFilter import PackageFilter -from .ExecContext import ExecContext -from .base import Result, InputMode -from .Package import Package -from .log import * - class Distro(abc.ABC): def __init__( - self, - ec: ExecContext, - id: str|None=None, - os_release_str: str|None=None, - default_pkg_filter: PackageFilter|None=None, - ) -> None: + self, + ec: ExecContext, + id: str | None = None, + os_release_str: str | None = None, + default_pkg_filter: PackageFilter | None = None, + ) -> None: if id is None: - raise ValueError(f'Tried to instaniate Distro without id') + raise ValueError('Tried to instaniate Distro without id') if ec is None: - raise ValueError(f'Tried to instaniate Distro "{id}" without execution context') + raise ValueError( + f'Tried to instaniate Distro "{id}" without execution context' + ) self.__exec_context = ec - self.__id: str|None = None - self.__os_release_str: str|None = os_release_str + self.__id: str | None = None + self.__os_release_str: str | None = os_release_str self.__default_pkg_filter = default_pkg_filter # Names that can be used by code outside this class to retrieve @@ -52,47 +55,60 @@ class Distro(abc.ABC): # == Load @classmethod - async def read_os_release_str(cls, ec: ExecContext) -> None: + async def read_os_release_str(cls, ec: ExecContext) -> str: release_file = '/etc/os-release' try: - result = await ec.get(release_file, throw=True) - ret = result.stdout.decode().strip() + result = await ec.get(release_file, throw = True) + return result.stdout_str except Exception as e: - log(INFO, f'Failed to read {release_file} ({str(e)}), falling back to uname') + log( + INFO, + f'Failed to read {release_file} ({str(e)}), falling back to uname' + ) result = await ec.run( - ['uname', '-s'], - throw=False, - cmd_input=InputMode.NonInteractive - ) + ['uname', '-s'], throw = False, cmd_input = InputMode.NonInteractive + ) if result.status != 0: - log(ERR, f'/etc/os-release and uname both failed, the latter with exit status {result.status}') + log( + ERR, + ( + '/etc/os-release and uname both failed, ' + f'the latter with {result.summary}' + ), + ) raise - uname = result.decode().stdout.strip().lower() + uname = result.stdout_str.lower() ret = f'ID={uname}\nVERSION_CODENAME=unknown' return ret @classmethod - def parse_os_release_field(self, key: str, os_release_str: str, throw: bool=False) -> str: - m = re.search(r'^\s*' + key + r'\s*=\s*("?)([^"\n]+)\1\s*$', os_release_str, re.MULTILINE) + def parse_os_release_field(cls, key: str, os_release_str: str) -> str: + m = re.search( + r'^\s*' + key + r'\s*=\s*("?)([^"\n]+)\1\s*$', os_release_str, re.MULTILINE + ) if m is None: - if throw: - raise Exception(f'Could not read "{key}=" from /etc/os-release') - return None + raise Exception(f'Could not read "{key}=" from /etc/os-release') return m.group(2) @classmethod - def parse_os_release_field_id(cls, os_release_str: str, throw: bool=False) -> str: - ret = cls.parse_os_release_field('ID', os_release_str, throw=throw) + def parse_os_release_field_id(cls, os_release_str: str) -> str: + ret = cls.parse_os_release_field('ID', os_release_str) match ret: case 'opensuse-tumbleweed': return 'suse' return ret @classmethod - async def instantiate(cls, ec: ExecContext, *args, id: str|None=None, os_release_str: str|None=None, **kwargs): + async def instantiate( + cls, + ec: ExecContext, + id: str | None = None, + os_release_str: str | None = None, + **kwargs, + ): if id is None: os_release_str = await cls.read_os_release_str(ec) - id = cls.parse_os_release_field_id(os_release_str, throw=True) + id = cls.parse_os_release_field_id(os_release_str) backend_id = id.lower().replace('-', '_') match backend_id: case 'ubuntu' | 'raspbian' | 'kali': @@ -108,11 +124,11 @@ class Distro(abc.ABC): log(ERR, f'Failed to import Distro module {module_path} ({str(e)})') raise cls = getattr(module, 'Distro') - ret = cls(ec, *args, id=id, os_release_str=os_release_str, **kwargs) + ret = cls(ec, id = id, os_release_str = os_release_str, **kwargs) return ret - def os_release_field(self, key: str, throw: bool=False) -> str: - return self.parse_os_release_field(key, self.os_release_str, throw) + def os_release_field(self, key: str) -> str: + return self.parse_os_release_field(key, self.os_release_str) async def cache(self) -> None: if self.__os_release_str is None: @@ -120,10 +136,12 @@ class Distro(abc.ABC): @cached_property def os_cascade(self) -> list[str]: + def __append(entry: str): - if not entry in ret: + if entry not in ret: ret.append(entry) - ret = [ 'os' ] + + ret = ['os'] match self.id: case 'centos': __append('linux') @@ -177,27 +195,32 @@ class Distro(abc.ABC): @property def os_release_str(self) -> str: if self.__os_release_str is None: - raise Exception(f'Tried to access OS release from an incompletely loaded Distro instance. Call reacache() before') + raise Exception( + 'Tried to access OS release from an incompletely loaded Distro ' + 'instance. Call cache() before' + ) return self.__os_release_str @cached_property def name(self) -> str: - return self.os_release_field('NAME', throw=True) + return self.os_release_field('NAME') @cached_property def id(self) -> str: - return self.parse_os_release_field_id(self.__os_release_str, throw=True) + return self.parse_os_release_field_id(self.os_release_str) @cached_property def codename(self) -> str: match self.id: case 'suse': - return self.os_release_field('ID', throw=True).split('-')[1] + return self.os_release_field('ID').split('-')[1] case 'kali': - return self.os_release_field('VERSION_CODENAME', throw=True).split('-')[1] + return self.os_release_field('VERSION_CODENAME').split('-')[1] case _: - return self.os_release_field('VERSION_CODENAME', throw=True) - raise NotImplementedError(f'Can\'t determine code name from distribution ID {self.id}') + return self.os_release_field('VERSION_CODENAME') + raise NotImplementedError( + f"Can't determine code name from distribution ID {self.id}" + ) @cached_property def os(self) -> str: @@ -214,33 +237,35 @@ class Distro(abc.ABC): @cached_property def gnu_triplet(self) -> str: - import sysconfig import shutil import subprocess + import sysconfig # Best: GNU host triplet Python was built for - for key in ("HOST_GNU_TYPE", "BUILD_GNU_TYPE"): # BUILD_GNU_TYPE can exist too + for key in ('HOST_GNU_TYPE', 'BUILD_GNU_TYPE'): # BUILD_GNU_TYPE can exist too ret = sysconfig.get_config_var(key) if isinstance(ret, str) and ret: return ret # Common on Debian/Ubuntu: multiarch component (often looks like a triplet) - ret = sysconfig.get_config_var("MULTIARCH") + ret = sysconfig.get_config_var('MULTIARCH') if isinstance(ret, str) and ret: return ret # Sometimes exposed (privately) by CPython - ret = getattr(sys.implementation, "_multiarch", None) + ret = getattr(sys.implementation, '_multiarch', None) if isinstance(ret, str) and ret: return ret # Last resort: ask the system compiler - for cc in ("gcc", "cc", "clang"): + for cc in ('gcc', 'cc', 'clang'): path = shutil.which(cc) if not path: continue try: - ret = subprocess.check_output([path, "-dumpmachine"], text=True, stderr=subprocess.DEVNULL).strip() + ret = subprocess.check_output( + [path, '-dumpmachine'], text = True, stderr = subprocess.DEVNULL + ).strip() if ret: return ret except Exception: @@ -252,14 +277,21 @@ class Distro(abc.ABC): def macros(cls) -> list[str]: return ['%%{' + name + '}' for name in cls.macro_names] - def expand_macros(self, fmt: str|Iterable) -> str|Iterable: + def expand_macros(self, fmt: str | Iterable) -> str | list[str]: + ret: str | list[str] if not isinstance(fmt, str): - ret: list[str] = [] + ret = [] for entry in fmt: - ret.append(self.expand_macros(entry)) + rv = self.expand_macros(entry) + if isinstance(rv, str): + ret.append(rv) + continue + raise NotImplementedError( + f'Expanding macros in nested lists is not supported: {rv}' + ) return ret ret = fmt - for macro in re.findall("%{([A-Za-z_-]+)}", fmt): + for macro in re.findall('%{([A-Za-z_-]+)}', fmt): try: name = macro.replace('-', '_') val = getattr(self, name) @@ -279,7 +311,7 @@ class Distro(abc.ABC): return self.__exec_context @property - def default_pkg_filter(self) -> str: + def default_pkg_filter(self) -> PackageFilter | None: return self.__default_pkg_filter async def run(self, *args, **kwargs) -> Result: @@ -289,7 +321,7 @@ class Distro(abc.ABC): return await self.__exec_context.sudo(*args, **kwargs) @property - def interactive(self) -> bool: + def interactive(self) -> bool | None: return self.__exec_context.interactive # == Distribution abstraction methods @@ -309,8 +341,8 @@ class Distro(abc.ABC): async def _dup(self, download_only: bool) -> None: pass - async def dup(self, download_only: bool=False) -> None: - return await self._dup(download_only=download_only) + async def dup(self, download_only: bool = False) -> None: + return await self._dup(download_only = download_only) # -- reboot_required @@ -318,10 +350,10 @@ class Distro(abc.ABC): async def _reboot_required(self, verbose: bool) -> bool: pass - async def reboot_required(self, verbose: bool|None=None) -> bool: + async def reboot_required(self, verbose: bool | None = None) -> bool: if verbose is None: verbose = self.ctx.verbose_default - return await self._reboot_required(verbose=verbose) + return await self._reboot_required(verbose = verbose) # -- select @@ -329,11 +361,16 @@ class Distro(abc.ABC): async def _select_by_name(self, names: Iterable[str]) -> Iterable[Package]: pass - async def _select(self, names: Iterable[str], filter: PackageFilter) -> Iterable[Package]: - assert filter, "No filter in _select()" + async def _select(self, names: Iterable[str], + filter: PackageFilter) -> Iterable[Package]: + assert filter, 'No filter in _select()' return [p for p in await self._select_by_name(names) if filter.match(p)] - async def select(self, names: Iterable[str] = [], filter: PackageFilter|None=None) -> Iterable[Package]: + async def select( + self, + names: Iterable[str] = [], + filter: PackageFilter | None = None + ) -> Iterable[Package]: if not filter: filter = self.__default_pkg_filter if not filter: @@ -349,17 +386,28 @@ class Distro(abc.ABC): # Default implementation assumes package manager can handle local files. # Not true for all distros. Override if Distro knows better. - async def _install_local_files(self, paths: Iterable[str], only_update: bool) -> None: - await self._install(paths, only_update=only_update) + async def _install_local_files( + self, paths: Iterable[str], only_update: bool + ) -> None: + await self._install(paths, only_update = only_update) # Download first and then install. Override if Distro knows better. async def _install_urls(self, urls: Iterable[str], only_update: bool) -> None: from .util import copy - tmp: str|None = None + + tmp: str | None = None try: - tmp = await self.__exec_context.mktemp('/tmp/jw-pkg-XXXXXX', directory=True) - paths = await copy(urls, self.__exec_context.uri.scheme_plus_authority + tmp) - await self._install_local_files(paths, only_update=only_update) + tmp = await self.__exec_context.mktemp( + '/tmp/jw-pkg-XXXXXX', directory = True + ) + paths = await copy( + urls, self.__exec_context.uri.scheme_plus_authority + tmp + ) + if isinstance(paths, Exception): + raise paths + if isinstance(paths, str): + paths = [paths] + await self._install_local_files(paths, only_update = only_update) finally: if tmp is not None: await self.__exec_context.erase(tmp) @@ -368,7 +416,9 @@ class Distro(abc.ABC): # - Download URLs into local directories and install # - Pass names to package manager # Override if Distro knows better. - async def _install_urls_and_names(self, packages: Iterable[str], only_update: bool) -> None: + async def _install_urls_and_names( + self, packages: Iterable[str], only_update: bool + ) -> None: urls: list[str] = [] names: list[str] = [] for package in packages: @@ -380,15 +430,15 @@ class Distro(abc.ABC): continue names.append(package) if urls: - await self._install_urls(urls, only_update=only_update) + await self._install_urls(urls, only_update = only_update) if names: - await self._install(names, only_update=only_update) + await self._install(names, only_update = only_update) - async def install(self, names: Iterable[str], only_update: bool=False) -> None: + async def install(self, names: Iterable[str], only_update: bool = False) -> None: if not names: - log(WARNING, f'No packages specified for installation') + log(WARNING, 'No packages specified for installation') return - await self._install_urls_and_names(names, only_update=only_update) + await self._install_urls_and_names(names, only_update = only_update) # -- delete @@ -398,7 +448,7 @@ class Distro(abc.ABC): async def delete(self, names: Iterable[str]) -> None: if not names: - log(WARNING, f'No packages specified for deletion') + log(WARNING, 'No packages specified for deletion') return return await self._delete(names) @@ -410,6 +460,6 @@ class Distro(abc.ABC): async def pkg_files(self, name: str) -> Iterable[str]: if not name: - log(WARNING, f'No package specified for inspection') + log(WARNING, 'No package specified for inspection') return [] return await self._pkg_files(name) diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 082f9362..8e501275 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -1,42 +1,43 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -import abc, re, sys, errno -from enum import Enum, auto -from typing import NamedTuple, TYPE_CHECKING -from decimal import Decimal, ROUND_FLOOR +import abc +import errno +import sys + +from decimal import ROUND_FLOOR, Decimal +from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: - from typing import Self, Type + from typing import Type from types import TracebackType -from .log import * from .base import Input, InputMode, Result, StatResult from .FileContext import FileContext as Base +from .log import DEBUG, ERR, NOTICE, log -_US = "\x1f" # unlikely to appear in numeric output +_US = '\x1f' # unlikely to appear in numeric output _BILLION = Decimal(1_000_000_000) -def _looks_like_option_error(stderr: str) -> bool: +def _looks_like_option_error(stderr: str | None) -> bool: + if stderr is None: + return False s = stderr.lower() return any( - needle in s - for needle in ( - "unrecognized option", - "illegal option", - "unknown option", - "invalid option", - "option requires an argument", - ) + needle in s for needle in ( + 'unrecognized option', + 'illegal option', + 'unknown option', + 'invalid option', + 'option requires an argument', ) ) -def _raise_stat_error(path: str, stderr: str, returncode: int) -> None: - msg = (stderr or "").strip() or f"stat exited with status {returncode}" +def _raise_stat_error(path: str, result: Result) -> None: + stderr = result.stderr_str_or_none or f'stat exited with status {result.status}' + msg = stderr.strip() lower = msg.lower() - if "no such file" in lower: + if 'no such file' in lower: raise FileNotFoundError(errno.ENOENT, msg, path) - if "permission denied" in lower or "operation not permitted" in lower: + if 'permission denied' in lower or 'operation not permitted' in lower: raise PermissionError(errno.EACCES, msg, path) raise OSError(errno.EIO, msg, path) @@ -46,14 +47,14 @@ def _parse_epoch(value: str) -> tuple[int, float, int]: (integer seconds for tuple slot, float seconds for attribute, ns for *_ns) """ dec = Decimal(value.strip()) - sec = int(dec.to_integral_value(rounding=ROUND_FLOOR)) - ns = int((dec * _BILLION).to_integral_value(rounding=ROUND_FLOOR)) + sec = int(dec.to_integral_value(rounding = ROUND_FLOOR)) + ns = int((dec * _BILLION).to_integral_value(rounding = ROUND_FLOOR)) return sec, float(dec), ns def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: if len(fields) != 13: raise ValueError( - f"unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}" + f'unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}' ) ( @@ -73,9 +74,9 @@ def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: ) = fields st_mode = int(mode_s, mode_base) - st_ino = int(ino_s) - st_dev = int(dev_s) - st_nlink = int(nlink_s) + # st_ino = int(ino_s) + # st_dev = int(dev_s) + # st_nlink = int(nlink_s) st_uid = uid_s st_gid = gid_s st_size = int(size_s) @@ -84,9 +85,9 @@ def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: st_mtime_i, st_mtime_f, st_mtime_ns = _parse_epoch(mtime_s) st_ctime_i, st_ctime_f, st_ctime_ns = _parse_epoch(ctime_s) - st_blksize = int(blksize_s) - st_blocks = int(blocks_s) - st_rdev = int(rdev_s) + # st_blksize = int(blksize_s) + # st_blocks = int(blocks_s) + # st_rdev = int(rdev_s) return StatResult( mode = st_mode, @@ -103,38 +104,38 @@ class ExecContext(Base): class CallContext: def __init__( - self, - parent: ExecContext, - title: str|None, - cmd: list[str], - cmd_input: Input, - mod_env: dict[str, str]|None, - wd: str|None, - log_prefix: str, - throw: bool, - verbose: bool, - ) -> None: + self, + parent: ExecContext, + title: str | None, + cmd: list[str], + cmd_input: Input, + mod_env: dict[str, str] | None, + wd: str | None, + log_prefix: str, + throw: bool, + verbose: bool | None, + ) -> None: self.__cmd = cmd self.__wd = wd self.__log_prefix = log_prefix self.__parent = parent self.__title = title - self.__pretty_cmd: str|None = None - self.__delim = title if title is not None else f'---- {parent.uri}: Running {self.pretty_cmd} -' + self.__pretty_cmd: str | None = None + self.__delim = ( + title if title is not None else + f'---- {parent.uri}: Running {self.pretty_cmd} -' + ) delim_len = 120 self.__delim += '-' * max(0, delim_len - len(self.__delim)) self.__mod_env = {'LC_ALL': 'C'} if mod_env is None else mod_env + self.__cmd_input: bytes | None = None # -- At the end of this dance, interactive needs to be either True # or False - interactive: bool|None = None - if not isinstance(cmd_input, InputMode): - interactive = False - self.__cmd_input = ( - cmd_input if isinstance(cmd_input, bytes) else - cmd_input.encode(sys.stdout.encoding or "utf-8") - ) - else: + interactive: bool | None = None + cmd_input_bytes: None | bytes + if isinstance(cmd_input, InputMode): + cmd_input_bytes = None match cmd_input: case InputMode.Interactive: interactive = True @@ -148,25 +149,34 @@ class ExecContext(Base): interactive = parent.interactive if interactive is None: interactive = sys.stdin.isatty() - self.__cmd_input = None - assert interactive in [ True, False ], f'Invalid: interactive = {invalid}' + else: + interactive = False + if cmd_input is None: + cmd_input_bytes = None + elif isinstance(cmd_input, str): + cmd_input_bytes = cmd_input.encode(sys.stdout.encoding or 'utf-8') + else: + cmd_input_bytes = cmd_input + self.__cmd_input = cmd_input_bytes + + assert interactive in [True, False], f'Invalid: interactive = {interactive}' self.__interactive = interactive - self.__cmd_input = cmd_input if not isinstance(cmd_input, InputMode) else None self.__throw = throw self.__verbose = verbose if verbose is not None else parent.verbose_default - def __enter__(self) -> CallContext: - self.log_delim(start=True) + def __enter__(self) -> ExecContext.CallContext: + self.log_delim(start = True) return self def __exit__( self, - exc_type: Type[BaseException]|None, - exc_value: BaseException|None, - traceback: TracebackType|None + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, ) -> bool: - self.log_delim(start=False) + self.log_delim(start = False) + return True @property def log_prefix(self) -> str: @@ -181,7 +191,7 @@ class ExecContext(Base): return self.__verbose @property - def cmd_input(self) -> bytes|None: + def cmd_input(self) -> bytes | None: return self.__cmd_input @property @@ -193,7 +203,7 @@ class ExecContext(Base): return self.__throw @property - def wd(self) -> str|None: + def wd(self) -> str | None: return self.__wd @property @@ -204,16 +214,17 @@ class ExecContext(Base): def pretty_cmd(self) -> str: if self.__pretty_cmd is None: from .util import pretty_cmd + self.__pretty_cmd = pretty_cmd(self.__cmd, self.__wd) return self.__pretty_cmd - def log(prio: int, *args, **kwargs) -> None: + def log(self, prio: int, *args, **kwargs) -> None: log(prio, self.__log_prefix, *args, **kwargs) def log_delim(self, start: bool) -> None: if not self.__verbose: return None - if self.__interactive: # Don't log footer in interative mode + if self.__interactive: # Don't log footer in interative mode if start: log(NOTICE, self.__delim) return @@ -223,12 +234,9 @@ class ExecContext(Base): def check_exit_code(self, result: Result) -> None: if result.status == 0: return - if (self.__throw or self.__verbose): - msg = f'Command exited with status {result.status}: {self.pretty_cmd}' - if result.stderr: - msg += ': ' + result.decode().stderr.strip() + if self.__throw or self.__verbose: if self.__throw: - raise RuntimeError(msg) + raise RuntimeError(result.summary) def exception(self, result: Result, e: Exception) -> Result: log(ERR, self.__log_prefix, f'Failed to run {self.pretty_cmd}') @@ -243,15 +251,35 @@ class ExecContext(Base): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + @classmethod + def create(cls, *args, **kwargs) -> ExecContext: + ret = super().create(*args, **kwargs) + if not isinstance(ret, cls): + raise TypeError(f'Expected {cls.__name__}, got {type(ret).__name__}') + return ret + + @abc.abstractmethod + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + raise NotImplementedError('Called pure virtual method _run()') + async def run( self, cmd: list[str], - wd: str|None = None, + wd: str | None = None, throw: bool = True, - verbose: bool|None = None, + verbose: bool | None = None, cmd_input: Input = InputMode.OptInteractive, - mod_env: dict[str, str]|None = None, - title: str = None + mod_env: dict[str, str] | None = None, + title: str | None = None, ) -> Result: """ Run a command asynchronously and return its output @@ -262,13 +290,16 @@ class ExecContext(Base): throw: Raise an exception on non-zero exit status if True verbose: Emit log output while the command runs cmd_input: - - "InputMode.OptInteractive" -> Let --interactive govern how to handle interactivity (default) + - "InputMode.OptInteractive" -> Let --interactive govern how to handle + interactivity (default) - "InputMode.Interactive" -> Inherit terminal stdin - "InputMode.Auto" -> Inherit terminal stdin if it is a TTY - "InputMode.NonInteractive" -> stdin from /dev/null - None -> Alias for InputMode.NonInteractive - otherwise -> Feed cmd_input to stdin - mod_env: Change set to command's environment. key: val adds a variable, key: None removes it + mod_env: Change set to command's environment: + - key: val adds a variable, + - key: None removes it Returns: A Result instance @@ -285,8 +316,17 @@ class ExecContext(Base): try: ret = Result(None, None, 1) - with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, mod_env=mod_env, wd=wd, - log_prefix='|', throw=throw, verbose=verbose) as cc: + with self.CallContext( + self, + title = title, + cmd = cmd, + cmd_input = cmd_input, + mod_env = mod_env, + wd = wd, + log_prefix = '|', + throw = throw, + verbose = verbose, + ) as cc: try: ret = await self._run( cmd = cc.cmd, @@ -295,7 +335,7 @@ class ExecContext(Base): cmd_input = cc.cmd_input, mod_env = cc.mod_env, interactive = cc.interactive, - log_prefix = cc.log_prefix + log_prefix = cc.log_prefix, ) except Exception as e: return cc.exception(ret, e) @@ -308,11 +348,11 @@ class ExecContext(Base): async def _sudo( self, cmd: list[str], - opts: list[str]|None, - wd: str|None, - mod_env_sudo: dict[str, str]|None, - mod_env_cmd: dict[str, str]|None, - cmd_input: bytes|None, + opts: list[str] | None, + wd: str | None, + mod_env_sudo: dict[str, str] | None, + mod_env_cmd: dict[str, str] | None, + cmd_input: bytes | None, verbose: bool, interactive: bool, log_prefix: str, @@ -320,20 +360,22 @@ class ExecContext(Base): def __check_equal_values(d1: dict[str, str], d2: dict[str, str]) -> None: for key, val in d1.items(): - if not d2.get(key, None) in [None, val]: - raise ValueError(f'Outer and inner environments differ at least for {key}: "{val}" != "{d2.get(key)}"') + if d2.get(key, None) not in [None, val]: + raise ValueError( + 'Outer and inner environments differ at least for ' + f'{key}: "{val}" != "{d2.get(key)}"' + ) fw_cmd: list[str] = [] fw_env: dict[str, str] = {} if opts is None: - opts = {} + opts = [] if mod_env_cmd: fw_env.update(mod_env_cmd) if self.username != 'root': - if mod_env_sudo and mod_env_cmd: __check_equal_values(mod_env_sudo, mod_env_cmd) __check_equal_values(mod_env_cmd, mod_env_sudo) @@ -345,7 +387,7 @@ class ExecContext(Base): fw_cmd.append('--preserve-env=' + ','.join(mod_env_cmd.keys())) if wd is not None: - opts.extend('-D', wd) + opts.extend(['-D', wd]) wd = None fw_cmd.extend(opts) @@ -361,20 +403,20 @@ class ExecContext(Base): verbose = verbose, cmd_input = cmd_input, interactive = interactive, - log_prefix = log_prefix + log_prefix = log_prefix, ) async def sudo( self, cmd: list[str], - opts: list[str]|None = None, - wd: str|None = None, - mod_env_sudo: dict[str, str]|None = None, - mod_env_cmd: dict[str, str]|None = None, + opts: list[str] | None = None, + wd: str | None = None, + mod_env_sudo: dict[str, str] | None = None, + mod_env_cmd: dict[str, str] | None = None, throw: bool = True, - verbose: bool|None = None, + verbose: bool | None = None, cmd_input: Input = InputMode.OptInteractive, - title: str = None + title: str | None = None, ) -> Result: # Note that in the calls to the wrapped method, cmd_input == None can @@ -382,9 +424,17 @@ class ExecContext(Base): assert cmd_input is not None, 'Invalid: cmd_input is None' ret = Result(None, None, 1) - with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, - mod_env=mod_env_cmd, wd=wd, - log_prefix='|', throw=throw, verbose=verbose) as cc: + with self.CallContext( + self, + title = title, + cmd = cmd, + cmd_input = cmd_input, + mod_env = mod_env_cmd, + wd = wd, + log_prefix = '|', + throw = throw, + verbose = verbose, + ) as cc: try: ret = await self._sudo( cmd = cc.cmd, @@ -416,19 +466,22 @@ class ExecContext(Base): ) async def _get( - self, - path: str, - wd: str|None, - throw: bool, - verbose: bool|None, - title: str + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str ) -> Result: ret = Result(None, None, 1) if wd is not None: path = wd + '/' + path - with self.CallContext(self, title=title, cmd=['cat', path], - cmd_input=InputMode.NonInteractive, wd=None, mod_env=None, - log_prefix='|', throw=throw, verbose=verbose) as cc: + with self.CallContext( + self, + title = title, + cmd = ['cat', path], + cmd_input = InputMode.NonInteractive, + wd = None, + mod_env = None, + log_prefix = '|', + throw = throw, + verbose = verbose, + ) as cc: try: ret = await self._run( cmd = cc.cmd, @@ -437,12 +490,12 @@ class ExecContext(Base): cmd_input = cc.cmd_input, mod_env = cc.mod_env, interactive = cc.interactive, - log_prefix = cc.log_prefix + log_prefix = cc.log_prefix, ) except Exception as e: return cc.exception(ret, e) - if ret.status != 0 and ret.stderr.decode().find('No such file') != -1: - raise FileNotFoundError(ret.stderr) + if ret.matches_error('No such file'): + raise FileNotFoundError(ret.summarize(cc.cmd, wd = cc.wd)) cc.check_exit_code(ret) return ret @@ -450,138 +503,176 @@ class ExecContext(Base): self, path: str, content: bytes, - wd: str|None, + wd: str | None, throw: bool, - verbose: bool|None, + verbose: bool | None, title: str, - owner: str|None, - group: str|None, - mode: str|None, + owner: str | None, + group: str | None, + mode: str | None, atomic: bool, ) -> Result: from .util import pretty_cmd - async def __run(cmd: list[str], cmd_input: Input=InputMode.NonInteractive, **kwargs) -> Result: - return await self.run(cmd, cmd_input=cmd_input, **kwargs) + async def __run( + cmd: list[str], + cmd_input: Input = InputMode.NonInteractive, + **kwargs + ) -> Result: + return await self.run(cmd, cmd_input = cmd_input, **kwargs) ret = Result(None, None, 1) try: + + class RemoteCmd(NamedTuple): + cmd: list[str] + cmd_input: Input = InputMode.NonInteractive + if wd is not None: path = wd + '/' + path - cmds: list[dict[str, str|list[str]|bool]] = [] - out = (await __run(['mktemp', path + '.XXXXXX'])).stdout.decode().strip() if atomic else path - cmds.append({'cmd': ['tee', out], 'cmd_input': content}) + cmds: list[RemoteCmd] = [] + stdout = (await __run(['mktemp', path + '.XXXXXX'])).stdout_str + if stdout is None: + raise Exception(f'Failed to create tmp-directory on {self.root}') + out = stdout.strip() if atomic else path + cmds.append(RemoteCmd( + cmd = ['tee', out], + cmd_input = content, + )) if owner is not None and group is not None: - cmds.append({'cmd': ['chown', f'{owner}:{group}', out]}) + cmds.append(RemoteCmd( + cmd = ['chown', f'{owner}:{group}', out], + )) elif owner is not None: - cmds.append({'cmd': ['chown', owner, out]}) + cmds.append(RemoteCmd( + cmd = ['chown', owner, out], + )) elif group is not None: - cmds.append({'cmd': ['chgrp', group, out]}) + cmds.append(RemoteCmd( + cmd = ['chgrp', group, out], + )) if mode is not None: - cmds.append({'cmd': ['chmod', mode, out]}) + cmds.append(RemoteCmd( + cmd = ['chmod', mode, out], + )) if atomic: - cmds.append({'cmd': ['mv', out, path]}) + cmds.append(RemoteCmd( + cmd = ['mv', out, path], + )) await self.open() try: for cmd in cmds: - log(DEBUG, f'{self.log_name}: Running {pretty_cmd(cmd['cmd'], wd)}') - ret = await __run(**cmd) + log(DEBUG, f'{self.log_name}: Running {pretty_cmd(cmd.cmd, wd)}') + ret = await __run(cmd.cmd) return ret finally: await self.close() - except: + except Exception as e: + msg = f'Failed to get {path} from {self.root} ({str(e)})' if throw: - raise - return cc.exception(ret, e) + raise Exception(msg) + log(ERR, msg) return ret async def _unlink(self, path: str) -> None: cmd = ['rm', '-f', path] - await self.run(cmd, cmd_input=InputMode.NonInteractive) + await self.run(cmd, cmd_input = InputMode.NonInteractive) async def _erase(self, path: str) -> None: cmd = ['rm', '-rf', path] - await self.run(cmd, cmd_input=InputMode.NonInteractive) + await self.run(cmd, cmd_input = InputMode.NonInteractive) async def _rename(self, src: str, dst: str) -> None: cmd = ['mv', src, dst] - await self.run(cmd, cmd_input=InputMode.NonInteractive) + await self.run(cmd, cmd_input = InputMode.NonInteractive) async def _mkdir(self, name: str, mode: int) -> None: cmd = ['mkdir', name, '-m', self.__mode_str(mode)] - await self.run(cmd, cmd_input=InputMode.NonInteractive) + await self.run(cmd, cmd_input = InputMode.NonInteractive) async def _mktemp(self, tmpl: str, directory: bool) -> str: cmd = ['mktemp'] if directory: cmd.append('-d') cmd.append(tmpl) - result = await self.run(cmd, cmd_input=InputMode.NonInteractive) - return result.stdout.strip().decode() + result = await self.run(cmd, cmd_input = InputMode.NonInteractive, throw = True) + if result.status != 0 or result.stdout is None: + raise Exception( + f'Failed to create temporary file on {self.root}: {result.summary}' + ) + return result.stdout_str async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: - async def __stat(opts: list[str]) -> str: - mod_env = { - 'LC_ALL': 'C' - } + async def __stat(opts: list[str]) -> Result: + mod_env = {'LC_ALL': 'C'} cmd = ['stat'] if follow_symlinks: cmd.append('-L') cmd.extend(opts) cmd.append(path) - return (await self.run(cmd, mod_env=mod_env, throw=False, - cmd_input=InputMode.NonInteractive)).decode() + return await self.run( + cmd, + mod_env = mod_env, + throw = False, + cmd_input = InputMode.NonInteractive + ) # GNU coreutils stat - gnu_format = _US.join([ - "%f", # st_mode in hex - "%i", # st_ino - "%d", # st_dev - "%h", # st_nlink - "%U", # st_uid - "%G", # st_gid - "%s", # st_size - "%.9X", # st_atime - "%.9Y", # st_mtime - "%.9Z", # st_ctime - "%o", # st_blksize hint - "%b", # st_blocks - "%r", # st_rdev - ]) + gnu_format = _US.join( + [ + '%f', # st_mode in hex + '%i', # st_ino + '%d', # st_dev + '%h', # st_nlink + '%U', # st_uid + '%G', # st_gid + '%s', # st_size + '%.9X', # st_atime + '%.9Y', # st_mtime + '%.9Z', # st_ctime + '%o', # st_blksize hint + '%b', # st_blocks + '%r', # st_rdev + ] + ) result = await __stat(['--printf', gnu_format]) - if result.status == 0: - return _build_stat_result(result.stdout.split(_US), mode_base=16) + if result.status == 0 and result.stdout is not None: + return _build_stat_result(result.stdout_str.split(_US), mode_base = 16) - if not _looks_like_option_error(result.stderr): + if not _looks_like_option_error(result.stderr_str_or_none): # log(DEBUG, f'GNU stat attempt failed on "{path}" ({str(e)})') - _raise_stat_error(path, result.stderr, result.status) + _raise_stat_error(path, result) # BSD / macOS / OpenBSD / NetBSD stat - bsd_format = _US.join([ - "%p", # st_mode in octal - "%i", # st_ino - "%d", # st_dev - "%l", # st_nlink - "%U", # st_uid - "%G", # st_gid - "%z", # st_size - "%.9Fa", # st_atime - "%.9Fm", # st_mtime - "%.9Fc", # st_ctime - "%k", # st_blksize - "%b", # st_blocks - "%r", # st_rdev - ]) + bsd_format = _US.join( + [ + '%p', # st_mode in octal + '%i', # st_ino + '%d', # st_dev + '%l', # st_nlink + '%U', # st_uid + '%G', # st_gid + '%z', # st_size + '%.9Fa', # st_atime + '%.9Fm', # st_mtime + '%.9Fc', # st_ctime + '%k', # st_blksize + '%b', # st_blocks + '%r', # st_rdev + ] + ) - result = await __stat(['-n', '-f', bst_format]) - if proc.returncode == 0: - return _build_stat_result(proc.stdout.rstrip('\n').split(_US), mode_base=8) - _raise_stat_error(path, result.stderr, result.status) + result = await __stat(['-n', '-f', bsd_format]) + stdout = result.stdout_str_or_none + if result.status != 0 or stdout is None: + _raise_stat_error(path, result) + assert stdout is not None # Just there to pacify the linter + return _build_stat_result(stdout.rstrip('\n').split(_US), mode_base = 8) - async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + async def _chown(self, path: str, owner: str | None, group: str | None) -> None: if owner is None and group is None: raise ValueError(f'Tried to chown("{path}") without owner and group') if group is None: @@ -590,7 +681,11 @@ class ExecContext(Base): ownership = ':' + group else: ownership = owner + ':' + group - await self.run(['chown', ownership, path], cmd_input=InputMode.NonInteractive) + assert ownership is not None # Impossible, just there to calm the linter + await self.run(['chown', ownership, path], cmd_input = InputMode.NonInteractive) async def _chmod(self, path: str, mode: int) -> None: - await self.run(['chmod', self.__mode_str(mode), path], cmd_input=InputMode.NonInteractive) + await self.run( + ['chmod', self.__mode_str(mode), path], + cmd_input = InputMode.NonInteractive + ) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 828da829..462cdf98 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -1,19 +1,17 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -import abc, re +import abc + from enum import Enum, auto -from typing import TYPE_CHECKING, Self -from functools import cached_property, cache +from functools import cached_property +from typing import TYPE_CHECKING + +from .log import DEBUG, ERR, log +from .Uri import Uri if TYPE_CHECKING: - from typing import Self - -from .log import * -from .base import Input, InputMode, Result, StatResult -from .Uri import Uri -from .ProcFilter import ProcPipeline + from .base import Result, StatResult + from .ProcFilter import ProcFilter, ProcPipeline class FileContext(abc.ABC): @@ -22,24 +20,27 @@ class FileContext(abc.ABC): Out = auto() def __init__( - self, - uri: str|Uri, - interactive: bool|None = None, - verbose_default = False, - chroot: bool = False, - in_pipe: ProcPipeline|None = None, - out_pipe: ProcPipeline|None = None, - ): + self, + uri: str | Uri, + interactive: bool | None = None, + verbose_default = False, + chroot: bool = False, + in_pipe: ProcPipeline | None = None, + out_pipe: ProcPipeline | None = None, + ): self.__uri = Uri.pimp(uri) self.__chroot = chroot self.__interactive = interactive self.__verbose_default = verbose_default - self.__log_name: str|None = None + self.__log_name: str | None = None self.__in_pipe = in_pipe self.__out_pipe = out_pipe self.__open_count = 0 - if not verbose_default in [True, False]: - raise ValueError(f'Tried to instantiate FileContext with verbose_default = "{verbose_default}"') + if verbose_default not in [True, False]: + raise ValueError( + 'Tried to instantiate FileContext with verbose_default ' + f'= "{verbose_default}"' + ) async def __aenter__(self): await self.open() @@ -91,7 +92,9 @@ class FileContext(abc.ABC): if self.__open_count == 1: await self._close() self.__open_count -= 1 - assert self.__open_count >= 0, f'Closed file context "{self}" more often than opened' + assert self.__open_count >= 0, ( + f'Closed file context "{self}" more often than opened' + ) @property def uri(self) -> Uri: @@ -106,7 +109,7 @@ class FileContext(abc.ABC): return self.__uri.path @property - def username(self) -> str|None: + def username(self) -> str | None: return self.__uri.username @property @@ -114,7 +117,7 @@ class FileContext(abc.ABC): return self.__uri.id @property - def interactive(self) -> bool|None: + def interactive(self) -> bool | None: return self.__interactive @property @@ -123,29 +126,24 @@ class FileContext(abc.ABC): @abc.abstractmethod async def _get( - self, - path: str, - wd: str|None, - throw: bool, - verbose: bool|None, - title: str + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str ) -> Result: raise NotImplementedError() async def get( self, path: str, - wd: str|None = None, + wd: str | None = None, throw: bool = True, - verbose: bool|None = None, - title: str=None, + verbose: bool | None = None, + title: str | None = None, ) -> Result: ret = await self._get( self._chroot(path), wd = wd, throw = throw, verbose = verbose, - title = title, + title = title or f'Fetching {path} from {self.uri}', ) return await self.__in_pipe.run(ret) if self.__in_pipe else ret @@ -153,13 +151,13 @@ class FileContext(abc.ABC): self, path: str, content: bytes, - wd: str|None, + wd: str | None, throw: bool, - verbose: bool|None, + verbose: bool | None, title: str, - owner: str|None, - group: str|None, - mode: str|None, + owner: str | None, + group: str | None, + mode: str | None, atomic: bool, ) -> Result: raise NotImplementedError() @@ -167,26 +165,26 @@ class FileContext(abc.ABC): async def put( self, path: str, - content: str, - wd: str|None = None, + content: bytes, + wd: str | None = None, throw: bool = True, - verbose: bool|None = None, - title: str = None, - owner: str|None = None, - group: str|None = None, - mode: int|None = None, - atomic: bool = False + verbose: bool | None = None, + title: str | None = None, + owner: str | None = None, + group: str | None = None, + mode: int | None = None, + atomic: bool = False, ) -> Result: mode_str = None if mode is None else oct(mode).replace('0o', '0') if self.__out_pipe is not None: - content = self.__out_pipe.run(content).stdout + result = await self.__out_pipe.run(content) return await self._put( self._chroot(path), - content, + result.stdout, wd = wd, throw = throw, verbose = verbose, - title = title, + title = title or f'Pushing content to {path} on {self.uri}', owner = owner, group = group, mode = mode_str, @@ -194,55 +192,73 @@ class FileContext(abc.ABC): ) async def _unlink(self, path: str) -> None: - raise NotImplementedError(f'{self.log_name}: unlink("{path}") is not implemented') + raise NotImplementedError( + f'{self.log_name}: unlink("{path}") is not implemented' + ) async def unlink(self, path: str) -> None: return await self._unlink(self._chroot(path)) async def _erase(self, path: str) -> None: - raise NotImplementedError(f'{self.log_name}: erase("{path}") is not implemented') + raise NotImplementedError( + f'{self.log_name}: erase("{path}") is not implemented' + ) async def erase(self, path: str) -> None: return await self._erase(self._chroot(path)) async def _rename(self, src: str, dst: str) -> None: - raise NotImplementedError(f'{self.log_name}: rename("{path}") is not implemented') + raise NotImplementedError( + f'{self.log_name}: rename("{src}" -> "{dst}") is not implemented' + ) async def rename(self, src: str, dst: str) -> None: return await self._rename(src, dst) async def _mkdir(self, path: str, mode: int) -> None: - raise NotImplementedError(f'{self.log_path}: mkdir({path}) is not implemented') + raise NotImplementedError(f'{self.log_name}: mkdir({path}) is not implemented') - async def mkdir(self, path: str, mode: int=0o777) -> None: + async def mkdir(self, path: str, mode: int = 0o777) -> None: return await self._mkdir(path, mode) - async def _mktemp(self, tmpl: str, directory: bool) -> None: - raise NotImplementedError(f'{self.log_name}: mktemp("{path}") is not implemented') + async def _mktemp(self, tmpl: str, directory: bool) -> str: + raise NotImplementedError( + f'{self.log_name}: mktemp("{tmpl}") is not implemented' + ) - async def mktemp(self, tmpl: str, directory: bool=False) -> None: + async def mktemp(self, tmpl: str, directory: bool = False) -> str: return await self._mktemp(self._chroot(tmpl), directory) - async def _chown(self, path: str, owner: str|None, group: str|None) -> None: - raise NotImplementedError(f'{self.log_name}: chown("{path}") is not implemented') + async def _chown(self, path: str, owner: str | None, group: str | None) -> None: + raise NotImplementedError( + f'{self.log_name}: chown("{path}") is not implemented' + ) - async def chown(self, path: str, owner: str|None=None, group: str|None=None) -> None: + async def chown( + self, path: str, owner: str | None = None, group: str | None = None + ) -> None: if owner is None and group is None: - raise ValueError(f'Tried to change ownership of {path} specifying neither owner nor group') + raise ValueError( + f'Tried to change ownership of {path} with neither owner nor group' + ) return await self._chown(self._chroot(path), owner, group) async def _chmod(self, path: str, mode: int) -> None: - raise NotImplementedError(f'{self.log_name}: chmod("{path}") is not implemented') + raise NotImplementedError( + f'{self.log_name}: chmod("{path}") is not implemented' + ) async def chmod(self, path: str, mode: int) -> None: return await self._chmod(self._chroot(path), mode) async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: - raise NotImplementedError(f'{self.log_name}: lstat("{path}") is not implemented') + raise NotImplementedError( + f'{self.log_name}: lstat("{path}") is not implemented' + ) - async def stat(self, path: str, follow_symlinks: bool=True) -> StatResult: + async def stat(self, path: str, follow_symlinks: bool = True) -> StatResult: if not isinstance(path, str): - raise TypeError(f"path must be str, got {type(path).__name__}") + raise TypeError(f'path must be str, got {type(path).__name__}') return await self._stat(self._chroot(path), follow_symlinks) async def _file_exists(self, path: str) -> bool: @@ -261,10 +277,17 @@ class FileContext(abc.ABC): async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: import stat + try: return stat.S_ISDIR((await self._stat(path, follow_symlinks)).mode) except NotImplementedError: - log(DEBUG, f'{self.log_name} doesn\'t implement stat(), judging by trailing slash if {path} is a directory') + log( + DEBUG, + ( + f"{self.log_name} doesn't implement stat(), judging by trailing " + 'slash if {path} is a directory' + ), + ) return path[-1] == '/' except FileNotFoundError as e: log(DEBUG, f'{self.log_name}: Failed to stat({path}) ({str(e)})') @@ -274,22 +297,28 @@ class FileContext(abc.ABC): raise return False - async def is_dir(self, path: str, follow_symlinks=True) -> bool: - return await self._is_dir(self._chroot(path), follow_symlinks=follow_symlinks) + async def is_dir(self, path: str, follow_symlinks = True) -> bool: + return await self._is_dir(self._chroot(path), follow_symlinks = follow_symlinks) @classmethod - def create(cls, uri: str|Uri, *args, **kwargs) -> Self: + def create(cls, uri: str | Uri, *args, **kwargs) -> FileContext: uri = Uri.pimp(uri) match uri.protocol: case 'local' | 'file': from .ec.Local import Local + return Local(uri, *args, **kwargs) case 'ssh': from .ec.SSHClient import ssh_client + return ssh_client(uri, *args, **kwargs) case 'http' | 'https': from .ec.Curl import Curl + return Curl(uri, *args, **kwargs) case _: pass - raise Exception(f'Can\'t create file context instance for "{uri}" with unsupported protocol "{uri.protocol}"') + raise Exception( + f'Can\'t create file context instance for "{uri}" with unsupported ' + f'protocol "{uri.protocol}"' + ) diff --git a/src/python/jw/pkg/lib/Package.py b/src/python/jw/pkg/lib/Package.py index 5e448d9b..63c47ff8 100644 --- a/src/python/jw/pkg/lib/Package.py +++ b/src/python/jw/pkg/lib/Package.py @@ -1,32 +1,35 @@ -# -*- coding: utf-8 -*- - from typing import Any meta_tags = [ - "name", - "vendor", - "packager", - "url", - "maintainer", + 'name', + 'vendor', + 'packager', + 'url', + 'maintainer', ] class Package: - - name: str = None - vendor: str|None = None - packager: str|None = None - url: str|None = None - maintainer: str|None = None + name: str + vendor: str | None = None + packager: str | None = None + url: str | None = None + maintainer: str | None = None @classmethod - def parse_spec_str(cls, spec: str, delimiter='|'): + def parse_spec_str(cls, spec: str, delimiter = '|'): tags = spec.split(delimiter) if len(tags) != 5: raise ValueError(f'Invalid package spec string "{spec}"') - return cls(name=tags[0], vendor=tags[1], packager=tags[2], url=tags[3], maintainer=tags[4]) + return cls( + name = tags[0], + vendor = tags[1], + packager = tags[2], + url = tags[3], + maintainer = tags[4], + ) @classmethod - def parse_specs_str(cls, specs: str, delimiter='|'): + def parse_specs_str(cls, specs: str, delimiter = '|'): ret: list[Package] = [] for spec in specs.splitlines(): ret.append(cls.parse_spec_str(spec)) @@ -39,7 +42,14 @@ class Package: ret[tag] = mapping.get(tag, '') return ret - def __init__(self, name: str, vendor: str|None=None, packager: str|None=None, url: str|None=None, maintainer: str|None=None): + def __init__( + self, + name: str, + vendor: str | None = None, + packager: str | None = None, + url: str | None = None, + maintainer: str | None = None, + ): self.name = name self.vendor = vendor self.packager = packager diff --git a/src/python/jw/pkg/lib/PackageFilter.py b/src/python/jw/pkg/lib/PackageFilter.py index cbe02e1b..5d7e0e7b 100644 --- a/src/python/jw/pkg/lib/PackageFilter.py +++ b/src/python/jw/pkg/lib/PackageFilter.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- - -import abc, re +import abc +import re from .Package import Package @@ -23,4 +22,7 @@ class PackageFilterString(PackageFilter): self.__definition = url_rx_str def _match(self, package: Package) -> bool: - return re.search(self.__definition, package.url) is not None + url = package.url + if url is None: + return False + return re.search(self.__definition, url) is not None diff --git a/src/python/jw/pkg/lib/ProcFilter.py b/src/python/jw/pkg/lib/ProcFilter.py index 11e3128c..42cb0de3 100644 --- a/src/python/jw/pkg/lib/ProcFilter.py +++ b/src/python/jw/pkg/lib/ProcFilter.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING import abc +from typing import TYPE_CHECKING + from .base import Result if TYPE_CHECKING: @@ -13,37 +12,40 @@ if TYPE_CHECKING: class ProcFilter(abc.ABC): @abc.abstractmethod - async def _run(self, data: bytes) -> Result: + async def _run(self, data: bytes | None) -> Result: raise NotImplementedError() - async def run(self, data: bytes) -> Result: + async def run(self, data: bytes | None) -> Result: return await self._run(data) class ProcFilterIdentity(ProcFilter): - async def _run(self, data: bytes) -> Result: + async def _run(self, data: bytes | None) -> Result: return Result(data, None, 0) class ProcPipeline: - def __init__(self, f: Iterable[ProcFilter]|ProcFilter = []) -> None: + def __init__(self, f: Iterable[ProcFilter] | ProcFilter = []) -> None: self.__filters: list[ProcFilter] = [] self.append(f) - def append(self, f: ProcFilter|Iterable[ProcFilter]) -> None: + def append(self, f: ProcFilter | Iterable[ProcFilter]) -> None: if not isinstance(f, ProcFilter): for e in f: self.append(e) return self.__filters.append(f) - async def run(self, data: bytes|Result) -> Result: + async def run(self, data: None | bytes | Result) -> Result: ret = data if isinstance(data, Result) else Result(data, None, 0) for f in self.__filters: ret = await f.run(ret.stdout) return ret -async def run(data: bytes|Result, chain: ProcFilter|list[ProcFilter]|ProcPipeline|None = None) -> Result: +async def run( + data: bytes | Result, + chain: ProcFilter | list[ProcFilter] | ProcPipeline | None = None, +) -> Result: if chain is None: if isinstance(data, Result): return data diff --git a/src/python/jw/pkg/lib/ProcFilterGpg.py b/src/python/jw/pkg/lib/ProcFilterGpg.py index 813c1fef..f111d967 100644 --- a/src/python/jw/pkg/lib/ProcFilterGpg.py +++ b/src/python/jw/pkg/lib/ProcFilterGpg.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING -from .ProcFilter import ProcFilter from .base import Result +from .ProcFilter import ProcFilter if TYPE_CHECKING: from .ExecContext import ExecContext @@ -14,9 +13,12 @@ class ProcFilterGpg(ProcFilter): def __init__(self, ec: ExecContext) -> None: self.__ec = ec - async def _run(self, data: bytes) -> Result: - return await self.__ec.run([ - "gpg", + async def _run(self, data: bytes | None) -> Result: + if data is None: + raise Exception('No data for GPG to decrypt') + return await self.__ec.run( + [ + 'gpg', '--batch', '--yes', '--quiet', @@ -24,5 +26,5 @@ class ProcFilterGpg(ProcFilter): '--decrypt', ], cmd_input = data, - throw = True + throw = True, ) diff --git a/src/python/jw/pkg/lib/TarIo.py b/src/python/jw/pkg/lib/TarIo.py index 64968c4e..4e476f3d 100644 --- a/src/python/jw/pkg/lib/TarIo.py +++ b/src/python/jw/pkg/lib/TarIo.py @@ -1,29 +1,34 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Self - -import abc, io +import abc +import io import tarfile -from tarfile import TarFile +from tarfile import TarFile, TarInfo + +from .base import StatResult from .CopyContext import CopyContext -from .FileContext import FileContext -from .log import * +from .ExecContext import ExecContext +from .log import DEBUG, ERR, log class TarIo(CopyContext): def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs, chroot=False) + kwargs['chroot'] = False + super().__init__(*args, **kwargs) def _match(self, path: str, path_filter: list[str]) -> bool: return path in path_filter - def _filter_tar_file(self, blob: bytes, path_filter: list[str]|None=None, matched: list[str]|None=None) -> bytes: + def _filter_tar_file( + self, + blob: bytes, + path_filter: list[str] | None = None, + matched: list[str] | None = None, + ) -> bytes: ret = io.BytesIO() - with tarfile.open(fileobj=ret, mode='w') as tf_out: - tf_in = TarFile(fileobj=io.BytesIO(blob)) + with tarfile.open(fileobj = ret, mode = 'w') as tf_out: + tf_in = TarFile(fileobj = io.BytesIO(blob)) for info in tf_in.getmembers(): if path_filter is not None and not self._match(info.name, path_filter): continue @@ -34,13 +39,18 @@ class TarIo(CopyContext): tf_out.addfile(info, buf) return ret.getvalue() - async def _read_filtered(self, path, path_filter: list[str]|None=None, matched: list[str]|None=None) -> bytes: + async def _read_filtered( + self, + path, + path_filter: list[str] | None = None, + matched: list[str] | None = None, + ) -> bytes: try: blob = (await self.src.get(path)).stdout except Exception as e: log(ERR, f'Failed to read tar file "{path}" ({str(e)}') raise - return self._filter_tar_file(blob, path_filter, matched=matched) + return self._filter_tar_file(blob, path_filter, matched = matched) def _add(self, tf: TarFile, path: str, st: StatResult, contents: bytes) -> None: file_obj = io.BytesIO(contents) @@ -50,37 +60,34 @@ class TarIo(CopyContext): info.uname = st.owner info.gname = st.group info.size = st.size - info.atime = st.atime - info.mtime = st.mtime - info.ctime = st.ctime - tf.addfile(info, io.BytesIO(file_obj)) - - async def _add_from_path(self, src: FileContext, tf: TarFile, path: str) -> None: - contents = await src.get(path) - st = await self.stat(path) - self._add(tf, path, st, contents) + info.mtime = int(st.mtime) + tf.addfile(info, file_obj) @abc.abstractmethod - async def _extract(self, blob: bytes, root: str|None=None) -> None: + async def _extract(self, blob: bytes, root: str | None = None) -> None: raise NotImplementedError() - async def extract(self, root: str|None=None, path_filter: list[str]|None=None) -> list[str]: + async def extract( + self, + root: str | None = None, + path_filter: list[str] | None = None + ) -> list[str]: ret: list[str] = [] - filtered = await self._read_filtered(self.src.root, path_filter, matched=ret) - await self._extract(blob=filtered, root=root) + filtered = await self._read_filtered(self.src.root, path_filter, matched = ret) + await self._extract(blob = filtered, root = root) return ret @classmethod - def create(cls, *args, type: str=None, **kwargs): + def create(cls, *args, type: str | None = None, **kwargs): if type is not None: raise NotImplementedError - #return TarIoTarFile(*args, **kwargs) + # return TarIoTarFile(*args, **kwargs) return TarIoTarExec(*args, **kwargs) class TarIoTarFile(TarIo): - async def _extract(self, blob: bytes, root: str|None=None) -> None: - tf = TarFile(fileobj=io.BytesIO(blob)) + async def _extract(self, blob: bytes, root: str | None = None) -> None: + tf = TarFile(fileobj = io.BytesIO(blob)) for info in tf.getmembers(): log(DEBUG, f'Extracting {info.name}') path = root + '/' + info.name if root else info.name @@ -96,14 +103,24 @@ class TarIoTarFile(TarIo): buf.read(), owner = info.uname, group = info.gname, - mode = info.mode, + mode = info.mode, ) class TarIoTarExec(TarIo): - async def _extract(self, blob: bytes, root: str|None=None) -> None: + @property + def dst(self) -> ExecContext: + ret = super().dst + if not isinstance(ret, ExecContext): + raise Exception( + 'Tried to get executable destination context from copy ' + 'context, which only has a file context' + ) + return ret + + async def _extract(self, blob: bytes, root: str | None = None) -> None: cmd = ['tar'] if root is not None: cmd += ['-C', root] cmd += ['-x', '-f', '-'] - await self.dst.run(cmd, cmd_input=blob) + await self.dst.run(cmd, cmd_input = blob) diff --git a/src/python/jw/pkg/lib/Types.py b/src/python/jw/pkg/lib/Types.py index fe2418ef..e00b4e1e 100644 --- a/src/python/jw/pkg/lib/Types.py +++ b/src/python/jw/pkg/lib/Types.py @@ -1,24 +1,29 @@ -# -*- coding: utf-8 -*- +import abc +import os +import re +import sys -from typing import TypeVar, Generic -import abc, re, sys, os -from collections.abc import Iterable, Iterator +from collections.abc import Iterator +from typing import TYPE_CHECKING, Generic, Iterable, TypeVar -from .log import * +from .log import OFF, log, parse_log_level -T = TypeVar("T") +if TYPE_CHECKING: + from typing import Any -class Types(abc.ABC, Iterable[T], Generic[T]): # export +T = TypeVar('T') - def __iter__(self) -> Iterator[T]: +class Types(abc.ABC, Iterable[type[T]], Generic[T]): # export + + def __iter__(self) -> Iterator[type[T]]: return iter(self._classes()) @abc.abstractmethod - def _classes(self) -> Iterable[T]: + def _classes(self) -> Iterable[type[T]]: pass @property - def classes(self) -> Iterable[T]: + def classes(self) -> Iterable[type[T]]: return self._classes() @abc.abstractmethod @@ -27,14 +32,20 @@ class Types(abc.ABC, Iterable[T], Generic[T]): # export def dump(self, prio: int, *args, **kwargs) -> None: contents = self._stringify() - log(prio, ",--- ", *args, **kwargs) + log(prio, ',--- ', *args, **kwargs) for line in contents: - log(prio, "| " + line) - log(prio, "`--- ", *args, **kwargs) + log(prio, '| ' + line) + log(prio, '`--- ', *args, **kwargs) -class LoadTypes(Types): # export +class LoadTypes(Types[T]): # export - def __init__(self, mod_names: list[str], type_name_filter: str=None, type_filter: list[T]=[], debug_level=None): + def __init__( + self, + mod_names: list[str], + type_name_filter: str | None = None, + type_filter: list[type[T]] = [], + debug_level = None, + ): if debug_level is None: val = os.getenv('JW_LOG_LEVEL_LOAD_TYPES') if val is not None: @@ -45,7 +56,7 @@ class LoadTypes(Types): # export self.__type_name_filter = type_name_filter self.__type_filter = type_filter self.__mod_names = mod_names - self.__classes: list[type[Any]]|None = None + self.__classes: list[type[T]] | None = None def _debug(self, *args, **kwargs) -> None: if self.__debug_level != OFF: @@ -53,37 +64,56 @@ class LoadTypes(Types): # export def _stringify(self): return [ - "type_name_filter: " + str(self.__type_name_filter), - "type_filter: " + ', '.join([str(f) for f in self.__type_filter]), - "mod_names: " + ', '.join(self.__mod_names) + 'type_name_filter: ' + str(self.__type_name_filter), + 'type_filter: ' + ', '.join([str(f) for f in self.__type_filter]), + 'mod_names: ' + ', '.join(self.__mod_names), ] - def _classes(self) -> Iterable[T]: + def _classes(self) -> Iterable[type[T]]: + if self.__classes is None: - import importlib, inspect - rx: Any|None = None + import importlib + import inspect + + rx: Any | None = None if self.__type_name_filter is not None: rx = re.compile(self.__type_name_filter) ret: list[Any] = [] for mod_name in self.__mod_names: if mod_name != '__main__': importlib.import_module(mod_name) - for member_name, c in inspect.getmembers(sys.modules[mod_name], inspect.isclass): + for member_name, c in inspect.getmembers( + sys.modules[mod_name], inspect.isclass + ): if rx is not None and not re.match(rx, member_name): - self._debug('o "{}.{}" has wrong name'.format(mod_name, member_name)) + self._debug( + 'o "{}.{}" has wrong name'.format(mod_name, member_name) + ) continue if inspect.isabstract(c): - self._debug('o "{}.{}" is abstract'.format(mod_name, member_name)) + self._debug( + 'o "{}.{}" is abstract'.format(mod_name, member_name) + ) continue if self.__type_filter: for tp in self.__type_filter: if issubclass(c, tp): break - self._debug('o "{}.{}" is not of type {}'.format(mod_name, member_name, tp)) + self._debug( + 'o "{}.{}" is not of type {}'.format( + mod_name, member_name, tp + ) + ) else: - self._debug('o "{}.{}" doesn\'t match type filter'.format(mod_name, member_name)) + self._debug( + 'o "{}.{}" doesn\'t match type filter'.format( + mod_name, member_name + ) + ) continue - self._debug('o "{}.{}" is fine, adding'.format(mod_name, member_name)) + self._debug( + 'o "{}.{}" is fine, adding'.format(mod_name, member_name) + ) ret.append(c) self.__classes = ret return self.__classes diff --git a/src/python/jw/pkg/lib/Uri.py b/src/python/jw/pkg/lib/Uri.py index 883b0c7a..13e7d1a5 100644 --- a/src/python/jw/pkg/lib/Uri.py +++ b/src/python/jw/pkg/lib/Uri.py @@ -1,20 +1,21 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING -from functools import cached_property import copy +from functools import cached_property +from typing import TYPE_CHECKING + if TYPE_CHECKING: + import urllib.parse + from typing import Self - import urllib # Make sure URIs are interpreted indentically everywhere - class Uri: - def __assemble(self, scheme: bool, credentials: bool, secure: bool, path: bool) -> str: + def __assemble( + self, scheme: bool, credentials: bool, secure: bool, path: bool + ) -> str: ret = '' if scheme: ret += f'{self.protocol}://' @@ -33,8 +34,8 @@ class Uri: def __init__(self, string: str) -> None: self.__string = string - self.__username: str|None = None - self.__password: str|None = None + self.__username: str | None = None + self.__password: str | None = None def __repr__(self) -> str: return self.full @@ -45,10 +46,11 @@ class Uri: @cached_property def __p(self) -> urllib.parse.ParseResult: from urllib.parse import urlparse + return urlparse(self.__string) @classmethod - def pimp(cls, url: str|Self) -> Uri: + def pimp(cls, url: str | Self) -> Uri: if isinstance(url, Uri): return url return Uri(url) @@ -69,7 +71,7 @@ class Uri: return self.scheme.replace('://', '') @property - def username(self) -> str|None: + def username(self) -> str | None: if self.__username is None: return self.__p.username return self.__username @@ -78,7 +80,7 @@ class Uri: self.__username = username @property - def password(self) -> str|None: + def password(self) -> str | None: if self.__password is None: return self.__p.password return self.__password @@ -87,15 +89,15 @@ class Uri: self.__password = password @cached_property - def hostname(self) -> str|None: + def hostname(self) -> str | None: return self.__p.hostname @cached_property - def port(self) -> int|None: + def port(self) -> int | None: return self.__p.port @cached_property - def port_str(self) -> str|None: + def port_str(self) -> str | None: if self.port is None: return None return str(self.port) @@ -110,11 +112,15 @@ class Uri: @cached_property def authority(self) -> str: - return self.__assemble(scheme=False, credentials=True, secure=False, path=False) + return self.__assemble( + scheme = False, credentials = True, secure = False, path = False + ) @cached_property def origin(self) -> str: - return self.__assemble(scheme=False, credentials=False, secure=True, path=False) + return self.__assemble( + scheme = False, credentials = False, secure = True, path = False + ) @cached_property def scheme_plus_authority(self) -> str: @@ -122,15 +128,21 @@ class Uri: @cached_property def id(self) -> str: - return self.__assemble(scheme=True, credentials=True, secure=True, path=False) + return self.__assemble( + scheme = True, credentials = True, secure = True, path = False + ) @cached_property def full(self) -> str: - return self.__assemble(scheme=True, credentials=True, secure=False, path=True) + return self.__assemble( + scheme = True, credentials = True, secure = False, path = True + ) @cached_property def safe_full_with_username(self) -> str: - return self.__assemble(scheme=True, credentials=True, secure=True, path=True) + return self.__assemble( + scheme = True, credentials = True, secure = True, path = True + ) def __new_with_path(self, base: str, path: str) -> Self: ret = copy.deepcopy(self) @@ -155,4 +167,4 @@ class Uri: return self.__new_with_path(self.__string, path) def new_replace_path(self, path: str) -> Self: - return self.__new_with_path(self.schema_plus_authority, path) + return self.__new_with_path(self.scheme_plus_authority, path) diff --git a/src/python/jw/pkg/lib/base.py b/src/python/jw/pkg/lib/base.py index 1d0a336a..dd42f431 100644 --- a/src/python/jw/pkg/lib/base.py +++ b/src/python/jw/pkg/lib/base.py @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from enum import Enum, auto -from typing import NamedTuple, TypeAlias, TYPE_CHECKING +import os -if TYPE_CHECKING: - from typing import Type +from enum import Enum, auto +from typing import NamedTuple, TypeAlias class InputMode(Enum): Interactive = auto() @@ -16,32 +13,158 @@ class InputMode(Enum): Input: TypeAlias = InputMode | bytes | str -class Result(NamedTuple): +class Result: - stdout: str|None - stderr: str|None - status: int|None + def __init__( + self, + stdout: bytes | None, + stderr: bytes | None, + status: int, + encoding: str = 'UTF-8', + strip: bool = True, + cmd: list[str] | None = None, + wd: str | None = None, + ) -> None: + self.__stdout = stdout + self.__stderr = stderr + self.__status = status + self.__encoding = encoding + self.__strip = strip + self.__cmd = cmd + self.__wd = wd - def decode(self, encoding='UTF-8', errors='replace') -> Result: - return Result( - self.stdout.decode(encoding, errors=errors) if self.stdout is not None else None, - self.stderr.decode(encoding, errors=errors) if self.stderr is not None else None, - self.status - ) + def __decode(self, stdxxx: bytes | None) -> str | None: + if stdxxx is None: + return None + ret = stdxxx.decode(self.encoding) + if self.strip: + return ret.strip() + return ret + + @property + def status(self) -> int | None: + return self.__status + + @property + def encoding(self) -> str: + return self.__encoding + + @encoding.setter + def encoding(self, value: str) -> None: + self.__encoding = value + + @property + def strip(self) -> bool: + return self.__strip + + @strip.setter + def strip(self, value: bool) -> None: + self.__strip = value + + @property + def cmd(self) -> list[str] | None: + return self.__cmd + + @cmd.setter + def cmd(self, value: list[str]) -> None: + self.__cmd = value + + @property + def wd(self) -> str | None: + return self.__wd + + @wd.setter + def wd(self, value: str) -> None: + self.__wd = value + + def matches_error(self, pattern: str) -> bool: + if self.status == 0: + return False + err = self.stderr_str + if err is None: + return False + import re + + return re.search(pattern, err) is not None + + def __summarize(self, cmd: list[str] | None, wd: str | None = None) -> str: + if cmd is None: + cmd = self.__cmd + call = '' + if cmd is not None: + from .util import pretty_cmd + + if wd is None: + wd = self.__wd + call = f'"{pretty_cmd(cmd, wd)}" ' + ret = f'Command {call}has exited with status {self.__status}' + call = pretty_cmd(cmd, wd) + if self.status != 0: + ret += f' -> stderr="{self.__stderr!r}"' + else: + if self.__stdout: + ret += f' -> stdout has {len(self.__stdout)} bytes' + else: + ret += ' -> stdout = None' + return ret + + def summarize(self, cmd: list[str] | None = None, wd: str | None = None) -> str: + return self.__summarize(cmd, wd) + + @property + def summary(self) -> str: + return self.__summarize(None, None) + + @property + def stdout(self) -> bytes: + if self.__stdout is None: + raise Exception(f'Result has no standard output stream: {self.summary}') + return self.__stdout + + @property + def stdout_or_none(self) -> bytes | None: + return self.__stdout + + @property + def stdout_str_or_none(self) -> str | None: + return self.__decode(self.__stdout) + + @property + def stdout_str(self) -> str: + return self.stdout.decode(self.__encoding) + + @property + def stderr(self) -> bytes: + if self.__stderr is None: + raise Exception(f'Result has no standard error stream: {self.summary}') + return self.__stderr + + @property + def stderr_or_none(self) -> bytes | None: + return self.__stderr + + @property + def stderr_str_or_none(self) -> str | None: + return self.__decode(self.__stderr) + + @property + def stderr_str(self) -> str: + return self.stderr.decode(self.__encoding) class StatResult(NamedTuple): - mode: int owner: str group: str size: int - atime: int - mtime: int - ctime: int + atime: float + mtime: float + ctime: float @classmethod def from_os(cls, rhs: os.stat_result) -> StatResult: - import pwd, grp + import grp + import pwd + return StatResult( rhs.st_mode, pwd.getpwuid(rhs.st_uid).pw_name, diff --git a/src/python/jw/pkg/lib/distros/arch/Distro.py b/src/python/jw/pkg/lib/distros/arch/Distro.py index df7adfbb..dbcef69b 100644 --- a/src/python/jw/pkg/lib/distros/arch/Distro.py +++ b/src/python/jw/pkg/lib/distros/arch/Distro.py @@ -1,53 +1,65 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from ...Distro import Distro as Base if TYPE_CHECKING: from typing import Iterable + from ...base import Result from ...Package import Package class Distro(Base): - async def pacman(self, args: list[str], verbose: bool=True, sudo: bool=True) -> Result: + async def pacman( + self, args: list[str], verbose: bool = True, sudo: bool = True + ) -> Result: cmd = ['/usr/bin/pacman'] if not self.interactive: cmd.extend(['--noconfirm']) cmd.extend(args) if sudo: - return await self.sudo(cmd, verbose=verbose) - return await self.run(cmd, verbose=verbose) + return await self.sudo(cmd, verbose = verbose) + return await self.run(cmd, verbose = verbose) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def _ref(self) -> None: - raise NotImplementedError('distro refresh is not yet implemented for Arch-like distributions') + raise NotImplementedError( + 'distro refresh is not yet implemented for Arch-like distributions' + ) async def _dup(self, download_only: bool) -> None: args = ['-Su'] - if args.download_only: + if download_only: args.append('-w') - return await self.pacman(args) + await self.pacman(args) async def _reboot_required(self, verbose: bool) -> bool: - raise NotImplementedError('distro reboot-required is not yet implemented for Arch-like distributions') + raise NotImplementedError( + 'distro reboot-required is not yet implemented for Arch-like distributions' + ) async def _select_by_name(self, names: Iterable[str]) -> Iterable[Package]: - raise NotImplementedError('distro select is not yet implemented for Arch-like distributions') + raise NotImplementedError( + 'distro select is not yet implemented for Arch-like distributions' + ) async def _install(self, names: Iterable[str], only_update: bool) -> None: if only_update: raise NotImplementedError('--only-update is not yet implemented for pacman') args = ['-S', '--needed'] - args.extend(args.packages) + args.extend(names) await self.pacman(args) async def _delete(self, names: Iterable[str]) -> None: - raise NotImplementedError('distro delete not yet implemented for Arch-like distributions') + raise NotImplementedError( + 'distro delete not yet implemented for Arch-like distributions' + ) async def _pkg_files(self, name: str) -> Iterable[str]: - raise NotImplementedError('distro pkg ls yet implemented for Arch-like distributions') + raise NotImplementedError( + 'distro pkg ls yet implemented for Arch-like distributions' + ) diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index 3cb794cc..d2811f07 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -1,47 +1,52 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING import os -from ...log import * +from typing import TYPE_CHECKING + from ...Distro import Distro as Base -from ...pm.dpkg import run_dpkg, run_dpkg_query, query_packages, list_files +from ...log import NOTICE, log +from ...pm.dpkg import list_files, query_packages, run_dpkg if TYPE_CHECKING: from typing import Iterable + from ...base import Result from ...Package import Package class Distro(Base): - async def apt_get(self, args: list[str], verbose: bool=True, sudo: bool=True): + async def apt_get( + self, args: list[str], verbose: bool = True, sudo: bool = True + ) -> Result: cmd = ['/usr/bin/apt-get'] mod_env_cmd = None if not self.interactive: cmd.extend(['--yes', '--quiet']) - mod_env_cmd = { 'DEBIAN_FRONTEND': 'noninteractive' } + mod_env_cmd = {'DEBIAN_FRONTEND': 'noninteractive'} cmd.extend(args) - if sudo: - return await self.sudo(cmd, verbose=verbose, mod_env_cmd=mod_env_cmd) - return await self.run(cmd, verbose=verbose) + return ( + await + self.sudo(cmd, verbose = verbose, mod_env_cmd = mod_env_cmd, throw = True) + if sudo else await self.run(cmd, verbose = verbose) + ) - async def dpkg(self, *args, **kwargs): - return await run_dpkg(*args, ec=self.ctx, **kwargs) + async def dpkg(self, *args, **kwargs) -> str: + kwargs.setdefault('ec', self.ctx) + return await run_dpkg(*args, **kwargs) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def _ref(self) -> None: - return await self.apt_get(['update']) + await self.apt_get(['update']) async def _dup(self, download_only: bool) -> None: args: list[str] = [] if download_only: args.append('--download-only') args.append('upgrade') - return await self.apt_get(args) + await self.apt_get(args) async def _reboot_required(self, verbose: bool) -> bool: reboot_required = '/run/reboot_required' @@ -56,11 +61,11 @@ class Distro(Base): print(content.strip()) return True if verbose: - log(NOTICE, f'No. {reboot_required} doesn\'t exist.') + log(NOTICE, f"No. {reboot_required} doesn't exist.") return False async def _select_by_name(self, names: Iterable[str]) -> Iterable[Package]: - return await query_packages(names, ec=self.ctx) + return await query_packages(names, ec = self.ctx) async def _install(self, names: Iterable[str], only_update: bool) -> None: args = ['install'] @@ -68,10 +73,10 @@ class Distro(Base): args.append('--only-upgrade') args.append('--no-install-recommends') args.extend(names) - return await self.apt_get(args) + await self.apt_get(args) async def _delete(self, names: Iterable[str]) -> None: - return await self.dpkg(['-P', *names], sudo=True) + await self.dpkg(['-P', *names], sudo = True) async def _pkg_files(self, name: str) -> Iterable[str]: - return await list_files(name, ec=self.ctx) + return await list_files(name, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/distros/suse/Distro.py b/src/python/jw/pkg/lib/distros/suse/Distro.py index de881a03..328b7288 100644 --- a/src/python/jw/pkg/lib/distros/suse/Distro.py +++ b/src/python/jw/pkg/lib/distros/suse/Distro.py @@ -1,62 +1,71 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations + from typing import TYPE_CHECKING from ...Distro import Distro as Base -from ...pm.rpm import run_rpm, query_packages, list_files +from ...pm.rpm import list_files, query_packages, run_rpm if TYPE_CHECKING: from typing import Iterable + from ...base import Result + from ...ExecContext import ExecContext from ...Package import Package class Distro(Base): - async def zypper(self, args: list[str], verbose: bool=True, sudo: bool=True) -> Result: + async def zypper( + self, args: list[str], verbose: bool = True, sudo: bool = True + ) -> Result: cmd = ['/usr/bin/zypper'] if not self.interactive: - cmd.extend(['--non-interactive', '--gpg-auto-import-keys', '--no-gpg-checks']) + cmd.extend( + ['--non-interactive', '--gpg-auto-import-keys', '--no-gpg-checks'] + ) cmd.extend(args) - if sudo: - return await self.sudo(cmd, verbose=verbose) - return await self.run(cmd, verbose=verbose) + return ( + await self.sudo(cmd, verbose = verbose) + if sudo else await self.run(cmd, verbose = verbose) + ) - async def rpm(self, *args, **kwargs) -> Result: - return await run_rpm(*args, ec=self.ctx, **kwargs) + async def rpm(self, *args, ec: ExecContext | None = None, **kwargs) -> str: + if ec is None: + ec = self.ctx + kwargs['ec'] = ec + return await run_rpm(*args, **kwargs) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def _ref(self) -> None: - return await self.zypper(['refresh']) + await self.zypper(['refresh']) async def _dup(self, download_only: bool) -> None: args = ['dup', '--force-resolution', '--auto-agree-with-licenses'] if download_only: args.append('--download-only') - return await self.zypper(args) + await self.zypper(args) async def _reboot_required(self, verbose: bool) -> bool: opts = [] if not verbose: pass - #opts.append('--quiet') + # opts.append('--quiet') opts.append('needs-rebooting') - stdout, stderr, ret = await self.zypper(opts, sudo=False, verbose=verbose) + ret = await self.zypper(opts, sudo = False, verbose = verbose) if ret != 0: return True return False async def _select_by_name(self, names: Iterable[str]) -> Iterable[Package]: - return await query_packages(names, ec=self.ctx) + return await query_packages(names, ec = self.ctx) async def _install(self, names: Iterable[str], only_update: bool) -> None: cmd = 'update' if only_update else 'install' - return await self.zypper([cmd, *names]) + await self.zypper([cmd, *names]) async def _delete(self, names: Iterable[str]) -> None: - return await self.rpm(['-e', *names], sudo=True) + await self.rpm(['-e', *names], sudo = True) async def _pkg_files(self, name: str) -> Iterable[str]: - return await list_files(name, ec=self.ctx) + return await list_files(name, ec = self.ctx) diff --git a/src/python/jw/pkg/lib/ec/Curl.py b/src/python/jw/pkg/lib/ec/Curl.py index 7c07c93f..08984ecc 100644 --- a/src/python/jw/pkg/lib/ec/Curl.py +++ b/src/python/jw/pkg/lib/ec/Curl.py @@ -1,32 +1,35 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations from typing import TYPE_CHECKING -from ..FileContext import FileContext as Base from ..base import Result +from ..FileContext import FileContext as Base if TYPE_CHECKING: from ..ExecContext import ExecContext from ..Uri import Uri + from .Local import Local class Curl(Base): - def __init__(self, uri: str|Uri, *args, ec: ExecContext|None=None, **kwargs) -> None: - super().__init__(uri=uri, *args, **kwargs) - self.__ec: ExecContext|None = ec - if ec is None: + def __init__( + self, uri: str | Uri, *args, ec: ExecContext | None = None, **kwargs + ) -> None: + + def __local() -> Local: from .Local import Local - self.__ec = Local(interactive=False, *args, **kwargs) + + return Local(interactive = False, *args, **kwargs) + + # MyPy complains for reasons I don't understand: + # E: "__init__" of "FileContext" gets multiple values for keyword # argument + # "uri" [misc] + super().__init__(uri = uri, *args, **kwargs) # type: ignore[misc] + + self.__ec = ec if ec else __local() async def _get( - self, - path: str, - wd: str|None, - throw: bool, - verbose: bool|None, - title: str + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str ) -> Result: cmd = ['curl'] if verbose is None: @@ -38,5 +41,5 @@ class Curl(Base): path = wd + '/' + path if not len(path) or path[0] != '/': path = '/' + path - cmd.append(self.url.to_string + self._chroot(path)) - return await self.__ec.run(cmd, throw=throw, verbose=verbose) + cmd.append(self.uri.to_string + self._chroot(path)) + return await self.__ec.run(cmd, throw = throw, verbose = verbose) diff --git a/src/python/jw/pkg/lib/ec/Local.py b/src/python/jw/pkg/lib/ec/Local.py index c858ab23..8320f078 100644 --- a/src/python/jw/pkg/lib/ec/Local.py +++ b/src/python/jw/pkg/lib/ec/Local.py @@ -1,91 +1,96 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING -import os, sys, subprocess, asyncio, pwd, grp, stat +import asyncio +import grp +import os +import pwd +import sys from functools import cache +from typing import TYPE_CHECKING -from ..ExecContext import ExecContext as Base from ..base import Result, StatResult - -from ..log import * -from ..util import pretty_cmd +from ..ExecContext import ExecContext as Base +from ..log import ERR, NOTICE, log if TYPE_CHECKING: from ..Uri import Uri class Local(Base): - def __init__(self, uri: str|Uri='local', *args, **kwargs) -> None: + def __init__(self, uri: str | Uri = 'local', *args, **kwargs) -> None: super().__init__(uri, *args, **kwargs) @cache def _username(self) -> str: - return pwd.getpwuid(os.getuid()).pw_name, + return pwd.getpwuid(os.getuid()).pw_name async def _run( self, cmd: list[str], - wd: str|None, + wd: str | None, verbose: bool, - cmd_input: bytes|None, - mod_env: dict[str, str]|None, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, interactive: bool, - log_prefix: str + log_prefix: str, ) -> Result: - def __log(prio, *args, verbose=verbose): + def __log(prio, *args, verbose = verbose): if verbose: log(prio, log_prefix, *args) def __make_pty_reader(collector: list[bytes], enc_for_verbose: str): + def _read(fd): ret = os.read(fd, 1024) if not ret: return ret collector.append(ret) return ret + return _read - cwd: str|None = None + cwd: str | None = None if wd is not None: cwd = os.getcwd() os.chdir(wd) try: - # -- interactive mode if interactive: - import pty def _spawn(): - # Apply env in PTY mode by temporarily updating os.environ around spawn. + # Apply env in PTY mode by temporarily updating os.environ + # around spawn. if mod_env: old_env = os.environ.copy() try: os.environ.update(mod_env) - return pty.spawn(cmd, master_read=reader) + return pty.spawn(cmd, master_read = reader) finally: os.environ.clear() os.environ.update(old_env) - return pty.spawn(cmd, master_read=reader) + return pty.spawn(cmd, master_read = reader) stdout_chunks: list[bytes] = [] - enc_for_verbose = sys.stdout.encoding or "utf-8" + enc_for_verbose = sys.stdout.encoding or 'utf-8' reader = __make_pty_reader(stdout_chunks, enc_for_verbose) exit_code = await asyncio.to_thread(_spawn) # PTY merges stdout/stderr - stdout = b"".join(stdout_chunks) if stdout_chunks else None + stdout = b''.join(stdout_chunks) if stdout_chunks else None return Result(stdout, None, exit_code) # -- non-interactive mode - stdin = asyncio.subprocess.DEVNULL if cmd_input is None else asyncio.subprocess.PIPE + + stdin = ( + asyncio.subprocess.DEVNULL + if cmd_input is None else asyncio.subprocess.PIPE + ) if mod_env: new_env = os.environ.copy() @@ -94,21 +99,21 @@ class Local(Base): proc = await asyncio.create_subprocess_exec( *cmd, - stdin=stdin, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=mod_env, + stdin = stdin, + stdout = asyncio.subprocess.PIPE, + stderr = asyncio.subprocess.PIPE, + env = mod_env, ) stdout_parts: list[bytes] = [] stderr_parts: list[bytes] = [] # -- decoding for verbose output in pipe mode - stdout_log_enc = sys.stdout.encoding or "utf-8" - stderr_log_enc = sys.stderr.encoding or "utf-8" + stdout_log_enc = sys.stdout.encoding or 'utf-8' + stderr_log_enc = sys.stderr.encoding or 'utf-8' async def read_stream(stream, prio, collector: list[bytes], log_enc: str): - buf = b"" + buf = b'' while True: chunk = await stream.read(4096) if not chunk: @@ -116,12 +121,12 @@ class Local(Base): collector.append(chunk) if verbose: buf += chunk - while b"\n" in buf: - line, buf = buf.split(b"\n", 1) - __log(prio, line.decode(log_enc, errors="replace")) + while b'\n' in buf: + line, buf = buf.split(b'\n', 1) + __log(prio, line.decode(log_enc, errors = 'replace')) if verbose and buf: # flush trailing partial line (no newline) - __log(prio, buf.decode(log_enc, errors="replace")) + __log(prio, buf.decode(log_enc, errors = 'replace')) tasks = [ asyncio.create_task( @@ -132,7 +137,8 @@ class Local(Base): ), ] - if stdin is asyncio.subprocess.PIPE: + if (cmd_input is not None and stdin is asyncio.subprocess.PIPE + and proc.stdin is not None): proc.stdin.write(cmd_input) await proc.stdin.drain() proc.stdin.close() @@ -140,8 +146,8 @@ class Local(Base): exit_code = await proc.wait() await asyncio.gather(*tasks) - stdout = b"".join(stdout_parts) if stdout_parts else None - stderr = b"".join(stderr_parts) if stderr_parts else None + stdout = b''.join(stdout_parts) if stdout_parts else None + stderr = b''.join(stderr_parts) if stderr_parts else None return Result(stdout, stderr, exit_code) @@ -153,7 +159,9 @@ class Local(Base): os.unlink(path) async def _erase(self, path: str) -> None: - if os.isdir(path): + if os.path.isdir(path): + import shutil + shutil.rmtree(path) return os.unlink(path) @@ -165,12 +173,12 @@ class Local(Base): os.mkdir(name, mode) async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: - return StatResult.from_os(os.stat(path, follow_symlinks=follow_symlinks)) + return StatResult.from_os(os.stat(path, follow_symlinks = follow_symlinks)) async def _file_exists(self, path: str) -> bool: return os.path.exists(path) - async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + async def _chown(self, path: str, owner: str | None, group: str | None) -> None: uid = pwd.getpwnam(owner).pw_uid if owner else -1 gid = grp.getgrnam(group).gr_gid if group else -1 os.chown(path, uid, gid) @@ -179,6 +187,6 @@ class Local(Base): os.chmod(path, mode) async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: - if (not follow_symlinks) and os.islink(path): + if (not follow_symlinks) and os.path.islink(path): return False return os.path.isdir(path) diff --git a/src/python/jw/pkg/lib/ec/SSHClient.py b/src/python/jw/pkg/lib/ec/SSHClient.py index c8534dff..c04cbd08 100644 --- a/src/python/jw/pkg/lib/ec/SSHClient.py +++ b/src/python/jw/pkg/lib/ec/SSHClient.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Any, TYPE_CHECKING +import abc +import os +import pwd +import sys -import os, abc, sys, pwd from enum import Flag, auto +from typing import TYPE_CHECKING -from ..util import pretty_cmd -from ..log import * from ..base import Result from ..ExecContext import ExecContext +from ..log import DEBUG, ERR, INFO, NOTICE, WARNING, log from ..Uri import Uri if TYPE_CHECKING: @@ -24,48 +24,50 @@ class SSHClient(ExecContext): ModEnv = auto() Wd = auto() - def __init__(self, uri: Uri|str, caps: Caps=Caps(0), *args, **kwargs) -> None: + def __init__(self, uri: Uri | str, caps: Caps = Caps(0), *args, **kwargs) -> None: uri = Uri.pimp(uri) if uri.username is None: uri.set_username(pwd.getpwuid(os.getuid()).pw_name) - super().__init__(uri=uri, *args, **kwargs) + super().__init__(uri = uri, *args, **kwargs) self.__caps = caps @abc.abstractmethod async def _run_ssh( + self, cmd: list[str], - wd: str|None, + wd: str | None, verbose: bool, - cmd_input: bytes|None, - mod_env: dict[str, str]|None, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, interactive: bool, - log_prefix: str + log_prefix: str, ) -> Result: pass async def _run( self, cmd: list[str], - wd: str|None, + wd: str | None, verbose: bool, - cmd_input: bytes|None, - mod_env: dict[str, str]|None, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, interactive: bool, - log_prefix: str + log_prefix: str, ) -> Result: def __log(prio: int, *args): log(prio, log_prefix, *args) - def __log_block(prio: int, title: str, block: str): + def __log_block(prio: int, title: str, block: bytes | str | None): if self.__caps & self.Caps.LogOutput: return if not block: return - encoding = sys.stdout.encoding or 'utf-8' - block = block.decode(encoding).strip() - if not block: - return + if isinstance(block, bytes): + encoding = sys.stdout.encoding or 'utf-8' + block = block.decode(encoding).strip() + # Needed to pacify pyright: block can't be anything else at this point + assert isinstance(block, str) delim = f'---- {title} ----' __log(prio, f',{delim}') for line in block.splitlines(): @@ -79,44 +81,49 @@ class SSHClient(ExecContext): raise NotImplementedError('Interactive SSH is not yet implemented') if mod_env is not None and not self.__caps & self.Caps.ModEnv: - raise NotImplementedError('Passing an environment to SSH commands is not yet implemented') + raise NotImplementedError( + 'Passing an environment to SSH commands is not yet implemented' + ) ret = await self._run_ssh( - cmd=cmd, - wd=wd, - verbose=verbose, - cmd_input=cmd_input, - mod_env=mod_env, - interactive=interactive, - log_prefix=log_prefix + cmd = cmd, + wd = wd, + verbose = verbose, + cmd_input = cmd_input, + mod_env = mod_env, + interactive = interactive, + log_prefix = log_prefix, ) if verbose: - __log_block(NOTICE, 'stdout', ret.stdout) - __log_block(NOTICE, 'stderr', ret.stderr) + __log_block(NOTICE, 'stdout', ret.stdout_str_or_none) + __log_block(NOTICE, 'stderr', ret.stderr_str_or_none) if ret.status != 0: __log(WARNING, f'Exit code {ret.status}') return ret @property - def hostname(self) -> str|None: + def hostname(self) -> str | None: return self.uri.hostname @property - def port(self) -> int|None: + def port(self) -> int | None: return self.uri.port @property - def username(self) -> str|None: + def username(self) -> str | None: return self.uri.username @property - def password(self) -> str|None: + def password(self) -> str | None: return self.uri.password -def ssh_client(*args, type: str|list[str]|None=None, **kwargs) -> SSHClient: # export +def ssh_client( + *args, type: str | list[str] | None = None, **kwargs +) -> SSHClient: # export from importlib import import_module + errors: list[str] = [] if type is None: val = os.getenv('JW_DEFAULT_SSH_CLIENT') @@ -128,11 +135,12 @@ def ssh_client(*args, type: str|list[str]|None=None, **kwargs) -> SSHClient: # e type = [type] for name in type: try: - ret = getattr(import_module(f'jw.pkg.lib.ec.ssh.{name}'), name)(*args, **kwargs) + ret = getattr(import_module(f'jw.pkg.lib.ec.ssh.{name}'), + name)(*args, **kwargs) log(INFO, f'Using SSH-client "{name}"') return ret except Exception as e: - msg = f'Can\'t instantiate SSH client class {name} ({str(e)})' + msg = f"Can't instantiate SSH client class {name} ({str(e)})" errors.append(msg) log(DEBUG, f'{msg}, trying next') msg = f'No working SSH clients for {" ".join([str(arg) for arg in args])}' diff --git a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py index b0159e3c..8f01a147 100644 --- a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py +++ b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py @@ -1,11 +1,15 @@ -# -*- coding: utf-8 -*- +import asyncio +import os +import shlex +import shutil +import signal +import sys -import os, sys, shlex, asyncio, asyncssh, shutil, signal +import asyncssh -from ...log import * from ...base import Result +from ...log import DEBUG, ERR, NOTICE, log from ..SSHClient import SSHClient as Base - from .util import join_cmd _USE_DEFAULT_KNOWN_HOSTS = object() @@ -25,15 +29,18 @@ class AsyncSSH(Base): super().__init__( uri, - caps = self.Caps.LogOutput | self.Caps.Wd | self.Caps.Interactive | self.Caps.ModEnv, - **kwargs + caps = self.Caps.LogOutput + | self.Caps.Wd + | self.Caps.Interactive + | self.Caps.ModEnv, + **kwargs, ) self.__client_keys = client_keys self.__known_hosts = known_hosts self.__term_type = term_type or os.environ.get('TERM', 'xterm') self.__connect_timeout = connect_timeout - self.__conn: asyncssh.SSHClientConnection|None = None + self.__conn: asyncssh.SSHClientConnection | None = None async def _open(self) -> None: await super()._open() @@ -48,7 +55,7 @@ class AsyncSSH(Base): log(DEBUG, f'Failed to close connection ({str(e)}, ignored)') self.__conn = None - def _connect_kwargs(self, hide_secrets: bool=False) -> dict: + def _connect_kwargs(self, hide_secrets: bool = False) -> dict: kwargs: dict = { 'host': self.hostname, 'port': self.port, @@ -72,7 +79,7 @@ class AsyncSSH(Base): except Exception as e: msg = f'-------------------- Failed to connect ({str(e)})' log(ERR, ',', msg) - for key, val in self._connect_kwargs(hide_secrets=True).items(): + for key, val in self._connect_kwargs(hide_secrets = True).items(): log(ERR, f'| {key:<20} = {val}') log(ERR, '`', msg) raise @@ -94,10 +101,13 @@ class AsyncSSH(Base): @staticmethod def _get_local_term_size() -> tuple[int, int, int, int]: - cols, rows = shutil.get_terminal_size(fallback=(80, 24)) + cols, rows = shutil.get_terminal_size(fallback = (80, 24)) xpixel = ypixel = 0 try: - import fcntl, termios, struct + import fcntl + import struct + import termios + packed = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, b'\0' * 8) rows2, cols2, xpixel, ypixel = struct.unpack('HHHH', packed) if cols2 > 0 and rows2 > 0: @@ -126,9 +136,9 @@ class AsyncSSH(Base): buf += chunk while b'\n' in buf: line, buf = buf.split(b'\n', 1) - log(prio, log_prefix, line.decode(log_enc, errors='replace')) + log(prio, log_prefix, line.decode(log_enc, errors = 'replace')) if verbose and buf: - log(prio, log_prefix, buf.decode(log_enc, errors='replace')) + log(prio, log_prefix, buf.decode(log_enc, errors = 'replace')) async def _run_interactive_on_conn( self, @@ -222,7 +232,8 @@ class AsyncSSH(Base): sys.stderr.flush() try: - import termios, tty + import termios + import tty old_tty_state = termios.tcgetattr(stdin_fd) tty.setraw(stdin_fd) @@ -257,7 +268,9 @@ class AsyncSSH(Base): exit_code = completed.exit_status if exit_code is None: - exit_code = completed.returncode if completed.returncode is not None else -1 + exit_code = ( + completed.returncode if completed.returncode is not None else -1 + ) stdout = b''.join(stdout_parts) if stdout_parts else None return Result(stdout, None, exit_code) @@ -278,6 +291,7 @@ class AsyncSSH(Base): if old_tty_state is not None: try: import termios + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_tty_state) except Exception: pass @@ -331,7 +345,7 @@ class AsyncSSH(Base): await proc.stdin.drain() proc.stdin.write_eof() - completed = await proc.wait(check=False) + completed = await proc.wait(check = False) await task exit_code = completed.exit_status @@ -346,14 +360,13 @@ class AsyncSSH(Base): cmd: list[str], wd: str | None, verbose: bool, - cmd_input: str | None, + cmd_input: bytes | None, mod_env: dict[str, str] | None, interactive: bool, log_prefix: str, ) -> Result: try: - if interactive: if self._has_local_tty(): return await self._run_interactive_on_conn( @@ -421,7 +434,7 @@ class AsyncSSH(Base): await proc.stdin.drain() proc.stdin.write_eof() - completed = await proc.wait(check=False) + completed = await proc.wait(check = False) await asyncio.gather(*tasks) stdout = b''.join(stdout_parts) if stdout_parts else None @@ -429,7 +442,9 @@ class AsyncSSH(Base): exit_code = completed.exit_status if exit_code is None: - exit_code = completed.returncode if completed.returncode is not None else -1 + exit_code = ( + completed.returncode if completed.returncode is not None else -1 + ) return Result(stdout, stderr, exit_code) diff --git a/src/python/jw/pkg/lib/ec/ssh/Exec.py b/src/python/jw/pkg/lib/ec/ssh/Exec.py index 23b95881..b902d6e4 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Exec.py +++ b/src/python/jw/pkg/lib/ec/ssh/Exec.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os + from typing import TYPE_CHECKING from ...base import InputMode @@ -8,18 +10,14 @@ from ..SSHClient import SSHClient as Base from .util import join_cmd if TYPE_CHECKING: - from ...base import Result + from ...base import Input, Result class Exec(Base): def __init__(self, uri, *args, **kwargs) -> None: - self.__askpass: str|None = None - self.__askpass_orig: dict[str, str|None] = dict() - super().__init__( - uri = uri, - caps = self.Caps.ModEnv, - **kwargs - ) + self.__askpass: str | None = None + self.__askpass_orig: dict[str, str | None] = dict() + super().__init__(uri = uri, caps = self.Caps.ModEnv, **kwargs) def __del__(self): for key, val in self.__askpass_orig.items(): @@ -32,36 +30,53 @@ class Exec(Base): def __init_askpass(self): if self.__askpass is None and self.password is not None: - import sys, tempfile + import sys + import tempfile + prefix = os.path.basename(sys.argv[0]) + '-' - f = tempfile.NamedTemporaryFile(mode='w+t', prefix=prefix, delete=False) + f = tempfile.NamedTemporaryFile( + mode = 'w+t', prefix = prefix, delete = False + ) os.chmod(f.name, 0o0700) self.__askpass = f.name f.write(f'#!/bin/bash\n\necho -n "{self.password}\n"') f.close() - for key, val in {'SSH_ASKPASS': self.__askpass, 'SSH_ASKPASS_REQUIRE': 'force'}.items(): + for key, val in { + 'SSH_ASKPASS': self.__askpass, + 'SSH_ASKPASS_REQUIRE': 'force', + }.items(): self.__askpass_orig[key] = os.getenv(key) os.environ[key] = val async def _run_ssh( self, cmd: list[str], - wd: str|None, + wd: str | None, verbose: bool, - cmd_input: bytes|None, - mod_env: dict[str, str]|None, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, interactive: bool, - log_prefix: str + log_prefix: str, ) -> Result: + + def __pub_cmd_input(cmd_input: bytes | None) -> Input: + if cmd_input is None: + if interactive: + return InputMode.Interactive + return InputMode.NonInteractive + return cmd_input + self.__init_askpass() - if cmd_input is None: - cmd_input = InputMode.Interactive if interactive else InputMode.NonInteractive - opts: dict[str, str] = [] + opts: list[str] = [] if mod_env: for key, val in mod_env.items(): opts.extend(['-o', f'SetEnv {key}="{val}"']) if self.username: opts.extend(['-l', self.username]) if self.port is not None: - pots.extend(['-p', str(self.port)]) - return await run_cmd(['ssh', *opts, self.hostname, join_cmd(cmd)], cmd_input=cmd_input, throw=False) + opts.extend(['-p', str(self.port)]) + return await run_cmd( + ['ssh', *opts, self.hostname, join_cmd(cmd)], + cmd_input = __pub_cmd_input(cmd_input), + throw = False, + ) diff --git a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py index b60be30c..93c8ec38 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py +++ b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py @@ -2,44 +2,46 @@ from __future__ import annotations from typing import TYPE_CHECKING -import paramiko # type: ignore # error: Library stubs not installed for "paramiko" +import paramiko # type: ignore[import-untyped] # error: Library stubs not installed for "paramiko" -from ...log import * from ...base import Result +from ...log import ERR, log from ..SSHClient import SSHClient as Base - from .util import join_cmd if TYPE_CHECKING: from typing import Any + import paramiko.agent # type: ignore[import-untyped] + import paramiko.SCPClient # type: ignore[import-untyped] + class Paramiko(Base): def __init__(self, uri, *args, **kwargs) -> None: - super().__init__( - uri, - *args, - caps = self.Caps.ModEnv, - **kwargs - ) - self.__timeout: float|None = None # Untested - self.___client: Any|None = None + kwargs['caps'] = (self.Caps.ModEnv, ) + super().__init__(uri, *args, **kwargs) + self.__timeout: float | None = None # Untested + self.___client: Any | None = None @property def __client(self) -> Any: if self.___client is None: ret = paramiko.SSHClient() ret.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + hostname = self.hostname + if hostname is None: + raise Exception('Tried to run connect without target hostname') try: ret.connect( - hostname = self.hostname, - username = self.username, - allow_agent = True + hostname = hostname, username = self.username, allow_agent = True ) except Exception as e: log(ERR, f'Failed to connect to {self.hostname} ({str(e)})') raise - s = ret.get_transport().open_session() + transport = ret.get_transport() + if transport is None: + raise Exception(f'Failed to get SSH transport for {hostname}') + s = transport.open_session() # set up the agent request handler to handle agent requests from the server paramiko.agent.AgentRequestHandler(s) self.___client = ret @@ -47,7 +49,7 @@ class Paramiko(Base): @property def __scp(self) -> Any: - return SCPClient(self.__client.get_transport()) + return paramiko.SCPClient(self.__client.get_transport()) async def _open(self) -> None: await super()._open() @@ -63,13 +65,13 @@ class Paramiko(Base): cmd: list[str], wd: str | None, verbose: bool, - cmd_input: str | None, + cmd_input: bytes | None, mod_env: dict[str, str] | None, interactive: bool, log_prefix: str, ) -> Result: try: - kwargs: [str, Any] = {} + kwargs: dict[str, Any] = {} if mod_env is not None: kwargs['environment'] = mod_env stdin, stdout, stderr = self.__client.exec_command( @@ -78,7 +80,7 @@ class Paramiko(Base): **kwargs, ) except Exception as e: - log(ERR, f'Command failed for {self.uri}: "{join_cmd(cmd)}"') + log(ERR, f'Command failed for {self.uri}: "{join_cmd(cmd)}" ({str(e)})') raise if cmd_input is not None: stdin.write(cmd_input) diff --git a/src/python/jw/pkg/lib/ec/ssh/util.py b/src/python/jw/pkg/lib/ec/ssh/util.py index b7cd6a9b..854423e8 100644 --- a/src/python/jw/pkg/lib/ec/ssh/util.py +++ b/src/python/jw/pkg/lib/ec/ssh/util.py @@ -1,19 +1,29 @@ -# -*- coding: utf-8 -*- +import shlex from typing import Iterable -import shlex - DEFAULT_SHELL_OPERATORS = { # redirections - ">", ">>", "<", "<<", "<<-", "<&", ">&", "<>", ">|", - "1>", "1>>", "2>", "2>>", - - # pipelines / control - "|", "||", "&", "&&", ";", - - # grouping - "(", ")", + '>', + '>>', + '<', + '<<', + '<<-', + '<&', + '>&', + '<>', + '>|', + '1>', + '1>>', + '2>', + '2>>', # pipelines / control + '|', + '||', + '&', + '&&', + ';', # grouping + '(', + ')', } def join_cmd( diff --git a/src/python/jw/pkg/lib/log.py b/src/python/jw/pkg/lib/log.py index 5e4e03a2..59c82913 100644 --- a/src/python/jw/pkg/lib/log.py +++ b/src/python/jw/pkg/lib/log.py @@ -1,38 +1,41 @@ -# -*- coding: utf-8 -*- +import datetime +import sys +import syslog -import sys, syslog, datetime +# fmt: disable # don't conflate +EMERG = int(syslog.LOG_EMERG) +ALERT = int(syslog.LOG_ALERT) +CRIT = int(syslog.LOG_CRIT) +ERR = int(syslog.LOG_ERR) +WARNING = int(syslog.LOG_WARNING) +NOTICE = int(syslog.LOG_NOTICE) +INFO = int(syslog.LOG_INFO) +DEBUG = int(syslog.LOG_DEBUG) +DEVEL = int(syslog.LOG_DEBUG + 1) +OFF = DEVEL + 1 -EMERG = int(syslog.LOG_EMERG) -ALERT = int(syslog.LOG_ALERT) -CRIT = int(syslog.LOG_CRIT) -ERR = int(syslog.LOG_ERR) -WARNING = int(syslog.LOG_WARNING) -NOTICE = int(syslog.LOG_NOTICE) -INFO = int(syslog.LOG_INFO) -DEBUG = int(syslog.LOG_DEBUG) -DEVEL = int(syslog.LOG_DEBUG + 1) -OFF = DEVEL + 1 - -_log_level = NOTICE -_last_tstamp = datetime.datetime.now() -_first_tstamp = _last_tstamp +_log_level = NOTICE +_last_tstamp = datetime.datetime.now() +_first_tstamp = _last_tstamp +# fmt: enable def _log_level_name_by_value(): if _log_level_name_by_value.map is None: _log_level_name_by_value.map = { - EMERG: "EMERG", - ALERT: "ALERT", - CRIT: "CRIT", - ERR: "ERR", - WARNING: "WARNING", - NOTICE: "NOTICE", - INFO: "INFO", - DEBUG: "DEBUG", - DEVEL: "DEVEL", - OFF: "OFF" + EMERG: 'EMERG', + ALERT: 'ALERT', + CRIT: 'CRIT', + ERR: 'ERR', + WARNING: 'WARNING', + NOTICE: 'NOTICE', + INFO: 'INFO', + DEBUG: 'DEBUG', + DEVEL: 'DEVEL', + OFF: 'OFF', } return _log_level_name_by_value.map -_log_level_name_by_value.map: dict[int, str]|None = None + +_log_level_name_by_value.map: dict[int, str] | None = None # type: ignore def _log_level_value_by_name(): if _log_level_value_by_name.map is None: @@ -41,21 +44,22 @@ def _log_level_value_by_name(): _log_level_value_by_name.map[name] = value _log_level_value_by_name.map[name.lower()] = value return _log_level_value_by_name.map -_log_level_value_by_name.map: dict[str, int]|None = None + +_log_level_value_by_name.map: dict[str, int] | None = None # type: ignore def get_log_level_name(level: int) -> str: return _log_level_name_by_value()[level] -def parse_log_level(level: str|int) -> int: +def parse_log_level(level: str | int) -> int: try: ret = int(level) if ret >= 0 and ret <= DEVEL: return ret except ValueError: - return _log_level_value_by_name()[level] - raise Exception("Invalid log level ", level) + return _log_level_value_by_name()[level] + raise Exception('Invalid log level ', level) -def set_log_level(level: int|None=None) -> int: +def set_log_level(level: str | int | None = None) -> int: global _log_level ret = _log_level if level is not None: diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index 88074d87..6ed15fb9 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -1,50 +1,66 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Iterable, TYPE_CHECKING +from typing import TYPE_CHECKING, Iterable if TYPE_CHECKING: from ..ExecContext import ExecContext from ..base import InputMode +from ..Package import Package from ..util import run_cmd, run_sudo -from ..Package import Package, meta_tags -_meta_map: dict[str, str]|None = None +_meta_map: dict[str, str] | None = None def meta_map(): global _meta_map if _meta_map is None: - _meta_map = Package.order_tags({ - 'name': 'binary:Package', - 'vendor': None, # deb doesn't have vendor field - 'packager': None, # -- packager -- - 'url': 'Homepage', - 'maintainer': 'Maintainer', - }) + _meta_map = Package.order_tags( + { + 'name': 'binary:Package', + 'vendor': None, # deb doesn't have vendor field + 'packager': None, # -- packager -- + 'url': 'Homepage', + 'maintainer': 'Maintainer', + } + ) return _meta_map -async def run_dpkg(args: list[str], sudo: bool=False, ec: ExecContext=None): # export +async def _run( + cmd: list[str], sudo: bool = False, ec: ExecContext | None = None +) -> str: + return ( + await run_sudo(cmd) + if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive) + ).stdout_str + +async def run_dpkg( + args: list[str], + sudo: bool = False, + ec: ExecContext | None = None +) -> str: # export cmd = ['/usr/bin/dpkg'] cmd.extend(args) - if sudo: - return await run_sudo(cmd, ec=ec) - return (await run_cmd(cmd, ec=ec)).decode() + return await _run(cmd, sudo, ec) -async def run_dpkg_query(args: list[str], sudo: bool=False, ec: ExecContext=None): # export +async def run_dpkg_query( + args: list[str], + sudo: bool = False, + ec: ExecContext | None = None +) -> str: # export cmd = ['/usr/bin/dpkg-query'] cmd.extend(args) - if sudo: - return await run_sudo(cmd) - return (await run_cmd(cmd, ec=ec, cmd_input=InputMode.NonInteractive)).decode() + return await _run(cmd, sudo, ec) -async def query_packages(names: Iterable[str] = [], ec: ExecContext=None) -> Iterable[Package]: - fmt_str = '|'.join([(f'${{{tag}}}' if tag else '') for tag in meta_map().values()]) + r'\n' +async def query_packages(names: Iterable[str] = [], + ec: ExecContext | None = None) -> Iterable[Package]: + fmt_str = ( + '|'.join([(f'${{{tag}}}' if tag else '') + for tag in meta_map().values()]) + r'\n' + ) # dpkg-query -W -f='${binary:Package}|${Maintainer}| ... \n' - specs, stderr, status = await run_dpkg_query(['-W', '-f=' + fmt_str, *names], sudo=False, ec=ec) + specs = await run_dpkg_query(['-W', '-f=' + fmt_str, *names], sudo = False, ec = ec) return Package.parse_specs_str(specs) -async def list_files(pkg: str, ec: ExecContext=None) -> list[str]: - file_list_str, stderr, status = await run_dpkg(['-L', pkg], sudo=False, ec=ec) +async def list_files(pkg: str, ec: ExecContext | None = None) -> list[str]: + file_list_str = await run_dpkg(['-L', pkg], sudo = False, ec = ec) return file_list_str.splitlines() diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index 24f36f49..b52c8271 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -1,45 +1,71 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import Iterable, TYPE_CHECKING +from typing import TYPE_CHECKING, Iterable + +from ..base import InputMode +from ..Package import Package +from ..util import run_cmd, run_sudo if TYPE_CHECKING: from ..ExecContext import ExecContext -from ..util import run_cmd, run_sudo -from ..base import InputMode -from ..Package import Package, meta_tags - -_meta_map: dict[str, str]|None = None +_meta_map: dict[str, str] | None = None def meta_map(): global _meta_map if _meta_map is None: - _meta_map = Package.order_tags({ - 'name': 'Name', - 'vendor': 'Vendor', - 'packager': 'Packager', - 'url': 'URL', - 'maintainer': None, # RPM doesn't have a maintainer field - }) + _meta_map = Package.order_tags( + { + 'name': 'Name', + 'vendor': 'Vendor', + 'packager': 'Packager', + 'url': 'URL', + 'maintainer': None, # RPM doesn't have a maintainer field + } + ) return _meta_map -async def run_rpm(args: list[str], sudo: bool=False, ec: ExecContext=None, mode: InputMode=InputMode.OptInteractive, **kwargs): # export +async def run_rpm( + args: list[str], + sudo: bool = False, + ec: ExecContext | None = None, + mode: InputMode = InputMode.OptInteractive, + **kwargs, +) -> str: # export cmd = ['/usr/bin/rpm'] cmd.extend(args) - if sudo: - return await run_sudo(cmd, ec=ec, cmd_input=mode, **kwargs) - return await run_cmd(cmd, ec=ec, cmd_input=mode, **kwargs) + result = ( + await run_sudo(cmd, ec = ec, cmd_input = mode, **kwargs) + if sudo else await run_cmd(cmd, ec = ec, cmd_input = mode, **kwargs) + ) + return result.stdout_str -async def query_packages(names: Iterable[str] = [], ec: ExecContext=None) -> Iterable[Package]: - fmt_str = '|'.join([(f'%{{{tag}}}' if tag else '') for tag in meta_map().values()]) + r'\n' +async def query_packages( + names: Iterable[str] = [], + ec: ExecContext | None = None, +) -> Iterable[Package]: # export + fmt_str = ( + '|'.join([(f'%{{{tag}}}' if tag else '') + for tag in meta_map().values()]) + r'\n' + ) opts = ['-q', '--queryformat', fmt_str] if not names: opts.append('-a') - specs, stderr, status = await run_rpm([*opts, *names], throw=True, sudo=False, mode=InputMode.NonInteractive, ec=ec) - return Package.parse_specs_str(specs.decode()) + specs = await run_rpm( + [*opts, *names], + throw = True, + sudo = False, + mode = InputMode.NonInteractive, + ec = ec + ) + return Package.parse_specs_str(specs) -async def list_files(pkg: str, ec: ExecContext=None) -> list[str]: - stdout, stderr, status = await run_rpm(['-ql', pkg], throw=True, sudo=False, mode=InputMode.NonInteractive, ec=ec) - return stdout.decode().splitlines() +async def list_files(pkg: str, ec: ExecContext | None = None) -> list[str]: + stdout = await run_rpm( + ['-ql', pkg], + throw = True, + sudo = False, + mode = InputMode.NonInteractive, + ec = ec + ) + return stdout.splitlines() diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index e1a71ba7..ac57121d 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -1,28 +1,31 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -from typing import TYPE_CHECKING, Iterable - -if TYPE_CHECKING: - from typing import Sequence - from .ExecContext import ExecContext - from .ProcFilter import ProcFilter, ProcPipeline - -import os, sys, json +import json +import os +import sys from argparse import Namespace from enum import Enum, auto +from typing import TYPE_CHECKING, Iterable, TypeVar, cast -from .log import * -from .base import InputMode +from .base import Input, InputMode, Result +from .log import DEBUG, ERR, log from .Uri import Uri +if TYPE_CHECKING: + from .ExecContext import ExecContext + from .FileContext import FileContext + from .ProcFilter import ProcFilter, ProcPipeline + +T = TypeVar('T') + class AskpassKey(Enum): Username = auto() Password = auto() -def pretty_cmd(cmd: list[str], wd=None): +def pretty_cmd(cmd: list[str] | None = None, wd = None): + if cmd is None: + cmd = sys.argv tokens = [cmd[0]] for token in cmd[1:]: if token.find(' ') != -1: @@ -34,44 +37,72 @@ def pretty_cmd(cmd: list[str], wd=None): return ret # See ExecContext.run() for what this function does -async def run_cmd(*args, ec: ExecContext|None=None, verbose: bool|None=None, cmd_input: Input=InputMode.NonInteractive, **kwargs) -> Result: +async def run_cmd( + *args, + ec: ExecContext | None = None, + verbose: bool | None = None, + cmd_input: Input = InputMode.NonInteractive, + **kwargs, +) -> Result: if verbose is None: verbose = False if ec is None else ec.verbose_default if ec is None: from .ec.Local import Local - interactive = cmd_input == InputMode.Interactive - ec = Local(verbose_default=verbose, interactive=interactive) - return await ec.run(verbose=verbose, *args, **kwargs) -async def run_curl(args: list[str], parse_json: bool=False, wd=None, throw=None, verbose=None, cmd_input=InputMode.NonInteractive, ec: ExecContext|None=None, decode=False) -> dict|str: # export + interactive = cmd_input == InputMode.Interactive + ec = Local(verbose_default = verbose, interactive = interactive) + kwargs['verbose'] = verbose + return await ec.run(*args, **kwargs) + +async def run_curl( + args: list[str], + wd = None, + throw = None, + verbose = None, + cmd_input = InputMode.NonInteractive, + ec: ExecContext | None = None, + decode = False, +) -> Result: if verbose is None: verbose = False if ec is None else ec.verbose_default cmd = ['curl'] if not verbose: cmd.append('-s') cmd.extend(args) - if parse_json: - decode = True - output = await run_cmd(cmd, wd=wd, throw=throw, verbose=verbose, cmd_input=cmd_input, ec=ec) - stdout, stderr, status = output.decode() if decode else output - if not parse_json: - ret = stdout - else: - try: - ret = json.loads(stdout) - except Exception as e: - size = 'unknown number of' - try: - size = len(stdout) - except: - pass - log(ERR, f'Failed to parse {size} bytes output of command ' - + f'>{pretty_cmd(cmd, wd)}< ({str(e)}): "{stdout}"', file=sys.stderr) - raise - return ret, stderr, status + return await run_cmd( + cmd, wd = wd, throw = throw, verbose = verbose, cmd_input = cmd_input, ec = ec + ) -async def run_askpass(askpass_env: list[str], key: AskpassKey, host: str|None=None, ec: ExecContext|None=None): - if host is not None: # Currently unsupported +async def run_curl_into( + expected_type: type[T], + args: list[str], + **kwargs, +) -> T: + result = await run_curl(args, **kwargs) + stdout = result.stdout_str + try: + ret = json.loads(stdout) + except Exception as e: + log( + ERR, + f'Failed to parse {len(stdout)} bytes of Curl output ({str(e)})', + file = sys.stderr, + ) + raise + if not isinstance(ret, expected_type): + raise TypeError( + f'Expected {expected_type.__name__}, got {type(ret).__name__} from Curl' + ) + return cast(T, ret) + +async def run_askpass( + askpass_env: list[str], + key: AskpassKey, + host: str | None = None, + ec: ExecContext | None = None, + throw: bool = False, +) -> str | None: + if host is not None: # Currently unsupported raise NotImplementedError(f'Tried to run askpass with host "{host}"') for var in askpass_env: exe = os.getenv(var) @@ -88,50 +119,88 @@ async def run_askpass(askpass_env: list[str], key: AskpassKey, host: str|None=No case 'SSH_ASKPASS': match key: case AskpassKey.Username: - continue # Can't get user name from SSH_ASKPASS + continue # Can't get user name from SSH_ASKPASS case AskpassKey.Password: exe_arg += 'Password' - ret, stderr, status = await run_cmd([exe, exe_arg], throw=False, ec=ec).decode() - if ret is not None: - return ret + result = await run_cmd([exe, exe_arg], throw = throw, ec = ec) + if result.status == 0 and result.stdout_or_none is not None: + ret = result.stdout_str_or_none + if ret: + return ret + msg = ( + f"Trying to get user data from {', '.join(askpass_env)} didn't produce anything" + ) + if throw: + raise Exception(msg) + log(DEBUG, msg) return None -async def run_sudo(cmd: list[str], *args, interactive: bool=True, ec: ExecContext|None=None, **kwargs): +async def run_sudo( + cmd: list[str], + *args, + interactive: bool = True, + ec: ExecContext | None = None, + **kwargs, +): if ec is None: from .ec.Local import Local - ec = Local(interactive=interactive) + + ec = Local(interactive = interactive) return await ec.sudo(cmd, *args, **kwargs) async def get( - uri: str|Uri, - *args, - ctx: FileContext|None=None, - content_filter: ProcFilter|list[ProcFilter]|ProcPipeline|None = None, - **kwargs - ) -> Result: + uri: str | Uri, + *args, + ctx: FileContext | None = None, + content_filter: ProcFilter | list[ProcFilter] | ProcPipeline | None = None, + **kwargs, +) -> Result: uri = Uri.pimp(uri) if ctx is None or uri.id != ctx.uri.id: from .FileContext import FileContext + ctx = FileContext.create(uri) from .ProcFilter import run as run_pipeline + return await run_pipeline(await ctx.get(uri.path, *args, **kwargs), content_filter) -async def copy(src_uri: str|Iterable[str], dst: str|FileContext, owner: str|None=None, group: str|None=None, mode: int|None=None, throw=True) -> Exception|str|list[str]: +async def copy( + src_uri: str | Iterable[str], + dst: str | FileContext, + owner: str | None = None, + group: str | None = None, + mode: int | None = None, + throw = True, +) -> Exception | str | list[str]: if not isinstance(src_uri, str): ret: list[str] = [] - for uri in src_uri: # TODO: Group identical netlocs into one CopyContext - rr = ret.append(await copy(uri, dst, owner, group, mode, throw)) + for uri in src_uri: # TODO: Group identical netlocs into one CopyContext + rr = await copy(uri, dst, owner, group, mode, throw) if isinstance(rr, Exception): return rr + if isinstance(rr, list): + ret.extend(rr) + if isinstance(rr, str): + ret.append(rr) + else: + raise Exception(f'copy() returned unexpected type {type(rr)}') return ret from .CopyContext import CopyContext + async with CopyContext(src_uri, dst) as ctx: try: - content = (await ctx.src.get(ctx.src.root, throw=True)).stdout + result = await ctx.src.get(ctx.src.root, throw = True) dst_path = ctx.dst.root if await ctx.dst.is_dir(ctx.dst.root): dst_path += '/' + os.path.basename(src_uri) - await ctx.dst.put(path=dst_path, content=content, owner=owner, group=group, mode=mode, throw=True) + await ctx.dst.put( + path = dst_path, + content = result.stdout, + owner = owner, + group = group, + mode = mode, + throw = True, + ) return dst_path except Exception as e: if throw: @@ -140,21 +209,39 @@ async def copy(src_uri: str|Iterable[str], dst: str|FileContext, owner: str|None return e assert False, 'Unreachable code' -async def get_username(args: Namespace|None=None, url: str|None=None, askpass_env: list[str]=[], ec: ExecContext|None=None) -> str: # export +async def get_username( + args: Namespace | None = None, + url: str | None = None, + askpass_env: list[str] = [], + ec: ExecContext | None = None, +) -> str | None: # export url_user = None if url is None else Uri(url).username if args is not None: if args.username is not None: if url_user is not None and url_user != args.username: - raise Exception(f'Username mismatch: called with --username="{args.username}", URL has user name "{url_user}"') + raise Exception( + f'Username mismatch: called with --username="{args.username}", ' + f'URL has user name "{url_user}"' + ) return args.username if url_user is not None: return url_user - return await run_askpass(askpass_env, AskpassKey.Username, ec=ec) + return await run_askpass(askpass_env, AskpassKey.Username, ec = ec) -async def get_password(args: Namespace|None=None, url: str|None=None, askpass_env: list[str]=[], ec: ExecContext|None=None) -> str: # export +async def get_password( + args: Namespace | None = None, + url: str | None = None, + askpass_env: list[str] = [], + ec: ExecContext | None = None, +) -> str | None: # export if args is None and url is None and not askpass_env: - raise Exception(f'Neither URL nor command-line arguments nor askpass environment variable available, can\'t get password') - if args is not None and hasattr(args, 'password'): # use getattr(), because we don't necessarily want to have insecure --password among options + raise Exception( + 'Neither URL nor command-line arguments nor askpass environment variable ' + "available, can't get password" + ) + if args is not None and hasattr(args, 'password'): + # use getattr(), because we don't necessarily want to have insecure + # --password among options ret = getattr(args, 'password') if ret is not None: return ret @@ -162,9 +249,13 @@ async def get_password(args: Namespace|None=None, url: str|None=None, askpass_en ret = Uri(url).password if ret is not None: return ret - return await run_askpass(askpass_env, AskpassKey.Password, ec=ec) + return await run_askpass(askpass_env, AskpassKey.Password, ec = ec) -async def get_profile_env(throw: bool=True, keep: Iterable[str]|bool=False, ec: ExecContext|None=None) -> dict[str, str]: # export +async def get_profile_env( + throw: bool = True, + keep: Iterable[str] | bool = False, + ec: ExecContext | None = None, +) -> dict[str, str]: # export """ Get a fresh environment from /etc/profile @@ -177,22 +268,28 @@ async def get_profile_env(throw: bool=True, keep: Iterable[str]|bool=False, ec: Returns: Dictionary with fresh environment """ - mod_env: dict[str,str]|None = None - if keep == False or isinstance(keep, Iterable): + mod_env: dict[str, str] | None = None + if (not keep) or isinstance(keep, Iterable): mod_env = { 'HOME': os.environ.get('HOME', '/'), 'USER': os.environ.get('USER', ''), 'PATH': '/usr/bin:/bin', } - # Run bash as a login shell, which sources /etc/profile, then print environment as NUL-separated key=value pairs + + # Run bash as a login shell, which sources /etc/profile, then print + # environment as NUL-separated key=value pairs cmd = ['/usr/bin/env', '-i', '/bin/bash', '-lc', 'env -0'] - result = await run_cmd(cmd, throw=throw, verbose=True, mod_env=mod_env, ec=ec) + result = await run_cmd( + cmd, throw = throw, verbose = True, mod_env = mod_env, ec = ec + ) ret: dict[str, str] = {} - for entry in result.stdout.rstrip(b"\0").split(b"\0"): - if not entry: - continue - key, val = entry.split(b"=", 1) - ret[key.decode()] = val.decode() + stdout = result.stdout_or_none + if stdout is not None: + for entry in stdout.rstrip(b'\0').split(b'\0'): + if not entry: + continue + bkey, bval = entry.split(b'=', 1) + ret[bkey.decode()] = bval.decode() if isinstance(keep, Iterable): for key in keep: val = os.getenv(key)