diff --git a/src/python/jw/pkg/App.py b/src/python/jw/pkg/App.py index 7abc3040..bcec244e 100644 --- a/src/python/jw/pkg/App.py +++ b/src/python/jw/pkg/App.py @@ -61,7 +61,8 @@ class App(Base): if m is None: raise Exception( f'Can\'t interpret "{fmt}" as valid topdir reference, ' - 'expecting "unaltered", "absolute", or "make:"' + 'expecting "absolute", "relative", "unaltered", ' + 'or "make:"' ) return '$(' + m.group(1) + ')' @@ -199,25 +200,24 @@ class App(Base): ), ) vals_list = vals.split(',') if vals else [] - match scope: - case Scope.Self: - buf += vals_list - case Scope.One | Scope.Subtree: - subscope = scope.Self if scope == Scope.One else scope - for val in vals_list: - val = val.strip() - if not (len(val)): - continue - self.__get_project_refs( - buf, - visited, - val, - section, - key, - add_self = True, - scope = subscope, - names_only = names_only, - ) + if scope != Scope.Self: + # Scope.Self adds the node itself (if add_self), but not its + # children, so stop recursing here + subscope = scope.Self if scope == Scope.One else scope + for val in vals_list: + val = val.strip() + if not (len(val)): + continue + self.__get_project_refs( + buf, + visited, + val, + section, + key, + add_self = True, + scope = subscope, + names_only = names_only, + ) if add_self: buf.append(spec) @@ -232,18 +232,17 @@ class App(Base): for project in projects: if project in graph: continue - for section in sections: - deps = self.get_project_refs( - [project], - ['pkg.requires.jw'], - sections, - scope = Scope.One, - add_self = False, - names_only = True, - ) - graph[project] = set(deps) - for dep in deps: - self.__read_dep_graph([dep], sections, graph) + deps = self.get_project_refs( + [project], + ['pkg.requires.jw'], + sections, + scope = Scope.One, + add_self = False, + names_only = True, + ) + graph[project] = set(deps) + for dep in deps: + self.__read_dep_graph([dep], sections, graph) def __flip_dep_graph(self, graph: Graph) -> Graph: ret: Graph = {} @@ -259,45 +258,43 @@ class App(Base): project: str, graph: Graph, unvisited: list[str], - temp: set[str], - path: list[str], - ) -> str | None: - if project in temp: + stack: list[str], + ) -> list[str] | None: + if project in stack: log(DEBUG, 'found circular dependency at project', project) - return project + idx = stack.index(project) + return stack[idx:] + [project] if project not in unvisited: return None - temp.add(project) + stack.append(project) if project in graph: for dep in graph[project]: - last = self.__find_circular_deps_recursive( - dep, graph, unvisited, temp, path + cycle = self.__find_circular_deps_recursive( + dep, graph, unvisited, stack ) - if last is not None: - path.insert(0, dep) - return last + if cycle is not None: + return cycle unvisited.remove(project) - temp.remove(project) + stack.pop() return None def __find_circular_deps(self, projects: list[str], flavours: list[str]) -> list[str]: graph: Graph = {} - ret: list[str] = [] self.__read_dep_graph(projects, flavours, graph) unvisited = list(graph.keys()) - temp: set[str] = set() flipped = self.__flip_dep_graph(graph) while unvisited: project = unvisited[0] log(DEBUG, 'Checking circular dependency of', project) - last = self.__find_circular_deps_recursive( - project, flipped, unvisited, temp, ret - ) - if last is not None: - log(DEBUG, f'Found circular dependency below {project}, last is {last}') - ret.append(last) - return ret + cycle = self.__find_circular_deps_recursive(project, flipped, unvisited, []) + if cycle is not None: + # An edge a -> b in the flipped graph means that b + # depends on a, so reverse to report the cycle in the + # original direction + cycle = list(reversed(cycle)) + log(DEBUG, f'Found circular dependency: {" -> ".join(cycle)}') + return cycle return [] def __init__(self, distro: Distro | None = None) -> None: diff --git a/src/python/jw/pkg/lib/App.py b/src/python/jw/pkg/lib/App.py index a0febcc7..858c7666 100644 --- a/src/python/jw/pkg/lib/App.py +++ b/src/python/jw/pkg/lib/App.py @@ -513,9 +513,8 @@ class App: # export return self.__parser def run(self, argv: list[str] | None = None) -> None: - previous_eloop: asyncio.AbstractEventLoop | None = None + previous_eloop = _get_current_event_loop() if self.__eloop is None: - previous_eloop = _get_current_event_loop() eloop = asyncio.new_event_loop() asyncio.set_event_loop(eloop) self.__eloop = eloop diff --git a/src/python/jw/pkg/lib/ExecApp.py b/src/python/jw/pkg/lib/ExecApp.py new file mode 100644 index 00000000..992b2cea --- /dev/null +++ b/src/python/jw/pkg/lib/ExecApp.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import sys + +from typing import TYPE_CHECKING, Any, override + +from .App import App as Base + +if TYPE_CHECKING: + + from argparse import ArgumentParser + + from .ExecContext import ExecContext + +class ExecApp(Base): # export + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.__opt_interactive: bool | None = None + self.__opt_verbose: bool | None = None + self.__exec_context: ExecContext | None = None + + @override + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + if self.__exec_context is not None: + await self.__exec_context.close() + self.__exec_context = None + + @override + def _add_arguments(self, parser: ArgumentParser) -> None: + super()._add_arguments(parser) + parser.add_argument( + '--interactive', + choices = ['true', 'false', 'auto'], + default = 'true', + help = 'Wait for user input or try to proceed unattended', + ) + parser.add_argument( + '--verbose', + action = 'store_true', + default = False, + help = "Be verbose on stderr about what's being done on the distro level", + ) + parser.add_argument( + '--target', default = 'local', help = 'Run commands on this host' + ) + + @property + def interactive(self) -> bool: + if self.__opt_interactive is None: + match self.args.interactive: + case 'true': + self.__opt_interactive = True + case 'false': + self.__opt_interactive = False + case 'auto': + self.__opt_interactive = sys.stdin.isatty() + case _: + raise ValueError( + f'Unknown --interactive value: {self.args.interactive}' + ) + # Not logically possible to fail, but this keeps pyright happy + assert self.__opt_interactive is not None + return self.__opt_interactive + + @property + def verbose(self) -> bool: + if self.__opt_verbose is None: + self.__opt_verbose = self.args.verbose + # Not logically possible to fail, but this keeps pyright happy + assert self.__opt_verbose is not None + return self.__opt_verbose + + @property + def exec_context(self) -> ExecContext: + if self.__exec_context is None: + from .ExecContext import ExecContext + + self.__exec_context = ExecContext.create( + self.args.target, + interactive = self.interactive, + verbose_default = self.verbose, + ) + return self.__exec_context diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 35685cc7..55faf2c7 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -553,14 +553,14 @@ class ExecContext(Base): for cmd in cmds: log(DEBUG, f'{self.log_name}: Running {pretty_cmd(cmd.cmd, wd)}') ret = await __run(cmd.cmd, cmd_input = cmd.cmd_input) - tmp_file = None # Has been successfully moved at this point + tmp_file = None # All commands, including the final mv, succeeded return ret finally: if tmp_file is not None: await self.erase(tmp_file) await self.close() except Exception as e: - msg = f'Failed to get {path} from {self.root} ({str(e)})' + msg = f'Failed to put content to {path} on {self.root} ({str(e)})' if throw: raise Exception(msg) log(ERR, msg) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 059152d7..2622e7e8 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -287,7 +287,7 @@ class FileContext(abc.ABC): DEBUG, ( f"{self.log_name} doesn't implement stat(), judging by trailing " - 'slash if {path} is a directory' + f'slash if {path} is a directory' ), ) return path[-1] == '/' diff --git a/src/python/jw/pkg/lib/TarIo.py b/src/python/jw/pkg/lib/TarIo.py index e0aee4da..9853dace 100644 --- a/src/python/jw/pkg/lib/TarIo.py +++ b/src/python/jw/pkg/lib/TarIo.py @@ -51,7 +51,7 @@ class TarIo(CopyContext): try: blob = (await self.src.get(path)).stdout except Exception as e: - log(ERR, f'Failed to read tar file "{path}" ({str(e)}') + log(ERR, f'Failed to read tar file "{path}" ({str(e)})') raise return self._filter_tar_file(blob, path_filter, matched = matched) diff --git a/src/python/jw/pkg/lib/Uri.py b/src/python/jw/pkg/lib/Uri.py index cada582e..11faa6d6 100644 --- a/src/python/jw/pkg/lib/Uri.py +++ b/src/python/jw/pkg/lib/Uri.py @@ -156,6 +156,12 @@ class Uri: ret.__password = None if not path: return ret + if ret.__string.endswith('://'): + # -- Empty authority: the trailing '/' is part of the '://' + # separator, so a leading slash in path must be kept to stay + # an absolute path + ret.__string += path if path.startswith('/') else '/' + path + return ret if ret.__string[-1] == '/': if path[0] == '/': ret.__string += path[1:] diff --git a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py index a6ce7af5..cd7d53d6 100644 --- a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py +++ b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py @@ -72,10 +72,9 @@ class AsyncSSH(Base): } if self.__known_hosts is not _USE_DEFAULT_KNOWN_HOSTS: kwargs['known_hosts'] = self.__known_hosts - ret = {k: v for k, v in kwargs.items() if v is not None} if hide_secrets and 'password' in kwargs: kwargs['password'] = '' - return ret + return {k: v for k, v in kwargs.items() if v is not None} @property async def _conn(self) -> asyncssh.SSHClientConnection: diff --git a/src/python/jw/pkg/lib/ec/ssh/Exec.py b/src/python/jw/pkg/lib/ec/ssh/Exec.py index 4e169551..ac405346 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Exec.py +++ b/src/python/jw/pkg/lib/ec/ssh/Exec.py @@ -39,7 +39,7 @@ class Exec(Base): ) os.chmod(f.name, 0o0700) self.__askpass = f.name - f.write(f'#!/bin/bash\n\necho -n "{self.password}\n"') + f.write(f'#!/bin/bash\n\necho -n "{self.password}"\n') f.close() for key, val in { 'SSH_ASKPASS': self.__askpass, diff --git a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py index 84ac0d9e..d1171d6f 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py +++ b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py @@ -32,10 +32,16 @@ class Paramiko(Base): hostname = self.hostname if hostname is None: raise Exception('Tried to run connect without target hostname') + kwargs: dict[str, Any] = { + 'hostname': hostname, + 'username': self.username, + 'password': self.password, + 'allow_agent': True, + } + if self.port is not None: + kwargs['port'] = self.port try: - ret.connect( - hostname = hostname, username = self.username, allow_agent = True - ) + ret.connect(**kwargs) except Exception as e: log(ERR, f'Failed to connect to {self.hostname} ({str(e)})') raise @@ -88,5 +94,8 @@ class Paramiko(Base): raise if cmd_input is not None: stdin.write(cmd_input) + stdin.channel.shutdown_write() + stdout_data = stdout.read() + stderr_data = stderr.read() exit_status = stdout.channel.recv_exit_status() - return Result(stdout.read(), stderr.read(), exit_status, cmd = cmd) + return Result(stdout_data, stderr_data, exit_status, cmd = cmd) diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index a7b30813..8b2857e9 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -29,7 +29,7 @@ async def _run( cmd: list[str], sudo: bool = False, ec: ExecContext | None = None ) -> str: return ( - await run_sudo(cmd) + await run_sudo(cmd, ec = ec, cmd_input = InputMode.NonInteractive) if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive) ).stdout_str