src/python/jw/pkg/App.py

@@ -61,7 +61,8 @@ class App(Base):
                 if m is None:
                     raise Exception(
                         f'Can\'t interpret "{fmt}" as valid topdir reference, '
-                        'expecting "unaltered", "absolute", or "make:<variable-name>"'
+                        'expecting "absolute", "relative", "unaltered", '
+                        'or "make:<variable-name>"'
                     )
                 return '$(' + m.group(1) + ')'
 
@@ -199,25 +200,24 @@ class App(Base):
             ),
         )
         vals_list = vals.split(',') if vals else []
-        match scope:
+        if scope != Scope.Self:
-            case Scope.Self:
+            # Scope.Self adds the node itself (if add_self), but not its
-                buf += vals_list
+            # children, so stop recursing here
-            case Scope.One | Scope.Subtree:
+            subscope = scope.Self if scope == Scope.One else scope
-                subscope = scope.Self if scope == Scope.One else scope
+            for val in vals_list:
-                for val in vals_list:
+                val = val.strip()
-                    val = val.strip()
+                if not (len(val)):
-                    if not (len(val)):
+                    continue
-                        continue
+                self.__get_project_refs(
-                    self.__get_project_refs(
+                    buf,
-                        buf,
+                    visited,
-                        visited,
+                    val,
-                        val,
+                    section,
-                        section,
+                    key,
-                        key,
+                    add_self = True,
-                        add_self = True,
+                    scope = subscope,
-                        scope = subscope,
+                    names_only = names_only,
-                        names_only = names_only,
+                )
-                    )
         if add_self:
             buf.append(spec)
 
@@ -232,18 +232,17 @@ class App(Base):
         for project in projects:
             if project in graph:
                 continue
-            for section in sections:
+            deps = self.get_project_refs(
-                deps = self.get_project_refs(
+                [project],
-                    [project],
+                ['pkg.requires.jw'],
-                    ['pkg.requires.jw'],
+                sections,
-                    sections,
+                scope = Scope.One,
-                    scope = Scope.One,
+                add_self = False,
-                    add_self = False,
+                names_only = True,
-                    names_only = True,
+            )
-                )
+            graph[project] = set(deps)
-                graph[project] = set(deps)
+            for dep in deps:
-                for dep in deps:
+                self.__read_dep_graph([dep], sections, graph)
-                    self.__read_dep_graph([dep], sections, graph)
 
     def __flip_dep_graph(self, graph: Graph) -> Graph:
         ret: Graph = {}
@@ -259,45 +258,43 @@ class App(Base):
         project: str,
         graph: Graph,
         unvisited: list[str],
-        temp: set[str],
+        stack: list[str],
-        path: list[str],
+    ) -> list[str] | None:
-    ) -> str | None:
+        if project in stack:
-        if project in temp:
             log(DEBUG, 'found circular dependency at project', project)
-            return project
+            idx = stack.index(project)
+            return stack[idx:] + [project]
         if project not in unvisited:
             return None
-        temp.add(project)
+        stack.append(project)
         if project in graph:
             for dep in graph[project]:
-                last = self.__find_circular_deps_recursive(
+                cycle = self.__find_circular_deps_recursive(
-                    dep, graph, unvisited, temp, path
+                    dep, graph, unvisited, stack
                 )
-                if last is not None:
+                if cycle is not None:
-                    path.insert(0, dep)
+                    return cycle
-                    return last
         unvisited.remove(project)
-        temp.remove(project)
+        stack.pop()
         return None
 
     def __find_circular_deps(self, projects: list[str],
                              flavours: list[str]) -> list[str]:
         graph: Graph = {}
-        ret: list[str] = []
         self.__read_dep_graph(projects, flavours, graph)
         unvisited = list(graph.keys())
-        temp: set[str] = set()
         flipped = self.__flip_dep_graph(graph)
         while unvisited:
             project = unvisited[0]
             log(DEBUG, 'Checking circular dependency of', project)
-            last = self.__find_circular_deps_recursive(
+            cycle = self.__find_circular_deps_recursive(project, flipped, unvisited, [])
-                project, flipped, unvisited, temp, ret
+            if cycle is not None:
-            )
+                # An edge a -> b in the flipped graph means that b
-            if last is not None:
+                # depends on a, so reverse to report the cycle in the
-                log(DEBUG, f'Found circular dependency below {project}, last is {last}')
+                # original direction
-                ret.append(last)
+                cycle = list(reversed(cycle))
-                return ret
+                log(DEBUG, f'Found circular dependency: {" -> ".join(cycle)}')
+                return cycle
         return []
 
     def __init__(self, distro: Distro | None = None) -> None:

src/python/jw/pkg/lib/App.py

@@ -513,9 +513,8 @@ class App:  # export
         return self.__parser
 
     def run(self, argv: list[str] | None = None) -> None:
-        previous_eloop: asyncio.AbstractEventLoop | None = None
+        previous_eloop = _get_current_event_loop()
         if self.__eloop is None:
-            previous_eloop = _get_current_event_loop()
             eloop = asyncio.new_event_loop()
             asyncio.set_event_loop(eloop)
             self.__eloop = eloop

src/python/jw/pkg/lib/ExecApp.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import sys
+
+from typing import TYPE_CHECKING, Any, override
+
+from .App import App as Base
+
+if TYPE_CHECKING:
+
+    from argparse import ArgumentParser
+
+    from .ExecContext import ExecContext
+
+class ExecApp(Base):  # export
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.__opt_interactive: bool | None = None
+        self.__opt_verbose: bool | None = None
+        self.__exec_context: ExecContext | None = None
+
+    @override
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        if self.__exec_context is not None:
+            await self.__exec_context.close()
+            self.__exec_context = None
+
+    @override
+    def _add_arguments(self, parser: ArgumentParser) -> None:
+        super()._add_arguments(parser)
+        parser.add_argument(
+            '--interactive',
+            choices = ['true', 'false', 'auto'],
+            default = 'true',
+            help = 'Wait for user input or try to proceed unattended',
+        )
+        parser.add_argument(
+            '--verbose',
+            action = 'store_true',
+            default = False,
+            help = "Be verbose on stderr about what's being done on the distro level",
+        )
+        parser.add_argument(
+            '--target', default = 'local', help = 'Run commands on this host'
+        )
+
+    @property
+    def interactive(self) -> bool:
+        if self.__opt_interactive is None:
+            match self.args.interactive:
+                case 'true':
+                    self.__opt_interactive = True
+                case 'false':
+                    self.__opt_interactive = False
+                case 'auto':
+                    self.__opt_interactive = sys.stdin.isatty()
+                case _:
+                    raise ValueError(
+                        f'Unknown --interactive value: {self.args.interactive}'
+                    )
+            # Not logically possible to fail, but this keeps pyright happy
+            assert self.__opt_interactive is not None
+        return self.__opt_interactive
+
+    @property
+    def verbose(self) -> bool:
+        if self.__opt_verbose is None:
+            self.__opt_verbose = self.args.verbose
+            # Not logically possible to fail, but this keeps pyright happy
+            assert self.__opt_verbose is not None
+        return self.__opt_verbose
+
+    @property
+    def exec_context(self) -> ExecContext:
+        if self.__exec_context is None:
+            from .ExecContext import ExecContext
+
+            self.__exec_context = ExecContext.create(
+                self.args.target,
+                interactive = self.interactive,
+                verbose_default = self.verbose,
+            )
+        return self.__exec_context

src/python/jw/pkg/lib/ExecContext.py

@@ -553,14 +553,14 @@ class ExecContext(Base):
                 for cmd in cmds:
                     log(DEBUG, f'{self.log_name}: Running {pretty_cmd(cmd.cmd, wd)}')
                     ret = await __run(cmd.cmd, cmd_input = cmd.cmd_input)
-                    tmp_file = None  # Has been successfully moved at this point
+                tmp_file = None  # All commands, including the final mv, succeeded
                 return ret
             finally:
                 if tmp_file is not None:
                     await self.erase(tmp_file)
                 await self.close()
         except Exception as e:
-            msg = f'Failed to get {path} from {self.root} ({str(e)})'
+            msg = f'Failed to put content to {path} on {self.root} ({str(e)})'
             if throw:
                 raise Exception(msg)
             log(ERR, msg)

src/python/jw/pkg/lib/FileContext.py

@@ -287,7 +287,7 @@ class FileContext(abc.ABC):
                 DEBUG,
                 (
                     f"{self.log_name} doesn't implement stat(), judging by trailing "
-                    'slash if {path} is a directory'
+                    f'slash if {path} is a directory'
                 ),
             )
             return path[-1] == '/'

src/python/jw/pkg/lib/TarIo.py

@@ -51,7 +51,7 @@ class TarIo(CopyContext):
         try:
             blob = (await self.src.get(path)).stdout
         except Exception as e:
-            log(ERR, f'Failed to read tar file "{path}" ({str(e)}')
+            log(ERR, f'Failed to read tar file "{path}" ({str(e)})')
             raise
         return self._filter_tar_file(blob, path_filter, matched = matched)
 

src/python/jw/pkg/lib/Uri.py

@@ -156,6 +156,12 @@ class Uri:
         ret.__password = None
         if not path:
             return ret
+        if ret.__string.endswith('://'):
+            # -- Empty authority: the trailing '/' is part of the '://'
+            # separator, so a leading slash in path must be kept to stay
+            # an absolute path
+            ret.__string += path if path.startswith('/') else '/' + path
+            return ret
         if ret.__string[-1] == '/':
             if path[0] == '/':
                 ret.__string += path[1:]

src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py

@@ -72,10 +72,9 @@ class AsyncSSH(Base):
         }
         if self.__known_hosts is not _USE_DEFAULT_KNOWN_HOSTS:
             kwargs['known_hosts'] = self.__known_hosts
-        ret = {k: v for k, v in kwargs.items() if v is not None}
         if hide_secrets and 'password' in kwargs:
             kwargs['password'] = '<hidden>'
-        return ret
+        return {k: v for k, v in kwargs.items() if v is not None}
 
     @property
     async def _conn(self) -> asyncssh.SSHClientConnection:

src/python/jw/pkg/lib/ec/ssh/Exec.py

@@ -39,7 +39,7 @@ class Exec(Base):
             )
             os.chmod(f.name, 0o0700)
             self.__askpass = f.name
-            f.write(f'#!/bin/bash\n\necho -n "{self.password}\n"')
+            f.write(f'#!/bin/bash\n\necho -n "{self.password}"\n')
             f.close()
             for key, val in {
                 'SSH_ASKPASS': self.__askpass,

src/python/jw/pkg/lib/ec/ssh/Paramiko.py

@@ -32,10 +32,16 @@ class Paramiko(Base):
             hostname = self.hostname
             if hostname is None:
                 raise Exception('Tried to run connect without target hostname')
+            kwargs: dict[str, Any] = {
+                'hostname': hostname,
+                'username': self.username,
+                'password': self.password,
+                'allow_agent': True,
+            }
+            if self.port is not None:
+                kwargs['port'] = self.port
             try:
-                ret.connect(
+                ret.connect(**kwargs)
-                    hostname = hostname, username = self.username, allow_agent = True
-                )
             except Exception as e:
                 log(ERR, f'Failed to connect to {self.hostname} ({str(e)})')
                 raise
@@ -88,5 +94,8 @@ class Paramiko(Base):
             raise
         if cmd_input is not None:
             stdin.write(cmd_input)
+        stdin.channel.shutdown_write()
+        stdout_data = stdout.read()
+        stderr_data = stderr.read()
         exit_status = stdout.channel.recv_exit_status()
-        return Result(stdout.read(), stderr.read(), exit_status, cmd = cmd)
+        return Result(stdout_data, stderr_data, exit_status, cmd = cmd)

src/python/jw/pkg/lib/pm/dpkg.py

@@ -29,7 +29,7 @@ async def _run(
     cmd: list[str], sudo: bool = False, ec: ExecContext | None = None
 ) -> str:
     return (
-        await run_sudo(cmd)
+        await run_sudo(cmd, ec = ec, cmd_input = InputMode.NonInteractive)
         if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive)
     ).stdout_str