diff --git a/src/python/jw/pkg/cmds/projects/CmdBuild.py b/src/python/jw/pkg/cmds/projects/CmdBuild.py index d7eb5ac1..2d83d6f3 100644 --- a/src/python/jw/pkg/cmds/projects/CmdBuild.py +++ b/src/python/jw/pkg/cmds/projects/CmdBuild.py @@ -101,7 +101,7 @@ class CmdBuild(Cmd): # export wd = self.app.find_dir(module, pretty=False) title = '---- [%d/%d]: Running "%s" in %s -' % (cur_project, num_projects, ' '.join(make_cmd), wd) - env = None + mod_env = None if args.env_reinit: keep: bool|list[str] = False if args.env_keep is not None: @@ -112,7 +112,7 @@ class CmdBuild(Cmd): # export keep=False case _: keep = args.env_keep.split(',') - env = await get_profile_env(keep=keep) + mod_env = await get_profile_env(keep=keep) try: await self.app.exec_context.run( @@ -120,7 +120,7 @@ class CmdBuild(Cmd): # export wd=wd, throw=True, verbose=True, - env=env, + mod_env=mod_env, title=title ) except Exception as e: diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 73c06c32..2b74c821 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -108,7 +108,7 @@ class ExecContext(Base): title: str|None, cmd: list[str], cmd_input: Input, - env: dict[str, str]|None, + mod_env: dict[str, str]|None, wd: str|None, log_prefix: str, throw: bool, @@ -123,7 +123,7 @@ class ExecContext(Base): self.__delim = title if title is not None else f'---- {parent.uri}: Running {self.pretty_cmd} -' delim_len = 120 self.__delim += '-' * max(0, delim_len - len(self.__delim)) - self.__env = {'LC_ALL': 'C'} if env is None else env + self.__mod_env = {'LC_ALL': 'C'} if mod_env is None else mod_env # -- At the end of this dance, interactive needs to be either True # or False @@ -185,8 +185,8 @@ class ExecContext(Base): return self.__cmd_input @property - def env(self) -> dict[str, str]: - return self.__env + def mod_env(self) -> dict[str, str]: + return self.__mod_env @property def throw(self) -> bool: @@ -254,8 +254,8 @@ class ExecContext(Base): throw: bool = True, verbose: bool|None = None, cmd_input: Input = InputMode.OptInteractive, - env: dict[str, str]|None = None, - title: str=None + mod_env: dict[str, str]|None = None, + title: str = None ) -> Result: """ Run a command asynchronously and return its output @@ -272,7 +272,7 @@ class ExecContext(Base): - "InputMode.NonInteractive" -> stdin from /dev/null - None -> Alias for InputMode.NonInteractive - otherwise -> Feed cmd_input to stdin - env: The environment the command should be run in + mod_env: Change set to command's environment. key: val adds a variable, key: None removes it Returns: A Result instance @@ -284,7 +284,7 @@ class ExecContext(Base): assert cmd_input is not None ret = Result(None, None, 1) - with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, env=env, wd=wd, + with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, mod_env=mod_env, wd=wd, log_prefix='|', throw=throw, verbose=verbose) as cc: try: ret = await self._run( @@ -292,7 +292,7 @@ class ExecContext(Base): wd=wd, verbose=cc.verbose, cmd_input=cc.cmd_input, - env=cc.env, + mod_env=cc.mod_env, interactive=cc.interactive, log_prefix=cc.log_prefix ) @@ -301,21 +301,76 @@ class ExecContext(Base): cc.check_exit_code(ret) return ret - @abc.abstractmethod - async def _sudo(self, *args, **kwargs) -> Result: - pass + async def _sudo( + self, + cmd: list[str], + opts: list[str]|None, + wd: str|None, + mod_env_sudo: dict[str, str]|None, + mod_env_cmd: dict[str, str]|None, + cmd_input: bytes|None, + verbose: bool, + interactive: bool, + log_prefix: str, + ) -> Result: + + def __check_equal_values(d1: dict[str, str], d2: dict[str, str]) -> None: + for key, val in d1.items(): + if not d2.get(key, None) in [None, val]: + raise ValueError(f'Outer and inner environments differ at least for {key}: "{val}" != "{d2.get(key)}"') + + fw_cmd: list[str] = [] + fw_env: dict[str, str] = {} + + if opts is None: + opts = {} + + if mod_env_cmd: + fw_env.update(mod_env_cmd) + + if self.username != 'root': + + if mod_env_sudo and mod_env_cmd: + __check_equal_values(mod_env_sudo, mod_env_cmd) + __check_equal_values(mod_env_cmd, mod_env_sudo) + + fw_cmd.append('/usr/bin/sudo') + if mod_env_sudo: + fw_env.update(mod_env_sudo) + if mod_env_cmd: + fw_cmd.append('--preserve-env=' + ','.join(mod_env_cmd.keys())) + + if wd is not None: + opts.extend('-D', wd) + wd = None + + fw_cmd.extend(opts) + + mod_env = fw_env if fw_env else None + + fw_cmd.extend(cmd) + + return await self._run( + fw_cmd, + wd = wd, + mod_env = mod_env, + verbose = verbose, + cmd_input = cmd_input, + interactive = interactive, + log_prefix = log_prefix + ) async def sudo( self, cmd: list[str], - mod_env: dict[str, str]|None=None, - opts: list[str]|None=None, + opts: list[str]|None = None, wd: str|None = None, + mod_env_sudo: dict[str, str]|None = None, + mod_env_cmd: dict[str, str]|None = None, throw: bool = True, verbose: bool|None = None, cmd_input: Input = InputMode.OptInteractive, - env: dict[str, str]|None = None, - title: str=None, + title: str = None ) -> Result: # Note that in the calls to the wrapped method, cmd_input == None can @@ -323,27 +378,39 @@ class ExecContext(Base): assert cmd_input is not None ret = Result(None, None, 1) - if opts is None: - opts = {} - with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, env=env, wd=wd, + with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, + mod_env=mod_env_cmd, wd=wd, log_prefix='|', throw=throw, verbose=verbose) as cc: try: ret = await self._sudo( - cmd=cc.cmd, - mod_env=mod_env, - opts=opts, - wd=wd, - verbose=cc.verbose, - cmd_input=cc.cmd_input, - env=cc.env, - interactive=cc.interactive, - log_prefix=cc.log_prefix, + cmd = cc.cmd, + opts = opts, + wd = cc.wd, + mod_env_sudo = mod_env_sudo, + mod_env_cmd = cc.mod_env, + verbose = cc.verbose, + cmd_input = cc.cmd_input, + interactive = cc.interactive, + log_prefix = cc.log_prefix, ) + except Exception as e: return cc.exception(ret, e) cc.check_exit_code(ret) return ret + return await self._sudo( + cmd, + opts = opts, + wd = wd, + mod_env_sudo = mod_env_sudo, + mod_env_cmd = mod_env_cmd, + throw = throw, + verbose = verbose, + cmd_input = cmd_input, + title = title, + ) + async def _get( self, path: str, @@ -356,7 +423,7 @@ class ExecContext(Base): if wd is not None: path = wd + '/' + path with self.CallContext(self, title=title, cmd=['cat', path], - cmd_input=InputMode.NonInteractive, wd=None, env=None, + cmd_input=InputMode.NonInteractive, wd=None, mod_env=None, log_prefix='|', throw=throw, verbose=verbose) as cc: try: ret = await self._run( @@ -364,7 +431,7 @@ class ExecContext(Base): wd=wd, verbose=cc.verbose, cmd_input=cc.cmd_input, - env=cc.env, + mod_env=cc.mod_env, interactive=cc.interactive, log_prefix=cc.log_prefix ) @@ -444,7 +511,7 @@ class ExecContext(Base): async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: async def __stat(opts: list[str]) -> str: - env = { + mod_env = { 'LC_ALL': 'C' } cmd = ['stat'] @@ -452,7 +519,7 @@ class ExecContext(Base): cmd.append('-L') cmd.extend(opts) cmd.append(path) - return (await self.run(cmd, env=env, throw=False, + return (await self.run(cmd, mod_env=mod_env, throw=False, cmd_input=InputMode.NonInteractive)).decode() # GNU coreutils stat diff --git a/src/python/jw/pkg/lib/distros/debian/Distro.py b/src/python/jw/pkg/lib/distros/debian/Distro.py index 31112d32..d0dd3fef 100644 --- a/src/python/jw/pkg/lib/distros/debian/Distro.py +++ b/src/python/jw/pkg/lib/distros/debian/Distro.py @@ -18,13 +18,13 @@ class Distro(Base): async def apt_get(self, args: list[str], verbose: bool=True, sudo: bool=True): cmd = ['/usr/bin/apt-get'] - mod_env = None + mod_env_cmd = None if not self.interactive: cmd.extend(['--yes', '--quiet']) - mod_env = { 'DEBIAN_FRONTEND': 'noninteractive' } + mod_env_cmd = { 'DEBIAN_FRONTEND': 'noninteractive' } cmd.extend(args) if sudo: - return await self.sudo(cmd, verbose=verbose, mod_env=mod_env) + return await self.sudo(cmd, verbose=verbose, mod_env_cmd=mod_env_cmd) return await self.run(cmd, verbose=verbose) async def dpkg(self, *args, **kwargs): diff --git a/src/python/jw/pkg/lib/ec/Local.py b/src/python/jw/pkg/lib/ec/Local.py index e4193d84..90154057 100644 --- a/src/python/jw/pkg/lib/ec/Local.py +++ b/src/python/jw/pkg/lib/ec/Local.py @@ -25,7 +25,7 @@ class Local(Base): wd: str|None, verbose: bool, cmd_input: bytes|None, - env: dict[str, str]|None, + mod_env: dict[str, str]|None, interactive: bool, log_prefix: str ) -> Result: @@ -58,10 +58,10 @@ class Local(Base): def _spawn(): # Apply env in PTY mode by temporarily updating os.environ around spawn. - if env: + if mod_env: old_env = os.environ.copy() try: - os.environ.update(env) + os.environ.update(mod_env) return pty.spawn(cmd, master_read=reader) finally: os.environ.clear() @@ -81,17 +81,17 @@ class Local(Base): # -- non-interactive mode stdin = asyncio.subprocess.DEVNULL if cmd_input is None else asyncio.subprocess.PIPE - if env: + if mod_env: new_env = os.environ.copy() - new_env.update(env) - env = new_env + new_env.update(mod_env) + mod_env = new_env proc = await asyncio.create_subprocess_exec( *cmd, stdin=stdin, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - env=env, + env=mod_env, ) stdout_parts: list[bytes] = [] @@ -143,21 +143,6 @@ class Local(Base): if cwd is not None: os.chdir(cwd) - async def _sudo(self, cmd: list[str], mod_env: dict[str, str], opts: list[str], *args, **kwargs) -> Result: - env: dict[str, str]|None = None - cmd_input: bytes|None = None - if mod_env: - env = os.environ.copy() - env.update(mod_env) - cmdline = [] - if os.getuid() != 0: - cmdline.append('/usr/bin/sudo') - if env is not None: - cmdline.append('--preserve-env=' + ','.join(mod_env.keys())) - cmdline.extend(opts) - cmdline.extend(cmd) - return await self._run(cmdline, *args, **kwargs) - async def _unlink(self, path: str) -> None: os.unlink(path) diff --git a/src/python/jw/pkg/lib/ec/SSHClient.py b/src/python/jw/pkg/lib/ec/SSHClient.py index aca3648b..eac2daa8 100644 --- a/src/python/jw/pkg/lib/ec/SSHClient.py +++ b/src/python/jw/pkg/lib/ec/SSHClient.py @@ -16,7 +16,7 @@ class SSHClient(ExecContext): class Caps(Flag): LogOutput = auto() Interactive = auto() - Env = auto() + ModEnv = auto() Wd = auto() def __init__(self, uri: str, caps: Caps=Caps(0), *args, **kwargs) -> None: @@ -41,7 +41,7 @@ class SSHClient(ExecContext): wd: str|None, verbose: bool, cmd_input: bytes|None, - env: dict[str, str]|None, + mod_env: dict[str, str]|None, interactive: bool, log_prefix: str ) -> Result: @@ -53,7 +53,7 @@ class SSHClient(ExecContext): wd: str|None, verbose: bool, cmd_input: bytes|None, - env: dict[str, str]|None, + mod_env: dict[str, str]|None, interactive: bool, log_prefix: str ) -> Result: @@ -82,7 +82,7 @@ class SSHClient(ExecContext): if interactive and not self.__caps & self.Caps.Interactive: raise NotImplementedError('Interactive SSH is not yet implemented') - if env is not None and not self.__caps & self.Caps.Env: + if mod_env is not None and not self.__caps & self.Caps.ModEnv: raise NotImplementedError('Passing an environment to SSH commands is not yet implemented') ret = await self._run_ssh( @@ -90,7 +90,7 @@ class SSHClient(ExecContext): wd=wd, verbose=verbose, cmd_input=cmd_input, - env=env, + mod_env=mod_env, interactive=interactive, log_prefix=log_prefix ) @@ -103,13 +103,6 @@ class SSHClient(ExecContext): return ret - async def _sudo(self, cmd: list[str], mod_env: dict[str, str], opts: list[str], *args, **kwargs) -> Result: - if self.username != 'root': - cmd = ['sudo', *opts, *cmd] - if mod_env: - log(WARNING, f'Modifying environment over SSH is not implemented, ignored') - return await self._run(cmd, *args, **kwargs) - @property def hostname(self) -> str|None: return self.__hostname diff --git a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py index d44c2de9..dde417f3 100644 --- a/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py +++ b/src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py @@ -25,7 +25,7 @@ class AsyncSSH(Base): super().__init__( uri, - caps=self.Caps.LogOutput | self.Caps.Wd | self.Caps.Interactive | self.Caps.Env, + caps = self.Caps.LogOutput | self.Caps.Wd | self.Caps.Interactive | self.Caps.ModEnv, **kwargs ) @@ -34,7 +34,8 @@ class AsyncSSH(Base): self.term_type = term_type or os.environ.get("TERM", "xterm") self.connect_timeout = connect_timeout - def _connect_kwargs(self) -> dict: + def _connect_kwargs(self, hide_secrets: bool=False) -> dict: + kwargs: dict = { "host": self.hostname, "port": self.port, @@ -47,7 +48,10 @@ class AsyncSSH(Base): if self.known_hosts is not _USE_DEFAULT_KNOWN_HOSTS: kwargs["known_hosts"] = self.known_hosts - return {k: v for k, v in kwargs.items() if v is not None} + ret = {k: v for k, v in kwargs.items() if v is not None} + if hide_secrets and 'password' in kwargs: + kwargs['password'] = '' + return ret @staticmethod def _build_remote_command(cmd: list[str], wd: str | None) -> str: @@ -61,29 +65,6 @@ class AsyncSSH(Base): return f"/bin/sh -lc {shlex.quote(inner)}" - @staticmethod - def _merge_env_into_forwarded_args( - args: tuple, - kwargs: dict, - mod_env: dict[str, str], - ) -> tuple[tuple, dict]: - args = list(args) - kwargs = dict(kwargs) - - if "env" in kwargs: - base_env = kwargs["env"] - merged_env = dict(base_env or {}) - merged_env.update(mod_env) - kwargs["env"] = merged_env or None - elif len(args) >= 4: - base_env = args[3] - merged_env = dict(base_env or {}) - merged_env.update(mod_env) - args[3] = merged_env or None - else: - kwargs["env"] = dict(mod_env) if mod_env else None - return tuple(args), kwargs - @staticmethod def _has_local_tty() -> bool: try: @@ -143,14 +124,15 @@ class AsyncSSH(Base): cmd: list[str], wd: str | None, cmd_input: bytes | None, - env: dict[str, str] | None, + mod_env: dict[str, str] | None, ) -> Result: + command = self._build_remote_command(cmd, wd) stdout_parts: list[bytes] = [] proc = await conn.create_process( command=command, - env=env, + env=mod_env, stdin=asyncssh.PIPE, stdout=asyncssh.PIPE, stderr=asyncssh.STDOUT, @@ -301,7 +283,7 @@ class AsyncSSH(Base): wd: str | None, verbose: bool, cmd_input: bytes | None, - env: dict[str, str] | None, + mod_env: dict[str, str] | None, log_prefix: str, ) -> Result: command = self._build_remote_command(cmd, wd) @@ -311,7 +293,7 @@ class AsyncSSH(Base): proc = await conn.create_process( command=command, - env=env, + env=mod_env, stdin=asyncssh.PIPE if cmd_input is not None else asyncssh.DEVNULL, stdout=asyncssh.PIPE, stderr=asyncssh.STDOUT, @@ -353,7 +335,7 @@ class AsyncSSH(Base): wd: str | None, verbose: bool, cmd_input: bytes | None, - env: dict[str, str] | None, + mod_env: dict[str, str] | None, interactive: bool, log_prefix: str, ) -> Result: @@ -364,7 +346,7 @@ class AsyncSSH(Base): cmd=cmd, wd=wd, cmd_input=cmd_input, - env=env, + mod_env=mod_env, ) return await self._run_captured_pty_on_conn( @@ -373,7 +355,7 @@ class AsyncSSH(Base): wd=wd, verbose=verbose, cmd_input=cmd_input, - env=env, + mod_env=mod_env, log_prefix=log_prefix, ) @@ -389,7 +371,7 @@ class AsyncSSH(Base): proc = await conn.create_process( command=command, - env=env, + env=mod_env, stdin=stdin_mode, stdout=asyncssh.PIPE, stderr=asyncssh.PIPE, @@ -443,48 +425,26 @@ class AsyncSSH(Base): wd: str | None, verbose: bool, cmd_input: str | None, - env: dict[str, str] | None, + mod_env: dict[str, str] | None, interactive: bool, log_prefix: str, ) -> Result: - async with asyncssh.connect(**self._connect_kwargs()) as conn: - return await self._run_on_conn( - conn=conn, - cmd=cmd, - wd=wd, - verbose=verbose, - cmd_input=cmd_input, - env=env, - interactive=interactive, - log_prefix=log_prefix, - ) - - async def _sudo( - self, - cmd: list[str], - mod_env: dict[str, str], - opts: list[str], - *args, - **kwargs, - ) -> Result: - args, kwargs = self._merge_env_into_forwarded_args(args, kwargs, mod_env) - - async with asyncssh.connect(**self._connect_kwargs()) as conn: - uid_result = await conn.run("id -u", check=False) - is_root = ( - uid_result.exit_status == 0 - and isinstance(uid_result.stdout, str) - and uid_result.stdout.strip() == "0" - ) - - cmdline: list[str] = [] - - if not is_root: - cmdline.append("/usr/bin/sudo") - if mod_env: - cmdline.append("--preserve-env=" + ",".join(mod_env.keys())) - cmdline.extend(opts) - - cmdline.extend(cmd) - - return await self._run_on_conn(conn, cmdline, *args, **kwargs) + try: + async with asyncssh.connect(**self._connect_kwargs()) as conn: + return await self._run_on_conn( + conn=conn, + cmd=cmd, + wd=wd, + verbose=verbose, + cmd_input=cmd_input, + mod_env=mod_env, + interactive=interactive, + log_prefix=log_prefix, + ) + except Exception as e: + msg = f'-------------------- Failed to run command {" ".join(cmd)} ({e})' + log(ERR, ',', msg) + for key, val in self._connect_kwargs(hide_secrets=True).items(): + log(ERR, f'| {key:<20} = {val}') + log(ERR, '`', msg) + raise diff --git a/src/python/jw/pkg/lib/ec/ssh/Exec.py b/src/python/jw/pkg/lib/ec/ssh/Exec.py index f9a208b4..23b95881 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Exec.py +++ b/src/python/jw/pkg/lib/ec/ssh/Exec.py @@ -17,7 +17,7 @@ class Exec(Base): self.__askpass_orig: dict[str, str|None] = dict() super().__init__( uri = uri, - caps = self.Caps.Env, + caps = self.Caps.ModEnv, **kwargs ) @@ -49,7 +49,7 @@ class Exec(Base): wd: str|None, verbose: bool, cmd_input: bytes|None, - env: dict[str, str]|None, + mod_env: dict[str, str]|None, interactive: bool, log_prefix: str ) -> Result: @@ -57,8 +57,8 @@ class Exec(Base): if cmd_input is None: cmd_input = InputMode.Interactive if interactive else InputMode.NonInteractive opts: dict[str, str] = [] - if env: - for key, val in env.items(): + if mod_env: + for key, val in mod_env.items(): opts.extend(['-o', f'SetEnv {key}="{val}"']) if self.username: opts.extend(['-l', self.username]) diff --git a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py index a6fc7c2a..176ffb08 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py +++ b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py @@ -16,7 +16,7 @@ class Paramiko(Base): super().__init__( uri, *args, - caps = self.Caps.Env, + caps = self.Caps.ModEnv, **kwargs ) self.__timeout: float|None = None # Untested @@ -55,14 +55,14 @@ class Paramiko(Base): wd: str | None, verbose: bool, cmd_input: str | None, - env: dict[str, str] | None, + mod_env: dict[str, str] | None, interactive: bool, log_prefix: str, ) -> Result: try: kwargs: [str, Any] = {} - if env is not None: - kwargs['environment'] = env + if mod_env is not None: + kwargs['environment'] = mod_env stdin, stdout, stderr = self.__ssh.exec_command( join_cmd(cmd), timeout=self.__timeout, diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 7f92126c..e4f5a8c0 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -161,16 +161,16 @@ async def get_profile_env(throw: bool=True, keep: Iterable[str]|bool=False, ec: Returns: Dictionary with fresh environment """ - env: dict[str,str]|None = None + mod_env: dict[str,str]|None = None if keep == False or isinstance(keep, Iterable): - env = { + mod_env = { 'HOME': os.environ.get('HOME', '/'), 'USER': os.environ.get('USER', ''), 'PATH': '/usr/bin:/bin', } # Run bash as a login shell, which sources /etc/profile, then print environment as NUL-separated key=value pairs cmd = ['/usr/bin/env', '-i', '/bin/bash', '-lc', 'env -0'] - result = await run_cmd(cmd, throw=throw, verbose=True, env=env, ec=ec) + result = await run_cmd(cmd, throw=throw, verbose=True, mod_env=mod_env, ec=ec) ret: dict[str, str] = {} for entry in result.stdout.rstrip(b"\0").split(b"\0"): if not entry: