janware/jw-pkg
1
mirror of ssh://git.janware.com/janware/proj/jw-pkg synced 2026-04-24 09:13:37 +02:00
Code Activity Actions

lib.ExecContext.run(), .sudo(): Rename env

Browse source 
The name of the env parameter to ExecContext.run() and .sudo() is not
descriptive enough for which environment is supposed to be modified
and how, so rename and split it up as follows:

  - .run(): env -> mod_env

  - .sudo(): env -> mod_env_sudo and mod_env_cmd

The parameters have the following meaning:

  - "mod_env*" means that the environment is modified, not replaced

  - "mod_env" and "mod_env_cmd" modify the environment "cmd" runs in

  - "mod_env_sudo" modifies the environment sudo runs in

Fix the fallout of the API change all over jw-pkg.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-04-19 14:04:35 +02:00
commit 54aecff8e4
9 changed files with 164 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/python/jw/pkg/cmds/projects/CmdBuild.py
View file

@ -101,7 +101,7 @@ class CmdBuild(Cmd): # export
wd = self.app.find_dir(module, pretty=False)
title = '---- [%d/%d]: Running "%s" in %s -' % (cur_project, num_projects, ' '.join(make_cmd), wd)
env = None
mod_env = None
if args.env_reinit:
keep: bool|list[str] = False
if args.env_keep is not None:
@ -112,7 +112,7 @@ class CmdBuild(Cmd): # export
keep=False
case _:
keep = args.env_keep.split(',')
env = await get_profile_env(keep=keep)
mod_env = await get_profile_env(keep=keep)
try:
await self.app.exec_context.run(
@ -120,7 +120,7 @@ class CmdBuild(Cmd): # export
wd=wd,
throw=True,
verbose=True,
env=env,
mod_env=mod_env,
title=title
)
except Exception as e:

131
src/python/jw/pkg/lib/ExecContext.py
View file

@ -108,7 +108,7 @@ class ExecContext(Base):
title: str|None,
cmd: list[str],
cmd_input: Input,
env: dict[str, str]|None,
mod_env: dict[str, str]|None,
wd: str|None,
log_prefix: str,
throw: bool,
@ -123,7 +123,7 @@ class ExecContext(Base):
self.__delim = title if title is not None else f'---- {parent.uri}: Running {self.pretty_cmd} -'
delim_len = 120
self.__delim += '-' * max(0, delim_len - len(self.__delim))
self.__env = {'LC_ALL': 'C'} if env is None else env
self.__mod_env = {'LC_ALL': 'C'} if mod_env is None else mod_env
# -- At the end of this dance, interactive needs to be either True
# or False
@ -185,8 +185,8 @@ class ExecContext(Base):
return self.__cmd_input
@property
def env(self) -> dict[str, str]:
return self.__env
def mod_env(self) -> dict[str, str]:
return self.__mod_env
@property
def throw(self) -> bool:
@ -254,8 +254,8 @@ class ExecContext(Base):
throw: bool = True,
verbose: bool|None = None,
cmd_input: Input = InputMode.OptInteractive,
env: dict[str, str]|None = None,
title: str=None
mod_env: dict[str, str]|None = None,
title: str = None
) -> Result:
"""
Run a command asynchronously and return its output
@ -272,7 +272,7 @@ class ExecContext(Base):
- "InputMode.NonInteractive" -> stdin from /dev/null
- None -> Alias for InputMode.NonInteractive
- otherwise -> Feed cmd_input to stdin
env: The environment the command should be run in
mod_env: Change set to command's environment. key: val adds a variable, key: None removes it
Returns:
A Result instance
@ -284,7 +284,7 @@ class ExecContext(Base):
assert cmd_input is not None
ret = Result(None, None, 1)
with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, env=env, wd=wd,
with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, mod_env=mod_env, wd=wd,
log_prefix='|', throw=throw, verbose=verbose) as cc:
try:
ret = await self._run(
@ -292,7 +292,7 @@ class ExecContext(Base):
wd=wd,
verbose=cc.verbose,
cmd_input=cc.cmd_input,
env=cc.env,
mod_env=cc.mod_env,
interactive=cc.interactive,
log_prefix=cc.log_prefix
)
@ -301,21 +301,76 @@ class ExecContext(Base):
cc.check_exit_code(ret)
return ret
@abc.abstractmethod
async def _sudo(self, *args, **kwargs) -> Result:
pass
async def _sudo(
self,
cmd: list[str],
opts: list[str]|None,
wd: str|None,
mod_env_sudo: dict[str, str]|None,
mod_env_cmd: dict[str, str]|None,
cmd_input: bytes|None,
verbose: bool,
interactive: bool,
log_prefix: str,
) -> Result:
def __check_equal_values(d1: dict[str, str], d2: dict[str, str]) -> None:
for key, val in d1.items():
if not d2.get(key, None) in [None, val]:
raise ValueError(f'Outer and inner environments differ at least for {key}: "{val}" != "{d2.get(key)}"')
fw_cmd: list[str] = []
fw_env: dict[str, str] = {}
if opts is None:
opts = {}
if mod_env_cmd:
fw_env.update(mod_env_cmd)
if self.username != 'root':
if mod_env_sudo and mod_env_cmd:
__check_equal_values(mod_env_sudo, mod_env_cmd)
__check_equal_values(mod_env_cmd, mod_env_sudo)
fw_cmd.append('/usr/bin/sudo')
if mod_env_sudo:
fw_env.update(mod_env_sudo)
if mod_env_cmd:
fw_cmd.append('--preserve-env=' + ','.join(mod_env_cmd.keys()))
if wd is not None:
opts.extend('-D', wd)
wd = None
fw_cmd.extend(opts)
mod_env = fw_env if fw_env else None
fw_cmd.extend(cmd)
return await self._run(
fw_cmd,
wd = wd,
mod_env = mod_env,
verbose = verbose,
cmd_input = cmd_input,
interactive = interactive,
log_prefix = log_prefix
)
async def sudo(
self,
cmd: list[str],
mod_env: dict[str, str]|None=None,
opts: list[str]|None=None,
opts: list[str]|None = None,
wd: str|None = None,
mod_env_sudo: dict[str, str]|None = None,
mod_env_cmd: dict[str, str]|None = None,
throw: bool = True,
verbose: bool|None = None,
cmd_input: Input = InputMode.OptInteractive,
env: dict[str, str]|None = None,
title: str=None,
title: str = None
) -> Result:
# Note that in the calls to the wrapped method, cmd_input == None can
@ -323,27 +378,39 @@ class ExecContext(Base):
assert cmd_input is not None
ret = Result(None, None, 1)
if opts is None:
opts = {}
with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input, env=env, wd=wd,
with self.CallContext(self, title=title, cmd=cmd, cmd_input=cmd_input,
mod_env=mod_env_cmd, wd=wd,
log_prefix='|', throw=throw, verbose=verbose) as cc:
try:
ret = await self._sudo(
cmd=cc.cmd,
mod_env=mod_env,
opts=opts,
wd=wd,
verbose=cc.verbose,
cmd_input=cc.cmd_input,
env=cc.env,
interactive=cc.interactive,
log_prefix=cc.log_prefix,
cmd = cc.cmd,
opts = opts,
wd = cc.wd,
mod_env_sudo = mod_env_sudo,
mod_env_cmd = cc.mod_env,
verbose = cc.verbose,
cmd_input = cc.cmd_input,
interactive = cc.interactive,
log_prefix = cc.log_prefix,
)
except Exception as e:
return cc.exception(ret, e)
cc.check_exit_code(ret)
return ret
return await self._sudo(
cmd,
opts = opts,
wd = wd,
mod_env_sudo = mod_env_sudo,
mod_env_cmd = mod_env_cmd,
throw = throw,
verbose = verbose,
cmd_input = cmd_input,
title = title,
)
async def _get(
self,
path: str,
@ -356,7 +423,7 @@ class ExecContext(Base):
if wd is not None:
path = wd + '/' + path
with self.CallContext(self, title=title, cmd=['cat', path],
cmd_input=InputMode.NonInteractive, wd=None, env=None,
cmd_input=InputMode.NonInteractive, wd=None, mod_env=None,
log_prefix='|', throw=throw, verbose=verbose) as cc:
try:
ret = await self._run(
@ -364,7 +431,7 @@ class ExecContext(Base):
wd=wd,
verbose=cc.verbose,
cmd_input=cc.cmd_input,
env=cc.env,
mod_env=cc.mod_env,
interactive=cc.interactive,
log_prefix=cc.log_prefix
)
@ -444,7 +511,7 @@ class ExecContext(Base):
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
async def __stat(opts: list[str]) -> str:
env = {
mod_env = {
'LC_ALL': 'C'
}
cmd = ['stat']
@ -452,7 +519,7 @@ class ExecContext(Base):
cmd.append('-L')
cmd.extend(opts)
cmd.append(path)
return (await self.run(cmd, env=env, throw=False,
return (await self.run(cmd, mod_env=mod_env, throw=False,
cmd_input=InputMode.NonInteractive)).decode()
# GNU coreutils stat

6
src/python/jw/pkg/lib/distros/debian/Distro.py
View file

@ -18,13 +18,13 @@ class Distro(Base):
async def apt_get(self, args: list[str], verbose: bool=True, sudo: bool=True):
cmd = ['/usr/bin/apt-get']
mod_env = None
mod_env_cmd = None
if not self.interactive:
cmd.extend(['--yes', '--quiet'])
mod_env = { 'DEBIAN_FRONTEND': 'noninteractive' }
mod_env_cmd = { 'DEBIAN_FRONTEND': 'noninteractive' }
cmd.extend(args)
if sudo:
return await self.sudo(cmd, verbose=verbose, mod_env=mod_env)
return await self.sudo(cmd, verbose=verbose, mod_env_cmd=mod_env_cmd)
return await self.run(cmd, verbose=verbose)
async def dpkg(self, *args, **kwargs):

29
src/python/jw/pkg/lib/ec/Local.py
View file

@ -25,7 +25,7 @@ class Local(Base):
wd: str|None,
verbose: bool,
cmd_input: bytes|None,
env: dict[str, str]|None,
mod_env: dict[str, str]|None,
interactive: bool,
log_prefix: str
) -> Result:
@ -58,10 +58,10 @@ class Local(Base):
def _spawn():
# Apply env in PTY mode by temporarily updating os.environ around spawn.
if env:
if mod_env:
old_env = os.environ.copy()
try:
os.environ.update(env)
os.environ.update(mod_env)
return pty.spawn(cmd, master_read=reader)
finally:
os.environ.clear()
@ -81,17 +81,17 @@ class Local(Base):
# -- non-interactive mode
stdin = asyncio.subprocess.DEVNULL if cmd_input is None else asyncio.subprocess.PIPE
if env:
if mod_env:
new_env = os.environ.copy()
new_env.update(env)
env = new_env
new_env.update(mod_env)
mod_env = new_env
proc = await asyncio.create_subprocess_exec(
*cmd,
stdin=stdin,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=env,
env=mod_env,
)
stdout_parts: list[bytes] = []
@ -143,21 +143,6 @@ class Local(Base):
if cwd is not None:
os.chdir(cwd)
async def _sudo(self, cmd: list[str], mod_env: dict[str, str], opts: list[str], *args, **kwargs) -> Result:
env: dict[str, str]|None = None
cmd_input: bytes|None = None
if mod_env:
env = os.environ.copy()
env.update(mod_env)
cmdline = []
if os.getuid() != 0:
cmdline.append('/usr/bin/sudo')
if env is not None:
cmdline.append('--preserve-env=' + ','.join(mod_env.keys()))
cmdline.extend(opts)
cmdline.extend(cmd)
return await self._run(cmdline, *args, **kwargs)
async def _unlink(self, path: str) -> None:
os.unlink(path)

17
src/python/jw/pkg/lib/ec/SSHClient.py
View file

@ -16,7 +16,7 @@ class SSHClient(ExecContext):
class Caps(Flag):
LogOutput = auto()
Interactive = auto()
Env = auto()
ModEnv = auto()
Wd = auto()
def __init__(self, uri: str, caps: Caps=Caps(0), *args, **kwargs) -> None:
@ -41,7 +41,7 @@ class SSHClient(ExecContext):
wd: str|None,
verbose: bool,
cmd_input: bytes|None,
env: dict[str, str]|None,
mod_env: dict[str, str]|None,
interactive: bool,
log_prefix: str
) -> Result:
@ -53,7 +53,7 @@ class SSHClient(ExecContext):
wd: str|None,
verbose: bool,
cmd_input: bytes|None,
env: dict[str, str]|None,
mod_env: dict[str, str]|None,
interactive: bool,
log_prefix: str
) -> Result:
@ -82,7 +82,7 @@ class SSHClient(ExecContext):
if interactive and not self.__caps & self.Caps.Interactive:
raise NotImplementedError('Interactive SSH is not yet implemented')
if env is not None and not self.__caps & self.Caps.Env:
if mod_env is not None and not self.__caps & self.Caps.ModEnv:
raise NotImplementedError('Passing an environment to SSH commands is not yet implemented')
ret = await self._run_ssh(
@ -90,7 +90,7 @@ class SSHClient(ExecContext):
wd=wd,
verbose=verbose,
cmd_input=cmd_input,
env=env,
mod_env=mod_env,
interactive=interactive,
log_prefix=log_prefix
)
@ -103,13 +103,6 @@ class SSHClient(ExecContext):
return ret
async def _sudo(self, cmd: list[str], mod_env: dict[str, str], opts: list[str], *args, **kwargs) -> Result:
if self.username != 'root':
cmd = ['sudo', *opts, *cmd]
if mod_env:
log(WARNING, f'Modifying environment over SSH is not implemented, ignored')
return await self._run(cmd, *args, **kwargs)
@property
def hostname(self) -> str|None:
return self.__hostname

112
src/python/jw/pkg/lib/ec/ssh/AsyncSSH.py
View file

@ -25,7 +25,7 @@ class AsyncSSH(Base):
super().__init__(
uri,
caps=self.Caps.LogOutput | self.Caps.Wd | self.Caps.Interactive | self.Caps.Env,
caps = self.Caps.LogOutput | self.Caps.Wd | self.Caps.Interactive | self.Caps.ModEnv,
**kwargs
)
@ -34,7 +34,8 @@ class AsyncSSH(Base):
self.term_type = term_type or os.environ.get("TERM", "xterm")
self.connect_timeout = connect_timeout
def _connect_kwargs(self) -> dict:
def _connect_kwargs(self, hide_secrets: bool=False) -> dict:
kwargs: dict = {
"host": self.hostname,
"port": self.port,
@ -47,7 +48,10 @@ class AsyncSSH(Base):
if self.known_hosts is not _USE_DEFAULT_KNOWN_HOSTS:
kwargs["known_hosts"] = self.known_hosts
return {k: v for k, v in kwargs.items() if v is not None}
ret = {k: v for k, v in kwargs.items() if v is not None}
if hide_secrets and 'password' in kwargs:
kwargs['password'] = '<hidden>'
return ret
@staticmethod
def _build_remote_command(cmd: list[str], wd: str | None) -> str:
@ -61,29 +65,6 @@ class AsyncSSH(Base):
return f"/bin/sh -lc {shlex.quote(inner)}"
@staticmethod
def _merge_env_into_forwarded_args(
args: tuple,
kwargs: dict,
mod_env: dict[str, str],
) -> tuple[tuple, dict]:
args = list(args)
kwargs = dict(kwargs)
if "env" in kwargs:
base_env = kwargs["env"]
merged_env = dict(base_env or {})
merged_env.update(mod_env)
kwargs["env"] = merged_env or None
elif len(args) >= 4:
base_env = args[3]
merged_env = dict(base_env or {})
merged_env.update(mod_env)
args[3] = merged_env or None
else:
kwargs["env"] = dict(mod_env) if mod_env else None
return tuple(args), kwargs
@staticmethod
def _has_local_tty() -> bool:
try:
@ -143,14 +124,15 @@ class AsyncSSH(Base):
cmd: list[str],
wd: str | None,
cmd_input: bytes | None,
env: dict[str, str] | None,
mod_env: dict[str, str] | None,
) -> Result:
command = self._build_remote_command(cmd, wd)
stdout_parts: list[bytes] = []
proc = await conn.create_process(
command=command,
env=env,
env=mod_env,
stdin=asyncssh.PIPE,
stdout=asyncssh.PIPE,
stderr=asyncssh.STDOUT,
@ -301,7 +283,7 @@ class AsyncSSH(Base):
wd: str | None,
verbose: bool,
cmd_input: bytes | None,
env: dict[str, str] | None,
mod_env: dict[str, str] | None,
log_prefix: str,
) -> Result:
command = self._build_remote_command(cmd, wd)
@ -311,7 +293,7 @@ class AsyncSSH(Base):
proc = await conn.create_process(
command=command,
env=env,
env=mod_env,
stdin=asyncssh.PIPE if cmd_input is not None else asyncssh.DEVNULL,
stdout=asyncssh.PIPE,
stderr=asyncssh.STDOUT,
@ -353,7 +335,7 @@ class AsyncSSH(Base):
wd: str | None,
verbose: bool,
cmd_input: bytes | None,
env: dict[str, str] | None,
mod_env: dict[str, str] | None,
interactive: bool,
log_prefix: str,
) -> Result:
@ -364,7 +346,7 @@ class AsyncSSH(Base):
cmd=cmd,
wd=wd,
cmd_input=cmd_input,
env=env,
mod_env=mod_env,
)
return await self._run_captured_pty_on_conn(
@ -373,7 +355,7 @@ class AsyncSSH(Base):
wd=wd,
verbose=verbose,
cmd_input=cmd_input,
env=env,
mod_env=mod_env,
log_prefix=log_prefix,
)
@ -389,7 +371,7 @@ class AsyncSSH(Base):
proc = await conn.create_process(
command=command,
env=env,
env=mod_env,
stdin=stdin_mode,
stdout=asyncssh.PIPE,
stderr=asyncssh.PIPE,
@ -443,48 +425,26 @@ class AsyncSSH(Base):
wd: str | None,
verbose: bool,
cmd_input: str | None,
env: dict[str, str] | None,
mod_env: dict[str, str] | None,
interactive: bool,
log_prefix: str,
) -> Result:
async with asyncssh.connect(**self._connect_kwargs()) as conn:
return await self._run_on_conn(
conn=conn,
cmd=cmd,
wd=wd,
verbose=verbose,
cmd_input=cmd_input,
env=env,
interactive=interactive,
log_prefix=log_prefix,
)
async def _sudo(
self,
cmd: list[str],
mod_env: dict[str, str],
opts: list[str],
*args,
**kwargs,
) -> Result:
args, kwargs = self._merge_env_into_forwarded_args(args, kwargs, mod_env)
async with asyncssh.connect(**self._connect_kwargs()) as conn:
uid_result = await conn.run("id -u", check=False)
is_root = (
uid_result.exit_status == 0
and isinstance(uid_result.stdout, str)
and uid_result.stdout.strip() == "0"
)
cmdline: list[str] = []
if not is_root:
cmdline.append("/usr/bin/sudo")
if mod_env:
cmdline.append("--preserve-env=" + ",".join(mod_env.keys()))
cmdline.extend(opts)
cmdline.extend(cmd)
return await self._run_on_conn(conn, cmdline, *args, **kwargs)
try:
async with asyncssh.connect(**self._connect_kwargs()) as conn:
return await self._run_on_conn(
conn=conn,
cmd=cmd,
wd=wd,
verbose=verbose,
cmd_input=cmd_input,
mod_env=mod_env,
interactive=interactive,
log_prefix=log_prefix,
)
except Exception as e:
msg = f'-------------------- Failed to run command {" ".join(cmd)} ({e})'
log(ERR, ',', msg)
for key, val in self._connect_kwargs(hide_secrets=True).items():
log(ERR, f'| {key:<20} = {val}')
log(ERR, '`', msg)
raise

8
src/python/jw/pkg/lib/ec/ssh/Exec.py
View file

@ -17,7 +17,7 @@ class Exec(Base):
self.__askpass_orig: dict[str, str|None] = dict()
super().__init__(
uri = uri,
caps = self.Caps.Env,
caps = self.Caps.ModEnv,
**kwargs
)
@ -49,7 +49,7 @@ class Exec(Base):
wd: str|None,
verbose: bool,
cmd_input: bytes|None,
env: dict[str, str]|None,
mod_env: dict[str, str]|None,
interactive: bool,
log_prefix: str
) -> Result:
@ -57,8 +57,8 @@ class Exec(Base):
if cmd_input is None:
cmd_input = InputMode.Interactive if interactive else InputMode.NonInteractive
opts: dict[str, str] = []
if env:
for key, val in env.items():
if mod_env:
for key, val in mod_env.items():
opts.extend(['-o', f'SetEnv {key}="{val}"'])
if self.username:
opts.extend(['-l', self.username])

8
src/python/jw/pkg/lib/ec/ssh/Paramiko.py
View file

@ -16,7 +16,7 @@ class Paramiko(Base):
super().__init__(
uri,
*args,
caps = self.Caps.Env,
caps = self.Caps.ModEnv,
**kwargs
)
self.__timeout: float|None = None # Untested
@ -55,14 +55,14 @@ class Paramiko(Base):
wd: str | None,
verbose: bool,
cmd_input: str | None,
env: dict[str, str] | None,
mod_env: dict[str, str] | None,
interactive: bool,
log_prefix: str,
) -> Result:
try:
kwargs: [str, Any] = {}
if env is not None:
kwargs['environment'] = env
if mod_env is not None:
kwargs['environment'] = mod_env
stdin, stdout, stderr = self.__ssh.exec_command(
join_cmd(cmd),
timeout=self.__timeout,

6
src/python/jw/pkg/lib/util.py
View file

@ -161,16 +161,16 @@ async def get_profile_env(throw: bool=True, keep: Iterable[str]|bool=False, ec:
Returns:
Dictionary with fresh environment
"""
env: dict[str,str]|None = None
mod_env: dict[str,str]|None = None
if keep == False or isinstance(keep, Iterable):
env = {
mod_env = {
'HOME': os.environ.get('HOME', '/'),
'USER': os.environ.get('USER', ''),
'PATH': '/usr/bin:/bin',
}
# Run bash as a login shell, which sources /etc/profile, then print environment as NUL-separated key=value pairs
cmd = ['/usr/bin/env', '-i', '/bin/bash', '-lc', 'env -0']
result = await run_cmd(cmd, throw=throw, verbose=True, env=env, ec=ec)
result = await run_cmd(cmd, throw=throw, verbose=True, mod_env=mod_env, ec=ec)
ret: dict[str, str] = {}
for entry in result.stdout.rstrip(b"\0").split(b"\0"):
if not entry: