From 3cf5b2264e7a90bb952e6b1905d15e935a8c796f Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 17 Apr 2026 09:15:06 +0200 Subject: [PATCH] lib.FileContext: Add file methods Add the following methods, meant to do the obvious: unlink(self, path: str) -> None erase(self, path: str) -> None rename(self, src: str, dst: str) -> None mktemp(self, tmpl: str, directory: bool=False) -> None chown(self, path: str, owner: str|None=None, group: str|None=None) -> None chmod(self, path: str, mode: int) -> None stat(self, path: str, follow_symlinks: bool=True) -> StatResult file_exists(self, path: str) -> bool is_dir(self, path: str) -> bool All methods are async and call their protected counterpart, which is designed to be overridden. If possible, default implementations do something meaningful, if not, they just raise plain NotImplementedError. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/ExecContext.py | 182 ++++++++++++++++++++++++++- src/python/jw/pkg/lib/FileContext.py | 81 +++++++++++- src/python/jw/pkg/lib/base.py | 5 +- src/python/jw/pkg/lib/ec/Local.py | 33 ++++- src/python/jw/pkg/lib/util.py | 4 +- 5 files changed, 295 insertions(+), 10 deletions(-) diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 45148852..0a29205b 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -2,18 +2,102 @@ from __future__ import annotations -import abc, re, sys +import abc, re, sys, errno from enum import Enum, auto from typing import NamedTuple, TYPE_CHECKING +from decimal import Decimal, ROUND_FLOOR if TYPE_CHECKING: from typing import Self, Type from types import TracebackType from .log import * -from .base import Input, InputMode, Result +from .base import Input, InputMode, Result, StatResult from .FileContext import FileContext as Base +_US = "\x1f" # unlikely to appear in numeric output +_BILLION = Decimal(1_000_000_000) + +def _looks_like_option_error(stderr: str) -> bool: + s = stderr.lower() + return any( + needle in s + for needle in ( + "unrecognized option", + "illegal option", + "unknown option", + "invalid option", + "option requires an argument", + ) + ) + +def _raise_stat_error(path: str, stderr: str, returncode: int) -> None: + msg = (stderr or "").strip() or f"stat exited with status {returncode}" + lower = msg.lower() + if "no such file" in lower: + raise FileNotFoundError(errno.ENOENT, msg, path) + if "permission denied" in lower or "operation not permitted" in lower: + raise PermissionError(errno.EACCES, msg, path) + raise OSError(errno.EIO, msg, path) + +def _parse_epoch(value: str) -> tuple[int, float, int]: + """ + Convert a decimal epoch timestamp string into: + (integer seconds for tuple slot, float seconds for attribute, ns for *_ns) + """ + dec = Decimal(value.strip()) + sec = int(dec.to_integral_value(rounding=ROUND_FLOOR)) + ns = int((dec * _BILLION).to_integral_value(rounding=ROUND_FLOOR)) + return sec, float(dec), ns + +def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: + if len(fields) != 13: + raise ValueError( + f"unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}" + ) + + ( + mode_s, + ino_s, + dev_s, + nlink_s, + uid_s, + gid_s, + size_s, + atime_s, + mtime_s, + ctime_s, + blksize_s, + blocks_s, + rdev_s, + ) = fields + + st_mode = int(mode_s, mode_base) + st_ino = int(ino_s) + st_dev = int(dev_s) + st_nlink = int(nlink_s) + st_uid = uid_s + st_gid = gid_s + st_size = int(size_s) + + st_atime_i, st_atime_f, st_atime_ns = _parse_epoch(atime_s) + st_mtime_i, st_mtime_f, st_mtime_ns = _parse_epoch(mtime_s) + st_ctime_i, st_ctime_f, st_ctime_ns = _parse_epoch(ctime_s) + + st_blksize = int(blksize_s) + st_blocks = int(blocks_s) + st_rdev = int(rdev_s) + + return StatResult( + mode = st_mode, + owner = st_uid, + group = st_gid, + size = st_size, + atime = st_atime_i, + mtime = st_mtime_i, + ctime = st_ctime_i, + ) + class ExecContext(Base): class CallContext: @@ -332,3 +416,97 @@ class ExecContext(Base): raise return cc.exception(ret, e) return ret + + async def _unlink(self, path: str) -> None: + cmd = ['rm', '-f', path] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _erase(self, path: str) -> None: + cmd = ['rm', '-rf', path] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _rename(self, src: str, dst: str) -> None: + cmd = ['mv', src, dst] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _mktemp(self, tmpl: str, directory: bool) -> str: + cmd = ['mktemp'] + if directory: + cmd.append('-d') + cmd.append(tmpl) + result = await self.run(cmd, cmd_input=InputMode.NonInteractive) + return result.stdout.strip().decode() + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + + async def __stat(opts: list[str]) -> str: + env = { + 'LC_ALL': 'C' + } + cmd = ['stat'] + if follow_symlinks: + cmd.append('-L') + cmd.extend(opts) + cmd.append(path) + return (await self.run(cmd, env=env, throw=False, + cmd_input=InputMode.NonInteractive)).decode() + + # GNU coreutils stat + gnu_format = _US.join([ + "%f", # st_mode in hex + "%i", # st_ino + "%d", # st_dev + "%h", # st_nlink + "%U", # st_uid + "%G", # st_gid + "%s", # st_size + "%.9X", # st_atime + "%.9Y", # st_mtime + "%.9Z", # st_ctime + "%o", # st_blksize hint + "%b", # st_blocks + "%r", # st_rdev + ]) + + result = await __stat(['--printf', gnu_format]) + if result.status == 0: + return _build_stat_result(result.stdout.split(_US), mode_base=16) + + if not _looks_like_option_error(result.stderr.decode()): + # log(DEBUG, f'GNU stat attempt failed on "{path}" ({str(e)})') + _raise_stat_error(path, result.stderr, result.status) + + # BSD / macOS / OpenBSD / NetBSD stat + bsd_format = _US.join([ + "%p", # st_mode in octal + "%i", # st_ino + "%d", # st_dev + "%l", # st_nlink + "%U", # st_uid + "%G", # st_gid + "%z", # st_size + "%.9Fa", # st_atime + "%.9Fm", # st_mtime + "%.9Fc", # st_ctime + "%k", # st_blksize + "%b", # st_blocks + "%r", # st_rdev + ]) + + result = await __stat(['-n', '-f', bst_format]) + if proc.returncode == 0: + return _build_stat_result(proc.stdout.rstrip('\n').split(_US), mode_base=8) + _raise_stat_error(path, result.stderr, result.status) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + assert owner is not None or group is not None + if group is None: + ownership = owner + elif owner is None: + ownership = ':' + group + else: + ownership = owner + ':' + group + await self.run(['chown', ownership, path], cmd_input=InputMode.NonInteractive) + + async def _chmod(self, path: str, mode: int) -> None: + await self.run(['chmod', oct(mode), path], cmd_input=InputMode.NonInteractive) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 7d967383..33720770 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from typing import Self from .log import * -from .base import Input, InputMode, Result +from .base import Input, InputMode, Result, StatResult class FileContext(abc.ABC): @@ -109,6 +109,85 @@ class FileContext(abc.ABC): return await self._put(path, content, wd=wd, throw=throw, verbose=verbose, title=title, owner=owner, group=group, mode=mode_str, atomic=atomic) + async def _unlink(self, path: str) -> None: + raise NotImplementedError(f'{self.log_name}: unlink() is not implemented') + + async def unlink(self, path: str) -> None: + return await self._unlink(path) + + async def _erase(self, path: str) -> None: + raise NotImplementedError(f'{self.log_name}: erase() is not implemented') + + async def erase(self, path: str) -> None: + return await self._erase(path) + + async def _rename(self, src: str, dst: str) -> None: + raise NotImplementedError(f'{self.log_name}: rename() is not implemented') + + async def rename(self, src: str, dst: str) -> None: + return await self._rename(src, dst) + + async def _mktemp(self, tmpl: str, directory: bool) -> None: + raise NotImplementedError(f'{self.log_name}: mktemp() is not implemented') + + async def mktemp(self, tmpl: str, directory: bool=False) -> None: + return await self._mktemp(tmpl, directory) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + raise NotImplementedError(f'{self.log_name}: chown() is not implemented') + + async def chown(self, path: str, owner: str|None=None, group: str|None=None) -> None: + if owner is None and group is None: + raise ValueError(f'Tried to change ownership of {path} specifying neither owner nor group') + return await self._chown(path, owner, group) + + async def _chmod(self, path: str, mode: int) -> None: + raise NotImplementedError(f'{self.log_name}: chmod() is not implemented') + + async def chmod(self, path: str, mode: int) -> None: + return await self._chmod(path, mode) + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + raise NotImplementedError(f'{self.log_name}: lstat() is not implemented') + + async def stat(self, path: str, follow_symlinks: bool=True) -> StatResult: + if not isinstance(path, str): + raise TypeError(f"path must be str, got {type(path).__name__}") + return await self._stat(path, follow_symlinks) + + async def _file_exists(self, path: str) -> bool: + try: + self._stat(path, False) + except FileNotFoundError as e: + log(DEBUG, f'Could not stat file {path} ({str(e)}), ignored') + return False + except Exception as e: + log(ERR, f'Could not stat file {path} ({str(e)}), ignored') + raise + return True + + async def file_exists(self, path: str) -> bool: + return self._file_exists(path) + + async def _is_dir(self, path: str) -> bool: + try: + return S_ISDIR(await self._stat(path).st_mode) + except NotImplementedError: + log(DEBUG, f'{self.log_name} doesn\'t implement stat(), judging by trailing slash if {path} is a directory') + return path[-1] == '/' + except FileNotFoundError as e: + log(DEBUG, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + return False + except Exception as e: + log(ERR, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + raise + return False + + async def is_dir(self, path: str) -> bool: + if not path: + raise ValueError('Tried to investigate file system resource with empty path') + return self._is_dir(path) + async def _close(self) -> None: pass diff --git a/src/python/jw/pkg/lib/base.py b/src/python/jw/pkg/lib/base.py index 81c232e5..1d0a336a 100644 --- a/src/python/jw/pkg/lib/base.py +++ b/src/python/jw/pkg/lib/base.py @@ -44,9 +44,10 @@ class StatResult(NamedTuple): import pwd, grp return StatResult( rhs.st_mode, - pwd.getpwuid(rhs.pw_uid).pw_name, - grp.getgrgid(gid).gr_name, + pwd.getpwuid(rhs.st_uid).pw_name, + grp.getgrgid(rhs.st_gid).gr_name, rhs.st_size, + rhs.st_atime, rhs.st_mtime, rhs.st_ctime, ) diff --git a/src/python/jw/pkg/lib/ec/Local.py b/src/python/jw/pkg/lib/ec/Local.py index 697872e8..f0b449c3 100644 --- a/src/python/jw/pkg/lib/ec/Local.py +++ b/src/python/jw/pkg/lib/ec/Local.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import os, sys, subprocess, asyncio +import os, sys, subprocess, asyncio, pwd, grp, stat from ..ExecContext import ExecContext as Base -from ..base import Result +from ..base import Result, StatResult from ..log import * from ..util import pretty_cmd @@ -151,3 +151,32 @@ class Local(Base): cmdline.extend(opts) cmdline.extend(cmd) return await self._run(cmdline, *args, **kwargs) + + async def _unlink(self, path: str) -> None: + os.unlink(path) + + async def _erase(self, path: str) -> None: + if os.isdir(path): + shutil.rmtree(path) + return + os.unlink(path) + + async def _rename(self, src: str, dst: str) -> None: + os.rename(src, dst) + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + return StatResult.from_os(os.stat(path, follow_symlinks=follow_symlinks)) + + async def _file_exists(self, path: str) -> bool: + return os.path.exists(path) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + uid = pwd.getpwnam(owner).pw_uid if owner else -1 + gid = grp.getgrnam(group).gr_gid if group else -1 + os.chown(path, uid, gid) + + async def _chmod(self, path: str, mode: int) -> None: + os.chmod(path, mode) + + async def _is_dir(self, path: str) -> bool: + return os.path.isdir(path) diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 329ff593..7f92126c 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -110,9 +110,7 @@ async def copy(src_uri: str, dst_uri: str, owner: str|None=None, group: str|None src, src_path = __ec(src_uri) content = (await src.get(src_path, throw=True)).stdout dst, dst_path = __ec(dst_uri) - if os.path.isdir(dst_path) and not dst_path[-1] == '/': - dst_path += '/' - if dst_path[-1] == '/': + if await dst.is_dir(path): dst_path += os.path.basename(src_path) await dst.put(path=dst_path, content=content, owner=owner, group=group, mode=mode, throw=True) except Exception as e: