diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index 45148852..0a29205b 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -2,18 +2,102 @@ from __future__ import annotations -import abc, re, sys +import abc, re, sys, errno from enum import Enum, auto from typing import NamedTuple, TYPE_CHECKING +from decimal import Decimal, ROUND_FLOOR if TYPE_CHECKING: from typing import Self, Type from types import TracebackType from .log import * -from .base import Input, InputMode, Result +from .base import Input, InputMode, Result, StatResult from .FileContext import FileContext as Base +_US = "\x1f" # unlikely to appear in numeric output +_BILLION = Decimal(1_000_000_000) + +def _looks_like_option_error(stderr: str) -> bool: + s = stderr.lower() + return any( + needle in s + for needle in ( + "unrecognized option", + "illegal option", + "unknown option", + "invalid option", + "option requires an argument", + ) + ) + +def _raise_stat_error(path: str, stderr: str, returncode: int) -> None: + msg = (stderr or "").strip() or f"stat exited with status {returncode}" + lower = msg.lower() + if "no such file" in lower: + raise FileNotFoundError(errno.ENOENT, msg, path) + if "permission denied" in lower or "operation not permitted" in lower: + raise PermissionError(errno.EACCES, msg, path) + raise OSError(errno.EIO, msg, path) + +def _parse_epoch(value: str) -> tuple[int, float, int]: + """ + Convert a decimal epoch timestamp string into: + (integer seconds for tuple slot, float seconds for attribute, ns for *_ns) + """ + dec = Decimal(value.strip()) + sec = int(dec.to_integral_value(rounding=ROUND_FLOOR)) + ns = int((dec * _BILLION).to_integral_value(rounding=ROUND_FLOOR)) + return sec, float(dec), ns + +def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: + if len(fields) != 13: + raise ValueError( + f"unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}" + ) + + ( + mode_s, + ino_s, + dev_s, + nlink_s, + uid_s, + gid_s, + size_s, + atime_s, + mtime_s, + ctime_s, + blksize_s, + blocks_s, + rdev_s, + ) = fields + + st_mode = int(mode_s, mode_base) + st_ino = int(ino_s) + st_dev = int(dev_s) + st_nlink = int(nlink_s) + st_uid = uid_s + st_gid = gid_s + st_size = int(size_s) + + st_atime_i, st_atime_f, st_atime_ns = _parse_epoch(atime_s) + st_mtime_i, st_mtime_f, st_mtime_ns = _parse_epoch(mtime_s) + st_ctime_i, st_ctime_f, st_ctime_ns = _parse_epoch(ctime_s) + + st_blksize = int(blksize_s) + st_blocks = int(blocks_s) + st_rdev = int(rdev_s) + + return StatResult( + mode = st_mode, + owner = st_uid, + group = st_gid, + size = st_size, + atime = st_atime_i, + mtime = st_mtime_i, + ctime = st_ctime_i, + ) + class ExecContext(Base): class CallContext: @@ -332,3 +416,97 @@ class ExecContext(Base): raise return cc.exception(ret, e) return ret + + async def _unlink(self, path: str) -> None: + cmd = ['rm', '-f', path] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _erase(self, path: str) -> None: + cmd = ['rm', '-rf', path] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _rename(self, src: str, dst: str) -> None: + cmd = ['mv', src, dst] + await self.run(cmd, cmd_input=InputMode.NonInteractive) + + async def _mktemp(self, tmpl: str, directory: bool) -> str: + cmd = ['mktemp'] + if directory: + cmd.append('-d') + cmd.append(tmpl) + result = await self.run(cmd, cmd_input=InputMode.NonInteractive) + return result.stdout.strip().decode() + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + + async def __stat(opts: list[str]) -> str: + env = { + 'LC_ALL': 'C' + } + cmd = ['stat'] + if follow_symlinks: + cmd.append('-L') + cmd.extend(opts) + cmd.append(path) + return (await self.run(cmd, env=env, throw=False, + cmd_input=InputMode.NonInteractive)).decode() + + # GNU coreutils stat + gnu_format = _US.join([ + "%f", # st_mode in hex + "%i", # st_ino + "%d", # st_dev + "%h", # st_nlink + "%U", # st_uid + "%G", # st_gid + "%s", # st_size + "%.9X", # st_atime + "%.9Y", # st_mtime + "%.9Z", # st_ctime + "%o", # st_blksize hint + "%b", # st_blocks + "%r", # st_rdev + ]) + + result = await __stat(['--printf', gnu_format]) + if result.status == 0: + return _build_stat_result(result.stdout.split(_US), mode_base=16) + + if not _looks_like_option_error(result.stderr.decode()): + # log(DEBUG, f'GNU stat attempt failed on "{path}" ({str(e)})') + _raise_stat_error(path, result.stderr, result.status) + + # BSD / macOS / OpenBSD / NetBSD stat + bsd_format = _US.join([ + "%p", # st_mode in octal + "%i", # st_ino + "%d", # st_dev + "%l", # st_nlink + "%U", # st_uid + "%G", # st_gid + "%z", # st_size + "%.9Fa", # st_atime + "%.9Fm", # st_mtime + "%.9Fc", # st_ctime + "%k", # st_blksize + "%b", # st_blocks + "%r", # st_rdev + ]) + + result = await __stat(['-n', '-f', bst_format]) + if proc.returncode == 0: + return _build_stat_result(proc.stdout.rstrip('\n').split(_US), mode_base=8) + _raise_stat_error(path, result.stderr, result.status) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + assert owner is not None or group is not None + if group is None: + ownership = owner + elif owner is None: + ownership = ':' + group + else: + ownership = owner + ':' + group + await self.run(['chown', ownership, path], cmd_input=InputMode.NonInteractive) + + async def _chmod(self, path: str, mode: int) -> None: + await self.run(['chmod', oct(mode), path], cmd_input=InputMode.NonInteractive) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 7d967383..33720770 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from typing import Self from .log import * -from .base import Input, InputMode, Result +from .base import Input, InputMode, Result, StatResult class FileContext(abc.ABC): @@ -109,6 +109,85 @@ class FileContext(abc.ABC): return await self._put(path, content, wd=wd, throw=throw, verbose=verbose, title=title, owner=owner, group=group, mode=mode_str, atomic=atomic) + async def _unlink(self, path: str) -> None: + raise NotImplementedError(f'{self.log_name}: unlink() is not implemented') + + async def unlink(self, path: str) -> None: + return await self._unlink(path) + + async def _erase(self, path: str) -> None: + raise NotImplementedError(f'{self.log_name}: erase() is not implemented') + + async def erase(self, path: str) -> None: + return await self._erase(path) + + async def _rename(self, src: str, dst: str) -> None: + raise NotImplementedError(f'{self.log_name}: rename() is not implemented') + + async def rename(self, src: str, dst: str) -> None: + return await self._rename(src, dst) + + async def _mktemp(self, tmpl: str, directory: bool) -> None: + raise NotImplementedError(f'{self.log_name}: mktemp() is not implemented') + + async def mktemp(self, tmpl: str, directory: bool=False) -> None: + return await self._mktemp(tmpl, directory) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + raise NotImplementedError(f'{self.log_name}: chown() is not implemented') + + async def chown(self, path: str, owner: str|None=None, group: str|None=None) -> None: + if owner is None and group is None: + raise ValueError(f'Tried to change ownership of {path} specifying neither owner nor group') + return await self._chown(path, owner, group) + + async def _chmod(self, path: str, mode: int) -> None: + raise NotImplementedError(f'{self.log_name}: chmod() is not implemented') + + async def chmod(self, path: str, mode: int) -> None: + return await self._chmod(path, mode) + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + raise NotImplementedError(f'{self.log_name}: lstat() is not implemented') + + async def stat(self, path: str, follow_symlinks: bool=True) -> StatResult: + if not isinstance(path, str): + raise TypeError(f"path must be str, got {type(path).__name__}") + return await self._stat(path, follow_symlinks) + + async def _file_exists(self, path: str) -> bool: + try: + self._stat(path, False) + except FileNotFoundError as e: + log(DEBUG, f'Could not stat file {path} ({str(e)}), ignored') + return False + except Exception as e: + log(ERR, f'Could not stat file {path} ({str(e)}), ignored') + raise + return True + + async def file_exists(self, path: str) -> bool: + return self._file_exists(path) + + async def _is_dir(self, path: str) -> bool: + try: + return S_ISDIR(await self._stat(path).st_mode) + except NotImplementedError: + log(DEBUG, f'{self.log_name} doesn\'t implement stat(), judging by trailing slash if {path} is a directory') + return path[-1] == '/' + except FileNotFoundError as e: + log(DEBUG, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + return False + except Exception as e: + log(ERR, f'{self.log_name}: Failed to stat({path}) ({str(e)})') + raise + return False + + async def is_dir(self, path: str) -> bool: + if not path: + raise ValueError('Tried to investigate file system resource with empty path') + return self._is_dir(path) + async def _close(self) -> None: pass diff --git a/src/python/jw/pkg/lib/base.py b/src/python/jw/pkg/lib/base.py index 81c232e5..1d0a336a 100644 --- a/src/python/jw/pkg/lib/base.py +++ b/src/python/jw/pkg/lib/base.py @@ -44,9 +44,10 @@ class StatResult(NamedTuple): import pwd, grp return StatResult( rhs.st_mode, - pwd.getpwuid(rhs.pw_uid).pw_name, - grp.getgrgid(gid).gr_name, + pwd.getpwuid(rhs.st_uid).pw_name, + grp.getgrgid(rhs.st_gid).gr_name, rhs.st_size, + rhs.st_atime, rhs.st_mtime, rhs.st_ctime, ) diff --git a/src/python/jw/pkg/lib/ec/Local.py b/src/python/jw/pkg/lib/ec/Local.py index 697872e8..f0b449c3 100644 --- a/src/python/jw/pkg/lib/ec/Local.py +++ b/src/python/jw/pkg/lib/ec/Local.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import os, sys, subprocess, asyncio +import os, sys, subprocess, asyncio, pwd, grp, stat from ..ExecContext import ExecContext as Base -from ..base import Result +from ..base import Result, StatResult from ..log import * from ..util import pretty_cmd @@ -151,3 +151,32 @@ class Local(Base): cmdline.extend(opts) cmdline.extend(cmd) return await self._run(cmdline, *args, **kwargs) + + async def _unlink(self, path: str) -> None: + os.unlink(path) + + async def _erase(self, path: str) -> None: + if os.isdir(path): + shutil.rmtree(path) + return + os.unlink(path) + + async def _rename(self, src: str, dst: str) -> None: + os.rename(src, dst) + + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + return StatResult.from_os(os.stat(path, follow_symlinks=follow_symlinks)) + + async def _file_exists(self, path: str) -> bool: + return os.path.exists(path) + + async def _chown(self, path: str, owner: str|None, group: str|None) -> None: + uid = pwd.getpwnam(owner).pw_uid if owner else -1 + gid = grp.getgrnam(group).gr_gid if group else -1 + os.chown(path, uid, gid) + + async def _chmod(self, path: str, mode: int) -> None: + os.chmod(path, mode) + + async def _is_dir(self, path: str) -> bool: + return os.path.isdir(path) diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 329ff593..7f92126c 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -110,9 +110,7 @@ async def copy(src_uri: str, dst_uri: str, owner: str|None=None, group: str|None src, src_path = __ec(src_uri) content = (await src.get(src_path, throw=True)).stdout dst, dst_path = __ec(dst_uri) - if os.path.isdir(dst_path) and not dst_path[-1] == '/': - dst_path += '/' - if dst_path[-1] == '/': + if await dst.is_dir(path): dst_path += os.path.basename(src_path) await dst.put(path=dst_path, content=content, owner=owner, group=group, mode=mode, throw=True) except Exception as e: