From cda9e210daa41e154939603edf6c0b23a1ff08f4 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 9 Oct 2026 10:48:17 +0200 Subject: [PATCH 1/2] lib.FcWrapper: Add a delegating file context wrapper Add a file context that sits in front of another one and delegates every primitive call to it. The wrapper adopts the identity of the wrapped context: uri, root, chroot flag, and the chroot path composition are the wrapped context's, so a path composed by the wrapper composes identically in the wrapped context. Every primitive is forwarded to the wrapped context's primitives, which receive the already-composed path; open() and close() are forwarded to the wrapped context's public methods, so the open count is kept in one place. The point is to give derived contexts a stable base: a caching context, or any other filter, overrides the primitives it cares about and inherits the rest, instead of re-deriving the chroot bookkeeping of every method. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/FcWrapper.py | 122 +++++++++++++ .../unit/python/jw/pkg/lib/FcWrapper/Makefile | 7 + test/unit/python/jw/pkg/lib/FcWrapper/test.py | 161 ++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 src/python/jw/pkg/lib/FcWrapper.py create mode 100644 test/unit/python/jw/pkg/lib/FcWrapper/Makefile create mode 100644 test/unit/python/jw/pkg/lib/FcWrapper/test.py diff --git a/src/python/jw/pkg/lib/FcWrapper.py b/src/python/jw/pkg/lib/FcWrapper.py new file mode 100644 index 00000000..1973870a --- /dev/null +++ b/src/python/jw/pkg/lib/FcWrapper.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Iterable, override + +if TYPE_CHECKING: + from .base import Result, StatResult + from .file_types import FileType + +from .FileContext import FileContext + +class FcWrapper(FileContext): # export + """A file context that delegates to a wrapped context. + + The wrapper adopts the identity of the wrapped context: uri, + root, chroot flag, and the chroot path composition are all the + wrapped context's, so a path composed by the wrapper composes + identically in the wrapped context. Every primitive call is + forwarded to the wrapped context's primitives, which receive + the already-composed path; open and close are forwarded to the + wrapped context's public methods, so the open count is kept in + one place. + """ + + def __init__(self, wrapped: FileContext) -> None: + super().__init__( + wrapped.uri, + interactive = wrapped.interactive, + verbose_default = wrapped.verbose_default, + chroot = wrapped.chroot, + ) + self.__wrapped = wrapped + + @property + def wrapped(self) -> FileContext: + return self.__wrapped + + @override + async def _open(self) -> None: + await self.__wrapped.open() + + @override + async def _close(self) -> None: + await self.__wrapped.close() + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + return await self.__wrapped._get(path, wd, throw, verbose, title) + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + return await self.__wrapped._put( + path, + content, + wd, + throw, + verbose, + title, + owner, + group, + mode, + atomic, + ftype, + ) + + @override + async def _unlink(self, path: str) -> None: + await self.__wrapped._unlink(path) + + @override + async def _erase(self, path: str) -> None: + await self.__wrapped._erase(path) + + @override + async def _rename(self, src: str, dst: str) -> None: + await self.__wrapped._rename(src, dst) + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + await self.__wrapped._mkdir(path, mode, ftype) + + @override + async def _mktemp(self, tmpl: str, directory: bool) -> str: + return await self.__wrapped._mktemp(tmpl, directory) + + @override + async def _chown(self, path: str, owner: str | None, group: str | None) -> None: + await self.__wrapped._chown(path, owner, group) + + @override + async def _chmod(self, path: str, mode: int) -> None: + await self.__wrapped._chmod(path, mode) + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + return await self.__wrapped._stat(path, follow_symlinks) + + @override + async def _file_exists(self, path: str) -> bool: + return await self.__wrapped._file_exists(path) + + @override + async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: + return await self.__wrapped._is_dir(path, follow_symlinks) + + @override + async def _readdir(self, path: str) -> Iterable[str]: + return await self.__wrapped._readdir(path) diff --git a/test/unit/python/jw/pkg/lib/FcWrapper/Makefile b/test/unit/python/jw/pkg/lib/FcWrapper/Makefile new file mode 100644 index 00000000..de693650 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/FcWrapper/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/FcWrapper/test.py b/test/unit/python/jw/pkg/lib/FcWrapper/test.py new file mode 100644 index 00000000..6e158013 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/FcWrapper/test.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import asyncio + +from typing import TYPE_CHECKING, Iterable, override + +if TYPE_CHECKING: + from jw.pkg.lib.file_types import FileType + +from jw.pkg.lib.base import StatResult +from jw.pkg.lib.FcWrapper import FcWrapper +from jw.pkg.lib.file_types import FT_DIR, FT_REG +from jw.pkg.lib.FileContext import FileContext +from jw.pkg.lib.Result import Result + +class FakeCtx(FileContext): + """An in-memory FileContext that counts its primitive calls""" + + def __init__(self, uri: str, chroot: bool = False) -> None: + super().__init__(uri, chroot = chroot) + self.files: dict[str, bytes] = {} + self.dirs: set[str] = set() + self.counts: dict[str, int] = {} + self.opens = 0 + + def count(self, op: str) -> None: + self.counts[op] = self.counts.get(op, 0) + 1 + + def _stat_of(self, path: str) -> StatResult: + if path in self.dirs: + return StatResult(0o40755, '0', '0', 40, 0.0, 0.0, 0.0) + if path in self.files: + return StatResult(0o100644, '0', '0', len(self.files[path]), 0.0, 0.0, 0.0) + raise FileNotFoundError(path) + + @override + async def _open(self) -> None: + self.opens += 1 + + @override + async def _close(self) -> None: + pass + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + self.count('get') + if wd is not None: + path = wd + '/' + path + if path not in self.files: + raise FileNotFoundError(path) + return Result(stdout = self.files[path]) + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + self.count('put') + if wd is not None: + path = wd + '/' + path + self.files[path] = content + return Result(stdout = b'') + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + self.count('mkdir') + self.dirs.add(path) + + @override + async def _unlink(self, path: str) -> None: + self.count('unlink') + self.files.pop(path, None) + self.dirs.discard(path) + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + self.count('stat') + return self._stat_of(path) + + @override + async def _file_exists(self, path: str) -> bool: + self.count('file_exists') + try: + self._stat_of(path) + return True + except FileNotFoundError: + return False + + @override + async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: + self.count('is_dir') + try: + st = self._stat_of(path) + return (st.mode & 0o170000) == 0o40000 + except FileNotFoundError: + return False + + @override + async def _readdir(self, path: str) -> Iterable[str]: + self.count('readdir') + prefix = path.rstrip('/') + '/' + return [ + p[len(prefix):] for p in sorted(self.files.keys() | self.dirs) + if p.startswith(prefix) + ] + +async def main() -> None: + fake = FakeCtx('local:///tmp/fc-wrapper-fake', chroot = True) + fc = FcWrapper(fake) + + # -- The wrapper adopts the identity of the wrapped context + assert fc.wrapped is fake + assert fc.uri is fake.uri + assert fc.root == fake.root + assert fc.chroot == fake.chroot + assert fc.username == fake.username + assert fc.interactive == fake.interactive + assert fc.verbose_default == fake.verbose_default + + # -- Put and get are forwarded; paths are composed once + await fc.put('etc/a.conf', b'x=1\n', ftype = FT_REG) + assert fake.files == {'/tmp/fc-wrapper-fake/etc/a.conf': b'x=1\n'} + assert fake.counts['put'] == 1 + + ret = await fc.get('etc/a.conf') + assert ret.stdout == b'x=1\n' + assert fake.counts['get'] == 1 + + # -- stat, file_exists, is_dir, and readdir are forwarded + st = await fc.stat('etc/a.conf') + assert st.size == 4 + assert await fc.file_exists('etc/a.conf') is True + assert await fc.file_exists('etc/missing.conf') is False + await fc.mkdir('etc/sub', 0o755, FT_DIR) + assert await fc.is_dir('etc/sub') is True + names = await fc.readdir('etc') + assert 'a.conf' in names and 'sub' in names + + # -- Unlink is forwarded + await fc.unlink('etc/a.conf') + assert '/tmp/fc-wrapper-fake/etc/a.conf' not in fake.files + assert fake.counts['unlink'] == 1 + + # -- Open and close reach the wrapped context's public methods + await fc.open() + await fc.close() + assert fake.opens == 1 + +asyncio.run(main()) -- 2.56.0 From e02f7248080d04adbd3bfb40a0a77f73d665f299 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 9 Oct 2026 10:49:37 +0200 Subject: [PATCH 2/2] lib.FcCache: Add a write-through read cache Add a write-through read cache in front of a file context. stat() and get() results are cached: stat per (path, follow_symlinks), get per (path, wd, throw). Successful stats and FileNotFoundErrors are cached; other errors are re-probed, so a transient transport failure does not poison the cache. Writes pass through to the wrapped context and drop the cache entries of the affected paths. The existence and directory checks are rerouted through the cached stat, so they pay the cache instead of the wrapped context's own fast path. warm() fills the stat cache of a set of paths, as file_exists() would stat them, without following symlinks. If the wrapped context is an exec context, one host command stats all of the paths; otherwise, one stat per path runs. The GNU stat format of ExecContext is lifted to the module constant STAT_GNU_FORMAT, and the field parser is exported as parse_stat_fields(), so warm() fills entries that are indistinguishable from a real stat(). Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/ExecContext.py | 47 ++-- src/python/jw/pkg/lib/FcCache.py | 234 +++++++++++++++++++ test/unit/python/jw/pkg/lib/FcCache/Makefile | 7 + test/unit/python/jw/pkg/lib/FcCache/test.py | 198 ++++++++++++++++ 4 files changed, 464 insertions(+), 22 deletions(-) create mode 100644 src/python/jw/pkg/lib/FcCache.py create mode 100644 test/unit/python/jw/pkg/lib/FcCache/Makefile create mode 100644 test/unit/python/jw/pkg/lib/FcCache/test.py diff --git a/src/python/jw/pkg/lib/ExecContext.py b/src/python/jw/pkg/lib/ExecContext.py index a282dc62..f4986c5e 100644 --- a/src/python/jw/pkg/lib/ExecContext.py +++ b/src/python/jw/pkg/lib/ExecContext.py @@ -17,6 +17,24 @@ from .FileContext import FileContext as Base from .log import DEBUG, ERR, NOTICE, log _US = '\x1f' # unlikely to appear in numeric output + +STAT_GNU_FORMAT = _US.join( + [ + '%f', # st_mode in hex + '%i', # st_ino + '%d', # st_dev + '%h', # st_nlink + '%U', # st_uid + '%G', # st_gid + '%s', # st_size + '%.9X', # st_atime + '%.9Y', # st_mtime + '%.9Z', # st_ctime + '%o', # st_blksize hint + '%b', # st_blocks + '%r', # st_rdev + ] +) _BILLION = Decimal(1_000_000_000) def _looks_like_option_error(stderr: str | None) -> bool: @@ -54,7 +72,10 @@ def _parse_epoch(value: str) -> tuple[int, float, int]: ns = int((dec * _BILLION).to_integral_value(rounding = ROUND_FLOOR)) return sec, float(dec), ns -def _build_stat_result(fields: list[str], mode_base: int) -> StatResult: +def parse_stat_fields(stdout: str, mode_base: int = 16) -> StatResult: # export + return build_stat_result(stdout.split(_US), mode_base = mode_base) + +def build_stat_result(fields: list[str], mode_base: int) -> StatResult: # export if len(fields) != 13: raise ValueError( f'unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}' @@ -621,27 +642,9 @@ class ExecContext(Base): ) # GNU coreutils stat - gnu_format = _US.join( - [ - '%f', # st_mode in hex - '%i', # st_ino - '%d', # st_dev - '%h', # st_nlink - '%U', # st_uid - '%G', # st_gid - '%s', # st_size - '%.9X', # st_atime - '%.9Y', # st_mtime - '%.9Z', # st_ctime - '%o', # st_blksize hint - '%b', # st_blocks - '%r', # st_rdev - ] - ) - - result = await __stat(['--printf', gnu_format]) + result = await __stat(['--printf', STAT_GNU_FORMAT]) if result.status == 0 and result.stdout is not None: - return _build_stat_result(result.stdout_str.split(_US), mode_base = 16) + return parse_stat_fields(result.stdout_str, mode_base = 16) if not _looks_like_option_error(result.stderr_str_or_none): # log(DEBUG, f'GNU stat attempt failed on "{path}" ({str(e)})') @@ -671,7 +674,7 @@ class ExecContext(Base): if result.status != 0 or stdout is None: _raise_stat_error(path, result) assert stdout is not None # Just there to pacify the linter - return _build_stat_result(stdout.rstrip('\n').split(_US), mode_base = 8) + return parse_stat_fields(stdout.rstrip('\n'), mode_base = 8) @override async def _chown(self, path: str, owner: str | None, group: str | None) -> None: diff --git a/src/python/jw/pkg/lib/FcCache.py b/src/python/jw/pkg/lib/FcCache.py new file mode 100644 index 00000000..8f52e963 --- /dev/null +++ b/src/python/jw/pkg/lib/FcCache.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import errno +import stat + +from typing import TYPE_CHECKING, Iterable, override + +if TYPE_CHECKING: + from .base import Result + from .FileContext import FileContext + from .file_types import FileType + +from .base import StatResult +from .ExecContext import STAT_GNU_FORMAT, ExecContext, parse_stat_fields +from .FcWrapper import FcWrapper + +class FcCache(FcWrapper): # export + """A write-through read cache in front of a file context. + + stat and get results are cached: stat per (path, + follow_symlinks), get per (path, wd, throw). Successful stats + and FileNotFoundErrors are cached; other errors are re-probed, + so a transient transport failure does not poison the cache. + Writes pass through to the wrapped context and drop the cache + entries of the affected paths. + + warm() probes a set of paths without following symlinks and + fills the stat cache with the results: an absent path is + absent, and a non-link is the same node, for both follow + variants, so the result is cached for both; a link is cached + without following, its followed variant probed on first use. + If the wrapped context is an exec context, one host command + stats all of the paths; otherwise, one stat per path runs. + Paths containing tab or newline characters are not supported. + """ + + def __init__(self, wrapped: FileContext) -> None: + super().__init__(wrapped) + self.__stat_cache: dict[tuple[str, bool], StatResult | Exception] = {} + self.__get_cache: dict[tuple[str, str | None, bool], Result] = {} + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + key = (path, follow_symlinks) + hit = self.__stat_cache.get(key) + if hit is not None: + if isinstance(hit, StatResult): + return hit + raise hit + try: + st = await super()._stat(path, follow_symlinks) + except FileNotFoundError as e: + self.__stat_cache[key] = e + raise + self.__stat_cache[key] = st + return st + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + key = (path, wd, throw) + hit = self.__get_cache.get(key) + if hit is None: + hit = await super()._get(path, wd, throw, verbose, title) + self.__get_cache[key] = hit + return hit + + @override + async def _file_exists(self, path: str) -> bool: + # -- Reroute the existence check through the cached stat + # instead of the wrapped context's own fast path. A path + # below a file cannot exist: the exec contexts report it + # as absent, so the local one is treated the same + try: + await self._stat(path, False) + except FileNotFoundError: + return False + except NotADirectoryError: + return False + return True + + @override + async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: + try: + return stat.S_ISDIR((await self._stat(path, follow_symlinks)).mode) + except FileNotFoundError: + return False + except NotADirectoryError: + return False + + async def warm(self, paths: Iterable[str]) -> None: + """Probe a set of paths without following symlinks and fill + the stat cache, as the class docstring describes. + + A path whose both follow variants are already in the cache + is skipped, so a warm of a set that is cached already does + nothing. If the wrapped context is an exec context, one + host command stats all of the remaining paths; otherwise, + one stat per path runs. + """ + composed = [] + for p in paths: + if '\t' in p or '\n' in p: + raise ValueError(f'warm: path {p!r} holds a tab or newline') + c = self._chroot(p) + if (c, False) not in self.__stat_cache or \ + (c, True) not in self.__stat_cache: + composed.append(c) + if not composed: + return + ec = self.wrapped + if not isinstance(ec, ExecContext): + for p in composed: + try: + st = await super()._stat(p, False) + except FileNotFoundError as e: + self.__cache_probe(p, e) + except NotADirectoryError: + # -- A path below a file is not statable; it is + # not cached, and file_exists() judges it + # absent + pass + else: + self.__cache_probe(p, st) + return + script = ( + 'while IFS= read -r p; do ' + "printf '%s\\t' \"$p\"; " + f"stat --printf '{STAT_GNU_FORMAT}\\n' \"$p\" 2>/dev/null || " + "printf 'ABSENT\\n'; " + 'done' + ) + ret = await ec.run( + ['sh', '-c', script], + cmd_input = ('\n'.join(composed) + '\n').encode(), + mod_env = {'LC_ALL': 'C'}, + throw = True, + ) + self.__parse_probe(ret.stdout_str, composed) + + def __cache_probe(self, path: str, result: StatResult | FileNotFoundError) -> None: + # -- Cache a probe that did not follow symlinks: an absent + # path is absent, and a non-link is the same node, for + # both follow variants; a link is cached without + # following, its followed variant probed on first use + self.__stat_cache[(path, False)] = result + if isinstance(result, FileNotFoundError) or not stat.S_ISLNK(result.mode): + self.__stat_cache[(path, True)] = result + + def __parse_probe(self, stdout: str, composed: list[str]) -> None: + # -- One line per path: the path, a tab, then the stat + # fields or ABSENT + lines = stdout.splitlines() + if len(lines) != len(composed): + raise RuntimeError( + f'warm: probe reported {len(lines)} lines for {len(composed)} paths' + ) + for p, line in zip(composed, lines): + name, sep, rest = line.partition('\t') + if name != p or not sep: + raise RuntimeError(f'warm: probe line does not name "{p}": {line!r}') + if rest == 'ABSENT': + self.__cache_probe(p, FileNotFoundError(errno.ENOENT, f'stat: {p}')) + else: + self.__cache_probe(p, parse_stat_fields(rest)) + + def __invalidate(self, *paths: str) -> None: + # -- A write changes the stat and get answers of its paths + for p in paths: + for fl in (True, False): + self.__stat_cache.pop((p, fl), None) + self.__get_cache = {k: r for k, r in self.__get_cache.items() if k[0] != p} + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + r = await super()._put( + path, + content, + wd, + throw, + verbose, + title, + owner, + group, + mode, + atomic, + ftype, + ) + self.__invalidate(path) + return r + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + await super()._mkdir(path, mode, ftype) + self.__invalidate(path) + + @override + async def _unlink(self, path: str) -> None: + await super()._unlink(path) + self.__invalidate(path) + + @override + async def _erase(self, path: str) -> None: + await super()._erase(path) + self.__invalidate(path) + + @override + async def _rename(self, src: str, dst: str) -> None: + await super()._rename(src, dst) + self.__invalidate(src, dst) + + @override + async def _chown(self, path: str, owner: str | None, group: str | None) -> None: + await super()._chown(path, owner, group) + self.__invalidate(path) + + @override + async def _chmod(self, path: str, mode: int) -> None: + await super()._chmod(path, mode) + self.__invalidate(path) diff --git a/test/unit/python/jw/pkg/lib/FcCache/Makefile b/test/unit/python/jw/pkg/lib/FcCache/Makefile new file mode 100644 index 00000000..de693650 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/FcCache/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/FcCache/test.py b/test/unit/python/jw/pkg/lib/FcCache/test.py new file mode 100644 index 00000000..8beee776 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/FcCache/test.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import asyncio +import os +import tempfile + +from typing import TYPE_CHECKING, override + +if TYPE_CHECKING: + from jw.pkg.lib.file_types import FileType + +from jw.pkg.lib.base import StatResult +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.ExecContext import ExecContext +from jw.pkg.lib.FcCache import FcCache +from jw.pkg.lib.file_types import FT_DIR, FT_REG +from jw.pkg.lib.FileContext import FileContext +from jw.pkg.lib.Result import Result + +class FakeCtx(FileContext): + """An in-memory FileContext that counts its primitive calls""" + + def __init__(self, uri: str, chroot: bool = False) -> None: + super().__init__(uri, chroot = chroot) + self.files: dict[str, bytes] = {} + self.dirs: set[str] = set() + self.counts: dict[str, int] = {} + + def count(self, op: str) -> None: + self.counts[op] = self.counts.get(op, 0) + 1 + + def _stat_of(self, path: str) -> StatResult: + if path in self.dirs: + return StatResult(0o40755, '0', '0', 40, 0.0, 0.0, 0.0) + if path in self.files: + return StatResult(0o100644, '0', '0', len(self.files[path]), 0.0, 0.0, 0.0) + raise FileNotFoundError(path) + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + self.count('get') + if wd is not None: + path = wd + '/' + path + if path not in self.files: + raise FileNotFoundError(path) + return Result(stdout = self.files[path]) + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + self.count('put') + if wd is not None: + path = wd + '/' + path + self.files[path] = content + return Result(stdout = b'') + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + self.count('mkdir') + self.dirs.add(path) + + @override + async def _unlink(self, path: str) -> None: + self.count('unlink') + self.files.pop(path, None) + self.dirs.discard(path) + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + self.count('stat') + return self._stat_of(path) + +class CountingLocal(Local): + """A Local context that counts its executed commands""" + + def __init__(self) -> None: + super().__init__('local') + self.runs = 0 + + @override + async def _run( + self, + cmd: list[str], + wd: str | None, + verbose: bool, + cmd_input: bytes | None, + mod_env: dict[str, str] | None, + interactive: bool, + log_prefix: str, + ) -> Result: + self.runs += 1 + return await super()._run( + cmd, wd, verbose, cmd_input, mod_env, interactive, log_prefix + ) + +async def main() -> None: + fake = FakeCtx('local:///tmp/fc-cache-fake', chroot = True) + fc = FcCache(fake) + await fc.mkdir('d', 0o755, FT_DIR) + await fc.put('d/a', b'x', ftype = FT_REG) + + # -- Successful stats are cached + st = await fc.stat('d/a') + assert st.size == 1 + assert (await fc.stat('d/a')).size == 1 + assert fake.counts['stat'] == 1 + + # -- Absences are cached, too + assert await fc.file_exists('d/missing') is False + assert await fc.file_exists('d/missing') is False + assert fake.counts['stat'] == 2 + + # -- Gets are cached per (path, wd, throw) + ret = await fc.get('d/a') + assert (await fc.get('d/a')).stdout == b'x' + assert fake.counts['get'] == 1 + ret = await fc._get('a', '/tmp/fc-cache-fake/d', True, None, 'get') + assert fake.counts['get'] == 2 + assert ret.stdout == b'x' + + # -- Writes drop the cache entries of their paths + await fc.put('d/a', b'xy', ftype = FT_REG) + assert (await fc.get('d/a')).stdout == b'xy' + assert fake.counts['get'] == 3 + assert (await fc.stat('d/a')).size == 2 + assert fake.counts['stat'] == 3 + + # -- Unlink drops them as well + await fc.unlink('d/a') + assert await fc.file_exists('d/a') is False + assert fake.counts['stat'] == 4 + assert '/tmp/fc-cache-fake/d/a' not in fake.files + + # -- The wrapper itself is not an exec context + assert not isinstance(fc, ExecContext) + + # -- warm() on a non-exec context: one stat per path, then cached + fake2 = FakeCtx('local:///tmp/fc-cache-warm', chroot = True) + fc2 = FcCache(fake2) + await fc2.put('p/q', b'z', ftype = FT_REG) + await fc2.warm(['p', 'p/q', 'p/r']) + assert fake2.counts['stat'] == 3 + await fc2.warm(['p', 'p/q', 'p/r']) + assert fake2.counts['stat'] == 3 + assert await fc2.file_exists('p/r') is False + assert fake2.counts['stat'] == 3 + + # -- warm() on an exec context: one host command for all paths + with tempfile.TemporaryDirectory() as td: + f = os.path.join(td, 'there') + with open(f, 'w') as fh: + fh.write('hello\n') + cl = CountingLocal() + fc3 = FcCache(cl) + missing = os.path.join(td, 'not-there') + await fc3.warm([f, missing]) + assert cl.runs == 1 + + st = await fc3.stat(f) + assert cl.runs == 1 + assert st.size == 6 + assert await fc3.file_exists(missing) is False + assert cl.runs == 1 + try: + await fc3.stat(missing) + assert False, 'stat of a warmed absence must raise' + except FileNotFoundError: + pass + assert cl.runs == 1 + + # -- A put through the cache lands in the wrapped context and + # drops its cache entries, so the next stat is a fresh one. + # Local stats without a command, so file_exists adds no run + g = os.path.join(td, 'new') + await fc3.put(g, b'fresh\n', ftype = FT_REG) + assert cl.runs == 2 + assert (await fc3.get(g)).stdout == b'fresh\n' + assert cl.runs == 3 + assert await fc3.file_exists(g) is True + assert cl.runs == 3 + assert await fc3.file_exists(g) is True + assert cl.runs == 3 + +asyncio.run(main()) -- 2.56.0