lib.FcCache: Add a write-through read cache #145
7 changed files with 754 additions and 22 deletions
|
|
@ -17,6 +17,24 @@ from .FileContext import FileContext as Base
|
|||
from .log import DEBUG, ERR, NOTICE, log
|
||||
|
||||
_US = '\x1f' # unlikely to appear in numeric output
|
||||
|
||||
STAT_GNU_FORMAT = _US.join(
|
||||
[
|
||||
'%f', # st_mode in hex
|
||||
'%i', # st_ino
|
||||
'%d', # st_dev
|
||||
'%h', # st_nlink
|
||||
'%U', # st_uid
|
||||
'%G', # st_gid
|
||||
'%s', # st_size
|
||||
'%.9X', # st_atime
|
||||
'%.9Y', # st_mtime
|
||||
'%.9Z', # st_ctime
|
||||
'%o', # st_blksize hint
|
||||
'%b', # st_blocks
|
||||
'%r', # st_rdev
|
||||
]
|
||||
)
|
||||
_BILLION = Decimal(1_000_000_000)
|
||||
|
||||
def _looks_like_option_error(stderr: str | None) -> bool:
|
||||
|
|
@ -54,7 +72,10 @@ def _parse_epoch(value: str) -> tuple[int, float, int]:
|
|||
ns = int((dec * _BILLION).to_integral_value(rounding = ROUND_FLOOR))
|
||||
return sec, float(dec), ns
|
||||
|
||||
def _build_stat_result(fields: list[str], mode_base: int) -> StatResult:
|
||||
def parse_stat_fields(stdout: str, mode_base: int = 16) -> StatResult: # export
|
||||
return build_stat_result(stdout.split(_US), mode_base = mode_base)
|
||||
|
||||
def build_stat_result(fields: list[str], mode_base: int) -> StatResult: # export
|
||||
if len(fields) != 13:
|
||||
raise ValueError(
|
||||
f'unexpected stat output: expected 13 fields, got {len(fields)}: {fields!r}'
|
||||
|
|
@ -621,27 +642,9 @@ class ExecContext(Base):
|
|||
)
|
||||
|
||||
# GNU coreutils stat
|
||||
gnu_format = _US.join(
|
||||
[
|
||||
'%f', # st_mode in hex
|
||||
'%i', # st_ino
|
||||
'%d', # st_dev
|
||||
'%h', # st_nlink
|
||||
'%U', # st_uid
|
||||
'%G', # st_gid
|
||||
'%s', # st_size
|
||||
'%.9X', # st_atime
|
||||
'%.9Y', # st_mtime
|
||||
'%.9Z', # st_ctime
|
||||
'%o', # st_blksize hint
|
||||
'%b', # st_blocks
|
||||
'%r', # st_rdev
|
||||
]
|
||||
)
|
||||
|
||||
result = await __stat(['--printf', gnu_format])
|
||||
result = await __stat(['--printf', STAT_GNU_FORMAT])
|
||||
if result.status == 0 and result.stdout is not None:
|
||||
return _build_stat_result(result.stdout_str.split(_US), mode_base = 16)
|
||||
return parse_stat_fields(result.stdout_str, mode_base = 16)
|
||||
|
||||
if not _looks_like_option_error(result.stderr_str_or_none):
|
||||
# log(DEBUG, f'GNU stat attempt failed on "{path}" ({str(e)})')
|
||||
|
|
@ -671,7 +674,7 @@ class ExecContext(Base):
|
|||
if result.status != 0 or stdout is None:
|
||||
_raise_stat_error(path, result)
|
||||
assert stdout is not None # Just there to pacify the linter
|
||||
return _build_stat_result(stdout.rstrip('\n').split(_US), mode_base = 8)
|
||||
return parse_stat_fields(stdout.rstrip('\n'), mode_base = 8)
|
||||
|
||||
@override
|
||||
async def _chown(self, path: str, owner: str | None, group: str | None) -> None:
|
||||
|
|
|
|||
234
src/python/jw/pkg/lib/FcCache.py
Normal file
234
src/python/jw/pkg/lib/FcCache.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import stat
|
||||
|
||||
from typing import TYPE_CHECKING, Iterable, override
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import Result
|
||||
from .FileContext import FileContext
|
||||
from .file_types import FileType
|
||||
|
||||
from .base import StatResult
|
||||
from .ExecContext import STAT_GNU_FORMAT, ExecContext, parse_stat_fields
|
||||
from .FcWrapper import FcWrapper
|
||||
|
||||
class FcCache(FcWrapper): # export
|
||||
"""A write-through read cache in front of a file context.
|
||||
|
||||
stat and get results are cached: stat per (path,
|
||||
follow_symlinks), get per (path, wd, throw). Successful stats
|
||||
and FileNotFoundErrors are cached; other errors are re-probed,
|
||||
so a transient transport failure does not poison the cache.
|
||||
Writes pass through to the wrapped context and drop the cache
|
||||
entries of the affected paths.
|
||||
|
||||
warm() probes a set of paths without following symlinks and
|
||||
fills the stat cache with the results: an absent path is
|
||||
absent, and a non-link is the same node, for both follow
|
||||
variants, so the result is cached for both; a link is cached
|
||||
without following, its followed variant probed on first use.
|
||||
If the wrapped context is an exec context, one host command
|
||||
stats all of the paths; otherwise, one stat per path runs.
|
||||
Paths containing tab or newline characters are not supported.
|
||||
"""
|
||||
|
||||
def __init__(self, wrapped: FileContext) -> None:
|
||||
super().__init__(wrapped)
|
||||
self.__stat_cache: dict[tuple[str, bool], StatResult | Exception] = {}
|
||||
self.__get_cache: dict[tuple[str, str | None, bool], Result] = {}
|
||||
|
||||
@override
|
||||
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
|
||||
key = (path, follow_symlinks)
|
||||
hit = self.__stat_cache.get(key)
|
||||
if hit is not None:
|
||||
if isinstance(hit, StatResult):
|
||||
return hit
|
||||
raise hit
|
||||
try:
|
||||
st = await super()._stat(path, follow_symlinks)
|
||||
except FileNotFoundError as e:
|
||||
self.__stat_cache[key] = e
|
||||
raise
|
||||
self.__stat_cache[key] = st
|
||||
return st
|
||||
|
||||
@override
|
||||
async def _get(
|
||||
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
|
||||
) -> Result:
|
||||
key = (path, wd, throw)
|
||||
hit = self.__get_cache.get(key)
|
||||
if hit is None:
|
||||
hit = await super()._get(path, wd, throw, verbose, title)
|
||||
self.__get_cache[key] = hit
|
||||
return hit
|
||||
|
||||
@override
|
||||
async def _file_exists(self, path: str) -> bool:
|
||||
# -- Reroute the existence check through the cached stat
|
||||
# instead of the wrapped context's own fast path. A path
|
||||
# below a file cannot exist: the exec contexts report it
|
||||
# as absent, so the local one is treated the same
|
||||
try:
|
||||
await self._stat(path, False)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except NotADirectoryError:
|
||||
return False
|
||||
return True
|
||||
|
||||
@override
|
||||
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
|
||||
try:
|
||||
return stat.S_ISDIR((await self._stat(path, follow_symlinks)).mode)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except NotADirectoryError:
|
||||
return False
|
||||
|
||||
async def warm(self, paths: Iterable[str]) -> None:
|
||||
"""Probe a set of paths without following symlinks and fill
|
||||
the stat cache, as the class docstring describes.
|
||||
|
||||
A path whose both follow variants are already in the cache
|
||||
is skipped, so a warm of a set that is cached already does
|
||||
nothing. If the wrapped context is an exec context, one
|
||||
host command stats all of the remaining paths; otherwise,
|
||||
one stat per path runs.
|
||||
"""
|
||||
composed = []
|
||||
for p in paths:
|
||||
if '\t' in p or '\n' in p:
|
||||
raise ValueError(f'warm: path {p!r} holds a tab or newline')
|
||||
c = self._chroot(p)
|
||||
if (c, False) not in self.__stat_cache or \
|
||||
(c, True) not in self.__stat_cache:
|
||||
composed.append(c)
|
||||
if not composed:
|
||||
return
|
||||
ec = self.wrapped
|
||||
if not isinstance(ec, ExecContext):
|
||||
for p in composed:
|
||||
try:
|
||||
st = await super()._stat(p, False)
|
||||
except FileNotFoundError as e:
|
||||
self.__cache_probe(p, e)
|
||||
except NotADirectoryError:
|
||||
# -- A path below a file is not statable; it is
|
||||
# not cached, and file_exists() judges it
|
||||
# absent
|
||||
pass
|
||||
else:
|
||||
self.__cache_probe(p, st)
|
||||
return
|
||||
script = (
|
||||
'while IFS= read -r p; do '
|
||||
"printf '%s\\t' \"$p\"; "
|
||||
f"stat --printf '{STAT_GNU_FORMAT}\\n' \"$p\" 2>/dev/null || "
|
||||
"printf 'ABSENT\\n'; "
|
||||
'done'
|
||||
)
|
||||
ret = await ec.run(
|
||||
['sh', '-c', script],
|
||||
cmd_input = ('\n'.join(composed) + '\n').encode(),
|
||||
mod_env = {'LC_ALL': 'C'},
|
||||
throw = True,
|
||||
)
|
||||
self.__parse_probe(ret.stdout_str, composed)
|
||||
|
||||
def __cache_probe(self, path: str, result: StatResult | FileNotFoundError) -> None:
|
||||
# -- Cache a probe that did not follow symlinks: an absent
|
||||
# path is absent, and a non-link is the same node, for
|
||||
# both follow variants; a link is cached without
|
||||
# following, its followed variant probed on first use
|
||||
self.__stat_cache[(path, False)] = result
|
||||
if isinstance(result, FileNotFoundError) or not stat.S_ISLNK(result.mode):
|
||||
self.__stat_cache[(path, True)] = result
|
||||
|
||||
def __parse_probe(self, stdout: str, composed: list[str]) -> None:
|
||||
# -- One line per path: the path, a tab, then the stat
|
||||
# fields or ABSENT
|
||||
lines = stdout.splitlines()
|
||||
if len(lines) != len(composed):
|
||||
raise RuntimeError(
|
||||
f'warm: probe reported {len(lines)} lines for {len(composed)} paths'
|
||||
)
|
||||
for p, line in zip(composed, lines):
|
||||
name, sep, rest = line.partition('\t')
|
||||
if name != p or not sep:
|
||||
raise RuntimeError(f'warm: probe line does not name "{p}": {line!r}')
|
||||
if rest == 'ABSENT':
|
||||
self.__cache_probe(p, FileNotFoundError(errno.ENOENT, f'stat: {p}'))
|
||||
else:
|
||||
self.__cache_probe(p, parse_stat_fields(rest))
|
||||
|
||||
def __invalidate(self, *paths: str) -> None:
|
||||
# -- A write changes the stat and get answers of its paths
|
||||
for p in paths:
|
||||
for fl in (True, False):
|
||||
self.__stat_cache.pop((p, fl), None)
|
||||
self.__get_cache = {k: r for k, r in self.__get_cache.items() if k[0] != p}
|
||||
|
||||
@override
|
||||
async def _put(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
wd: str | None,
|
||||
throw: bool,
|
||||
verbose: bool | None,
|
||||
title: str,
|
||||
owner: str | None,
|
||||
group: str | None,
|
||||
mode: str | None,
|
||||
atomic: bool,
|
||||
ftype: FileType,
|
||||
) -> Result:
|
||||
r = await super()._put(
|
||||
path,
|
||||
content,
|
||||
wd,
|
||||
throw,
|
||||
verbose,
|
||||
title,
|
||||
owner,
|
||||
group,
|
||||
mode,
|
||||
atomic,
|
||||
ftype,
|
||||
)
|
||||
self.__invalidate(path)
|
||||
return r
|
||||
|
||||
@override
|
||||
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
|
||||
await super()._mkdir(path, mode, ftype)
|
||||
self.__invalidate(path)
|
||||
|
||||
@override
|
||||
async def _unlink(self, path: str) -> None:
|
||||
await super()._unlink(path)
|
||||
self.__invalidate(path)
|
||||
|
||||
@override
|
||||
async def _erase(self, path: str) -> None:
|
||||
await super()._erase(path)
|
||||
self.__invalidate(path)
|
||||
|
||||
@override
|
||||
async def _rename(self, src: str, dst: str) -> None:
|
||||
await super()._rename(src, dst)
|
||||
self.__invalidate(src, dst)
|
||||
|
||||
@override
|
||||
async def _chown(self, path: str, owner: str | None, group: str | None) -> None:
|
||||
await super()._chown(path, owner, group)
|
||||
self.__invalidate(path)
|
||||
|
||||
@override
|
||||
async def _chmod(self, path: str, mode: int) -> None:
|
||||
await super()._chmod(path, mode)
|
||||
self.__invalidate(path)
|
||||
122
src/python/jw/pkg/lib/FcWrapper.py
Normal file
122
src/python/jw/pkg/lib/FcWrapper.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Iterable, override
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import Result, StatResult
|
||||
from .file_types import FileType
|
||||
|
||||
from .FileContext import FileContext
|
||||
|
||||
class FcWrapper(FileContext): # export
|
||||
"""A file context that delegates to a wrapped context.
|
||||
|
||||
The wrapper adopts the identity of the wrapped context: uri,
|
||||
root, chroot flag, and the chroot path composition are all the
|
||||
wrapped context's, so a path composed by the wrapper composes
|
||||
identically in the wrapped context. Every primitive call is
|
||||
forwarded to the wrapped context's primitives, which receive
|
||||
the already-composed path; open and close are forwarded to the
|
||||
wrapped context's public methods, so the open count is kept in
|
||||
one place.
|
||||
"""
|
||||
|
||||
def __init__(self, wrapped: FileContext) -> None:
|
||||
super().__init__(
|
||||
wrapped.uri,
|
||||
interactive = wrapped.interactive,
|
||||
verbose_default = wrapped.verbose_default,
|
||||
chroot = wrapped.chroot,
|
||||
)
|
||||
self.__wrapped = wrapped
|
||||
|
||||
@property
|
||||
def wrapped(self) -> FileContext:
|
||||
return self.__wrapped
|
||||
|
||||
@override
|
||||
async def _open(self) -> None:
|
||||
await self.__wrapped.open()
|
||||
|
||||
@override
|
||||
async def _close(self) -> None:
|
||||
await self.__wrapped.close()
|
||||
|
||||
@override
|
||||
async def _get(
|
||||
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
|
||||
) -> Result:
|
||||
return await self.__wrapped._get(path, wd, throw, verbose, title)
|
||||
|
||||
@override
|
||||
async def _put(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
wd: str | None,
|
||||
throw: bool,
|
||||
verbose: bool | None,
|
||||
title: str,
|
||||
owner: str | None,
|
||||
group: str | None,
|
||||
mode: str | None,
|
||||
atomic: bool,
|
||||
ftype: FileType,
|
||||
) -> Result:
|
||||
return await self.__wrapped._put(
|
||||
path,
|
||||
content,
|
||||
wd,
|
||||
throw,
|
||||
verbose,
|
||||
title,
|
||||
owner,
|
||||
group,
|
||||
mode,
|
||||
atomic,
|
||||
ftype,
|
||||
)
|
||||
|
||||
@override
|
||||
async def _unlink(self, path: str) -> None:
|
||||
await self.__wrapped._unlink(path)
|
||||
|
||||
@override
|
||||
async def _erase(self, path: str) -> None:
|
||||
await self.__wrapped._erase(path)
|
||||
|
||||
@override
|
||||
async def _rename(self, src: str, dst: str) -> None:
|
||||
await self.__wrapped._rename(src, dst)
|
||||
|
||||
@override
|
||||
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
|
||||
await self.__wrapped._mkdir(path, mode, ftype)
|
||||
|
||||
@override
|
||||
async def _mktemp(self, tmpl: str, directory: bool) -> str:
|
||||
return await self.__wrapped._mktemp(tmpl, directory)
|
||||
|
||||
@override
|
||||
async def _chown(self, path: str, owner: str | None, group: str | None) -> None:
|
||||
await self.__wrapped._chown(path, owner, group)
|
||||
|
||||
@override
|
||||
async def _chmod(self, path: str, mode: int) -> None:
|
||||
await self.__wrapped._chmod(path, mode)
|
||||
|
||||
@override
|
||||
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
|
||||
return await self.__wrapped._stat(path, follow_symlinks)
|
||||
|
||||
@override
|
||||
async def _file_exists(self, path: str) -> bool:
|
||||
return await self.__wrapped._file_exists(path)
|
||||
|
||||
@override
|
||||
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
|
||||
return await self.__wrapped._is_dir(path, follow_symlinks)
|
||||
|
||||
@override
|
||||
async def _readdir(self, path: str) -> Iterable[str]:
|
||||
return await self.__wrapped._readdir(path)
|
||||
7
test/unit/python/jw/pkg/lib/FcCache/Makefile
Normal file
7
test/unit/python/jw/pkg/lib/FcCache/Makefile
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
TOPDIR = ../../../../../../..
|
||||
|
||||
include $(TOPDIR)/make/proj.mk
|
||||
include $(JWBDIR)/make/py-run.mk
|
||||
|
||||
all:
|
||||
test: run
|
||||
198
test/unit/python/jw/pkg/lib/FcCache/test.py
Normal file
198
test/unit/python/jw/pkg/lib/FcCache/test.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from typing import TYPE_CHECKING, override
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from jw.pkg.lib.file_types import FileType
|
||||
|
||||
from jw.pkg.lib.base import StatResult
|
||||
from jw.pkg.lib.ec.Local import Local
|
||||
from jw.pkg.lib.ExecContext import ExecContext
|
||||
from jw.pkg.lib.FcCache import FcCache
|
||||
from jw.pkg.lib.file_types import FT_DIR, FT_REG
|
||||
from jw.pkg.lib.FileContext import FileContext
|
||||
from jw.pkg.lib.Result import Result
|
||||
|
||||
class FakeCtx(FileContext):
|
||||
"""An in-memory FileContext that counts its primitive calls"""
|
||||
|
||||
def __init__(self, uri: str, chroot: bool = False) -> None:
|
||||
super().__init__(uri, chroot = chroot)
|
||||
self.files: dict[str, bytes] = {}
|
||||
self.dirs: set[str] = set()
|
||||
self.counts: dict[str, int] = {}
|
||||
|
||||
def count(self, op: str) -> None:
|
||||
self.counts[op] = self.counts.get(op, 0) + 1
|
||||
|
||||
def _stat_of(self, path: str) -> StatResult:
|
||||
if path in self.dirs:
|
||||
return StatResult(0o40755, '0', '0', 40, 0.0, 0.0, 0.0)
|
||||
if path in self.files:
|
||||
return StatResult(0o100644, '0', '0', len(self.files[path]), 0.0, 0.0, 0.0)
|
||||
raise FileNotFoundError(path)
|
||||
|
||||
@override
|
||||
async def _get(
|
||||
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
|
||||
) -> Result:
|
||||
self.count('get')
|
||||
if wd is not None:
|
||||
path = wd + '/' + path
|
||||
if path not in self.files:
|
||||
raise FileNotFoundError(path)
|
||||
return Result(stdout = self.files[path])
|
||||
|
||||
@override
|
||||
async def _put(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
wd: str | None,
|
||||
throw: bool,
|
||||
verbose: bool | None,
|
||||
title: str,
|
||||
owner: str | None,
|
||||
group: str | None,
|
||||
mode: str | None,
|
||||
atomic: bool,
|
||||
ftype: FileType,
|
||||
) -> Result:
|
||||
self.count('put')
|
||||
if wd is not None:
|
||||
path = wd + '/' + path
|
||||
self.files[path] = content
|
||||
return Result(stdout = b'')
|
||||
|
||||
@override
|
||||
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
|
||||
self.count('mkdir')
|
||||
self.dirs.add(path)
|
||||
|
||||
@override
|
||||
async def _unlink(self, path: str) -> None:
|
||||
self.count('unlink')
|
||||
self.files.pop(path, None)
|
||||
self.dirs.discard(path)
|
||||
|
||||
@override
|
||||
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
|
||||
self.count('stat')
|
||||
return self._stat_of(path)
|
||||
|
||||
class CountingLocal(Local):
|
||||
"""A Local context that counts its executed commands"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__('local')
|
||||
self.runs = 0
|
||||
|
||||
@override
|
||||
async def _run(
|
||||
self,
|
||||
cmd: list[str],
|
||||
wd: str | None,
|
||||
verbose: bool,
|
||||
cmd_input: bytes | None,
|
||||
mod_env: dict[str, str] | None,
|
||||
interactive: bool,
|
||||
log_prefix: str,
|
||||
) -> Result:
|
||||
self.runs += 1
|
||||
return await super()._run(
|
||||
cmd, wd, verbose, cmd_input, mod_env, interactive, log_prefix
|
||||
)
|
||||
|
||||
async def main() -> None:
|
||||
fake = FakeCtx('local:///tmp/fc-cache-fake', chroot = True)
|
||||
fc = FcCache(fake)
|
||||
await fc.mkdir('d', 0o755, FT_DIR)
|
||||
await fc.put('d/a', b'x', ftype = FT_REG)
|
||||
|
||||
# -- Successful stats are cached
|
||||
st = await fc.stat('d/a')
|
||||
assert st.size == 1
|
||||
assert (await fc.stat('d/a')).size == 1
|
||||
assert fake.counts['stat'] == 1
|
||||
|
||||
# -- Absences are cached, too
|
||||
assert await fc.file_exists('d/missing') is False
|
||||
assert await fc.file_exists('d/missing') is False
|
||||
assert fake.counts['stat'] == 2
|
||||
|
||||
# -- Gets are cached per (path, wd, throw)
|
||||
ret = await fc.get('d/a')
|
||||
assert (await fc.get('d/a')).stdout == b'x'
|
||||
assert fake.counts['get'] == 1
|
||||
ret = await fc._get('a', '/tmp/fc-cache-fake/d', True, None, 'get')
|
||||
assert fake.counts['get'] == 2
|
||||
assert ret.stdout == b'x'
|
||||
|
||||
# -- Writes drop the cache entries of their paths
|
||||
await fc.put('d/a', b'xy', ftype = FT_REG)
|
||||
assert (await fc.get('d/a')).stdout == b'xy'
|
||||
assert fake.counts['get'] == 3
|
||||
assert (await fc.stat('d/a')).size == 2
|
||||
assert fake.counts['stat'] == 3
|
||||
|
||||
# -- Unlink drops them as well
|
||||
await fc.unlink('d/a')
|
||||
assert await fc.file_exists('d/a') is False
|
||||
assert fake.counts['stat'] == 4
|
||||
assert '/tmp/fc-cache-fake/d/a' not in fake.files
|
||||
|
||||
# -- The wrapper itself is not an exec context
|
||||
assert not isinstance(fc, ExecContext)
|
||||
|
||||
# -- warm() on a non-exec context: one stat per path, then cached
|
||||
fake2 = FakeCtx('local:///tmp/fc-cache-warm', chroot = True)
|
||||
fc2 = FcCache(fake2)
|
||||
await fc2.put('p/q', b'z', ftype = FT_REG)
|
||||
await fc2.warm(['p', 'p/q', 'p/r'])
|
||||
assert fake2.counts['stat'] == 3
|
||||
await fc2.warm(['p', 'p/q', 'p/r'])
|
||||
assert fake2.counts['stat'] == 3
|
||||
assert await fc2.file_exists('p/r') is False
|
||||
assert fake2.counts['stat'] == 3
|
||||
|
||||
# -- warm() on an exec context: one host command for all paths
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
f = os.path.join(td, 'there')
|
||||
with open(f, 'w') as fh:
|
||||
fh.write('hello\n')
|
||||
cl = CountingLocal()
|
||||
fc3 = FcCache(cl)
|
||||
missing = os.path.join(td, 'not-there')
|
||||
await fc3.warm([f, missing])
|
||||
assert cl.runs == 1
|
||||
|
||||
st = await fc3.stat(f)
|
||||
assert cl.runs == 1
|
||||
assert st.size == 6
|
||||
assert await fc3.file_exists(missing) is False
|
||||
assert cl.runs == 1
|
||||
try:
|
||||
await fc3.stat(missing)
|
||||
assert False, 'stat of a warmed absence must raise'
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
assert cl.runs == 1
|
||||
|
||||
# -- A put through the cache lands in the wrapped context and
|
||||
# drops its cache entries, so the next stat is a fresh one.
|
||||
# Local stats without a command, so file_exists adds no run
|
||||
g = os.path.join(td, 'new')
|
||||
await fc3.put(g, b'fresh\n', ftype = FT_REG)
|
||||
assert cl.runs == 2
|
||||
assert (await fc3.get(g)).stdout == b'fresh\n'
|
||||
assert cl.runs == 3
|
||||
assert await fc3.file_exists(g) is True
|
||||
assert cl.runs == 3
|
||||
assert await fc3.file_exists(g) is True
|
||||
assert cl.runs == 3
|
||||
|
||||
asyncio.run(main())
|
||||
7
test/unit/python/jw/pkg/lib/FcWrapper/Makefile
Normal file
7
test/unit/python/jw/pkg/lib/FcWrapper/Makefile
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
TOPDIR = ../../../../../../..
|
||||
|
||||
include $(TOPDIR)/make/proj.mk
|
||||
include $(JWBDIR)/make/py-run.mk
|
||||
|
||||
all:
|
||||
test: run
|
||||
161
test/unit/python/jw/pkg/lib/FcWrapper/test.py
Normal file
161
test/unit/python/jw/pkg/lib/FcWrapper/test.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from typing import TYPE_CHECKING, Iterable, override
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from jw.pkg.lib.file_types import FileType
|
||||
|
||||
from jw.pkg.lib.base import StatResult
|
||||
from jw.pkg.lib.FcWrapper import FcWrapper
|
||||
from jw.pkg.lib.file_types import FT_DIR, FT_REG
|
||||
from jw.pkg.lib.FileContext import FileContext
|
||||
from jw.pkg.lib.Result import Result
|
||||
|
||||
class FakeCtx(FileContext):
|
||||
"""An in-memory FileContext that counts its primitive calls"""
|
||||
|
||||
def __init__(self, uri: str, chroot: bool = False) -> None:
|
||||
super().__init__(uri, chroot = chroot)
|
||||
self.files: dict[str, bytes] = {}
|
||||
self.dirs: set[str] = set()
|
||||
self.counts: dict[str, int] = {}
|
||||
self.opens = 0
|
||||
|
||||
def count(self, op: str) -> None:
|
||||
self.counts[op] = self.counts.get(op, 0) + 1
|
||||
|
||||
def _stat_of(self, path: str) -> StatResult:
|
||||
if path in self.dirs:
|
||||
return StatResult(0o40755, '0', '0', 40, 0.0, 0.0, 0.0)
|
||||
if path in self.files:
|
||||
return StatResult(0o100644, '0', '0', len(self.files[path]), 0.0, 0.0, 0.0)
|
||||
raise FileNotFoundError(path)
|
||||
|
||||
@override
|
||||
async def _open(self) -> None:
|
||||
self.opens += 1
|
||||
|
||||
@override
|
||||
async def _close(self) -> None:
|
||||
pass
|
||||
|
||||
@override
|
||||
async def _get(
|
||||
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
|
||||
) -> Result:
|
||||
self.count('get')
|
||||
if wd is not None:
|
||||
path = wd + '/' + path
|
||||
if path not in self.files:
|
||||
raise FileNotFoundError(path)
|
||||
return Result(stdout = self.files[path])
|
||||
|
||||
@override
|
||||
async def _put(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
wd: str | None,
|
||||
throw: bool,
|
||||
verbose: bool | None,
|
||||
title: str,
|
||||
owner: str | None,
|
||||
group: str | None,
|
||||
mode: str | None,
|
||||
atomic: bool,
|
||||
ftype: FileType,
|
||||
) -> Result:
|
||||
self.count('put')
|
||||
if wd is not None:
|
||||
path = wd + '/' + path
|
||||
self.files[path] = content
|
||||
return Result(stdout = b'')
|
||||
|
||||
@override
|
||||
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
|
||||
self.count('mkdir')
|
||||
self.dirs.add(path)
|
||||
|
||||
@override
|
||||
async def _unlink(self, path: str) -> None:
|
||||
self.count('unlink')
|
||||
self.files.pop(path, None)
|
||||
self.dirs.discard(path)
|
||||
|
||||
@override
|
||||
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
|
||||
self.count('stat')
|
||||
return self._stat_of(path)
|
||||
|
||||
@override
|
||||
async def _file_exists(self, path: str) -> bool:
|
||||
self.count('file_exists')
|
||||
try:
|
||||
self._stat_of(path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
@override
|
||||
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
|
||||
self.count('is_dir')
|
||||
try:
|
||||
st = self._stat_of(path)
|
||||
return (st.mode & 0o170000) == 0o40000
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
@override
|
||||
async def _readdir(self, path: str) -> Iterable[str]:
|
||||
self.count('readdir')
|
||||
prefix = path.rstrip('/') + '/'
|
||||
return [
|
||||
p[len(prefix):] for p in sorted(self.files.keys() | self.dirs)
|
||||
if p.startswith(prefix)
|
||||
]
|
||||
|
||||
async def main() -> None:
|
||||
fake = FakeCtx('local:///tmp/fc-wrapper-fake', chroot = True)
|
||||
fc = FcWrapper(fake)
|
||||
|
||||
# -- The wrapper adopts the identity of the wrapped context
|
||||
assert fc.wrapped is fake
|
||||
assert fc.uri is fake.uri
|
||||
assert fc.root == fake.root
|
||||
assert fc.chroot == fake.chroot
|
||||
assert fc.username == fake.username
|
||||
assert fc.interactive == fake.interactive
|
||||
assert fc.verbose_default == fake.verbose_default
|
||||
|
||||
# -- Put and get are forwarded; paths are composed once
|
||||
await fc.put('etc/a.conf', b'x=1\n', ftype = FT_REG)
|
||||
assert fake.files == {'/tmp/fc-wrapper-fake/etc/a.conf': b'x=1\n'}
|
||||
assert fake.counts['put'] == 1
|
||||
|
||||
ret = await fc.get('etc/a.conf')
|
||||
assert ret.stdout == b'x=1\n'
|
||||
assert fake.counts['get'] == 1
|
||||
|
||||
# -- stat, file_exists, is_dir, and readdir are forwarded
|
||||
st = await fc.stat('etc/a.conf')
|
||||
assert st.size == 4
|
||||
assert await fc.file_exists('etc/a.conf') is True
|
||||
assert await fc.file_exists('etc/missing.conf') is False
|
||||
await fc.mkdir('etc/sub', 0o755, FT_DIR)
|
||||
assert await fc.is_dir('etc/sub') is True
|
||||
names = await fc.readdir('etc')
|
||||
assert 'a.conf' in names and 'sub' in names
|
||||
|
||||
# -- Unlink is forwarded
|
||||
await fc.unlink('etc/a.conf')
|
||||
assert '/tmp/fc-wrapper-fake/etc/a.conf' not in fake.files
|
||||
assert fake.counts['unlink'] == 1
|
||||
|
||||
# -- Open and close reach the wrapped context's public methods
|
||||
await fc.open()
|
||||
await fc.close()
|
||||
assert fake.opens == 1
|
||||
|
||||
asyncio.run(main())
|
||||
Loading…
Reference in a new issue