src/python/jw/pkg/cmds/secrets/lib/DistroContext.py

@@ -3,11 +3,21 @@ from __future__ import annotations
 import re
 import sys
 
+from contextlib import suppress
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from ....lib.ec.secret_store.Base import Base as SecretStore
+from ....lib.ec.secret_store.KvFile import KvFile
+from ....lib.ec.secret_store.layout import (
+    DEFAULT_PREFIX,
+    FT_SECRET,
+    FT_SECRET_KEY,
+    SECRET_SUFFIX,
+    host_prefix,
+)
+from ....lib.FileContext import FileContext
 from ....lib.log import DEBUG, NOTICE, WARNING, log
-from ....lib.ProcFilterGpg import ProcFilterGpg
 from .FilesContext import FilesContext
 
 if TYPE_CHECKING:
@@ -19,7 +29,7 @@ if TYPE_CHECKING:
 class DistroContext(FilesContext):
 
     def __init__(self, distro: Distro) -> None:
-        super().__init__(distro.ctx)
+        super().__init__(distro.ctx, distro = distro)
         self.__distro = distro
 
     async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@@ -40,7 +50,7 @@ class DistroContext(FilesContext):
     ) -> list[str]:
         ret = []
         for tmpl in await self.list_template_files(pkg_names):
-            path = str(Path(tmpl).with_suffix('.jw-secret'))
+            path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
             if ignore_missing and not await self.ctx.file_exists(path):
                 continue
             ret.append(path)
@@ -101,64 +111,60 @@ class DistroContext(FilesContext):
         verbose: bool = False,
     ) -> None:
 
-        async def _read_secret_tar_blob(src_uri: str) -> bytes:
-            ec = self.ctx
-            from ....lib.ec.Local import Local
-            from ....lib.util import get
-
-            if not isinstance(ec, Local):
-                # Security: Use a local exec context for decrypting and
-                # filtering secrets
-                ec = Local()
-            return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
-
-        def _is_needed_secret(path: str) -> bool:
-            return path in secret_paths
-
-        from .tar import extract as tar_extract
-        from .tar import filter as tar_filter
-        from .tar import merge as tar_merge
-        from .tar import rewrite as tar_rewrite
-
         if only_missing:
             raise NotImplementedError('--only-missing is not yet implemented')
 
-        secret_paths = await self.list_secret_paths(pkg_names)
+        fc = FileContext.create(src_uri, chroot = True)
+        await fc.open()
+        try:
             hostname = self.ctx.uri.hostname
             if not hostname:
-            raise Exception('Have no hostname to find secrets in tar file')
+                raise Exception('Have no hostname to find secrets in the store')
-        host_root_in_tar = '/'.join(reversed(hostname.split('.')))
+            if isinstance(fc, SecretStore):
-        host_rx = re.compile(r'^' + host_root_in_tar)
+                if not fc.whole:
-        default_rx = re.compile(r'^default')
+                    raise Exception(
-
+                        f'src {src_uri} must name the whole store, not a '
-        filtered_paths: list[str] = []
+                        f'rooted subtree'
-        extracted_paths: list[str] = []
-
-        blob_all = await _read_secret_tar_blob(src_uri)
-        if blob_all is None:
-            raise Exception(f'Tar blob {src_uri} is empty')
-        blob_host_filtered = tar_filter(
-            blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
                     )
-        blob_host_transformed = tar_rewrite(
+                src: FileContext = fc
-            blob_host_filtered, lambda p: re.sub(host_rx, '', p)
+                # -- Store candidates: the host tree, then the default
-        )
+                #    tree
-        blob_default_filtered = tar_filter(
+                prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
-            blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
-        )
-        blob_default_transformed = tar_rewrite(
-            blob_default_filtered, lambda p: re.sub(default_rx, '', p)
-        )
-        blob_secret_material = tar_merge(
-            [blob_host_transformed, blob_default_transformed], overwrite = False
-        )
-        blob_needed = tar_filter(
-            blob_secret_material, _is_needed_secret, extracted_paths
-        )
-
-        await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
-        for path in secret_paths:
-            if path not in extracted_paths:
-                log(NOTICE, f'not extracted: {path}')
             else:
-                log(NOTICE, f'extracted:     {path}')
+                # -- A file tree in the on-host layout: wrap it in the kvfile store
+                src = KvFile(fc, distro = self.__distro)
+                prefixes = ('', )
+
+            # -- The store and the kvfile source are alike addressed
+            #    by the on-host file name
+            suffix = SECRET_SUFFIX
+
+            for entry in await self.list_compilation_targets(pkg_names):
+                # -- A present entry, even an empty one, wins over the
+                #    fallback
+                entry_rel = f'{entry.lstrip("/")}{suffix}'
+                addr = entry_rel
+                keys: list[str] | None = None
+                for pfx in prefixes:
+                    addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
+                    try:
+                        keys = list(await src.readdir(addr))
+                        break
+                    except FileNotFoundError:
+                        continue
+                if keys is None:
+                    log(NOTICE, f'not extracted: {entry}')
+                    continue
+                log(NOTICE, f'extracted:     {entry}')
+                pairs: list[tuple[str, bytes]] = []
+                for key in keys:
+                    val = await src.get(f'{addr}/{key}')
+                    pairs.append((key, val.stdout))
+                dst = self.store
+                with suppress(FileNotFoundError):
+                    await dst.unlink(entry_rel)
+                await dst.mkdir(entry_rel, ftype = FT_SECRET)
+                for key, content in pairs:
+                    await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
+        finally:
+            await fc.close()

src/python/jw/pkg/cmds/secrets/lib/FilesContext.py

@@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable
-    from ....lib.FileContext import FileContext
 
+    from ....lib.Distro import Distro
+    from ....lib.FileContext import FileContext
+    from ....lib.ec.secret_store.Base import Base
+
+from ....lib.ec.secret_store.KvFile import KvFile
+from ....lib.ec.secret_store.layout import SECRET_SUFFIX
 from ....lib.log import DEBUG, NOTICE, WARNING, log
 from .base import Attrs
 
 class FilesContext:
 
-    def __init__(self, ctx: FileContext) -> None:
+    def __init__(
+        self,
+        ctx: FileContext,
+        store: Base | None = None,
+        distro: Distro | None = None,
+    ) -> None:
         self.__ctx = ctx
+        if store is None:
+            store = KvFile(ctx, distro = distro)
+        self.__store = store
 
     @property
     def ctx(self) -> FileContext:
         return self.__ctx
 
-    async def _read_key_value_file(
+    @property
-        self,
+    def store(self) -> Base:
-        path: str,
+        return self.__store
-        throw: bool = False,
-    ) -> dict[str, str]:
-        ret: dict[str, str] = {}
-        try:
-            result = await self.ctx.get(path)
-            for line in result.stdout_str.splitlines():
-                line = line.strip()
-                if not line or line.startswith('#'):
-                    continue
-                if '=' not in line:
-                    continue
-                key, val = line.split('=', 1)
-                key = key.strip()
-                val = val.strip()
-                if key:
-                    ret[key] = val
-        except FileNotFoundError:
-            log(DEBUG, f'File not found {path}')
-        return ret
 
     def _parse_attributes(self, content: str) -> Attrs | None:
 
@@ -143,6 +137,10 @@ class FilesContext:
         for key, val in replace.items():
             new_content = new_content.replace(key, val)
 
+        unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
+        if unresolved:
+            raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
+
         tmp_path: str | None = None
 
         try:
@@ -200,12 +198,21 @@ class FilesContext:
     ) -> bool:
         path_tmpl = target_path + '.jw-tmpl'
         path_secret_file = target_path + '.jw-secret-file'
-        path_secret = target_path + '.jw-secret'
+        path_secret = target_path + SECRET_SUFFIX
         attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
         attrs.update(
             await self._read_attributes([path_tmpl, path_secret_file, path_secret])
         )
-        replace = await self._read_key_value_file(path_secret)
+        entry = f'{target_path}{SECRET_SUFFIX}'
+        try:
+            keys = await self.__store.readdir(entry)
+        except FileNotFoundError:
+            log(DEBUG, f'No secrets for target {target_path}')
+            keys = []
+        replace: dict[str, str] = {}
+        for key in keys:
+            val = await self.__store.get(f'{entry}/{key}')
+            replace[key] = val.stdout.decode()
         for src in [path_secret_file, path_tmpl]:
             try:
                 await self._compile_one_template_file(

src/python/jw/pkg/cmds/secrets/lib/tar.py

@@ -1,83 +0,0 @@
-from __future__ import annotations
-
-import io
-import tarfile
-
-from tarfile import TarFile
-from typing import TYPE_CHECKING, Callable
-
-from ....lib.ExecContext import ExecContext
-from ....lib.log import DEBUG, log
-
-if TYPE_CHECKING:
-    from typing import Iterable
-
-    from ....lib.FileContext import FileContext
-
-def filter(
-    blob: bytes,
-    path_filter: Callable[[str], bool] | None,
-    matched: list[str] | None = None,
-) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        tf_in = TarFile(fileobj = io.BytesIO(blob))
-        for info in tf_in.getmembers():
-            if path_filter is not None and not path_filter(info.name):
-                continue
-            log(DEBUG, f'Adding {info.name}')
-            if matched is not None:
-                matched.append(info.name)
-            buf = tf_in.extractfile(info)
-            tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        tf_in = TarFile(fileobj = io.BytesIO(blob))
-        for info in tf_in.getmembers():
-            new_name = rewrite_filter(info.name)
-            log(DEBUG, f'Rewriting {info.name} -> {new_name}')
-            info.name = new_name
-            # -- A pax "path" supplement from the input would override
-            #    the rewritten name on re-serialization
-            info.pax_headers = {
-                k: v
-                for k, v in info.pax_headers.items() if k != 'path'
-            }
-            buf = tf_in.extractfile(info)
-            tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        for blob in blobs:
-            tf_in = TarFile(fileobj = io.BytesIO(blob))
-            existing_names = tf_out.getnames()
-            for info in tf_in.getmembers():
-                if not overwrite and info.name in existing_names:
-                    continue
-                buf = tf_in.extractfile(info)
-                tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-async def extract(
-    dst: FileContext,
-    blob: bytes,
-    root: str | None = None,
-    verbose: bool = False
-) -> None:
-    cmd = ['tar']
-    if root is not None:
-        cmd += ['-C', root]
-    if verbose:
-        cmd += '-v'
-    cmd += ['-x', '-f', '-']
-    if not isinstance(dst, ExecContext):
-        raise NotImplementedError(
-            'Extracting tar files to a non-executable '
-            f'context is not yet implemented: {dst}'
-        )
-    await dst.run(cmd, verbose = verbose, cmd_input = blob)

src/python/jw/pkg/lib/FileContext.py

@@ -360,6 +360,18 @@ class FileContext(abc.ABC):
                 from .ec.Curl import Curl
 
                 return Curl(uri, *args, **kwargs)
+            case 'targpg':
+                from .ec.secret_store.TarGpg import TarGpg
+
+                return TarGpg(uri, *args, **kwargs)
+            case 'kvfile':
+                from .ec.secret_store.KvFile import KvFile
+
+                scheme = 'ssh' if uri.hostname else 'local'
+                inner = FileContext.create(
+                    f'{scheme}://{uri.authority}', chroot = False
+                )
+                return KvFile(inner, uri = uri)
             case _:
                 pass
         raise Exception(

src/python/jw/pkg/lib/ec/secret_store/Base.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override
+
+from ...base import Result, StatResult
+from ...file_types import FT_DIR, FT_REG
+from ...FileContext import FileContext
+from .layout import DEFAULT_PREFIX, host_prefix
+
+if TYPE_CHECKING:
+    import os
+
+    from typing import Any, Iterable
+
+    from ...file_types import FileType
+    from ...Uri import Uri
+
+T = TypeVar('T')
+
+class Base(FileContext):
+    """A store of secret entries in the canonical store layout.
+
+    A store is a FileContext over the store namespace. The URI
+    addresses a subtree of that namespace: no authority means the
+    whole store, a host authority means that host's tree (its
+    reversed domain prefix), `default` names the default tree, and a
+    path narrows further. The subtree is the store's namespace
+    root; addresses passed to the methods are relative to it.
+
+    A store is a typed file store: a store path names a file, and
+    put() persists the content at the address as-is. The store does
+    not identify content by file name; what a file holds - an entry,
+    or a blob - is a property of its content, resolved at stat
+    time. A node is what its content makes it:
+
+    - an entry carries key-value secrets and behaves like a
+      directory: readdir() on the entry lists its keys, get() on
+      <entry>/<key> returns the value of the key, and get() on the
+      entry itself returns the serialized entry
+    - a branch is a path prefix: readdir() lists its children
+    - a blob is non-key-value content: get() returns its bytes
+
+    A store rooted at a host applies the default fallback: an
+    address that misses below the host is retried below the default
+    tree. A whole-store or default-rooted store is strict; callers
+    apply the fallback themselves.
+
+    The URI-addressable stores are created by FileContext.create()
+    from their scheme. The kvfile store is the store of the on-host
+    layout: its addresses are config paths of the host it is rooted
+    at, and it may be constructed directly over a host context.
+    """
+
+    def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
+        # -- A store is always chrooted. The flag states the
+        #    fact, and the chroot it arms is undone in the
+        #    protected layer by _unchroot(); an explicit value
+        #    is overridden.
+        kwargs['chroot'] = True
+        super().__init__(uri, *args, **kwargs)
+
+    @property
+    def __host_root(self) -> str:
+        # -- The root of the host tree this store is rooted at, '' if
+        #    the store is whole-store or rooted at the default tree:
+        #    then there is no host tree to fall back from, and the
+        #    caller applies the default fallback itself.
+        host = self.uri.hostname or ''
+        if host and host != 'default':
+            return host_prefix(host)
+        return ''
+
+    @property
+    def __root_in_storage(self) -> str:
+
+        # -- The namespace node this store is rooted at: the
+        #    reversed host prefix of the URI authority, if any
+        #    (`default` is a reserved word), followed by the URI
+        #    path.
+        host = self.uri.hostname or ''
+        path = self.uri.path.lstrip('/')
+        if host and host != 'default':
+            host = host_prefix(host)
+        return '/'.join(p for p in (host, path) if p)
+
+    def _path_in_storage(self, path: str) -> str:
+        path = path.lstrip('/')
+        if not self.__root_in_storage:
+            return path
+        if not path:
+            return self.__root_in_storage
+        return f'{self.__root_in_storage}/{path}'
+
+    def _default_path_in_storage(self, path: str) -> str | None:
+        host_root = self.__host_root
+        ns = self.__root_in_storage
+        if not host_root or not (path == ns or path.startswith(f'{ns}/')):
+            return None
+        return f'{DEFAULT_PREFIX}{path[len(host_root):]}'
+
+    def _unchroot(self, path: str) -> str:
+        # -- The inverse of _chroot(): the public wrappers hand the
+        #    protected layer addresses prefixed with the root; store
+        #    addresses are relative to it, and the root itself is the
+        #    empty address. Already-relative addresses pass through.
+        if not self.root:
+            return path.lstrip('/')
+        if path == self.root:
+            return ''
+        prefix = f'{self.root}/'
+        if path.startswith(prefix):
+            return path[len(prefix):]
+        return path
+
+    async def _call_with_fallback(
+        self,
+        path: str,
+        func: Callable[[str], Awaitable[T]],
+    ) -> T:
+        storage_path = self._path_in_storage(path)
+        try:
+            return await func(storage_path)
+        except FileNotFoundError:
+            default_storage_path = self._default_path_in_storage(storage_path)
+            if default_storage_path is None:
+                raise
+            return await func(default_storage_path)
+
+    @override
+    async def put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None = None,
+        throw: bool = True,
+        verbose: bool | None = None,
+        title: str | None = None,
+        owner: str | None = None,
+        group: str | None = None,
+        mode: int | None = None,
+        atomic: bool = False,
+        ftype: FileType = FT_REG,
+    ) -> Result:
+        # -- A put that changes nothing is a no-op: the pre-read is
+        #    fallback-aware, so a value already represented, below
+        #    the host tree or below the default tree, does not
+        #    trigger a store write. Metadata arguments opt out: they
+        #    may be the only change
+        if owner is None and group is None and mode is None:
+            try:
+                existing = await self.get(path)
+            except Exception:
+                pass
+            else:
+                if self._put_is_noop(existing.stdout, content, ftype):
+                    return Result()
+        return await super().put(
+            path,
+            content,
+            wd,
+            throw,
+            verbose,
+            title,
+            owner,
+            group,
+            mode,
+            atomic,
+            ftype,
+        )
+
+    def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
+        # -- A put that leaves the stored content byte-identical
+        #    changes nothing. A store that normalizes content on
+        #    read overrides this to compare the normalized forms
+        return existing == content
+
+    @override
+    async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
+        # -- An entry is a file on the backing store, yet it lists
+        #    its keys: answer from the node's type, not the
+        #    object's mode
+        try:
+            return (await self._stat(path, follow_symlinks)).is_dir
+        except FileNotFoundError:
+            return False
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        # -- A store is listable: a directory of keys, entries, or
+        #    blobs, whatever its nodes hold.
+        raise NotImplementedError()
+
+    @override
+    async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
+        # -- The base store materializes nothing: branches are
+        #    implicit namespace prefixes, and entries materialize on
+        #    their first write. A store whose nodes have a host home
+        #    overrides this
+        pass
+
+    def _stat_result(
+        self,
+        owner: str,
+        group: str,
+        mtime: float,
+        ftype: FileType,
+        mode: int | None = None,
+        size: int = 0,
+    ) -> StatResult:
+        # -- The metadata of a node: the mode and size, if given, are
+        #    the ones of the file, directory, or member behind the
+        #    node; a node without a backing object keeps the
+        #    synthesized type bits and a size of 0. The access and
+        #    change times follow the modification time
+        import stat
+
+        if mode is None:
+            mode = (
+                (stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else
+                (stat.S_IFREG | 0o400)
+            )
+        return StatResult(
+            mode = mode,
+            owner = owner,
+            group = group,
+            size = size,
+            atime = mtime,
+            mtime = mtime,
+            ctime = mtime,
+            ftype = ftype,
+        )
+
+    def _stat_result_from_os(
+        self,
+        st: os.stat_result,
+        ftype: FileType,
+        size: int | None = None,
+    ) -> StatResult:
+        return self._stat_result(
+            owner = str(st.st_uid),
+            group = str(st.st_gid),
+            mtime = st.st_mtime,
+            ftype = ftype,
+            mode = st.st_mode,
+            size = st.st_size if size is None else size,
+        )
+
+    @property
+    def whole(self) -> bool:
+        # -- True, if the store is rooted at the whole store namespace:
+        #    no authority, no path
+        return not self.__root_in_storage

src/python/jw/pkg/lib/ec/secret_store/KvFile.py

@@ -0,0 +1,439 @@
+from __future__ import annotations
+
+import errno
+import os
+
+from typing import TYPE_CHECKING, Iterable, override
+
+if TYPE_CHECKING:
+    from ...Distro import Distro
+    from ...FileContext import FileContext
+    from ...Uri import Uri
+    from ...base import StatResult
+    from ...file_types import FileType
+
+from ...FcCache import FcCache
+from ...file_types import FT_COMPOUND, FT_DIR
+from ...kv import parse_kv, serialize_kv
+from ...Result import Result
+from .Base import Base
+from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX
+
+class KvFile(Base):
+    """The on-host store of a host's package secrets.
+
+    A config path X is either an entry, the file X.jw-secret with
+    KEY=VALUE lines, whose children in the address space are its
+    keys, or a blob, the file X.jw-secret-file, whose content is the
+    file itself, a leaf. Any other path is a branch: a real
+    directory on the host, with no file of its own.
+
+    Rooted at the host: addresses are config paths relative to the
+    host root, absolute on the host with the leading slash. A URI
+    may root the store at a subtree of the host tree; its path is
+    composed into every address, and the URI is the store's
+    identity. A suffixed address names the on-host file of its
+    config path, the suffix picking the kind: .jw-secret the
+    entry, .jw-secret-file the blob.
+
+    get reads a key below an entry or the blob at its config path;
+    put writes a key into its entry file, or a blob as a
+    .jw-secret-file at its config path, and accepts entry, key, and
+    blob content only; unlink drops the file the address names. No
+    default fallback.
+
+    An entry lists its keys. A branch lists the config paths below
+    it, the root included: the host's package set is the bounded
+    base, the .jw-tmpl files of the installed packages, presence-
+    checked on the host. The distro is autodetected from the host
+    on first use, or given in the constructor; the autodetected
+    distro applies the JW_DEFAULT_PKG_FILTER package filter, if
+    set, so a URI-built context honors the app's documented
+    filter fallback. is_dir() answers from the entry file and the
+    host tree, so the write path never pays for the enumeration.
+
+    The on-host tree is a precondition: mkdir materializes a missing
+    entry file below an existing directory, and mkdir and put fail
+    with EROFS below a missing one. KvFile never creates
+    directories. Entry and blob files are mode 0400;
+    owner and group follow the put request, if given.
+    """
+
+    def __init__(
+        self,
+        ctx: FileContext,
+        uri: str | Uri | None = None,
+        distro: Distro | None = None,
+    ) -> None:
+        # -- The URI roots the namespace at the host tree's root, or
+        #    at a subtree of it: its path is composed into every
+        #    address, and it is the store's identity. A direct
+        #    construction names the context's host
+        if uri is None:
+            uri = f'kvfile://{ctx.uri.hostname or "target"}'
+        super().__init__(uri = uri)
+        self.__raw = ctx
+        self.__distro: Distro | None = distro
+        self.___config_paths: list[str] | None = None
+        self.__subtree = self.uri.path.lstrip('/')
+        # -- The context is read through a cache: the read path
+        #    probes the same handful of paths again and again, and
+        #    the cache pays the transport once per path
+        self.__ctx = FcCache(ctx)
+
+    @override
+    async def _open(self) -> None:
+        # -- The lifecycle propagates to the host context: an
+        #    open() pin on the store, the CopyContext of a copy
+        #    walk, keeps the host's transport open for its
+        #    duration, and the run()s within it share it
+        await self.__raw.open()
+
+    @override
+    async def _close(self) -> None:
+        await self.__raw.close()
+
+    def __addr(self, path: str) -> str:
+        # -- The host-relative address: the subtree the URI roots
+        #    the store at, then the address relative to it
+        path = self._unchroot(path)
+        if not path:
+            return self.__subtree
+        if not self.__subtree:
+            return path
+        return f'{self.__subtree}/{path}'
+
+    @staticmethod
+    def __resolve(path: str) -> tuple[str, bool]:
+        # -- The config path of the address, and whether the address
+        #    names a blob file: a suffixed address is the on-host
+        #    file name of its config path, the suffix picking the
+        #    kind
+        if path.endswith(SECRET_SUFFIX):
+            return path[:-len(SECRET_SUFFIX)], False
+        if path.endswith(BLOB_SUFFIX):
+            return path[:-len(BLOB_SUFFIX)], True
+        return path, False
+
+    async def __config_paths(self) -> list[str]:
+        # -- The config paths of the host's package secrets: the
+        #    templates of the installed packages, relative to the
+        #    host root. The package database is the bounded base, a
+        #    filesystem scan is not. Computed once: the package set
+        #    is stable for the lifetime of the session
+        if self.___config_paths is None:
+            if self.__distro is None:
+                from ...Distro import Distro
+                from ...ExecContext import ExecContext
+
+                # -- The distro probes the host through the raw
+                #    context: the cache is a plain file context, no
+                #    exec of its own
+                if not isinstance(self.__raw, ExecContext):
+                    raise ValueError(
+                        'KvFile: a distro must be given when the host'
+                        ' context is not an exec context'
+                    )
+                # -- A URI-built context has no app distro to
+                #    inherit the package filter from: apply the
+                #    app's documented fallback
+                from ...PackageFilter import PackageFilterString
+
+                pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER')
+                self.__distro = await Distro.instantiate(
+                    ec = self.__raw,
+                    default_pkg_filter = (
+                        PackageFilterString(pkg_filter_str) if pkg_filter_str else None
+                    ),
+                )
+            # -- The packages' file paths, filtered to the templates
+            ret: set[str] = set()
+            for f in await self.__distro.pkg_files():
+                if not f.endswith('.jw-tmpl'):
+                    continue
+                ret.add(f.lstrip('/')[:-len('.jw-tmpl')])
+            self.___config_paths = sorted(ret)
+            # -- Warm the stat cache with one host command, so the
+            #    per-level presence checks and the walk's node stats
+            #    hit memory: the root, the candidate files of both
+            #    kinds, the directories they live in, and the file
+            #    names a suffixed address would probe those
+            #    directories by
+            warm = ['/']
+            warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths]
+            warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths]
+            dirs: set[str] = set()
+            for p in self.___config_paths:
+                parts = p.split('/')
+                for i in range(1, len(parts)):
+                    dirs.add('/'.join(parts[:i]))
+            for d in sorted(dirs):
+                warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}']
+            await self.__ctx.warm(warm)
+        assert self.___config_paths is not None
+        return self.___config_paths
+
+    async def __entry_kv(self, entry: str) -> dict[str, str]:
+        path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}'
+        if not await self.__ctx.file_exists(path):
+            raise FileNotFoundError(entry)
+        ret = await self.__ctx.get(path)
+        return parse_kv(ret.stdout)
+
+    async def __is_entry(self, path: str) -> bool:
+        # -- The entry file on the host; KvFile has no default tree.
+        #    The path is host-relative, as from __addr()
+        if not path:
+            return False
+        return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}')
+
+    @override
+    async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
+        # -- The metadata of the host file behind the node, the type
+        #    overridden: an entry is a file that enumerates its
+        #    keys, a key a leaf below it; a branch is a plain
+        #    directory. A key has no file of its own, so it keeps
+        #    the size 0
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+
+        async def __stat_of(real: str, ftype: FileType) -> StatResult:
+            st = await self.__ctx.stat(real, follow_symlinks)
+            return self._stat_result(
+                st.owner,
+                st.group,
+                st.mtime,
+                ftype,
+                mode = st.mode,
+                size = st.size,
+            )
+
+        if not config:
+            return await __stat_of('/', FT_DIR)
+        if is_blob:
+            real = f'/{config}{BLOB_SUFFIX}'
+            if not await self.__ctx.file_exists(real):
+                raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
+            return await __stat_of(real, FT_SECRET_FILE)
+        if await self.__is_entry(config):
+            return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET)
+        parent, sep, key = config.rpartition('/')
+        if sep and key:
+            parent, _ = self.__resolve(parent)
+            if await self.__is_entry(parent) and key in await self.__entry_kv(parent):
+                # -- The type is synthesized, the mode the one of
+                #    the entry file
+                st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks)
+                return self._stat_result(
+                    st.owner,
+                    st.group,
+                    st.mtime,
+                    FT_SECRET_KEY,
+                    mode = st.mode,
+                )
+        blob = f'/{config}{BLOB_SUFFIX}'
+        if await self.__ctx.file_exists(blob):
+            return await __stat_of(blob, FT_SECRET_FILE)
+        if await self.__ctx.is_dir(f'/{config}'):
+            return await __stat_of(f'/{config}', FT_DIR)
+        raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
+
+    @override
+    async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
+        # -- An entry is a directory of its keys, a key below an
+        #    entry is a leaf: the entry is a file on the host, so
+        #    there is no host path below it to stat. A branch is a
+        #    real directory on the host. The package set is not
+        #    consulted: the write path never pays for the
+        #    enumeration
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if not config:
+            return True
+        parent, sep, _ = config.rpartition('/')
+        if sep:
+            parent, _ = self.__resolve(parent)
+            if await self.__is_entry(parent):
+                return False
+        if await self.__is_entry(config):
+            return True
+        if is_blob:
+            return False
+        return await self.__ctx.is_dir(f'/{config}')
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        # -- An entry lists its keys; a branch, the root included,
+        #    lists the config paths below it, presence-checked on
+        #    the host, under the on-host file name of the file that
+        #    backs it
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if config and await self.__is_entry(config):
+            return (await self.__entry_kv(config)).keys()
+        if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'):
+            raise NotADirectoryError(path)
+        children: set[str] = set()
+        prefix = f'{config}/' if config else ''
+        for cfg in await self.__config_paths():
+            if not cfg.startswith(prefix):
+                continue
+            rest = cfg[len(prefix):]
+            label, sep, _ = rest.partition('/')
+            if sep:
+                # -- A deeper candidate lists its branch label, if a
+                #    secret is present below it: the label names a
+                #    directory on the host, which a present secret
+                #    below implies
+                if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \
+                        await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
+                    children.add(label)
+                continue
+            # -- A direct child lists under the on-host file name of
+            #    the file that backs it, entry and blob alike
+            if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'):
+                children.add(f'{label}{SECRET_SUFFIX}')
+            if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
+                children.add(f'{label}{BLOB_SUFFIX}')
+        if config and not children:
+            # -- A branch the host lacks is absent; a real directory
+            #    with no secret children lists empty
+            if not await self.__ctx.is_dir(f'/{config}'):
+                raise FileNotFoundError(path)
+        return children
+
+    @override
+    async def _get(
+        self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
+    ) -> Result:
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if is_blob:
+            real = f'/{config}{BLOB_SUFFIX}'
+            if not await self.__ctx.file_exists(real):
+                raise FileNotFoundError(path)
+            return await self.__ctx.get(real)
+        real = f'/{config}'
+        if await self.__ctx.file_exists(real + SECRET_SUFFIX):
+            # -- The entry file is the serialized entry
+            return await self.__ctx.get(real + SECRET_SUFFIX)
+        if await self.__ctx.file_exists(real + BLOB_SUFFIX):
+            return await self.__ctx.get(real + BLOB_SUFFIX)
+        if '/' not in config:
+            raise FileNotFoundError(path)
+        entry, key = config.rsplit('/', 1)
+        entry, _ = self.__resolve(entry)
+        kv = await self.__entry_kv(entry)
+        try:
+            return Result(stdout = kv[key].encode())
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    @override
+    async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
+        # -- The address space has no directories of its own: a node
+        #    is a branch (a real directory) or an entry (a suffixed
+        #    file); the type says which to build. Below a missing
+        #    branch, mkdir fails with EROFS: the caller creates the
+        #    levels
+        path = self.__addr(path)
+        config, _ = self.__resolve(path)
+        real = f'/{config}'
+        if ftype.is_subtype(FT_COMPOUND):
+            # -- An entry: materialize the empty entry file
+            if await self.__ctx.file_exists(real + SECRET_SUFFIX):
+                return
+            if not await self.__ctx.is_dir(os.path.dirname(real)):
+                raise OSError(
+                    errno.EROFS, f'mkdir: "{path}" is below a missing directory'
+                )
+            await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True)
+        elif ftype.is_subtype(FT_DIR):
+            # -- A branch: a real directory
+            if await self.__ctx.is_dir(real):
+                return
+            if not await self.__ctx.is_dir(os.path.dirname(real)):
+                raise OSError(
+                    errno.EROFS, f'mkdir: "{path}" is below a missing directory'
+                )
+            await self.__ctx.mkdir(real, mode)
+        else:
+            raise ValueError(f'mkdir: "{path}" is not a directory type')
+
+    @override
+    async def _put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None,
+        throw: bool,
+        verbose: bool | None,
+        title: str,
+        owner: str | None,
+        group: str | None,
+        mode: str | None,
+        atomic: bool,
+        ftype: FileType,
+    ) -> Result:
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if ftype.is_subtype(FT_SECRET):
+            # -- The whole entry: the content is the serialized
+            #    entry, written over the entry file; the branch is
+            #    a precondition, as for a blob
+            if is_blob:
+                raise ValueError(
+                    f'put: "{path}" names a blob file, but the ftype is an entry'
+                )
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'/{config}{SECRET_SUFFIX}',
+                content,
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        if ftype.is_subtype(FT_SECRET_FILE):
+            # -- A blob: the content is the file itself, at its
+            #    on-host file name
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'/{config}{BLOB_SUFFIX}',
+                content,
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        entry, sep, key = config.rpartition('/')
+        if not sep or not entry or not key or '/' in key:
+            raise ValueError(f'put: "{path}" is not an entry/key path')
+        entry, _ = self.__resolve(entry)
+        real_entry = f'/{entry}'
+        if ftype.is_subtype(FT_SECRET_KEY):
+            # -- A key: merge into the entry file, which must exist;
+            #    mkdir() is the materialization. The put is atomic,
+            #    so it also overwrites an existing entry file, which
+            #    is mode 0400
+            if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'):
+                raise OSError(errno.EROFS, f'put: "{entry}" has no entry file')
+            kv = await self.__entry_kv(entry)
+            kv[key] = content.decode()
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'{real_entry}{SECRET_SUFFIX}',
+                serialize_kv(kv),
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        raise ValueError(f'put: ftype {ftype} is not a store content type')
+
+    @override
+    async def _unlink(self, path: str) -> None:
+        # -- Unlink the file the address names: the address is the
+        #    on-host file name below the subtree, so the host path
+        #    is the composed address itself
+        await self.__ctx.unlink(f'/{self.__addr(path)}')

src/python/jw/pkg/lib/ec/secret_store/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/pkg/lib/ec/secret_store/TarGpg.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import errno
+import io
+import os
+import stat
+import time
+
+from tarfile import TarFile, TarInfo
+from typing import TYPE_CHECKING, Iterable, override
+
+from ...file_types import FT_DIR
+from ...kv import parse_kv
+from ...Result import Result
+from .Base import Base
+from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+
+if TYPE_CHECKING:
+    from typing import Any
+
+    from ...base import StatResult
+    from ...Uri import Uri
+
+class TarGpg(Base):
+    """A store of secrets in a GPG-encrypted tarball in the canonical
+    layout.
+
+    The tarball location is deployment information, not part of the
+    address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The
+    store URI addresses a subtree of the namespace, as in the base
+    class.
+
+    A store path names a member, and optionally a key below an
+    entry. The member of an entry is its address, and its keys are
+    addressable below it; a blob is a member at its address holding
+    non-key-value content. The store does not identify content by
+    member name: a member whose content is a key-value file is an
+    entry, anything else a blob.
+
+    The tarball is fetched and decrypted once, then cached. Fetching,
+    decrypting, and filtering happen in a local exec context, so the
+    GPG key and the decrypted material stay on this machine.
+    """
+
+    def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
+        super().__init__(uri, *args, **kwargs)
+        self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '')
+        self.__tf: TarFile | None = None
+        self.__kv_cache: dict[str, dict[str, str]] = {}
+
+    async def __tar(self) -> TarFile:
+        if self.__tf is None:
+            from ...ProcFilterGpg import ProcFilterGpg
+            from ...util import get
+            from ..Local import Local
+
+            data = (
+                await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local()))
+            ).stdout
+            self.__tf = TarFile(fileobj = io.BytesIO(data))
+        return self.__tf
+
+    async def __members(self) -> dict[str, TarInfo]:
+        tf = await self.__tar()
+        return {m.name: m for m in tf.getmembers()}
+
+    async def __entry_kv(self, entry: str) -> dict[str, str]:
+        if entry not in self.__kv_cache:
+            tf = await self.__tar()
+            try:
+                info = tf.getmember(entry)
+            except KeyError:
+                raise FileNotFoundError(entry) from None
+            buf = tf.extractfile(info)
+            if buf is None:
+                raise Exception(f'Can\'t read store entry "{entry}"')
+            self.__kv_cache[entry] = parse_kv(buf.read())
+        return self.__kv_cache[entry]
+
+    async def __is_entry(self, entry: str) -> bool:
+        # -- The member at entry is an entry: its content is a
+        #    non-empty kv file
+        tf = await self.__tar()
+        try:
+            info = tf.getmember(entry)
+        except KeyError:
+            return False
+        if not info.isfile():
+            return False
+        buf = tf.extractfile(info)
+        if buf is None:
+            return False
+        return len(parse_kv(buf.read())) != 0
+
+    async def __tar_stat(self, storage_path: str) -> StatResult:
+        # -- The metadata of the member that stands behind the node;
+        #    a branch has no member of its own and is synthesized, a
+        #    key keeps the size 0
+        members = await self.__members()
+
+        def __meta(name: str) -> tuple[str, str, float, int, int]:
+            # -- A member's mode carries the permission bits only, so
+            #    the type bits are synthesized; a store member is
+            #    always a regular file
+            mi = members[name]
+            return (
+                mi.uname or str(mi.uid),
+                mi.gname or str(mi.gid),
+                mi.mtime,
+                stat.S_IFREG | (mi.mode & 0o777),
+                mi.size,
+            )
+
+        if await self.__is_entry(storage_path):
+            owner, group, mtime, mode, size = __meta(storage_path)
+            return self._stat_result(
+                owner,
+                group,
+                mtime,
+                FT_SECRET,
+                mode = mode,
+                size = size,
+            )
+        par, sep, key = storage_path.rpartition('/')
+        if sep and key and await self.__is_entry(par):
+            if key in await self.__entry_kv(par):
+                # -- The type is synthesized, the mode the one of the
+                #    entry member, the size stays 0
+                owner, group, mtime, mode, _ = __meta(par)
+                return self._stat_result(
+                    owner,
+                    group,
+                    mtime,
+                    FT_SECRET_KEY,
+                    mode = mode,
+                )
+        if storage_path in members:
+            owner, group, mtime, mode, size = __meta(storage_path)
+            return self._stat_result(
+                owner,
+                group,
+                mtime,
+                FT_SECRET_FILE,
+                mode = mode,
+                size = size,
+            )
+        if not storage_path or \
+                any(name.startswith(f'{storage_path}/') for name in members):
+            return self._stat_result(
+                str(os.getuid()),
+                str(os.getgid()),
+                time.time(),
+                FT_DIR,
+            )
+        raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
+
+    async def __tar_get(self, path: str) -> Result:
+        members = await self.__members()
+        if path == '':
+            raise IsADirectoryError(path)
+        if await self.__is_entry(path):
+            # -- The entry member is the serialized entry
+            tf = await self.__tar()
+            buf = tf.extractfile(members[path])
+            if buf is None:
+                raise Exception(f'Can\'t read store entry "{path}"')
+            return Result(stdout = buf.read())
+        if any(name.startswith(f'{path}/') for name in members):
+            raise IsADirectoryError(path)
+        if '/' in path:
+            entry, key = path.rsplit('/', 1)
+            if await self.__is_entry(entry):
+                kv = await self.__entry_kv(entry)
+                try:
+                    return Result(stdout = kv[key].encode())
+                except KeyError:
+                    raise FileNotFoundError(path) from None
+        info = members.get(path)
+        if info is not None and info.isfile():
+            tf = await self.__tar()
+            buf = tf.extractfile(info)
+            if buf is None:
+                raise Exception(f'Can\'t read store blob "{path}"')
+            return Result(stdout = buf.read())
+        raise FileNotFoundError(path)
+
+    async def __tar_readdir(self, path: str) -> Iterable[str]:
+        members = await self.__members()
+        if await self.__is_entry(path):
+            return (await self.__entry_kv(path)).keys()
+        if path in members:
+            raise NotADirectoryError(path)
+
+        def __names(prefix: str) -> set[str]:
+            # -- No name filter: every member is store content
+            ret: set[str] = set()
+            for name in members:
+                if not name.startswith(prefix):
+                    continue
+                rest = name[len(prefix):]
+                ret.add(rest.partition('/')[0])
+            return ret
+
+        ret = __names(f'{path}/' if path else '')
+        mirror = self._default_path_in_storage(path)
+        if mirror is not None:
+            ret |= __names(f'{mirror}/' if mirror else '')
+        if not ret:
+            raise FileNotFoundError(path)
+        return sorted(ret)
+
+    @override
+    async def _get(
+        self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
+    ) -> Result:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_get)
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_readdir)
+
+    @override
+    async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_stat)

src/python/jw/pkg/lib/ec/secret_store/layout.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from ...file_types import FT_COMPOUND, FT_REG, file_types
+
+# -- The canonical store layout, shared by every store implementation and
+# the install pipeline that consumes it:
+#
+#   default/<path>      entries applying to all hosts
+#   <host-root>/<path>  entries applying to a single host, where
+#                       <host-root> is the host's FQDN, reversed and split
+#                       on dots (com/example/host for host.example.com)
+#
+# Host-specific entries take precedence over default ones.
+
+# Suffix of the key-value secret file accompanying a compiled target
+SECRET_SUFFIX = '.jw-secret'
+
+# Suffix of a non-key-value secret file (certificate, key, ...)
+BLOB_SUFFIX = '.jw-secret-file'
+
+DEFAULT_PREFIX = 'default'
+
+def content_name(name: str) -> str | None:
+    # -- The config path of an on-host secret file name: the name
+    #    with its content suffix removed, or None if the name
+    #    carries neither suffix
+    for suffix in (SECRET_SUFFIX, BLOB_SUFFIX):
+        if name.endswith(suffix):
+            return name.removesuffix(suffix)
+    return None
+
+def host_prefix(hostname: str) -> str:
+    return '/'.join(reversed(hostname.split('.')))
+
+# -- The file types of the store content, registered with the global
+#    file type registry: an entry is a compound of its keys, a key a
+#    leaf below it, a blob a file
+FT_NAME_SECRET = 'secret'
+FT_NAME_SECRET_KEY = 'secret-key'
+FT_NAME_SECRET_FILE = 'secret-file'
+
+file_types.register_types(
+    {
+        FT_NAME_SECRET: FT_COMPOUND,
+        FT_NAME_SECRET_KEY: FT_REG,
+        FT_NAME_SECRET_FILE: FT_REG,
+    }
+)
+
+FT_SECRET = file_types.get(FT_NAME_SECRET)
+FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY)
+FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE)

src/python/jw/pkg/lib/kv.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+def parse_kv(data: bytes) -> dict[str, str]:
+    """Parse a dotenv-style key-value entry into a dict"""
+    ret: dict[str, str] = {}
+    for line in data.decode().splitlines():
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if '=' not in line:
+            continue
+        key, val = line.split('=', 1)
+        key = key.strip()
+        val = val.strip()
+        if key:
+            ret[key] = val
+    return ret
+
+def serialize_kv(kv: dict[str, str]) -> bytes:
+    """Serialize a dict back into a dotenv-style key-value entry"""
+    return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode()

test/integration/jw_pkg/secrets/test.py

@@ -45,6 +45,7 @@ from jw.pkg.lib.Package import Package
 HOST = 'jannet.example.com'
 HOST_PREFIX = '/'.join(reversed(HOST.split('.')))
 PKG = 'fixture-pkg'
+PKG2 = 'fixture-pkg2'
 USER = pwd.getpwuid(os.getuid()).pw_name
 GROUP = grp.getgrgid(os.getgid()).gr_name
 
@@ -189,12 +190,21 @@ def main() -> None:
         #    extract it, and must not leave an empty file behind
         with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f:
             f.write('x=X\n')
+        # -- qux references a macro that has no store entry: the
+        #    compile must fail rather than emit a literal macro
+        with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f:
+            f.write(
+                f'# conf: owner={USER}, group={GROUP}, mode=640\n'
+                'present=PRESENT\n'
+                'missing=__UNRESOLVED__\n'
+            )
         with open(target + '/etc/readme.txt', 'w') as f:
             f.write('plain package file\n')
 
         foo_tmpl = target + '/etc/foo.conf.jw-tmpl'
         bar_tmpl = target + '/etc/bar.conf.jw-tmpl'
         baz_tmpl = target + '/etc/baz.conf.jw-tmpl'
+        qux_tmpl = target + '/etc/qux.conf.jw-tmpl'
         foo_secret = target + '/etc/foo.conf.jw-secret'
         bar_secret = target + '/etc/bar.conf.jw-secret'
         baz_secret = target + '/etc/baz.conf.jw-secret'
@@ -231,7 +241,10 @@ def main() -> None:
 
         distro = FakeDistro(
             Local('local://' + HOST),
-            {PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt']},
+            {
+                PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'],
+                PKG2: [qux_tmpl],
+            },
         )
 
         # -- list-templates: only the .jw-tmpl files of the package
@@ -356,6 +369,37 @@ def main() -> None:
         with open(bar_out) as f:
             assert f.read() == 'plain=plain-value\n'
 
+        # -- compile-templates fails on an unresolved macro: qux
+        #    resolves PRESENT from its secret file, but the template
+        #    also references __UNRESOLVED__, which is not in the
+        #    store. The failure must name the macro, and no compiled
+        #    file may be written
+        qux_secret = target + '/etc/qux.conf.jw-secret'
+        qux_out = target + '/etc/qux.conf'
+        with open(qux_secret, 'w') as f:
+            f.write('PRESENT=there\n')
+        try:
+            run_cli(
+                distro,
+                [
+                    'secrets',
+                    'compile-templates',
+                    '--owner',
+                    USER,
+                    '--group',
+                    GROUP,
+                    '--mode',
+                    '400',
+                    PKG2,
+                ],
+            )
+        except RuntimeError as e:
+            assert 'Unresolved macros' in str(e), e
+            assert '__UNRESOLVED__' in str(e), e
+        else:
+            raise AssertionError('unresolved macro must fail the compile')
+        assert not os.path.exists(qux_out)
+
         out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
         assert out == [foo_out, bar_out, baz_out], out
 

test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py

@@ -0,0 +1,384 @@
+import asyncio
+import errno
+import os
+import stat
+import tempfile
+
+from typing import Any, override
+
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.ec.secret_store.KvFile import KvFile
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.Package import Package
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        target = os.path.join(tmp, 'etc', 'foo.conf')
+        os.makedirs(os.path.dirname(target))
+        with open(target + '.jw-secret', 'w') as f:
+            f.write(
+                '# a comment\n\n  key1 = value1  \nkey2=value2\n'
+                '=novalue\nkey3=\n'
+            )
+
+        # FileContext.create() resolves the kvfile scheme, keeping
+        # the URI as the store's identity
+        created = FileContext.create('kvfile://')
+        assert isinstance(created, KvFile)
+        assert created.uri.scheme == 'kvfile'
+
+        store = KvFile(Local())
+        assert set(await store.readdir(target)) == {'key1', 'key2', 'key3'}
+        assert (await store.get(f'{target}/key1')).stdout == b'value1'
+        assert (await store.get(f'{target}/key3')).stdout == b''
+
+        # The entry is gettable as a whole: the serialized file
+        assert (await store.get(target)).stdout == \
+            b'# a comment\n\n  key1 = value1  \nkey2=value2\n=novalue\nkey3=\n'
+
+        # -- stat() answers the node's type, the metadata of the
+        #    host file behind it; a key has no file of its own, and
+        #    keeps the size 0
+        blob = os.path.join(tmp, 'etc', 'bar.pem')
+        with open(blob + '.jw-secret-file', 'w') as f:
+            f.write('a blob\n')
+        st = await store.stat(target)
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == os.path.getsize(target + '.jw-secret')
+        st = await store.stat(f'{target}/key1')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == 0
+        st = await store.stat(blob)
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == os.path.getsize(blob + '.jw-secret-file')
+        st = await store.stat(os.path.dirname(target))
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert not st.is_gettable
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert not st.is_gettable
+        try:
+            await store.stat(f'{target}/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists(target)
+        assert not await store.file_exists(f'{target}/missing')
+
+        # A missing key and a missing entry raise FileNotFoundError
+        for path in (f'{target}/missing', os.path.join(tmp, 'etc', 'missing.conf')):
+            try:
+                await store.get(path)
+                raise AssertionError(f'expected FileNotFoundError for {path}')
+            except FileNotFoundError:
+                pass
+
+        # put writes a key into the entry file, merging with the
+        # existing keys
+        await store.put(f'{target}/key4', b'value4', ftype = FT_SECRET_KEY)
+        assert (await store.get(f'{target}/key4')).stdout == b'value4'
+        assert (await store.get(f'{target}/key1')).stdout == b'value1'
+        # -- a second put overwrites the entry file, which is mode 0400
+        await store.put(f'{target}/key1', b'value1-new', ftype = FT_SECRET_KEY)
+        assert (await store.get(f'{target}/key1')).stdout == b'value1-new'
+        assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
+        new_entry = os.path.join(tmp, 'etc', 'new.conf')
+        # -- a key put does not create: below a missing entry file
+        #    it fails with EROFS; mkdir() is the materialization
+        try:
+            await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
+            raise AssertionError('expected EROFS')
+        except OSError as e:
+            assert e.errno == errno.EROFS
+        await store.mkdir(new_entry, ftype = FT_SECRET)
+        await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
+        with open(new_entry + '.jw-secret') as f:
+            assert f.read() == 'k=v\n'
+
+        # -- a typed put replaces the entry: the serialized content
+        #    is written over the entry file, mode 0400
+        await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
+        assert set(await store.readdir(target)) == {'new1', 'new2'}
+        assert (await store.get(target)).stdout == b'new1=n1\nnew2=n2\n'
+        with open(target + '.jw-secret') as f:
+            assert f.read() == 'new1=n1\nnew2=n2\n'
+        assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
+
+        # -- a put that changes nothing does not rewrite the file
+        mtime = os.stat(target + '.jw-secret').st_mtime_ns
+        await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
+        assert os.stat(target + '.jw-secret').st_mtime_ns == mtime
+
+        # -- An untyped put is refused: the store content types are
+        #    entry, key, and blob
+        try:
+            await store.put('HOSTNAME', b'x')
+            raise AssertionError('expected ValueError')
+        except ValueError:
+            pass
+
+        # -- put: a blob below a branch lands as a .jw-secret-file at
+        #    its config path, mode 0400
+        pem = os.path.join(tmp, 'etc', 'cert.pem.jw-secret-file')
+        await store.put(
+            os.path.join(tmp, 'etc', 'cert.pem'),
+            b'-----BEGIN CERT-----\n',
+            ftype = FT_SECRET_FILE,
+        )
+        with open(pem) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert stat.S_IMODE(os.stat(pem).st_mode) == 0o400
+
+        # -- mkdir: the type says which node to build, the default
+        #    a branch; a branch is a real directory, an entry an
+        #    empty entry file, mode 0400
+        await store.mkdir(os.path.join(tmp, 'etc'))
+        assert not os.path.exists(os.path.join(tmp, 'etc.jw-secret'))
+        baz = os.path.join(tmp, 'etc', 'baz')
+        await store.mkdir(baz)
+        assert os.path.isdir(baz)
+        bar = os.path.join(tmp, 'etc', 'bar.conf')
+        await store.mkdir(bar, ftype = FT_SECRET)
+        assert os.path.isfile(bar + '.jw-secret')
+        assert open(bar + '.jw-secret').read() == ''
+        assert stat.S_IMODE(os.stat(bar + '.jw-secret').st_mode) == 0o400
+
+        # -- mkdir and put fail with EROFS below a missing directory
+        for fn in (
+                lambda: store.mkdir(os.path.join(tmp, 'no', 'such', 'entry')),
+                lambda: store.put(
+                    os.path.join(tmp, 'no', 'such', 'entry', 'key'),
+                    b'x',
+                    ftype = FT_SECRET_KEY, ),
+        ):
+            try:
+                await fn()
+                raise AssertionError('expected EROFS')
+            except OSError as e:
+                assert e.errno == errno.EROFS
+
+        # -- put forwards owner and group to the host context: an
+        #    unknown group fails the put
+        try:
+            await store.put(
+                f'{target}/key7', b'v7', ftype = FT_SECRET_KEY, group = 'no-such-group'
+            )
+        except AssertionError:
+            raise
+        except Exception as e:
+            assert 'no-such-group' in str(e)
+        else:
+            raise AssertionError('expected the put to fail')
+
+        # unlink drops the file the address names, and a missing
+        # file raises
+        await store.unlink(target + '.jw-secret')
+        assert not os.path.exists(target + '.jw-secret')
+        try:
+            await store.unlink(target + '.jw-secret')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+
+        # -- Branch listing: the config paths below a branch, the
+        #    root included, come from the host's package set, the
+        #    templates of the installed packages, presence-checked
+        #    on the host. A fake distro supplies the list
+        class FakeDistro(SuseDistro):
+
+            def __init__(self, files: list[str]) -> None:
+                super().__init__(Local(), id = 'suse')
+                self.files = files
+                self.queries = 0
+
+            @override
+            async def _pkg_files(self, names: list[str] | None) -> list[str]:
+                self.queries += 1
+                return self.files
+
+        tree = os.path.join(tmp, 'tree')
+        os.makedirs(os.path.join(tree, 'etc'))
+        with open(os.path.join(tree, 'etc', 'foo.conf.jw-secret'), 'w') as f:
+            f.write('k1=v1\nk2=v2\n')
+        with open(os.path.join(tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as f:
+            f.write('-----BEGIN CERT-----\n')
+        with open(os.path.join(tree, 'etc', 'other.conf.jw-secret'), 'w') as f:
+            f.write('o1=v1\n')
+        # -- A template without a secret file is not a config path;
+        #    a package file that is not a template is not one
+        #    either
+        fake = FakeDistro(
+            [
+                f'{tree}/etc/foo.conf.jw-tmpl',
+                f'{tree}/etc/cert.pem.jw-tmpl',
+                f'{tree}/etc/ghost.conf.jw-tmpl',
+                f'{tree}/etc/other.conf',
+            ]
+        )
+        # -- The address space is rooted at the host root; the
+        #    package file list reports host-absolute paths, so the
+        #    fixture tree sits at its path below the root
+        bstore = KvFile(Local(), distro = fake)
+        base = tree.lstrip('/')
+        # -- is_dir answers from the host state, without consulting
+        #    the package set
+        assert await bstore.is_dir('')
+        assert await bstore.is_dir(base)
+        assert await bstore.is_dir(f'{base}/etc/foo.conf')
+        assert not await bstore.is_dir(f'{base}/etc/cert.pem')
+        assert fake.queries == 0
+        # -- is_gettable answers from the node's type: an entry,
+        #    a key, a blob; not a branch, the root included
+        assert await bstore.is_gettable(f'{base}/etc/foo.conf')
+        assert await bstore.is_gettable(f'{base}/etc/foo.conf/k1')
+        assert await bstore.is_gettable(f'{base}/etc/cert.pem')
+        assert not await bstore.is_gettable(f'{base}/etc')
+        assert not await bstore.is_gettable('')
+        assert fake.queries == 0
+        # -- Listing consults it, once: each branch lists the secret
+        #    files below it, under their on-host file names,
+        #    presence-checked
+        assert set(await bstore.readdir('')) == {'tmp'}
+        assert fake.queries == 1
+        assert set(await bstore.readdir(f'{base}/etc')) == \
+            {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
+        assert fake.queries == 1
+        # -- An entry still lists its keys, without the package set
+        assert set(await bstore.readdir(f'{base}/etc/foo.conf')) == {'k1', 'k2'}
+        assert fake.queries == 1
+        # -- A branch without config paths raises FileNotFoundError
+        try:
+            await bstore.readdir('no/such')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- A real directory with no secret children lists empty
+        os.makedirs(os.path.join(tree, 'var', 'empty'))
+        assert set(await bstore.readdir(f'{base}/var/empty')) == set()
+        # -- The blob reads back through the store
+        assert (await bstore.get(f'{base}/etc/cert.pem')).stdout == \
+            b'-----BEGIN CERT-----\n'
+
+        # -- The entry's config path is also a deployed file on the
+        #    host, as on a real host: a key below it has no host
+        #    path, and is_dir() must not stat below the file
+        with open(os.path.join(tree, 'etc', 'foo.conf'), 'w') as f:
+            f.write('deployed\n')
+        assert not (await bstore.is_dir(f'{base}/etc/foo.conf/k1'))
+        assert (await bstore.get(f'{base}/etc/foo.conf/k1')).stdout == b'v1'
+
+        # -- A URI may root the store at a subtree of the host tree:
+        #    create() keeps the URI as the store's identity, and its
+        #    path is composed into every address. The inner context
+        #    is host-rooted, so the on-host files are reached
+        #    without a double path
+        created = FileContext.create(f'kvfile://{tree}/etc')
+        assert created.uri.path == f'{tree}/etc'
+        sstore = KvFile(Local(), uri = f'kvfile://{tree}/etc', distro = fake)
+        await sstore.open()
+        assert set(await sstore.readdir('')) == \
+            {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
+        # -- An entry lists its keys, suffixed and unsuffixed alike
+        assert set(await sstore.readdir('foo.conf.jw-secret')) == {'k1', 'k2'}
+        assert set(await sstore.readdir('foo.conf')) == {'k1', 'k2'}
+        assert (await sstore.get('foo.conf/k1')).stdout == b'v1'
+        assert (await sstore.get('cert.pem')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        # -- A suffixed address names the on-host file of its config
+        #    path, the suffix picking the kind
+        assert (await sstore.get('foo.conf.jw-secret')).stdout == \
+            b'k1=v1\nk2=v2\n'
+        assert (await sstore.get('cert.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        st = await sstore.stat('foo.conf.jw-secret')
+        assert st.ftype is FT_SECRET
+        st = await sstore.stat('foo.conf.jw-secret/k1')
+        assert st.ftype is FT_SECRET_KEY
+        st = await sstore.stat('cert.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE
+        # -- A blob address refuses to list
+        try:
+            await sstore.readdir('cert.pem.jw-secret-file')
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+        # -- A write through the subtree root lands in the subtree
+        await sstore.mkdir('sub')
+        await sstore.mkdir('sub/conf', ftype = FT_SECRET)
+        await sstore.put('sub/conf/k', b'v', ftype = FT_SECRET_KEY)
+        with open(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) as f:
+            assert f.read() == 'k=v\n'
+        await sstore.unlink('sub/conf.jw-secret')
+        assert not os.path.exists(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret'))
+        await sstore.close()
+
+        # -- The autodetected distro applies the app's documented
+        #    filter fallback: JW_DEFAULT_PKG_FILTER reaches
+        #    Distro.instantiate() as the default package filter
+        from unittest import mock
+
+        from jw.pkg.lib.Distro import Distro
+
+        captured: dict[str, Any] = {}
+
+        async def __instantiate(*args: Any, **kwargs: Any) -> FakeDistro:
+            captured['kwargs'] = kwargs
+            return fake
+
+        os.environ['JW_DEFAULT_PKG_FILTER'] = 'url=~jw.dev'
+        try:
+            with mock.patch.object(Distro,
+                                   'instantiate',
+                                   new = classmethod(__instantiate)):
+                astore = KvFile(Local())
+                assert set(await astore.readdir('')) == {'tmp'}
+                flt = captured['kwargs'].get('default_pkg_filter')
+                assert flt is not None
+                assert flt.match(Package('fixture', url = 'https://jw.dev/fixture.rpm'))
+                assert not flt.match(
+                    Package('other', url = 'https://elsewhere/other.rpm')
+                )
+        finally:
+            del os.environ['JW_DEFAULT_PKG_FILTER']
+
+        # -- The lifecycle propagates to the host context: an
+        #    open() pin on the store keeps the host's transport
+        #    open for its duration, and the run()s within it share
+        #    it
+        class CountingLocal(Local):
+
+            def __init__(self) -> None:
+                super().__init__()
+                self.opens = 0
+                self.closes = 0
+
+            @override
+            async def _open(self) -> None:
+                self.opens += 1
+                await super()._open()
+
+            @override
+            async def _close(self) -> None:
+                self.closes += 1
+                await super()._close()
+
+        cl = CountingLocal()
+        pstore = KvFile(cl)
+        await pstore.open()
+        assert cl.opens == 1
+        await cl.run(['true'])
+        await cl.run(['true'])
+        assert cl.opens == 1
+        assert cl.closes == 0
+        await pstore.close()
+        assert cl.closes == 1
+
+asyncio.run(run_tests())
+print('All secrets kvfile store tests passed')

test/unit/python/jw/pkg/lib/ec/secret_store/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/dirs.mk

test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py

@@ -0,0 +1,235 @@
+import asyncio
+import io
+import os
+import shutil
+import stat
+import subprocess
+import tarfile
+import tempfile
+
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.util import copy
+
+def make_tar(path: str, entries: dict[str, str]) -> None:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj = buf, mode = 'w') as tf:
+        for name, text in entries.items():
+            info = tarfile.TarInfo(name)
+            data = text.encode()
+            info.size = len(data)
+            info.mode = 0o400
+            tf.addfile(info, io.BytesIO(data))
+    with open(path, 'wb') as f:
+        f.write(buf.getvalue())
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        # -- A throw-away GPG home with a key for the test
+        gpghome = os.path.join(tmp, 'gnupg')
+        os.makedirs(gpghome, mode = 0o700)
+        env = dict(os.environ, GNUPGHOME = gpghome)
+        subprocess.run(
+            [
+                'gpg',
+                '--batch',
+                '--pinentry-mode',
+                'loopback',
+                '--passphrase',
+                '',
+                '--quick-gen-key',
+                'test@example.com',
+            ],
+            check = True,
+            capture_output = True,
+            env = env,
+        )
+
+        # -- A store in the canonical layout, GPG-encrypted tarball
+        tar_path = os.path.join(tmp, 'secrets.tar')
+        make_tar(
+            tar_path,
+            {
+                'default/etc/a.jw-secret': 'a=1\n',
+                'default/etc/d.jw-secret': 'd=4\n',
+                'com/example/host/etc/a.jw-secret': 'a=2\n',
+                'com/example/host/etc/b.jw-secret': 'b=3\n',
+                # Non-key-value content is stored as a .jw-secret-file
+                # and addressable as a blob
+                'com/example/host/etc/c.pem.jw-secret-file': \
+                    '-----BEGIN CERT-----\n',
+                # A member without a content suffix is store
+                # content too: non-key-value content is a blob
+                'com/example/host/etc/README.md': 'not key-value content\n',
+            }
+        )
+        enc_path = tar_path + '.gpg'
+        subprocess.run(
+            [
+                'gpg',
+                '--batch',
+                '--yes',
+                '--quiet',
+                '--no-tty',
+                '--trust-model',
+                'always',
+                '--recipient',
+                'test@example.com',
+                '--encrypt',
+                '--output',
+                enc_path,
+                tar_path,
+            ],
+            check = True,
+            capture_output = True,
+            env = env,
+        )
+
+        # The store decrypts in a local exec context, which inherits
+        # the test's GNUPGHOME; the tarball location is deployment
+        # information, passed via the environment
+        os.environ['GNUPGHOME'] = gpghome
+        os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path
+
+        # -- The whole store: strict, no fallback
+        store = FileContext.create('targpg://')
+        await store.open()
+        assert isinstance(store, TarGpg)
+        assert set(await store.readdir('')) == {'default', 'com'}
+        # -- The names as stored: the store does not strip or filter
+        #    by member name, a suffixed name being the address of
+        #    its node
+        assert set(await store.readdir('com/example/host/etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
+             'README.md'}
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
+            b'2'
+        assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
+            b'3'
+
+        # The entry is gettable as a whole: the member's content
+        assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
+
+        # A blob is a file: get() returns its bytes, readdir() refuses
+        pem = 'com/example/host/etc/c.pem.jw-secret-file'
+        assert (await store.get(pem)).stdout == b'-----BEGIN CERT-----\n'
+        try:
+            await store.readdir(pem)
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+
+        # A member without a content suffix is store content too: a
+        # non-key-value member is a blob, listed, statable, gettable
+        assert 'README.md' in set(await store.readdir('com/example/host/etc'))
+        st = await store.stat('com/example/host/etc/README.md')
+        assert st.ftype is FT_SECRET_FILE
+        assert (await store.get('com/example/host/etc/README.md')).stdout == \
+            b'not key-value content\n'
+
+        # A missing entry and a missing key raise FileNotFoundError
+        for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
+            try:
+                await store.get(path)
+                raise AssertionError(f'expected FileNotFoundError for {path}')
+            except FileNotFoundError:
+                pass
+
+        # -- stat() answers the node's type, the metadata of the
+        #    member behind it; a key has no member of its own, and a
+        #    branch none at all, so they keep the size 0
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        st = await store.stat('default/etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.size == len((await store.get('default/etc/a.jw-secret')).stdout)
+        st = await store.stat('default/etc/a.jw-secret/a')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        assert st.size == 0
+        st = await store.stat('com/example/host/etc/c.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        assert st.size == len(
+            (await store.get('com/example/host/etc/c.pem.jw-secret-file')).stdout
+        )
+        st = await store.stat('default/etc')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert st.size == 0
+        # -- is_dir answers the node's listability: an entry lists
+        #    its keys, a branch its children; a key is a leaf
+        assert await store.is_dir('default/etc/a.jw-secret')
+        assert await store.is_dir('default/etc')
+        assert not await store.is_dir('default/etc/a.jw-secret/a')
+        try:
+            await store.stat('default/etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists('default/etc/a.jw-secret')
+        assert not await store.file_exists('default/etc/missing')
+        await store.close()
+
+        # -- A host-rooted store applies the default fallback
+        store = FileContext.create('targpg://host.example.com')
+        assert isinstance(store, TarGpg)
+        await store.open()
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
+        # A directory listing is the union with the default tree: d
+        # exists only under default/, but is listable as it is
+        # gettable
+        assert set(await store.readdir('etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
+             'd.jw-secret', 'README.md'}
+        # A miss below the host is retried below the default tree
+        assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
+        # A key listing stays host-wins-wholesale
+        assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
+        # -- stat() distinguishes an entry from a branch and the root
+        st = await store.stat('etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        st = await store.stat('etc')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        await store.close()
+
+        # -- A store as a copy source: an entry lands as a single
+        #    file with its serialized content, a branch as a
+        #    directory, and a blob as a file; the default-only entry
+        #    d lands from the union listing
+        host = os.path.join(tmp, 'host-tree')
+        os.makedirs(host)
+        ret = await copy('targpg://host.example.com', host, recursive = True)
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            f'{host}/etc/a.jw-secret',
+            f'{host}/etc/b.jw-secret',
+            f'{host}/etc/c.pem.jw-secret-file',
+            f'{host}/etc/d.jw-secret',
+            f'{host}/etc/README.md',
+        }
+        with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
+            assert f.read() == 'a=2\n'
+        assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
+        with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        # -- A single object that is an entry: the serialized whole
+        #    lands at the destination
+        dst = os.path.join(tmp, 'copied', 'entry')
+        os.makedirs(os.path.dirname(dst))
+        ret = await copy('targpg://host.example.com/etc/a.jw-secret', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == 'a=2\n'
+
+def main() -> None:
+    if not shutil.which('gpg'):
+        print('Skipping secrets targpg store tests (gpg required)')
+        return
+    asyncio.run(run_tests())
+    print('All secrets targpg store tests passed')
+
+main()

test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py

@@ -0,0 +1,41 @@
+from jw.pkg.lib.ec.secret_store.layout import (
+    BLOB_SUFFIX,
+    DEFAULT_PREFIX,
+    FT_SECRET,
+    FT_SECRET_FILE,
+    FT_SECRET_KEY,
+    SECRET_SUFFIX,
+    content_name,
+    host_prefix,
+)
+from jw.pkg.lib.file_types import FT_COMPOUND, FT_DIR, FT_REG
+
+# The layout constants are the store contract
+assert DEFAULT_PREFIX == 'default'
+assert SECRET_SUFFIX == '.jw-secret'
+assert BLOB_SUFFIX == '.jw-secret-file'
+
+# content_name strips the content suffix, and names without a
+# content suffix are not store content
+assert content_name('etc/a.jw-secret') == 'etc/a'
+assert content_name('etc/c.pem.jw-secret-file') == 'etc/c.pem'
+assert content_name('etc/README.md') is None
+
+# An FQDN maps to its reversed, dot-split host root
+assert host_prefix('host.example.com') == 'com/example/host'
+assert host_prefix('a.b.c.d') == 'd/c/b/a'
+
+# A single-label hostname maps to itself
+assert host_prefix('localhost') == 'localhost'
+
+# The store content types are registered with the file types: an
+# entry is a kind of compound, a key and a blob a kind of file
+assert FT_SECRET.is_subtype(FT_COMPOUND)
+assert FT_SECRET.is_subtype(FT_REG)
+assert not FT_SECRET.is_subtype(FT_DIR)
+assert FT_SECRET_KEY.is_subtype(FT_REG)
+assert not FT_SECRET_KEY.is_subtype(FT_DIR)
+assert FT_SECRET_FILE.is_subtype(FT_REG)
+assert not FT_SECRET_FILE.is_subtype(FT_DIR)
+
+print('All secrets layout tests passed')