lib.ec.secret_store: Add and use framework

"jw-pkg secrets install" copies secret material from a GPG-encrypted tar
archive to dotenv .jw-secret files on a target host. This commit refactors
the underlying code and adds hooks for the inverse copy direction, and for
more secret sources and targets.

The peers between which secrets are copied are represented by the new
abstract secret_store.Base class, and the existing tar blob and host
storages, inheriting from that class, are TarGpg and KvFile. Moreover,
secret_store.Base is inheriting from FileContext, so all of jw-pkg's
file-like operations are available as hooks.

Secrets inside instances of two file contexts are addressable as
(targpg|kvfile)://target.host.com/path/to/file.conf.jw-secret. The on-host
KvFile will look either locally or via SSH for that file, and TarGpg will
look inside a tar archive specified in the JW_SECRETS_TARGPG_TAR
environment variable.

Uris with these schemas can be passed to FileContext.create(), which will
return a chrooted file context of the correct type.

KvFile.stat("/path/to/file.conf.jw-secret").ftype is FT_SECRET which
inherits from FT_COMPOUND - which means that readdir() works on such
animals. Not entirely sure if that's a good idea, but it feels pretty
generic. Let's see where that takes us.

DistroContext.install() after this refactor reads its source through the
store interface and writes the landing through the KvFile store.

Both secret stores are exercised through the existing unit test of jw-pkg
secrets install, and unit tests for both stores are also added by this
commit.

Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/jw/pkg/cmds/secrets/lib/DistroContext.py

@@ -3,11 +3,21 @@ from __future__ import annotations
 import re
 import sys
 
+from contextlib import suppress
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from ....lib.ec.secret_store.Base import Base as SecretStore
+from ....lib.ec.secret_store.KvFile import KvFile
+from ....lib.ec.secret_store.layout import (
+    DEFAULT_PREFIX,
+    FT_SECRET,
+    FT_SECRET_KEY,
+    SECRET_SUFFIX,
+    host_prefix,
+)
+from ....lib.FileContext import FileContext
 from ....lib.log import DEBUG, NOTICE, WARNING, log
-from ....lib.ProcFilterGpg import ProcFilterGpg
 from .FilesContext import FilesContext
 
 if TYPE_CHECKING:
@@ -19,7 +29,7 @@ if TYPE_CHECKING:
 class DistroContext(FilesContext):
 
     def __init__(self, distro: Distro) -> None:
-        super().__init__(distro.ctx)
+        super().__init__(distro.ctx, distro = distro)
         self.__distro = distro
 
     async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@@ -40,7 +50,7 @@ class DistroContext(FilesContext):
     ) -> list[str]:
         ret = []
         for tmpl in await self.list_template_files(pkg_names):
-            path = str(Path(tmpl).with_suffix('.jw-secret'))
+            path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
             if ignore_missing and not await self.ctx.file_exists(path):
                 continue
             ret.append(path)
@@ -101,64 +111,60 @@ class DistroContext(FilesContext):
         verbose: bool = False,
     ) -> None:
 
-        async def _read_secret_tar_blob(src_uri: str) -> bytes:
-            ec = self.ctx
-            from ....lib.ec.Local import Local
-            from ....lib.util import get
-
-            if not isinstance(ec, Local):
-                # Security: Use a local exec context for decrypting and
-                # filtering secrets
-                ec = Local()
-            return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
-
-        def _is_needed_secret(path: str) -> bool:
-            return path in secret_paths
-
-        from .tar import extract as tar_extract
-        from .tar import filter as tar_filter
-        from .tar import merge as tar_merge
-        from .tar import rewrite as tar_rewrite
-
         if only_missing:
             raise NotImplementedError('--only-missing is not yet implemented')
 
-        secret_paths = await self.list_secret_paths(pkg_names)
+        fc = FileContext.create(src_uri, chroot = True)
-        hostname = self.ctx.uri.hostname
+        await fc.open()
-        if not hostname:
+        try:
-            raise Exception('Have no hostname to find secrets in tar file')
+            hostname = self.ctx.uri.hostname
-        host_root_in_tar = '/'.join(reversed(hostname.split('.')))
+            if not hostname:
-        host_rx = re.compile(r'^' + host_root_in_tar)
+                raise Exception('Have no hostname to find secrets in the store')
-        default_rx = re.compile(r'^default')
+            if isinstance(fc, SecretStore):
-
+                if not fc.whole:
-        filtered_paths: list[str] = []
+                    raise Exception(
-        extracted_paths: list[str] = []
+                        f'src {src_uri} must name the whole store, not a '
-
+                        f'rooted subtree'
-        blob_all = await _read_secret_tar_blob(src_uri)
+                    )
-        if blob_all is None:
+                src: FileContext = fc
-            raise Exception(f'Tar blob {src_uri} is empty')
+                # -- Store candidates: the host tree, then the default
-        blob_host_filtered = tar_filter(
+                #    tree
-            blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
+                prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
-        )
-        blob_host_transformed = tar_rewrite(
-            blob_host_filtered, lambda p: re.sub(host_rx, '', p)
-        )
-        blob_default_filtered = tar_filter(
-            blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
-        )
-        blob_default_transformed = tar_rewrite(
-            blob_default_filtered, lambda p: re.sub(default_rx, '', p)
-        )
-        blob_secret_material = tar_merge(
-            [blob_host_transformed, blob_default_transformed], overwrite = False
-        )
-        blob_needed = tar_filter(
-            blob_secret_material, _is_needed_secret, extracted_paths
-        )
-
-        await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
-        for path in secret_paths:
-            if path not in extracted_paths:
-                log(NOTICE, f'not extracted: {path}')
             else:
-                log(NOTICE, f'extracted:     {path}')
+                # -- A file tree in the on-host layout: wrap it in the kvfile store
+                src = KvFile(fc, distro = self.__distro)
+                prefixes = ('', )
+
+            # -- The store and the kvfile source are alike addressed
+            #    by the on-host file name
+            suffix = SECRET_SUFFIX
+
+            for entry in await self.list_compilation_targets(pkg_names):
+                # -- A present entry, even an empty one, wins over the
+                #    fallback
+                entry_rel = f'{entry.lstrip("/")}{suffix}'
+                addr = entry_rel
+                keys: list[str] | None = None
+                for pfx in prefixes:
+                    addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
+                    try:
+                        keys = list(await src.readdir(addr))
+                        break
+                    except FileNotFoundError:
+                        continue
+                if keys is None:
+                    log(NOTICE, f'not extracted: {entry}')
+                    continue
+                log(NOTICE, f'extracted:     {entry}')
+                pairs: list[tuple[str, bytes]] = []
+                for key in keys:
+                    val = await src.get(f'{addr}/{key}')
+                    pairs.append((key, val.stdout))
+                dst = self.store
+                with suppress(FileNotFoundError):
+                    await dst.unlink(entry_rel)
+                await dst.mkdir(entry_rel, ftype = FT_SECRET)
+                for key, content in pairs:
+                    await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
+        finally:
+            await fc.close()

src/python/jw/pkg/cmds/secrets/lib/FilesContext.py

@@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable
-    from ....lib.FileContext import FileContext
 
+    from ....lib.Distro import Distro
+    from ....lib.FileContext import FileContext
+    from ....lib.ec.secret_store.Base import Base
+
+from ....lib.ec.secret_store.KvFile import KvFile
+from ....lib.ec.secret_store.layout import SECRET_SUFFIX
 from ....lib.log import DEBUG, NOTICE, WARNING, log
 from .base import Attrs
 
 class FilesContext:
 
-    def __init__(self, ctx: FileContext) -> None:
+    def __init__(
+        self,
+        ctx: FileContext,
+        store: Base | None = None,
+        distro: Distro | None = None,
+    ) -> None:
         self.__ctx = ctx
+        if store is None:
+            store = KvFile(ctx, distro = distro)
+        self.__store = store
 
     @property
     def ctx(self) -> FileContext:
         return self.__ctx
 
-    async def _read_key_value_file(
+    @property
-        self,
+    def store(self) -> Base:
-        path: str,
+        return self.__store
-        throw: bool = False,
-    ) -> dict[str, str]:
-        ret: dict[str, str] = {}
-        try:
-            result = await self.ctx.get(path)
-            for line in result.stdout_str.splitlines():
-                line = line.strip()
-                if not line or line.startswith('#'):
-                    continue
-                if '=' not in line:
-                    continue
-                key, val = line.split('=', 1)
-                key = key.strip()
-                val = val.strip()
-                if key:
-                    ret[key] = val
-        except FileNotFoundError:
-            log(DEBUG, f'File not found {path}')
-        return ret
 
     def _parse_attributes(self, content: str) -> Attrs | None:
 
@@ -143,6 +137,10 @@ class FilesContext:
         for key, val in replace.items():
             new_content = new_content.replace(key, val)
 
+        unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
+        if unresolved:
+            raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
+
         tmp_path: str | None = None
 
         try:
@@ -200,12 +198,21 @@ class FilesContext:
     ) -> bool:
         path_tmpl = target_path + '.jw-tmpl'
         path_secret_file = target_path + '.jw-secret-file'
-        path_secret = target_path + '.jw-secret'
+        path_secret = target_path + SECRET_SUFFIX
         attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
         attrs.update(
             await self._read_attributes([path_tmpl, path_secret_file, path_secret])
         )
-        replace = await self._read_key_value_file(path_secret)
+        entry = f'{target_path}{SECRET_SUFFIX}'
+        try:
+            keys = await self.__store.readdir(entry)
+        except FileNotFoundError:
+            log(DEBUG, f'No secrets for target {target_path}')
+            keys = []
+        replace: dict[str, str] = {}
+        for key in keys:
+            val = await self.__store.get(f'{entry}/{key}')
+            replace[key] = val.stdout.decode()
         for src in [path_secret_file, path_tmpl]:
             try:
                 await self._compile_one_template_file(

src/python/jw/pkg/cmds/secrets/lib/tar.py

@@ -1,83 +0,0 @@
-from __future__ import annotations
-
-import io
-import tarfile
-
-from tarfile import TarFile
-from typing import TYPE_CHECKING, Callable
-
-from ....lib.ExecContext import ExecContext
-from ....lib.log import DEBUG, log
-
-if TYPE_CHECKING:
-    from typing import Iterable
-
-    from ....lib.FileContext import FileContext
-
-def filter(
-    blob: bytes,
-    path_filter: Callable[[str], bool] | None,
-    matched: list[str] | None = None,
-) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        tf_in = TarFile(fileobj = io.BytesIO(blob))
-        for info in tf_in.getmembers():
-            if path_filter is not None and not path_filter(info.name):
-                continue
-            log(DEBUG, f'Adding {info.name}')
-            if matched is not None:
-                matched.append(info.name)
-            buf = tf_in.extractfile(info)
-            tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        tf_in = TarFile(fileobj = io.BytesIO(blob))
-        for info in tf_in.getmembers():
-            new_name = rewrite_filter(info.name)
-            log(DEBUG, f'Rewriting {info.name} -> {new_name}')
-            info.name = new_name
-            # -- A pax "path" supplement from the input would override
-            #    the rewritten name on re-serialization
-            info.pax_headers = {
-                k: v
-                for k, v in info.pax_headers.items() if k != 'path'
-            }
-            buf = tf_in.extractfile(info)
-            tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
-    ret = io.BytesIO()
-    with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
-        for blob in blobs:
-            tf_in = TarFile(fileobj = io.BytesIO(blob))
-            existing_names = tf_out.getnames()
-            for info in tf_in.getmembers():
-                if not overwrite and info.name in existing_names:
-                    continue
-                buf = tf_in.extractfile(info)
-                tf_out.addfile(info, buf)
-    return ret.getvalue()
-
-async def extract(
-    dst: FileContext,
-    blob: bytes,
-    root: str | None = None,
-    verbose: bool = False
-) -> None:
-    cmd = ['tar']
-    if root is not None:
-        cmd += ['-C', root]
-    if verbose:
-        cmd += '-v'
-    cmd += ['-x', '-f', '-']
-    if not isinstance(dst, ExecContext):
-        raise NotImplementedError(
-            'Extracting tar files to a non-executable '
-            f'context is not yet implemented: {dst}'
-        )
-    await dst.run(cmd, verbose = verbose, cmd_input = blob)

src/python/jw/pkg/lib/FileContext.py

@@ -360,6 +360,18 @@ class FileContext(abc.ABC):
                 from .ec.Curl import Curl
 
                 return Curl(uri, *args, **kwargs)
+            case 'targpg':
+                from .ec.secret_store.TarGpg import TarGpg
+
+                return TarGpg(uri, *args, **kwargs)
+            case 'kvfile':
+                from .ec.secret_store.KvFile import KvFile
+
+                scheme = 'ssh' if uri.hostname else 'local'
+                inner = FileContext.create(
+                    f'{scheme}://{uri.authority}', chroot = False
+                )
+                return KvFile(inner, uri = uri)
             case _:
                 pass
         raise Exception(

src/python/jw/pkg/lib/ec/secret_store/Base.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override
+
+from ...base import Result, StatResult
+from ...file_types import FT_DIR, FT_REG
+from ...FileContext import FileContext
+from .layout import DEFAULT_PREFIX, host_prefix
+
+if TYPE_CHECKING:
+    import os
+
+    from typing import Any, Iterable
+
+    from ...file_types import FileType
+    from ...Uri import Uri
+
+T = TypeVar('T')
+
+class Base(FileContext):
+    """A store of secret entries in the canonical store layout.
+
+    A store is a FileContext over the store namespace. The URI
+    addresses a subtree of that namespace: no authority means the
+    whole store, a host authority means that host's tree (its
+    reversed domain prefix), `default` names the default tree, and a
+    path narrows further. The subtree is the store's namespace
+    root; addresses passed to the methods are relative to it.
+
+    A store is a typed file store: a store path names a file, and
+    put() persists the content at the address as-is. The store does
+    not identify content by file name; what a file holds - an entry,
+    or a blob - is a property of its content, resolved at stat
+    time. A node is what its content makes it:
+
+    - an entry carries key-value secrets and behaves like a
+      directory: readdir() on the entry lists its keys, get() on
+      <entry>/<key> returns the value of the key, and get() on the
+      entry itself returns the serialized entry
+    - a branch is a path prefix: readdir() lists its children
+    - a blob is non-key-value content: get() returns its bytes
+
+    A store rooted at a host applies the default fallback: an
+    address that misses below the host is retried below the default
+    tree. A whole-store or default-rooted store is strict; callers
+    apply the fallback themselves.
+
+    The URI-addressable stores are created by FileContext.create()
+    from their scheme. The kvfile store is the store of the on-host
+    layout: its addresses are config paths of the host it is rooted
+    at, and it may be constructed directly over a host context.
+    """
+
+    def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
+        # -- A store is always chrooted. The flag states the
+        #    fact, and the chroot it arms is undone in the
+        #    protected layer by _unchroot(); an explicit value
+        #    is overridden.
+        kwargs['chroot'] = True
+        super().__init__(uri, *args, **kwargs)
+
+    @property
+    def __host_root(self) -> str:
+        # -- The root of the host tree this store is rooted at, '' if
+        #    the store is whole-store or rooted at the default tree:
+        #    then there is no host tree to fall back from, and the
+        #    caller applies the default fallback itself.
+        host = self.uri.hostname or ''
+        if host and host != 'default':
+            return host_prefix(host)
+        return ''
+
+    @property
+    def __root_in_storage(self) -> str:
+
+        # -- The namespace node this store is rooted at: the
+        #    reversed host prefix of the URI authority, if any
+        #    (`default` is a reserved word), followed by the URI
+        #    path.
+        host = self.uri.hostname or ''
+        path = self.uri.path.lstrip('/')
+        if host and host != 'default':
+            host = host_prefix(host)
+        return '/'.join(p for p in (host, path) if p)
+
+    def _path_in_storage(self, path: str) -> str:
+        path = path.lstrip('/')
+        if not self.__root_in_storage:
+            return path
+        if not path:
+            return self.__root_in_storage
+        return f'{self.__root_in_storage}/{path}'
+
+    def _default_path_in_storage(self, path: str) -> str | None:
+        host_root = self.__host_root
+        ns = self.__root_in_storage
+        if not host_root or not (path == ns or path.startswith(f'{ns}/')):
+            return None
+        return f'{DEFAULT_PREFIX}{path[len(host_root):]}'
+
+    def _unchroot(self, path: str) -> str:
+        # -- The inverse of _chroot(): the public wrappers hand the
+        #    protected layer addresses prefixed with the root; store
+        #    addresses are relative to it, and the root itself is the
+        #    empty address. Already-relative addresses pass through.
+        if not self.root:
+            return path.lstrip('/')
+        if path == self.root:
+            return ''
+        prefix = f'{self.root}/'
+        if path.startswith(prefix):
+            return path[len(prefix):]
+        return path
+
+    async def _call_with_fallback(
+        self,
+        path: str,
+        func: Callable[[str], Awaitable[T]],
+    ) -> T:
+        storage_path = self._path_in_storage(path)
+        try:
+            return await func(storage_path)
+        except FileNotFoundError:
+            default_storage_path = self._default_path_in_storage(storage_path)
+            if default_storage_path is None:
+                raise
+            return await func(default_storage_path)
+
+    @override
+    async def put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None = None,
+        throw: bool = True,
+        verbose: bool | None = None,
+        title: str | None = None,
+        owner: str | None = None,
+        group: str | None = None,
+        mode: int | None = None,
+        atomic: bool = False,
+        ftype: FileType = FT_REG,
+    ) -> Result:
+        # -- A put that changes nothing is a no-op: the pre-read is
+        #    fallback-aware, so a value already represented, below
+        #    the host tree or below the default tree, does not
+        #    trigger a store write. Metadata arguments opt out: they
+        #    may be the only change
+        if owner is None and group is None and mode is None:
+            try:
+                existing = await self.get(path)
+            except Exception:
+                pass
+            else:
+                if self._put_is_noop(existing.stdout, content, ftype):
+                    return Result()
+        return await super().put(
+            path,
+            content,
+            wd,
+            throw,
+            verbose,
+            title,
+            owner,
+            group,
+            mode,
+            atomic,
+            ftype,
+        )
+
+    def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
+        # -- A put that leaves the stored content byte-identical
+        #    changes nothing. A store that normalizes content on
+        #    read overrides this to compare the normalized forms
+        return existing == content
+
+    @override
+    async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
+        # -- An entry is a file on the backing store, yet it lists
+        #    its keys: answer from the node's type, not the
+        #    object's mode
+        try:
+            return (await self._stat(path, follow_symlinks)).is_dir
+        except FileNotFoundError:
+            return False
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        # -- A store is listable: a directory of keys, entries, or
+        #    blobs, whatever its nodes hold.
+        raise NotImplementedError()
+
+    @override
+    async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
+        # -- The base store materializes nothing: branches are
+        #    implicit namespace prefixes, and entries materialize on
+        #    their first write. A store whose nodes have a host home
+        #    overrides this
+        pass
+
+    def _stat_result(
+        self,
+        owner: str,
+        group: str,
+        mtime: float,
+        ftype: FileType,
+        mode: int | None = None,
+        size: int = 0,
+    ) -> StatResult:
+        # -- The metadata of a node: the mode and size, if given, are
+        #    the ones of the file, directory, or member behind the
+        #    node; a node without a backing object keeps the
+        #    synthesized type bits and a size of 0. The access and
+        #    change times follow the modification time
+        import stat
+
+        if mode is None:
+            mode = (
+                (stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else
+                (stat.S_IFREG | 0o400)
+            )
+        return StatResult(
+            mode = mode,
+            owner = owner,
+            group = group,
+            size = size,
+            atime = mtime,
+            mtime = mtime,
+            ctime = mtime,
+            ftype = ftype,
+        )
+
+    def _stat_result_from_os(
+        self,
+        st: os.stat_result,
+        ftype: FileType,
+        size: int | None = None,
+    ) -> StatResult:
+        return self._stat_result(
+            owner = str(st.st_uid),
+            group = str(st.st_gid),
+            mtime = st.st_mtime,
+            ftype = ftype,
+            mode = st.st_mode,
+            size = st.st_size if size is None else size,
+        )
+
+    @property
+    def whole(self) -> bool:
+        # -- True, if the store is rooted at the whole store namespace:
+        #    no authority, no path
+        return not self.__root_in_storage

src/python/jw/pkg/lib/ec/secret_store/KvFile.py

@@ -0,0 +1,439 @@
+from __future__ import annotations
+
+import errno
+import os
+
+from typing import TYPE_CHECKING, Iterable, override
+
+if TYPE_CHECKING:
+    from ...Distro import Distro
+    from ...FileContext import FileContext
+    from ...Uri import Uri
+    from ...base import StatResult
+    from ...file_types import FileType
+
+from ...FcCache import FcCache
+from ...file_types import FT_COMPOUND, FT_DIR
+from ...kv import parse_kv, serialize_kv
+from ...Result import Result
+from .Base import Base
+from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX
+
+class KvFile(Base):
+    """The on-host store of a host's package secrets.
+
+    A config path X is either an entry, the file X.jw-secret with
+    KEY=VALUE lines, whose children in the address space are its
+    keys, or a blob, the file X.jw-secret-file, whose content is the
+    file itself, a leaf. Any other path is a branch: a real
+    directory on the host, with no file of its own.
+
+    Rooted at the host: addresses are config paths relative to the
+    host root, absolute on the host with the leading slash. A URI
+    may root the store at a subtree of the host tree; its path is
+    composed into every address, and the URI is the store's
+    identity. A suffixed address names the on-host file of its
+    config path, the suffix picking the kind: .jw-secret the
+    entry, .jw-secret-file the blob.
+
+    get reads a key below an entry or the blob at its config path;
+    put writes a key into its entry file, or a blob as a
+    .jw-secret-file at its config path, and accepts entry, key, and
+    blob content only; unlink drops the file the address names. No
+    default fallback.
+
+    An entry lists its keys. A branch lists the config paths below
+    it, the root included: the host's package set is the bounded
+    base, the .jw-tmpl files of the installed packages, presence-
+    checked on the host. The distro is autodetected from the host
+    on first use, or given in the constructor; the autodetected
+    distro applies the JW_DEFAULT_PKG_FILTER package filter, if
+    set, so a URI-built context honors the app's documented
+    filter fallback. is_dir() answers from the entry file and the
+    host tree, so the write path never pays for the enumeration.
+
+    The on-host tree is a precondition: mkdir materializes a missing
+    entry file below an existing directory, and mkdir and put fail
+    with EROFS below a missing one. KvFile never creates
+    directories. Entry and blob files are mode 0400;
+    owner and group follow the put request, if given.
+    """
+
+    def __init__(
+        self,
+        ctx: FileContext,
+        uri: str | Uri | None = None,
+        distro: Distro | None = None,
+    ) -> None:
+        # -- The URI roots the namespace at the host tree's root, or
+        #    at a subtree of it: its path is composed into every
+        #    address, and it is the store's identity. A direct
+        #    construction names the context's host
+        if uri is None:
+            uri = f'kvfile://{ctx.uri.hostname or "target"}'
+        super().__init__(uri = uri)
+        self.__raw = ctx
+        self.__distro: Distro | None = distro
+        self.___config_paths: list[str] | None = None
+        self.__subtree = self.uri.path.lstrip('/')
+        # -- The context is read through a cache: the read path
+        #    probes the same handful of paths again and again, and
+        #    the cache pays the transport once per path
+        self.__ctx = FcCache(ctx)
+
+    @override
+    async def _open(self) -> None:
+        # -- The lifecycle propagates to the host context: an
+        #    open() pin on the store, the CopyContext of a copy
+        #    walk, keeps the host's transport open for its
+        #    duration, and the run()s within it share it
+        await self.__raw.open()
+
+    @override
+    async def _close(self) -> None:
+        await self.__raw.close()
+
+    def __addr(self, path: str) -> str:
+        # -- The host-relative address: the subtree the URI roots
+        #    the store at, then the address relative to it
+        path = self._unchroot(path)
+        if not path:
+            return self.__subtree
+        if not self.__subtree:
+            return path
+        return f'{self.__subtree}/{path}'
+
+    @staticmethod
+    def __resolve(path: str) -> tuple[str, bool]:
+        # -- The config path of the address, and whether the address
+        #    names a blob file: a suffixed address is the on-host
+        #    file name of its config path, the suffix picking the
+        #    kind
+        if path.endswith(SECRET_SUFFIX):
+            return path[:-len(SECRET_SUFFIX)], False
+        if path.endswith(BLOB_SUFFIX):
+            return path[:-len(BLOB_SUFFIX)], True
+        return path, False
+
+    async def __config_paths(self) -> list[str]:
+        # -- The config paths of the host's package secrets: the
+        #    templates of the installed packages, relative to the
+        #    host root. The package database is the bounded base, a
+        #    filesystem scan is not. Computed once: the package set
+        #    is stable for the lifetime of the session
+        if self.___config_paths is None:
+            if self.__distro is None:
+                from ...Distro import Distro
+                from ...ExecContext import ExecContext
+
+                # -- The distro probes the host through the raw
+                #    context: the cache is a plain file context, no
+                #    exec of its own
+                if not isinstance(self.__raw, ExecContext):
+                    raise ValueError(
+                        'KvFile: a distro must be given when the host'
+                        ' context is not an exec context'
+                    )
+                # -- A URI-built context has no app distro to
+                #    inherit the package filter from: apply the
+                #    app's documented fallback
+                from ...PackageFilter import PackageFilterString
+
+                pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER')
+                self.__distro = await Distro.instantiate(
+                    ec = self.__raw,
+                    default_pkg_filter = (
+                        PackageFilterString(pkg_filter_str) if pkg_filter_str else None
+                    ),
+                )
+            # -- The packages' file paths, filtered to the templates
+            ret: set[str] = set()
+            for f in await self.__distro.pkg_files():
+                if not f.endswith('.jw-tmpl'):
+                    continue
+                ret.add(f.lstrip('/')[:-len('.jw-tmpl')])
+            self.___config_paths = sorted(ret)
+            # -- Warm the stat cache with one host command, so the
+            #    per-level presence checks and the walk's node stats
+            #    hit memory: the root, the candidate files of both
+            #    kinds, the directories they live in, and the file
+            #    names a suffixed address would probe those
+            #    directories by
+            warm = ['/']
+            warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths]
+            warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths]
+            dirs: set[str] = set()
+            for p in self.___config_paths:
+                parts = p.split('/')
+                for i in range(1, len(parts)):
+                    dirs.add('/'.join(parts[:i]))
+            for d in sorted(dirs):
+                warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}']
+            await self.__ctx.warm(warm)
+        assert self.___config_paths is not None
+        return self.___config_paths
+
+    async def __entry_kv(self, entry: str) -> dict[str, str]:
+        path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}'
+        if not await self.__ctx.file_exists(path):
+            raise FileNotFoundError(entry)
+        ret = await self.__ctx.get(path)
+        return parse_kv(ret.stdout)
+
+    async def __is_entry(self, path: str) -> bool:
+        # -- The entry file on the host; KvFile has no default tree.
+        #    The path is host-relative, as from __addr()
+        if not path:
+            return False
+        return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}')
+
+    @override
+    async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
+        # -- The metadata of the host file behind the node, the type
+        #    overridden: an entry is a file that enumerates its
+        #    keys, a key a leaf below it; a branch is a plain
+        #    directory. A key has no file of its own, so it keeps
+        #    the size 0
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+
+        async def __stat_of(real: str, ftype: FileType) -> StatResult:
+            st = await self.__ctx.stat(real, follow_symlinks)
+            return self._stat_result(
+                st.owner,
+                st.group,
+                st.mtime,
+                ftype,
+                mode = st.mode,
+                size = st.size,
+            )
+
+        if not config:
+            return await __stat_of('/', FT_DIR)
+        if is_blob:
+            real = f'/{config}{BLOB_SUFFIX}'
+            if not await self.__ctx.file_exists(real):
+                raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
+            return await __stat_of(real, FT_SECRET_FILE)
+        if await self.__is_entry(config):
+            return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET)
+        parent, sep, key = config.rpartition('/')
+        if sep and key:
+            parent, _ = self.__resolve(parent)
+            if await self.__is_entry(parent) and key in await self.__entry_kv(parent):
+                # -- The type is synthesized, the mode the one of
+                #    the entry file
+                st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks)
+                return self._stat_result(
+                    st.owner,
+                    st.group,
+                    st.mtime,
+                    FT_SECRET_KEY,
+                    mode = st.mode,
+                )
+        blob = f'/{config}{BLOB_SUFFIX}'
+        if await self.__ctx.file_exists(blob):
+            return await __stat_of(blob, FT_SECRET_FILE)
+        if await self.__ctx.is_dir(f'/{config}'):
+            return await __stat_of(f'/{config}', FT_DIR)
+        raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
+
+    @override
+    async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
+        # -- An entry is a directory of its keys, a key below an
+        #    entry is a leaf: the entry is a file on the host, so
+        #    there is no host path below it to stat. A branch is a
+        #    real directory on the host. The package set is not
+        #    consulted: the write path never pays for the
+        #    enumeration
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if not config:
+            return True
+        parent, sep, _ = config.rpartition('/')
+        if sep:
+            parent, _ = self.__resolve(parent)
+            if await self.__is_entry(parent):
+                return False
+        if await self.__is_entry(config):
+            return True
+        if is_blob:
+            return False
+        return await self.__ctx.is_dir(f'/{config}')
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        # -- An entry lists its keys; a branch, the root included,
+        #    lists the config paths below it, presence-checked on
+        #    the host, under the on-host file name of the file that
+        #    backs it
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if config and await self.__is_entry(config):
+            return (await self.__entry_kv(config)).keys()
+        if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'):
+            raise NotADirectoryError(path)
+        children: set[str] = set()
+        prefix = f'{config}/' if config else ''
+        for cfg in await self.__config_paths():
+            if not cfg.startswith(prefix):
+                continue
+            rest = cfg[len(prefix):]
+            label, sep, _ = rest.partition('/')
+            if sep:
+                # -- A deeper candidate lists its branch label, if a
+                #    secret is present below it: the label names a
+                #    directory on the host, which a present secret
+                #    below implies
+                if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \
+                        await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
+                    children.add(label)
+                continue
+            # -- A direct child lists under the on-host file name of
+            #    the file that backs it, entry and blob alike
+            if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'):
+                children.add(f'{label}{SECRET_SUFFIX}')
+            if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
+                children.add(f'{label}{BLOB_SUFFIX}')
+        if config and not children:
+            # -- A branch the host lacks is absent; a real directory
+            #    with no secret children lists empty
+            if not await self.__ctx.is_dir(f'/{config}'):
+                raise FileNotFoundError(path)
+        return children
+
+    @override
+    async def _get(
+        self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
+    ) -> Result:
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if is_blob:
+            real = f'/{config}{BLOB_SUFFIX}'
+            if not await self.__ctx.file_exists(real):
+                raise FileNotFoundError(path)
+            return await self.__ctx.get(real)
+        real = f'/{config}'
+        if await self.__ctx.file_exists(real + SECRET_SUFFIX):
+            # -- The entry file is the serialized entry
+            return await self.__ctx.get(real + SECRET_SUFFIX)
+        if await self.__ctx.file_exists(real + BLOB_SUFFIX):
+            return await self.__ctx.get(real + BLOB_SUFFIX)
+        if '/' not in config:
+            raise FileNotFoundError(path)
+        entry, key = config.rsplit('/', 1)
+        entry, _ = self.__resolve(entry)
+        kv = await self.__entry_kv(entry)
+        try:
+            return Result(stdout = kv[key].encode())
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    @override
+    async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
+        # -- The address space has no directories of its own: a node
+        #    is a branch (a real directory) or an entry (a suffixed
+        #    file); the type says which to build. Below a missing
+        #    branch, mkdir fails with EROFS: the caller creates the
+        #    levels
+        path = self.__addr(path)
+        config, _ = self.__resolve(path)
+        real = f'/{config}'
+        if ftype.is_subtype(FT_COMPOUND):
+            # -- An entry: materialize the empty entry file
+            if await self.__ctx.file_exists(real + SECRET_SUFFIX):
+                return
+            if not await self.__ctx.is_dir(os.path.dirname(real)):
+                raise OSError(
+                    errno.EROFS, f'mkdir: "{path}" is below a missing directory'
+                )
+            await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True)
+        elif ftype.is_subtype(FT_DIR):
+            # -- A branch: a real directory
+            if await self.__ctx.is_dir(real):
+                return
+            if not await self.__ctx.is_dir(os.path.dirname(real)):
+                raise OSError(
+                    errno.EROFS, f'mkdir: "{path}" is below a missing directory'
+                )
+            await self.__ctx.mkdir(real, mode)
+        else:
+            raise ValueError(f'mkdir: "{path}" is not a directory type')
+
+    @override
+    async def _put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None,
+        throw: bool,
+        verbose: bool | None,
+        title: str,
+        owner: str | None,
+        group: str | None,
+        mode: str | None,
+        atomic: bool,
+        ftype: FileType,
+    ) -> Result:
+        path = self.__addr(path)
+        config, is_blob = self.__resolve(path)
+        if ftype.is_subtype(FT_SECRET):
+            # -- The whole entry: the content is the serialized
+            #    entry, written over the entry file; the branch is
+            #    a precondition, as for a blob
+            if is_blob:
+                raise ValueError(
+                    f'put: "{path}" names a blob file, but the ftype is an entry'
+                )
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'/{config}{SECRET_SUFFIX}',
+                content,
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        if ftype.is_subtype(FT_SECRET_FILE):
+            # -- A blob: the content is the file itself, at its
+            #    on-host file name
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'/{config}{BLOB_SUFFIX}',
+                content,
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        entry, sep, key = config.rpartition('/')
+        if not sep or not entry or not key or '/' in key:
+            raise ValueError(f'put: "{path}" is not an entry/key path')
+        entry, _ = self.__resolve(entry)
+        real_entry = f'/{entry}'
+        if ftype.is_subtype(FT_SECRET_KEY):
+            # -- A key: merge into the entry file, which must exist;
+            #    mkdir() is the materialization. The put is atomic,
+            #    so it also overwrites an existing entry file, which
+            #    is mode 0400
+            if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'):
+                raise OSError(errno.EROFS, f'put: "{entry}" has no entry file')
+            kv = await self.__entry_kv(entry)
+            kv[key] = content.decode()
+            m = 0o400 if mode is None else int(mode, 8)
+            return await self.__ctx.put(
+                f'{real_entry}{SECRET_SUFFIX}',
+                serialize_kv(kv),
+                owner = owner,
+                group = group,
+                mode = m,
+                atomic = True,
+            )
+        raise ValueError(f'put: ftype {ftype} is not a store content type')
+
+    @override
+    async def _unlink(self, path: str) -> None:
+        # -- Unlink the file the address names: the address is the
+        #    on-host file name below the subtree, so the host path
+        #    is the composed address itself
+        await self.__ctx.unlink(f'/{self.__addr(path)}')

src/python/jw/pkg/lib/ec/secret_store/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/pkg/lib/ec/secret_store/TarGpg.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import errno
+import io
+import os
+import stat
+import time
+
+from tarfile import TarFile, TarInfo
+from typing import TYPE_CHECKING, Iterable, override
+
+from ...file_types import FT_DIR
+from ...kv import parse_kv
+from ...Result import Result
+from .Base import Base
+from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+
+if TYPE_CHECKING:
+    from typing import Any
+
+    from ...base import StatResult
+    from ...Uri import Uri
+
+class TarGpg(Base):
+    """A store of secrets in a GPG-encrypted tarball in the canonical
+    layout.
+
+    The tarball location is deployment information, not part of the
+    address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The
+    store URI addresses a subtree of the namespace, as in the base
+    class.
+
+    A store path names a member, and optionally a key below an
+    entry. The member of an entry is its address, and its keys are
+    addressable below it; a blob is a member at its address holding
+    non-key-value content. The store does not identify content by
+    member name: a member whose content is a key-value file is an
+    entry, anything else a blob.
+
+    The tarball is fetched and decrypted once, then cached. Fetching,
+    decrypting, and filtering happen in a local exec context, so the
+    GPG key and the decrypted material stay on this machine.
+    """
+
+    def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
+        super().__init__(uri, *args, **kwargs)
+        self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '')
+        self.__tf: TarFile | None = None
+        self.__kv_cache: dict[str, dict[str, str]] = {}
+
+    async def __tar(self) -> TarFile:
+        if self.__tf is None:
+            from ...ProcFilterGpg import ProcFilterGpg
+            from ...util import get
+            from ..Local import Local
+
+            data = (
+                await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local()))
+            ).stdout
+            self.__tf = TarFile(fileobj = io.BytesIO(data))
+        return self.__tf
+
+    async def __members(self) -> dict[str, TarInfo]:
+        tf = await self.__tar()
+        return {m.name: m for m in tf.getmembers()}
+
+    async def __entry_kv(self, entry: str) -> dict[str, str]:
+        if entry not in self.__kv_cache:
+            tf = await self.__tar()
+            try:
+                info = tf.getmember(entry)
+            except KeyError:
+                raise FileNotFoundError(entry) from None
+            buf = tf.extractfile(info)
+            if buf is None:
+                raise Exception(f'Can\'t read store entry "{entry}"')
+            self.__kv_cache[entry] = parse_kv(buf.read())
+        return self.__kv_cache[entry]
+
+    async def __is_entry(self, entry: str) -> bool:
+        # -- The member at entry is an entry: its content is a
+        #    non-empty kv file
+        tf = await self.__tar()
+        try:
+            info = tf.getmember(entry)
+        except KeyError:
+            return False
+        if not info.isfile():
+            return False
+        buf = tf.extractfile(info)
+        if buf is None:
+            return False
+        return len(parse_kv(buf.read())) != 0
+
+    async def __tar_stat(self, storage_path: str) -> StatResult:
+        # -- The metadata of the member that stands behind the node;
+        #    a branch has no member of its own and is synthesized, a
+        #    key keeps the size 0
+        members = await self.__members()
+
+        def __meta(name: str) -> tuple[str, str, float, int, int]:
+            # -- A member's mode carries the permission bits only, so
+            #    the type bits are synthesized; a store member is
+            #    always a regular file
+            mi = members[name]
+            return (
+                mi.uname or str(mi.uid),
+                mi.gname or str(mi.gid),
+                mi.mtime,
+                stat.S_IFREG | (mi.mode & 0o777),
+                mi.size,
+            )
+
+        if await self.__is_entry(storage_path):
+            owner, group, mtime, mode, size = __meta(storage_path)
+            return self._stat_result(
+                owner,
+                group,
+                mtime,
+                FT_SECRET,
+                mode = mode,
+                size = size,
+            )
+        par, sep, key = storage_path.rpartition('/')
+        if sep and key and await self.__is_entry(par):
+            if key in await self.__entry_kv(par):
+                # -- The type is synthesized, the mode the one of the
+                #    entry member, the size stays 0
+                owner, group, mtime, mode, _ = __meta(par)
+                return self._stat_result(
+                    owner,
+                    group,
+                    mtime,
+                    FT_SECRET_KEY,
+                    mode = mode,
+                )
+        if storage_path in members:
+            owner, group, mtime, mode, size = __meta(storage_path)
+            return self._stat_result(
+                owner,
+                group,
+                mtime,
+                FT_SECRET_FILE,
+                mode = mode,
+                size = size,
+            )
+        if not storage_path or \
+                any(name.startswith(f'{storage_path}/') for name in members):
+            return self._stat_result(
+                str(os.getuid()),
+                str(os.getgid()),
+                time.time(),
+                FT_DIR,
+            )
+        raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
+
+    async def __tar_get(self, path: str) -> Result:
+        members = await self.__members()
+        if path == '':
+            raise IsADirectoryError(path)
+        if await self.__is_entry(path):
+            # -- The entry member is the serialized entry
+            tf = await self.__tar()
+            buf = tf.extractfile(members[path])
+            if buf is None:
+                raise Exception(f'Can\'t read store entry "{path}"')
+            return Result(stdout = buf.read())
+        if any(name.startswith(f'{path}/') for name in members):
+            raise IsADirectoryError(path)
+        if '/' in path:
+            entry, key = path.rsplit('/', 1)
+            if await self.__is_entry(entry):
+                kv = await self.__entry_kv(entry)
+                try:
+                    return Result(stdout = kv[key].encode())
+                except KeyError:
+                    raise FileNotFoundError(path) from None
+        info = members.get(path)
+        if info is not None and info.isfile():
+            tf = await self.__tar()
+            buf = tf.extractfile(info)
+            if buf is None:
+                raise Exception(f'Can\'t read store blob "{path}"')
+            return Result(stdout = buf.read())
+        raise FileNotFoundError(path)
+
+    async def __tar_readdir(self, path: str) -> Iterable[str]:
+        members = await self.__members()
+        if await self.__is_entry(path):
+            return (await self.__entry_kv(path)).keys()
+        if path in members:
+            raise NotADirectoryError(path)
+
+        def __names(prefix: str) -> set[str]:
+            # -- No name filter: every member is store content
+            ret: set[str] = set()
+            for name in members:
+                if not name.startswith(prefix):
+                    continue
+                rest = name[len(prefix):]
+                ret.add(rest.partition('/')[0])
+            return ret
+
+        ret = __names(f'{path}/' if path else '')
+        mirror = self._default_path_in_storage(path)
+        if mirror is not None:
+            ret |= __names(f'{mirror}/' if mirror else '')
+        if not ret:
+            raise FileNotFoundError(path)
+        return sorted(ret)
+
+    @override
+    async def _get(
+        self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
+    ) -> Result:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_get)
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_readdir)
+
+    @override
+    async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__tar_stat)

src/python/jw/pkg/lib/ec/secret_store/layout.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from ...file_types import FT_COMPOUND, FT_REG, file_types
+
+# -- The canonical store layout, shared by every store implementation and
+# the install pipeline that consumes it:
+#
+#   default/<path>      entries applying to all hosts
+#   <host-root>/<path>  entries applying to a single host, where
+#                       <host-root> is the host's FQDN, reversed and split
+#                       on dots (com/example/host for host.example.com)
+#
+# Host-specific entries take precedence over default ones.
+
+# Suffix of the key-value secret file accompanying a compiled target
+SECRET_SUFFIX = '.jw-secret'
+
+# Suffix of a non-key-value secret file (certificate, key, ...)
+BLOB_SUFFIX = '.jw-secret-file'
+
+DEFAULT_PREFIX = 'default'
+
+def content_name(name: str) -> str | None:
+    # -- The config path of an on-host secret file name: the name
+    #    with its content suffix removed, or None if the name
+    #    carries neither suffix
+    for suffix in (SECRET_SUFFIX, BLOB_SUFFIX):
+        if name.endswith(suffix):
+            return name.removesuffix(suffix)
+    return None
+
+def host_prefix(hostname: str) -> str:
+    return '/'.join(reversed(hostname.split('.')))
+
+# -- The file types of the store content, registered with the global
+#    file type registry: an entry is a compound of its keys, a key a
+#    leaf below it, a blob a file
+FT_NAME_SECRET = 'secret'
+FT_NAME_SECRET_KEY = 'secret-key'
+FT_NAME_SECRET_FILE = 'secret-file'
+
+file_types.register_types(
+    {
+        FT_NAME_SECRET: FT_COMPOUND,
+        FT_NAME_SECRET_KEY: FT_REG,
+        FT_NAME_SECRET_FILE: FT_REG,
+    }
+)
+
+FT_SECRET = file_types.get(FT_NAME_SECRET)
+FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY)
+FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE)

src/python/jw/pkg/lib/kv.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+def parse_kv(data: bytes) -> dict[str, str]:
+    """Parse a dotenv-style key-value entry into a dict"""
+    ret: dict[str, str] = {}
+    for line in data.decode().splitlines():
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if '=' not in line:
+            continue
+        key, val = line.split('=', 1)
+        key = key.strip()
+        val = val.strip()
+        if key:
+            ret[key] = val
+    return ret
+
+def serialize_kv(kv: dict[str, str]) -> bytes:
+    """Serialize a dict back into a dotenv-style key-value entry"""
+    return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode()

test/integration/jw_pkg/secrets/test.py

@@ -45,6 +45,7 @@ from jw.pkg.lib.Package import Package
 HOST = 'jannet.example.com'
 HOST_PREFIX = '/'.join(reversed(HOST.split('.')))
 PKG = 'fixture-pkg'
+PKG2 = 'fixture-pkg2'
 USER = pwd.getpwuid(os.getuid()).pw_name
 GROUP = grp.getgrgid(os.getgid()).gr_name
 
@@ -189,12 +190,21 @@ def main() -> None:
         #    extract it, and must not leave an empty file behind
         with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f:
             f.write('x=X\n')
+        # -- qux references a macro that has no store entry: the
+        #    compile must fail rather than emit a literal macro
+        with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f:
+            f.write(
+                f'# conf: owner={USER}, group={GROUP}, mode=640\n'
+                'present=PRESENT\n'
+                'missing=__UNRESOLVED__\n'
+            )
         with open(target + '/etc/readme.txt', 'w') as f:
             f.write('plain package file\n')
 
         foo_tmpl = target + '/etc/foo.conf.jw-tmpl'
         bar_tmpl = target + '/etc/bar.conf.jw-tmpl'
         baz_tmpl = target + '/etc/baz.conf.jw-tmpl'
+        qux_tmpl = target + '/etc/qux.conf.jw-tmpl'
         foo_secret = target + '/etc/foo.conf.jw-secret'
         bar_secret = target + '/etc/bar.conf.jw-secret'
         baz_secret = target + '/etc/baz.conf.jw-secret'
@@ -231,7 +241,10 @@ def main() -> None:
 
         distro = FakeDistro(
             Local('local://' + HOST),
-            {PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt']},
+            {
+                PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'],
+                PKG2: [qux_tmpl],
+            },
         )
 
         # -- list-templates: only the .jw-tmpl files of the package
@@ -356,6 +369,37 @@ def main() -> None:
         with open(bar_out) as f:
             assert f.read() == 'plain=plain-value\n'
 
+        # -- compile-templates fails on an unresolved macro: qux
+        #    resolves PRESENT from its secret file, but the template
+        #    also references __UNRESOLVED__, which is not in the
+        #    store. The failure must name the macro, and no compiled
+        #    file may be written
+        qux_secret = target + '/etc/qux.conf.jw-secret'
+        qux_out = target + '/etc/qux.conf'
+        with open(qux_secret, 'w') as f:
+            f.write('PRESENT=there\n')
+        try:
+            run_cli(
+                distro,
+                [
+                    'secrets',
+                    'compile-templates',
+                    '--owner',
+                    USER,
+                    '--group',
+                    GROUP,
+                    '--mode',
+                    '400',
+                    PKG2,
+                ],
+            )
+        except RuntimeError as e:
+            assert 'Unresolved macros' in str(e), e
+            assert '__UNRESOLVED__' in str(e), e
+        else:
+            raise AssertionError('unresolved macro must fail the compile')
+        assert not os.path.exists(qux_out)
+
         out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
         assert out == [foo_out, bar_out, baz_out], out
 

test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py

@@ -0,0 +1,384 @@
+import asyncio
+import errno
+import os
+import stat
+import tempfile
+
+from typing import Any, override
+
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.ec.secret_store.KvFile import KvFile
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.Package import Package
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        target = os.path.join(tmp, 'etc', 'foo.conf')
+        os.makedirs(os.path.dirname(target))
+        with open(target + '.jw-secret', 'w') as f:
+            f.write(
+                '# a comment\n\n  key1 = value1  \nkey2=value2\n'
+                '=novalue\nkey3=\n'
+            )
+
+        # FileContext.create() resolves the kvfile scheme, keeping
+        # the URI as the store's identity
+        created = FileContext.create('kvfile://')
+        assert isinstance(created, KvFile)
+        assert created.uri.scheme == 'kvfile'
+
+        store = KvFile(Local())
+        assert set(await store.readdir(target)) == {'key1', 'key2', 'key3'}
+        assert (await store.get(f'{target}/key1')).stdout == b'value1'
+        assert (await store.get(f'{target}/key3')).stdout == b''
+
+        # The entry is gettable as a whole: the serialized file
+        assert (await store.get(target)).stdout == \
+            b'# a comment\n\n  key1 = value1  \nkey2=value2\n=novalue\nkey3=\n'
+
+        # -- stat() answers the node's type, the metadata of the
+        #    host file behind it; a key has no file of its own, and
+        #    keeps the size 0
+        blob = os.path.join(tmp, 'etc', 'bar.pem')
+        with open(blob + '.jw-secret-file', 'w') as f:
+            f.write('a blob\n')
+        st = await store.stat(target)
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == os.path.getsize(target + '.jw-secret')
+        st = await store.stat(f'{target}/key1')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == 0
+        st = await store.stat(blob)
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        assert st.is_gettable
+        assert st.size == os.path.getsize(blob + '.jw-secret-file')
+        st = await store.stat(os.path.dirname(target))
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert not st.is_gettable
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert not st.is_gettable
+        try:
+            await store.stat(f'{target}/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists(target)
+        assert not await store.file_exists(f'{target}/missing')
+
+        # A missing key and a missing entry raise FileNotFoundError
+        for path in (f'{target}/missing', os.path.join(tmp, 'etc', 'missing.conf')):
+            try:
+                await store.get(path)
+                raise AssertionError(f'expected FileNotFoundError for {path}')
+            except FileNotFoundError:
+                pass
+
+        # put writes a key into the entry file, merging with the
+        # existing keys
+        await store.put(f'{target}/key4', b'value4', ftype = FT_SECRET_KEY)
+        assert (await store.get(f'{target}/key4')).stdout == b'value4'
+        assert (await store.get(f'{target}/key1')).stdout == b'value1'
+        # -- a second put overwrites the entry file, which is mode 0400
+        await store.put(f'{target}/key1', b'value1-new', ftype = FT_SECRET_KEY)
+        assert (await store.get(f'{target}/key1')).stdout == b'value1-new'
+        assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
+        new_entry = os.path.join(tmp, 'etc', 'new.conf')
+        # -- a key put does not create: below a missing entry file
+        #    it fails with EROFS; mkdir() is the materialization
+        try:
+            await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
+            raise AssertionError('expected EROFS')
+        except OSError as e:
+            assert e.errno == errno.EROFS
+        await store.mkdir(new_entry, ftype = FT_SECRET)
+        await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
+        with open(new_entry + '.jw-secret') as f:
+            assert f.read() == 'k=v\n'
+
+        # -- a typed put replaces the entry: the serialized content
+        #    is written over the entry file, mode 0400
+        await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
+        assert set(await store.readdir(target)) == {'new1', 'new2'}
+        assert (await store.get(target)).stdout == b'new1=n1\nnew2=n2\n'
+        with open(target + '.jw-secret') as f:
+            assert f.read() == 'new1=n1\nnew2=n2\n'
+        assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
+
+        # -- a put that changes nothing does not rewrite the file
+        mtime = os.stat(target + '.jw-secret').st_mtime_ns
+        await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
+        assert os.stat(target + '.jw-secret').st_mtime_ns == mtime
+
+        # -- An untyped put is refused: the store content types are
+        #    entry, key, and blob
+        try:
+            await store.put('HOSTNAME', b'x')
+            raise AssertionError('expected ValueError')
+        except ValueError:
+            pass
+
+        # -- put: a blob below a branch lands as a .jw-secret-file at
+        #    its config path, mode 0400
+        pem = os.path.join(tmp, 'etc', 'cert.pem.jw-secret-file')
+        await store.put(
+            os.path.join(tmp, 'etc', 'cert.pem'),
+            b'-----BEGIN CERT-----\n',
+            ftype = FT_SECRET_FILE,
+        )
+        with open(pem) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert stat.S_IMODE(os.stat(pem).st_mode) == 0o400
+
+        # -- mkdir: the type says which node to build, the default
+        #    a branch; a branch is a real directory, an entry an
+        #    empty entry file, mode 0400
+        await store.mkdir(os.path.join(tmp, 'etc'))
+        assert not os.path.exists(os.path.join(tmp, 'etc.jw-secret'))
+        baz = os.path.join(tmp, 'etc', 'baz')
+        await store.mkdir(baz)
+        assert os.path.isdir(baz)
+        bar = os.path.join(tmp, 'etc', 'bar.conf')
+        await store.mkdir(bar, ftype = FT_SECRET)
+        assert os.path.isfile(bar + '.jw-secret')
+        assert open(bar + '.jw-secret').read() == ''
+        assert stat.S_IMODE(os.stat(bar + '.jw-secret').st_mode) == 0o400
+
+        # -- mkdir and put fail with EROFS below a missing directory
+        for fn in (
+                lambda: store.mkdir(os.path.join(tmp, 'no', 'such', 'entry')),
+                lambda: store.put(
+                    os.path.join(tmp, 'no', 'such', 'entry', 'key'),
+                    b'x',
+                    ftype = FT_SECRET_KEY, ),
+        ):
+            try:
+                await fn()
+                raise AssertionError('expected EROFS')
+            except OSError as e:
+                assert e.errno == errno.EROFS
+
+        # -- put forwards owner and group to the host context: an
+        #    unknown group fails the put
+        try:
+            await store.put(
+                f'{target}/key7', b'v7', ftype = FT_SECRET_KEY, group = 'no-such-group'
+            )
+        except AssertionError:
+            raise
+        except Exception as e:
+            assert 'no-such-group' in str(e)
+        else:
+            raise AssertionError('expected the put to fail')
+
+        # unlink drops the file the address names, and a missing
+        # file raises
+        await store.unlink(target + '.jw-secret')
+        assert not os.path.exists(target + '.jw-secret')
+        try:
+            await store.unlink(target + '.jw-secret')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+
+        # -- Branch listing: the config paths below a branch, the
+        #    root included, come from the host's package set, the
+        #    templates of the installed packages, presence-checked
+        #    on the host. A fake distro supplies the list
+        class FakeDistro(SuseDistro):
+
+            def __init__(self, files: list[str]) -> None:
+                super().__init__(Local(), id = 'suse')
+                self.files = files
+                self.queries = 0
+
+            @override
+            async def _pkg_files(self, names: list[str] | None) -> list[str]:
+                self.queries += 1
+                return self.files
+
+        tree = os.path.join(tmp, 'tree')
+        os.makedirs(os.path.join(tree, 'etc'))
+        with open(os.path.join(tree, 'etc', 'foo.conf.jw-secret'), 'w') as f:
+            f.write('k1=v1\nk2=v2\n')
+        with open(os.path.join(tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as f:
+            f.write('-----BEGIN CERT-----\n')
+        with open(os.path.join(tree, 'etc', 'other.conf.jw-secret'), 'w') as f:
+            f.write('o1=v1\n')
+        # -- A template without a secret file is not a config path;
+        #    a package file that is not a template is not one
+        #    either
+        fake = FakeDistro(
+            [
+                f'{tree}/etc/foo.conf.jw-tmpl',
+                f'{tree}/etc/cert.pem.jw-tmpl',
+                f'{tree}/etc/ghost.conf.jw-tmpl',
+                f'{tree}/etc/other.conf',
+            ]
+        )
+        # -- The address space is rooted at the host root; the
+        #    package file list reports host-absolute paths, so the
+        #    fixture tree sits at its path below the root
+        bstore = KvFile(Local(), distro = fake)
+        base = tree.lstrip('/')
+        # -- is_dir answers from the host state, without consulting
+        #    the package set
+        assert await bstore.is_dir('')
+        assert await bstore.is_dir(base)
+        assert await bstore.is_dir(f'{base}/etc/foo.conf')
+        assert not await bstore.is_dir(f'{base}/etc/cert.pem')
+        assert fake.queries == 0
+        # -- is_gettable answers from the node's type: an entry,
+        #    a key, a blob; not a branch, the root included
+        assert await bstore.is_gettable(f'{base}/etc/foo.conf')
+        assert await bstore.is_gettable(f'{base}/etc/foo.conf/k1')
+        assert await bstore.is_gettable(f'{base}/etc/cert.pem')
+        assert not await bstore.is_gettable(f'{base}/etc')
+        assert not await bstore.is_gettable('')
+        assert fake.queries == 0
+        # -- Listing consults it, once: each branch lists the secret
+        #    files below it, under their on-host file names,
+        #    presence-checked
+        assert set(await bstore.readdir('')) == {'tmp'}
+        assert fake.queries == 1
+        assert set(await bstore.readdir(f'{base}/etc')) == \
+            {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
+        assert fake.queries == 1
+        # -- An entry still lists its keys, without the package set
+        assert set(await bstore.readdir(f'{base}/etc/foo.conf')) == {'k1', 'k2'}
+        assert fake.queries == 1
+        # -- A branch without config paths raises FileNotFoundError
+        try:
+            await bstore.readdir('no/such')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- A real directory with no secret children lists empty
+        os.makedirs(os.path.join(tree, 'var', 'empty'))
+        assert set(await bstore.readdir(f'{base}/var/empty')) == set()
+        # -- The blob reads back through the store
+        assert (await bstore.get(f'{base}/etc/cert.pem')).stdout == \
+            b'-----BEGIN CERT-----\n'
+
+        # -- The entry's config path is also a deployed file on the
+        #    host, as on a real host: a key below it has no host
+        #    path, and is_dir() must not stat below the file
+        with open(os.path.join(tree, 'etc', 'foo.conf'), 'w') as f:
+            f.write('deployed\n')
+        assert not (await bstore.is_dir(f'{base}/etc/foo.conf/k1'))
+        assert (await bstore.get(f'{base}/etc/foo.conf/k1')).stdout == b'v1'
+
+        # -- A URI may root the store at a subtree of the host tree:
+        #    create() keeps the URI as the store's identity, and its
+        #    path is composed into every address. The inner context
+        #    is host-rooted, so the on-host files are reached
+        #    without a double path
+        created = FileContext.create(f'kvfile://{tree}/etc')
+        assert created.uri.path == f'{tree}/etc'
+        sstore = KvFile(Local(), uri = f'kvfile://{tree}/etc', distro = fake)
+        await sstore.open()
+        assert set(await sstore.readdir('')) == \
+            {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
+        # -- An entry lists its keys, suffixed and unsuffixed alike
+        assert set(await sstore.readdir('foo.conf.jw-secret')) == {'k1', 'k2'}
+        assert set(await sstore.readdir('foo.conf')) == {'k1', 'k2'}
+        assert (await sstore.get('foo.conf/k1')).stdout == b'v1'
+        assert (await sstore.get('cert.pem')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        # -- A suffixed address names the on-host file of its config
+        #    path, the suffix picking the kind
+        assert (await sstore.get('foo.conf.jw-secret')).stdout == \
+            b'k1=v1\nk2=v2\n'
+        assert (await sstore.get('cert.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        st = await sstore.stat('foo.conf.jw-secret')
+        assert st.ftype is FT_SECRET
+        st = await sstore.stat('foo.conf.jw-secret/k1')
+        assert st.ftype is FT_SECRET_KEY
+        st = await sstore.stat('cert.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE
+        # -- A blob address refuses to list
+        try:
+            await sstore.readdir('cert.pem.jw-secret-file')
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+        # -- A write through the subtree root lands in the subtree
+        await sstore.mkdir('sub')
+        await sstore.mkdir('sub/conf', ftype = FT_SECRET)
+        await sstore.put('sub/conf/k', b'v', ftype = FT_SECRET_KEY)
+        with open(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) as f:
+            assert f.read() == 'k=v\n'
+        await sstore.unlink('sub/conf.jw-secret')
+        assert not os.path.exists(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret'))
+        await sstore.close()
+
+        # -- The autodetected distro applies the app's documented
+        #    filter fallback: JW_DEFAULT_PKG_FILTER reaches
+        #    Distro.instantiate() as the default package filter
+        from unittest import mock
+
+        from jw.pkg.lib.Distro import Distro
+
+        captured: dict[str, Any] = {}
+
+        async def __instantiate(*args: Any, **kwargs: Any) -> FakeDistro:
+            captured['kwargs'] = kwargs
+            return fake
+
+        os.environ['JW_DEFAULT_PKG_FILTER'] = 'url=~jw.dev'
+        try:
+            with mock.patch.object(Distro,
+                                   'instantiate',
+                                   new = classmethod(__instantiate)):
+                astore = KvFile(Local())
+                assert set(await astore.readdir('')) == {'tmp'}
+                flt = captured['kwargs'].get('default_pkg_filter')
+                assert flt is not None
+                assert flt.match(Package('fixture', url = 'https://jw.dev/fixture.rpm'))
+                assert not flt.match(
+                    Package('other', url = 'https://elsewhere/other.rpm')
+                )
+        finally:
+            del os.environ['JW_DEFAULT_PKG_FILTER']
+
+        # -- The lifecycle propagates to the host context: an
+        #    open() pin on the store keeps the host's transport
+        #    open for its duration, and the run()s within it share
+        #    it
+        class CountingLocal(Local):
+
+            def __init__(self) -> None:
+                super().__init__()
+                self.opens = 0
+                self.closes = 0
+
+            @override
+            async def _open(self) -> None:
+                self.opens += 1
+                await super()._open()
+
+            @override
+            async def _close(self) -> None:
+                self.closes += 1
+                await super()._close()
+
+        cl = CountingLocal()
+        pstore = KvFile(cl)
+        await pstore.open()
+        assert cl.opens == 1
+        await cl.run(['true'])
+        await cl.run(['true'])
+        assert cl.opens == 1
+        assert cl.closes == 0
+        await pstore.close()
+        assert cl.closes == 1
+
+asyncio.run(run_tests())
+print('All secrets kvfile store tests passed')

test/unit/python/jw/pkg/lib/ec/secret_store/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/dirs.mk

test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py

@@ -0,0 +1,235 @@
+import asyncio
+import io
+import os
+import shutil
+import stat
+import subprocess
+import tarfile
+import tempfile
+
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.util import copy
+
+def make_tar(path: str, entries: dict[str, str]) -> None:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj = buf, mode = 'w') as tf:
+        for name, text in entries.items():
+            info = tarfile.TarInfo(name)
+            data = text.encode()
+            info.size = len(data)
+            info.mode = 0o400
+            tf.addfile(info, io.BytesIO(data))
+    with open(path, 'wb') as f:
+        f.write(buf.getvalue())
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        # -- A throw-away GPG home with a key for the test
+        gpghome = os.path.join(tmp, 'gnupg')
+        os.makedirs(gpghome, mode = 0o700)
+        env = dict(os.environ, GNUPGHOME = gpghome)
+        subprocess.run(
+            [
+                'gpg',
+                '--batch',
+                '--pinentry-mode',
+                'loopback',
+                '--passphrase',
+                '',
+                '--quick-gen-key',
+                'test@example.com',
+            ],
+            check = True,
+            capture_output = True,
+            env = env,
+        )
+
+        # -- A store in the canonical layout, GPG-encrypted tarball
+        tar_path = os.path.join(tmp, 'secrets.tar')
+        make_tar(
+            tar_path,
+            {
+                'default/etc/a.jw-secret': 'a=1\n',
+                'default/etc/d.jw-secret': 'd=4\n',
+                'com/example/host/etc/a.jw-secret': 'a=2\n',
+                'com/example/host/etc/b.jw-secret': 'b=3\n',
+                # Non-key-value content is stored as a .jw-secret-file
+                # and addressable as a blob
+                'com/example/host/etc/c.pem.jw-secret-file': \
+                    '-----BEGIN CERT-----\n',
+                # A member without a content suffix is store
+                # content too: non-key-value content is a blob
+                'com/example/host/etc/README.md': 'not key-value content\n',
+            }
+        )
+        enc_path = tar_path + '.gpg'
+        subprocess.run(
+            [
+                'gpg',
+                '--batch',
+                '--yes',
+                '--quiet',
+                '--no-tty',
+                '--trust-model',
+                'always',
+                '--recipient',
+                'test@example.com',
+                '--encrypt',
+                '--output',
+                enc_path,
+                tar_path,
+            ],
+            check = True,
+            capture_output = True,
+            env = env,
+        )
+
+        # The store decrypts in a local exec context, which inherits
+        # the test's GNUPGHOME; the tarball location is deployment
+        # information, passed via the environment
+        os.environ['GNUPGHOME'] = gpghome
+        os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path
+
+        # -- The whole store: strict, no fallback
+        store = FileContext.create('targpg://')
+        await store.open()
+        assert isinstance(store, TarGpg)
+        assert set(await store.readdir('')) == {'default', 'com'}
+        # -- The names as stored: the store does not strip or filter
+        #    by member name, a suffixed name being the address of
+        #    its node
+        assert set(await store.readdir('com/example/host/etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
+             'README.md'}
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
+            b'2'
+        assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
+            b'3'
+
+        # The entry is gettable as a whole: the member's content
+        assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
+
+        # A blob is a file: get() returns its bytes, readdir() refuses
+        pem = 'com/example/host/etc/c.pem.jw-secret-file'
+        assert (await store.get(pem)).stdout == b'-----BEGIN CERT-----\n'
+        try:
+            await store.readdir(pem)
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+
+        # A member without a content suffix is store content too: a
+        # non-key-value member is a blob, listed, statable, gettable
+        assert 'README.md' in set(await store.readdir('com/example/host/etc'))
+        st = await store.stat('com/example/host/etc/README.md')
+        assert st.ftype is FT_SECRET_FILE
+        assert (await store.get('com/example/host/etc/README.md')).stdout == \
+            b'not key-value content\n'
+
+        # A missing entry and a missing key raise FileNotFoundError
+        for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
+            try:
+                await store.get(path)
+                raise AssertionError(f'expected FileNotFoundError for {path}')
+            except FileNotFoundError:
+                pass
+
+        # -- stat() answers the node's type, the metadata of the
+        #    member behind it; a key has no member of its own, and a
+        #    branch none at all, so they keep the size 0
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        st = await store.stat('default/etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.size == len((await store.get('default/etc/a.jw-secret')).stdout)
+        st = await store.stat('default/etc/a.jw-secret/a')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        assert st.size == 0
+        st = await store.stat('com/example/host/etc/c.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        assert st.size == len(
+            (await store.get('com/example/host/etc/c.pem.jw-secret-file')).stdout
+        )
+        st = await store.stat('default/etc')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert st.size == 0
+        # -- is_dir answers the node's listability: an entry lists
+        #    its keys, a branch its children; a key is a leaf
+        assert await store.is_dir('default/etc/a.jw-secret')
+        assert await store.is_dir('default/etc')
+        assert not await store.is_dir('default/etc/a.jw-secret/a')
+        try:
+            await store.stat('default/etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists('default/etc/a.jw-secret')
+        assert not await store.file_exists('default/etc/missing')
+        await store.close()
+
+        # -- A host-rooted store applies the default fallback
+        store = FileContext.create('targpg://host.example.com')
+        assert isinstance(store, TarGpg)
+        await store.open()
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
+        # A directory listing is the union with the default tree: d
+        # exists only under default/, but is listable as it is
+        # gettable
+        assert set(await store.readdir('etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
+             'd.jw-secret', 'README.md'}
+        # A miss below the host is retried below the default tree
+        assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
+        # A key listing stays host-wins-wholesale
+        assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
+        # -- stat() distinguishes an entry from a branch and the root
+        st = await store.stat('etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        st = await store.stat('etc')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        st = await store.stat('')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        await store.close()
+
+        # -- A store as a copy source: an entry lands as a single
+        #    file with its serialized content, a branch as a
+        #    directory, and a blob as a file; the default-only entry
+        #    d lands from the union listing
+        host = os.path.join(tmp, 'host-tree')
+        os.makedirs(host)
+        ret = await copy('targpg://host.example.com', host, recursive = True)
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            f'{host}/etc/a.jw-secret',
+            f'{host}/etc/b.jw-secret',
+            f'{host}/etc/c.pem.jw-secret-file',
+            f'{host}/etc/d.jw-secret',
+            f'{host}/etc/README.md',
+        }
+        with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
+            assert f.read() == 'a=2\n'
+        assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
+        with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        # -- A single object that is an entry: the serialized whole
+        #    lands at the destination
+        dst = os.path.join(tmp, 'copied', 'entry')
+        os.makedirs(os.path.dirname(dst))
+        ret = await copy('targpg://host.example.com/etc/a.jw-secret', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == 'a=2\n'
+
+def main() -> None:
+    if not shutil.which('gpg'):
+        print('Skipping secrets targpg store tests (gpg required)')
+        return
+    asyncio.run(run_tests())
+    print('All secrets targpg store tests passed')
+
+main()

test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py

@@ -0,0 +1,41 @@
+from jw.pkg.lib.ec.secret_store.layout import (
+    BLOB_SUFFIX,
+    DEFAULT_PREFIX,
+    FT_SECRET,
+    FT_SECRET_FILE,
+    FT_SECRET_KEY,
+    SECRET_SUFFIX,
+    content_name,
+    host_prefix,
+)
+from jw.pkg.lib.file_types import FT_COMPOUND, FT_DIR, FT_REG
+
+# The layout constants are the store contract
+assert DEFAULT_PREFIX == 'default'
+assert SECRET_SUFFIX == '.jw-secret'
+assert BLOB_SUFFIX == '.jw-secret-file'
+
+# content_name strips the content suffix, and names without a
+# content suffix are not store content
+assert content_name('etc/a.jw-secret') == 'etc/a'
+assert content_name('etc/c.pem.jw-secret-file') == 'etc/c.pem'
+assert content_name('etc/README.md') is None
+
+# An FQDN maps to its reversed, dot-split host root
+assert host_prefix('host.example.com') == 'com/example/host'
+assert host_prefix('a.b.c.d') == 'd/c/b/a'
+
+# A single-label hostname maps to itself
+assert host_prefix('localhost') == 'localhost'
+
+# The store content types are registered with the file types: an
+# entry is a kind of compound, a key and a blob a kind of file
+assert FT_SECRET.is_subtype(FT_COMPOUND)
+assert FT_SECRET.is_subtype(FT_REG)
+assert not FT_SECRET.is_subtype(FT_DIR)
+assert FT_SECRET_KEY.is_subtype(FT_REG)
+assert not FT_SECRET_KEY.is_subtype(FT_DIR)
+assert FT_SECRET_FILE.is_subtype(FT_REG)
+assert not FT_SECRET_FILE.is_subtype(FT_DIR)
+
+print('All secrets layout tests passed')