lib.ec.secret_store.Sops: Add store class

This commit adds a new class inheriting from secret_store.Base: Sops. The
currently available classes allow to store secrets offline in a
GPG-encrypted tarball, but those are not versioned. This commit allows to
use a SOPS / AGE-encrypted Git repository as storage backend instead.

The additions in detail:

- Sops: the store implementation; the repository location comes from
  JW_SECRETS_SOPS_REPO, get() on an entry returns the decrypted serialized
  content, and a typed put (FT_SECRET) re-encrypts the entry wholesale
- The age key comes from JW_SECRETS_SOPS_KEY_PATH only: the house variable
  is the sole input to the store, and SOPS_AGE_KEY_FILE is set on the sops
  subprocess per invocation, never read from the process environment;
  SopsKeyError is raised when the variable is not set
- FileContext.create(): the sops:// scheme
- The store's unit tests, covering the node types, the deploy and fetch
  round trip, the whole-entry get and put, the key resolution, and the
  default fallback

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0
Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0

src/python/jw/pkg/lib/FileContext.py

@@ -360,6 +360,10 @@ class FileContext(abc.ABC):
                 from .ec.Curl import Curl
 
                 return Curl(uri, *args, **kwargs)
+            case 'sops':
+                from .ec.secret_store.Sops import Sops
+
+                return Sops(uri, *args, **kwargs)
             case 'targpg':
                 from .ec.secret_store.TarGpg import TarGpg
 

src/python/jw/pkg/lib/ec/secret_store/Sops.py

@@ -0,0 +1,410 @@
+from __future__ import annotations
+
+import errno
+import hashlib
+import os
+
+from contextlib import suppress
+from typing import TYPE_CHECKING, Iterable, override
+
+if TYPE_CHECKING:
+    from typing import Any
+
+    from ...ExecContext import ExecContext
+    from ...Uri import Uri
+    from ...base import StatResult
+    from ...file_types import FileType
+
+from ...file_types import FT_DIR
+from ...kv import parse_kv, serialize_kv
+from ...Result import Result
+from .Base import Base
+from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+
+def _sops_dotenv_norm(content: bytes) -> bytes:
+    # -- The form sops' dotenv serialization produces: blank
+    #    lines dropped, a single trailing newline, everything
+    #    else verbatim. The identity for content already in
+    #    that form
+    if not content:
+        return content
+    return b'\n'.join(line for line in content.split(b'\n') if line) + b'\n'
+
+class SopsKeyError(Exception):
+    """The age key file of the sops store could not be resolved"""
+
+class Sops(Base):
+    """A secret store of SOPS-encrypted files in a canonical layout.
+
+    The repository location is deployment information, not part of
+    the address: JW_SECRETS_SOPS_REPO names a local directory (used
+    as-is) or a git URL (cloned into a temporary directory until
+    close()), and JW_SECRETS_SOPS_ROOT an optional base directory
+    within it. The store URI addresses a subtree of the namespace,
+    as in the base class.
+
+    A store path names a file, and optionally a key below an entry.
+    The file of an entry is its address, and its keys are
+    addressable below it; a blob is a file at its address holding
+    non-key-value content. The store does not identify content by
+    file name: a file that decrypts to a key-value file is an entry,
+    anything else a blob. The repository files are sops files in
+    dotenv format: an entry's keys are its data lines, a blob is
+    stored under a single data key.
+
+    put() writes the file at its address, sops-encrypting the
+    content for its type: an entry wholesale, a key merged into its
+    entry, a blob as raw bytes. The store accepts entry, key, and
+    blob content only; the encryption recipient is derived from the
+    SOPS age key, and committing the repository is the caller's
+    step.
+
+    Files are decrypted with sops in a local exec context, and the
+    age key file is resolved from JW_SECRETS_SOPS_KEY_PATH, so the
+    key and the decrypted material stay on this machine.
+    """
+
+    def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
+        super().__init__(uri, *args, **kwargs)
+        self.__repo = os.environ.get('JW_SECRETS_SOPS_REPO', '.')
+        self.__base = os.environ.get('JW_SECRETS_SOPS_ROOT', '').strip('/')
+        self.__kv_cache: dict[str, dict[str, str]] = {}
+        self.__decrypt_cache: dict[bytes, bytes] = {}
+        self.___keyfile: str | None = None
+        self.___recipient: str | None = None
+        self.___git_checkout: str | None = None
+        self.___crypt_ec: ExecContext | None = None
+
+    async def __crypt_ec(self) -> ExecContext:
+        if self.___crypt_ec is None:
+            from ..Local import Local
+
+            # -- Security: decrypt in a local exec context for now so
+            #    that the SOPS key and the decrypted material never
+            #    leave this machine
+            self.___crypt_ec = Local()
+        return self.___crypt_ec
+
+    async def __git_checkout(self) -> str:
+        if self.___git_checkout is None:
+            from ...base import InputMode
+
+            ec = await self.__crypt_ec()
+            if os.path.isdir(self.__repo):
+                self.___git_checkout = self.__repo
+            else:
+                ret = await ec.mktemp('/tmp/jw-pkg-sops-XXXXXX', directory = True)
+                try:
+                    await ec.run(
+                        ['git', 'clone', '--quiet', self.__repo, ret],
+                        cmd_input = InputMode.NonInteractive,
+                        throw = True,
+                    )
+                except Exception:
+                    await ec.erase(ret)
+                    raise
+                self.___git_checkout = ret
+        return self.___git_checkout
+
+    async def __git_subdir(self, path: str) -> str:
+        checkout = await self.__git_checkout()
+        base = f'{checkout}/{self.__base}' if self.__base else checkout
+        return f'{base}/{path}' if path else base
+
+    async def __git_get(self, path: str) -> Result:
+        if path == '' or os.path.isdir(await self.__git_subdir(path)):
+            raise IsADirectoryError(path)
+        file_path = await self.__git_subdir(path)
+        if await self.__is_entry(path):
+            # -- The entry file is the serialized entry
+            return Result(stdout = await self.__decrypt(file_path))
+        if '/' in path:
+            entry, key = path.rsplit('/', 1)
+            if await self.__is_entry(entry):
+                kv = await self.__decrypt_kv_content(entry)
+                try:
+                    return Result(stdout = kv[key].encode())
+                except KeyError:
+                    raise FileNotFoundError(path) from None
+        if os.path.isfile(file_path):
+            from ...base import InputMode
+
+            ec = await self.__crypt_ec()
+            result = await ec.run(
+                [
+                    'sops',
+                    '--decrypt',
+                    '--input-type',
+                    'dotenv',
+                    '--output-type',
+                    'binary',
+                    file_path,
+                ],
+                cmd_input = InputMode.NonInteractive,
+                mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()},
+                throw = True,
+            )
+            return Result(stdout = result.stdout)
+        raise FileNotFoundError(path)
+
+    async def __git_readdir(self, path: str) -> Iterable[str]:
+
+        def __names(d: str) -> set[str]:
+            # -- No name filter: every file is store content
+            return set(os.listdir(d))
+
+        if await self.__is_entry(path):
+            return (await self.__decrypt_kv_content(path)).keys()
+        os_path = await self.__git_subdir(path)
+        if os.path.isfile(os_path):
+            raise NotADirectoryError(path)
+        if os.path.isdir(os_path):
+            names = __names(os_path)
+            default_path = self._default_path_in_storage(path)
+            if default_path is not None:
+                d = await self.__git_subdir(default_path)
+                if os.path.isdir(d):
+                    names |= __names(d)
+            return sorted(names)
+        raise FileNotFoundError(path)
+
+    async def __git_stat(
+        self,
+        storage_path: str,
+        follow_symlinks: bool,
+    ) -> StatResult:
+        stat_fn = os.lstat if not follow_symlinks else os.stat
+        file_path = await self.__git_subdir(storage_path)
+        if await self.__is_entry(storage_path):
+            return self._stat_result_from_os(stat_fn(file_path), FT_SECRET)
+        entry, sep, key = storage_path.rpartition('/')
+        if sep and key and await self.__is_entry(entry):
+            if key in await self.__decrypt_kv_content(entry):
+                # -- A key has no file of its own: the size stays 0
+                st = stat_fn(await self.__git_subdir(entry))
+                return self._stat_result_from_os(st, FT_SECRET_KEY, size = 0)
+        if os.path.isfile(file_path):
+            return self._stat_result_from_os(stat_fn(file_path), FT_SECRET_FILE)
+        if os.path.isdir(file_path):
+            return self._stat_result_from_os(stat_fn(file_path), FT_DIR)
+        raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
+
+    async def __keyfile(self) -> str:
+        # -- The house variable is the sole input; SOPS_AGE_KEY_FILE
+        #    is set per invocation on the sops subprocess, never read
+        #    from the process environment
+        if self.___keyfile is None:
+            keyfile = os.environ.get('JW_SECRETS_SOPS_KEY_PATH')
+            if not keyfile:
+                raise SopsKeyError('JW_SECRETS_SOPS_KEY_PATH is not set')
+            self.___keyfile = keyfile
+        return self.___keyfile
+
+    async def __recipient(self) -> str:
+        if self.___recipient is None:
+            from ...base import InputMode
+
+            keyfile = await self.__keyfile()
+            ec = await self.__crypt_ec()
+            # -- Pipe the output: in interactive mode the pty relays
+            #    the child's stdout to the console as well
+            result = await ec.run(
+                ['age-keygen', '-y', keyfile],
+                cmd_input = InputMode.NonInteractive,
+                throw = True,
+            )
+            self.___recipient = result.stdout.decode().strip()
+        return self.___recipient
+
+    async def __put_key(self, entry: str, key: str, content: bytes) -> Result:
+        # -- Merge the key into the entry, replacing the entry's
+        #    existing value for the key
+        self.__kv_cache.pop(entry, None)
+        try:
+            kv = await self.__decrypt_kv_content(entry)
+        except FileNotFoundError:
+            kv = {}
+        kv[key] = content.decode()
+        await self.__encrypt(await self.__git_subdir(entry), serialize_kv(kv), 'dotenv')
+        return Result()
+
+    async def __put_entry(self, entry: str, content: bytes) -> Result:
+        # -- Replace the entry: the content is the serialized
+        #    entry, encrypted wholesale, so a key not in it is gone
+        self.__kv_cache.pop(entry, None)
+        await self.__encrypt(await self.__git_subdir(entry), content, 'dotenv')
+        return Result()
+
+    async def __put_blob(self, path: str, content: bytes) -> Result:
+        await self.__encrypt(await self.__git_subdir(path), content, 'binary')
+        return Result()
+
+    async def __encrypt(self, path: str, content: bytes, input_type: str) -> None:
+        # -- Encrypt the content to the repository file. The
+        #    plaintext lives in a temporary file next to it until the
+        #    encryption succeeds, and the recipient is derived from
+        #    the SOPS age key the store decrypts with
+        from ...base import InputMode
+
+        ec = await self.__crypt_ec()
+        recipient = await self.__recipient()
+        os.makedirs(os.path.dirname(path), exist_ok = True)
+        tmp = f'{path}.tmp'
+        try:
+            with open(tmp, 'wb') as f:
+                f.write(content)
+            await ec.run(
+                [
+                    'sops',
+                    '--input-type',
+                    input_type,
+                    '--output-type',
+                    'dotenv',
+                    '--encrypt',
+                    '--age',
+                    recipient,
+                    '--output',
+                    path,
+                    tmp,
+                ],
+                cmd_input = InputMode.NonInteractive,
+                mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()},
+                throw = True,
+            )
+        except Exception:
+            with suppress(FileNotFoundError):
+                os.unlink(tmp)
+            raise
+        os.unlink(tmp)
+
+    async def __decrypt(self, path: str) -> bytes:
+        # -- The decryption is a pure function of the file's content,
+        #    so the result is cached by the content's hash: the entry
+        #    check and the content fetch of one get() share one
+        #    decryption, a repeated read of an unchanged file is
+        #    free, and a changed file - after a put, or an external
+        #    update of the store - re-decrypts under its new hash
+        #    without any invalidation
+        if not os.path.isfile(path):
+            raise FileNotFoundError(path)
+        with open(path, 'rb') as f:
+            raw = f.read()
+        key = hashlib.sha256(raw).digest()
+        ret = self.__decrypt_cache.get(key)
+        if ret is None:
+            ec = await self.__crypt_ec()
+            ret = (
+                await ec.run(
+                    [
+                        'sops',
+                        'decrypt',
+                        '--input-type',
+                        'dotenv',
+                        '--output-type',
+                        'dotenv',
+                    ],
+                    cmd_input = raw,
+                    mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()},
+                    throw = True,
+                )
+            ).stdout or b''
+            self.__decrypt_cache[key] = ret
+        return ret
+
+    async def __decrypt_kv_content(self, entry: str) -> dict[str, str]:
+        ret = self.__kv_cache.get(entry)
+        if ret is None:
+            path = await self.__git_subdir(entry)
+            ret = parse_kv(await self.__decrypt(path))
+            self.__kv_cache[entry] = ret
+        return ret
+
+    async def __is_entry(self, entry: str) -> bool:
+        # -- The file at entry is an entry: it decrypts to a kv
+        #    file, and the single data key sops gives a blob is not
+        #    all of its content
+        path = await self.__git_subdir(entry)
+        if not os.path.isfile(path):
+            return False
+        try:
+            kv = await self.__decrypt_kv_content(entry)
+        except SopsKeyError:
+            raise
+        except Exception:
+            return False
+        return not (len(kv) == 1 and 'data' in kv)
+
+    # -- Override base class methods
+
+    @override
+    async def _close(self) -> None:
+        if self.___git_checkout is not None and self.___git_checkout != self.__repo:
+            ec = await self.__crypt_ec()
+            await ec.erase(self.___git_checkout)
+        self.___git_checkout = None
+
+    @override
+    async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(
+            path,
+            lambda p: self.__git_stat(p, follow_symlinks),
+        )
+
+    @override
+    async def _get(
+        self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
+    ) -> Result:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__git_get)
+
+    @override
+    async def _put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None,
+        throw: bool,
+        verbose: bool | None,
+        title: str,
+        owner: str | None,
+        group: str | None,
+        mode: str | None,
+        atomic: bool,
+        ftype: FileType,
+    ) -> Result:
+        # -- Resolve the address to the absolute namespace path in
+        #    the host tree; a write does not take the default
+        #    fallback.
+        path = self._unchroot(path)
+        if not path:
+            raise ValueError(f'put: "{path}" is not an entry/key or blob path')
+        storage_path = self._path_in_storage(path)
+        if ftype.is_subtype(FT_SECRET):
+            return await self.__put_entry(storage_path, content)
+        if ftype.is_subtype(FT_SECRET_KEY):
+            entry, sep, key = storage_path.rpartition('/')
+            if not sep or not key:
+                raise ValueError(f'put: "{path}" is not of the form path/key')
+            return await self.__put_key(entry, key, content)
+        if ftype.is_subtype(FT_SECRET_FILE):
+            return await self.__put_blob(storage_path, content)
+        raise ValueError(f'put: ftype {ftype} is not a store content type')
+
+    @override
+    def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
+        # -- sops re-serializes a dotenv entry on decrypt: it
+        #    drops blank lines and adds a trailing newline, so
+        #    the byte check of Base._put_is_noop() never fires
+        #    for content outside that form, and every put
+        #    re-encrypts the unchanged entry. Compare the
+        #    sops-canonical forms. A blob round-trips
+        #    byte-exact and keeps the byte check
+        if ftype.is_subtype(FT_SECRET):
+            return _sops_dotenv_norm(existing) == _sops_dotenv_norm(content)
+        return existing == content
+
+    @override
+    async def _readdir(self, path: str) -> Iterable[str]:
+        path = self._unchroot(path)
+        return await self._call_with_fallback(path, self.__git_readdir)

test/unit/python/jw/pkg/lib/ec/secret_store/Sops/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py

@@ -0,0 +1,645 @@
+import asyncio
+import os
+import shutil
+import stat
+import subprocess
+import tempfile
+
+from glob import glob
+from typing import override
+
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.ec.secret_store.KvFile import KvFile
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.ec.secret_store.Sops import Sops, SopsKeyError
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.kv import parse_kv
+from jw.pkg.lib.util import copy
+
+def encrypt(repo: str, rel: str, text: str, input_type: str, recipient: str) -> None:
+    path = os.path.join(repo, rel)
+    os.makedirs(os.path.dirname(path), exist_ok = True)
+    with open(path, 'w') as f:
+        f.write(text)
+    subprocess.run(
+        [
+            'sops',
+            '--input-type',
+            input_type,
+            '--output-type',
+            'dotenv',
+            '--encrypt',
+            '--age',
+            recipient,
+            '--output',
+            path,
+            path
+        ],
+        check = True,
+        capture_output = True,
+    )
+
+async def run_tests() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        # -- A throw-away age key, resolved via JW_SECRETS_SOPS_KEY_PATH
+        keyfile = os.path.join(tmp, 'key.txt')
+        subprocess.run(
+            ['age-keygen', '-o', keyfile], check = True, capture_output = True
+        )
+        recipient = subprocess.run(
+            ['age-keygen', '-y', keyfile], check = True, capture_output = True
+        ).stdout.decode().strip()
+        os.environ['JW_SECRETS_SOPS_KEY_PATH'] = keyfile
+
+        # -- A store in the canonical layout, SOPS-encrypted
+        repo = os.path.join(tmp, 'repo')
+        encrypt(repo, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient)
+        encrypt(repo, 'default/etc/d.jw-secret', 'd=4\n', 'dotenv', recipient)
+        encrypt(repo, 'com/example/host/etc/a.jw-secret', 'a=2\n', 'dotenv', recipient)
+        encrypt(repo, 'com/example/host/etc/b.jw-secret', 'b=3\n', 'dotenv', recipient)
+        # Non-key-value content is stored as a .jw-secret-file and
+        # addressable as a blob
+        encrypt(
+            repo,
+            'com/example/host/etc/c.pem.jw-secret-file',
+            '-----BEGIN CERT-----\n',
+            'binary',
+            recipient
+        )
+        # A branch with a blob, for the deploy: it must land as a
+        # real directory, not a phantom entry
+        encrypt(
+            repo,
+            'com/example/host/etc/new/x.pem.jw-secret-file',
+            '-----BEGIN NEW BRANCH CERT-----\n',
+            'binary',
+            recipient
+        )
+        # A file without a content suffix is store content too:
+        # non-key-value content is a blob
+        encrypt(
+            repo,
+            'com/example/host/etc/README.md',
+            'not key-value content\n',
+            'binary',
+            recipient
+        )
+
+        # -- The whole store: strict, no fallback
+        os.environ['JW_SECRETS_SOPS_REPO'] = repo
+        os.environ['JW_SECRETS_SOPS_ROOT'] = ''
+        store = FileContext.create('sops://')
+        await store.open()
+        assert set(await store.readdir('')) == {'default', 'com'}
+        # -- The names as stored: the store does not strip or filter
+        #    by file name, a suffixed name being the address of its
+        #    node
+        assert set(await store.readdir('com/example/host/etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', 'new',
+             'README.md'}
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
+            b'2'
+        assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
+            b'3'
+
+        # The entry is gettable as a whole: the decrypted dotenv
+        assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
+
+        # A blob is a file: get() returns its bytes, readdir() refuses
+        assert (await store.get(
+            'com/example/host/etc/c.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        try:
+            await store.readdir('com/example/host/etc/c.pem.jw-secret-file')
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+
+        # A file without a content suffix is store content too: a
+        # non-key-value file is a blob, listed, statable, gettable
+        assert 'README.md' in set(await store.readdir('com/example/host/etc'))
+        st = await store.stat('com/example/host/etc/README.md')
+        assert st.ftype is FT_SECRET_FILE
+        assert (await store.get('com/example/host/etc/README.md')).stdout == \
+            b'not key-value content\n'
+
+        # A missing entry and a missing key raise FileNotFoundError
+        for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
+            try:
+                await store.get(path)
+                raise AssertionError(f'expected FileNotFoundError for {path}')
+            except FileNotFoundError:
+                pass
+        await store.close()
+        # A local directory is used as-is; close() must not remove it
+        assert os.path.isdir(repo)
+
+        # -- A host-rooted store applies the default fallback
+        store = FileContext.create('sops://host.example.com')
+        assert isinstance(store, Sops)
+        await store.open()
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
+        # A directory listing is the union with the default tree: d
+        # exists only under default/, but is listable as it is
+        # gettable
+        assert set(await store.readdir('etc')) == \
+            {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
+             'd.jw-secret', 'new', 'README.md'}
+        # A miss below the host is retried below the default tree
+        assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
+        # A key listing stays host-wins-wholesale
+        assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
+
+        # -- stat() answers the node's type: an entry is a file that
+        #    enumerates its keys, a key a leaf below it, a blob a
+        #    file, a branch a plain directory; a key has no file of
+        #    its own, and keeps the size 0
+        st = await store.stat('etc')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        assert st.size == os.path.getsize(os.path.join(repo, 'com/example/host/etc'))
+        st = await store.stat('etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.size == os.path.getsize(
+            os.path.join(repo, 'com/example/host/etc/a.jw-secret')
+        )
+        st = await store.stat('etc/a.jw-secret/a')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        assert st.size == 0
+        st = await store.stat('etc/c.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        assert st.size == os.path.getsize(
+            os.path.join(repo, 'com/example/host/etc/c.pem.jw-secret-file')
+        )
+        # A miss below the host falls back to the default tree
+        st = await store.stat('etc/d.jw-secret/d')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        try:
+            await store.stat('etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists('etc/a.jw-secret')
+        assert not await store.file_exists('etc/missing')
+        await store.close()
+
+        # -- A store address as a copy source: FileContext.create()
+        #    resolves the scheme
+        dst = os.path.join(tmp, 'copied', 'a')
+        os.makedirs(os.path.dirname(dst))
+        ret = await copy('sops://host.example.com/etc/a.jw-secret/a', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == '2'
+
+        # A single object that is an entry: the serialized whole
+        # lands at the destination
+        dst = os.path.join(tmp, 'copied', 'entry')
+        os.makedirs(os.path.dirname(dst), exist_ok = True)
+        ret = await copy('sops://host.example.com/etc/a.jw-secret', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == 'a=2\n'
+
+        # -- A store entry as a recursive copy source: the entry is
+        #    gettable as a whole, and the walk terminates at it
+        tree = os.path.join(tmp, 'tree')
+        ret = await copy(
+            'sops://host.example.com/etc/a.jw-secret', tree, recursive = True
+        )
+        assert ret == tree
+        with open(tree) as f:
+            assert f.read() == 'a=2\n'
+
+        # -- Deploy: a recursive copy from the store to the on-host
+        #    layout; the kvfile scheme wraps the target context,
+        #    chrooted at the root of the on-host tree. The host tree
+        #    is a precondition: etc/ exists
+        host_tree = os.path.join(tmp, 'host-tree')
+        os.makedirs(os.path.join(host_tree, 'etc'))
+        ret = await copy(
+            'sops://host.example.com', f'kvfile://{host_tree}', recursive = True
+        )
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            'etc/a.jw-secret',
+            'etc/b.jw-secret',
+            'etc/c.pem.jw-secret-file',
+            'etc/d.jw-secret',
+            'etc/new/x.pem.jw-secret-file',
+            'etc/README.md',
+        }
+        # -- Entries land as entry files, mode 0400; the host
+        #    value wins over default, and the default-only entry d
+        #    lands from the union listing
+        for entry, kv in (
+            ('etc/a', {'a': '2'}),
+            ('etc/b', {'b': '3'}),
+            ('etc/d', {'d': '4'}),
+        ):
+            path = os.path.join(host_tree, *entry.split('/')) + '.jw-secret'
+            assert os.path.isfile(path)
+            assert stat.S_IMODE(os.stat(path).st_mode) == 0o400
+            with open(path, 'rb') as fb:
+                assert parse_kv(fb.read()) == kv
+        # -- The blob lands as a .jw-secret-file at its config path,
+        #    and no .jw-secret garbage is created for the branch
+        pem = os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')
+        with open(pem) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert not os.path.exists(os.path.join(host_tree, 'etc.jw-secret'))
+        # -- The unsuffixed blob lands suffixed at its config path
+        readme = os.path.join(host_tree, 'etc', 'README.md.jw-secret-file')
+        with open(readme) as f:
+            assert f.read() == 'not key-value content\n'
+        # -- A new branch is a real directory, not a phantom entry:
+        #    its blob lands suffixed below it
+        assert os.path.isdir(os.path.join(host_tree, 'etc', 'new'))
+        new_pem = os.path.join(host_tree, 'etc', 'new', 'x.pem.jw-secret-file')
+        with open(new_pem) as f:
+            assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
+        assert not os.path.exists(os.path.join(host_tree, 'etc', 'new.jw-secret'))
+
+        # -- A recursive copy onto a plain host tree: an entry
+        #    lands as a single file with its serialized content, a
+        #    branch as a directory, and a blob as a file
+        host = os.path.join(tmp, 'plain-host')
+        os.makedirs(host)
+        ret = await copy('sops://host.example.com', host, recursive = True)
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            f'{host}/etc/a.jw-secret',
+            f'{host}/etc/b.jw-secret',
+            f'{host}/etc/c.pem.jw-secret-file',
+            f'{host}/etc/d.jw-secret',
+            f'{host}/etc/new/x.pem.jw-secret-file',
+            f'{host}/etc/README.md',
+        }
+        with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
+            assert f.read() == 'a=2\n'
+        assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
+        with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert os.path.isdir(os.path.join(host, 'etc', 'new'))
+        with open(os.path.join(host, 'etc', 'new', 'x.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
+
+        # -- A single object into the on-host layout: with --parents,
+        #    the object lands at its source address
+        host_tree = os.path.join(tmp, 'host-tree2')
+        os.makedirs(os.path.join(host_tree, 'etc'))
+        ret = await copy(
+            'sops://host.example.com/etc/a.jw-secret/a',
+            f'kvfile://{host_tree}',
+            parents = True
+        )
+        assert ret == 'etc/a.jw-secret/a'
+        with open(os.path.join(host_tree, 'etc', 'a.jw-secret'), 'rb') as fb:
+            assert parse_kv(fb.read()) == {'a': '2'}
+        ret = await copy(
+            'sops://host.example.com/etc/c.pem.jw-secret-file',
+            f'kvfile://{host_tree}',
+            parents = True
+        )
+        assert ret == 'etc/c.pem.jw-secret-file'
+        with open(os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+
+        # -- Fetch: a recursive copy from the on-host layout into
+        #    the store. The kvfile source is handed the host's
+        #    package set, and lists its branches from it; the
+        #    entries and the blob land below the destination host's
+        #    prefix
+        host_tree = os.path.join(tmp, 'fetch-host')
+        os.makedirs(os.path.join(host_tree, 'etc'))
+        with open(os.path.join(host_tree, 'etc', 'foo.conf.jw-secret'), 'w') as fh:
+            fh.write('f1=1\nf2=2\n')
+        with open(os.path.join(host_tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh:
+            fh.write('-----BEGIN FETCHED CERT-----\n')
+        # -- As on a real host, the deployed config and cert are
+        #    present too: the key paths below the entry have no
+        #    host counterpart
+        with open(os.path.join(host_tree, 'etc', 'foo.conf'), 'w') as fh:
+            fh.write('deployed config\n')
+        with open(os.path.join(host_tree, 'etc', 'cert.pem'), 'w') as fh:
+            fh.write('deployed cert\n')
+
+        class FakeDistro(SuseDistro):
+
+            def __init__(self, files: list[str]) -> None:
+                super().__init__(Local(), id = 'suse')
+                self.files = files
+
+            @override
+            async def _pkg_files(self, names: list[str] | None) -> list[str]:
+                return self.files
+
+        fake = FakeDistro(
+            [
+                f'{host_tree}/etc/foo.conf.jw-tmpl',
+                f'{host_tree}/etc/cert.pem.jw-tmpl',
+            ]
+        )
+        # -- The package file list reports host-absolute paths, so
+        #    the fixture tree sits at its path below the host root
+        src = KvFile(Local(), distro = fake)
+        base = host_tree.lstrip('/')
+        ret = await copy(src, 'sops://fetch.example.com', recursive = True)
+        assert isinstance(ret, list)
+        # -- The copy transports the on-host file names unchanged
+        assert set(ret) == {
+            f'{base}/etc/foo.conf.jw-secret',
+            f'{base}/etc/cert.pem.jw-secret-file',
+        }
+        fstore = FileContext.create('sops://fetch.example.com')
+        await fstore.open()
+        assert set(await fstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \
+            {'f1', 'f2'}
+        assert (await fstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \
+            b'1'
+        assert (await fstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN FETCHED CERT-----\n'
+        await fstore.close()
+
+        # -- A host subtree: the fallback mirrors below the subtree.
+        #    root stays the URI path; the namespace root is separate
+        store = FileContext.create('sops://host.example.com/etc')
+        assert isinstance(store, Sops)
+        await store.open()
+        assert store.root == '/etc'
+        assert (await store.get('a.jw-secret/a')).stdout == b'2'
+        assert (await store.get('d.jw-secret/d')).stdout == b'4'
+        await store.close()
+
+        # -- A store is always chrooted: an explicit value is
+        #    overridden, the address space stays relative
+        store = Sops('sops://host.example.com/etc', chroot = False)
+        await store.open()
+        assert store.chroot
+        assert (await store.get('a.jw-secret/a')).stdout == b'2'
+        assert (await store.get('d.jw-secret/d')).stdout == b'4'
+        assert set(await store.readdir('a.jw-secret')) == {'a'}
+        await store.close()
+
+        # -- The default tree is terminal: no fallback from it
+        store = FileContext.create('sops://default')
+        await store.open()
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'1'
+        try:
+            await store.get('etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        await store.close()
+
+        # -- A store as a copy destination: the walk asks the source
+        #    which node is an entry; entries land as entry files
+        #    below the destination host's prefix, blobs at their
+        #    config path, sops-encrypted in the repository
+        ret = await copy(
+            'sops://host.example.com', 'sops://host2.example.com', recursive = True
+        )
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            'etc/a.jw-secret',
+            'etc/b.jw-secret',
+            'etc/c.pem.jw-secret-file',
+            'etc/d.jw-secret',
+            'etc/new/x.pem.jw-secret-file',
+            'etc/README.md',
+        }
+        store = FileContext.create('sops://host2.example.com')
+        await store.open()
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
+        assert (await store.get('etc/b.jw-secret/b')).stdout == b'3'
+        assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
+        assert (await store.get('etc/c.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        await store.close()
+        # -- The content lands in suffixed files below the host
+        #    prefix, and no entry is created for the branch
+        assert os.path.isfile(os.path.join(repo, 'com/example/host2/etc/a.jw-secret'))
+        assert os.path.isfile(
+            os.path.join(repo, 'com/example/host2/etc/c.pem.jw-secret-file')
+        )
+        assert not os.path.exists(os.path.join(repo, 'com/example/host2/etc.jw-secret'))
+
+        # -- put(): a key put merges into its entry, a leaf put
+        #    writes a blob, a leaf below an existing entry lands as
+        #    a key there, and a top-level object is refused
+        store = FileContext.create('sops://host.example.com')
+        await store.open()
+        await store.put('etc/e.jw-secret/e', b'5', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/e.jw-secret/e')).stdout == b'5'
+        await store.put(
+            'etc/f.pem.jw-secret-file',
+            b'-----BEGIN KEY-----\n',
+            ftype = FT_SECRET_FILE
+        )
+        assert (await store.get('etc/f.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN KEY-----\n'
+        await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY)
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
+        await store.put('etc/a.jw-secret/a', b'9', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
+        # -- a typed put replaces the entry: the serialized content
+        #    is written wholesale, keys not in it are gone
+        await store.put('etc/a.jw-secret', b'a=9\nextra=e\n', ftype = FT_SECRET)
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
+        assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\nextra=e\n'
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
+        # -- A one-component address is a blob at a top-level config
+        #    path, as on the read side
+        await store.put('top', b'1', ftype = FT_SECRET_FILE)
+        assert (await store.get('top')).stdout == b'1'
+        # -- An untyped put is refused: the store content types are
+        #    entry, key, and blob
+        try:
+            await store.put('etc/untyped', b'x')
+            raise AssertionError('expected ValueError')
+        except ValueError:
+            pass
+        # -- The repository files are sops files in dotenv format,
+        #    as in the pre-existing store: the flat sops_version
+        #    data line marks it
+        for rel in ('com/example/host/etc/e.jw-secret',
+                    'com/example/host/etc/f.pem.jw-secret-file'):
+            with open(os.path.join(repo, rel), 'rb') as data:
+                assert b'sops_version=' in data.read()
+        # -- The namespace root itself has no store home
+        try:
+            await store.put('', b'1')
+            raise AssertionError('expected ValueError')
+        except ValueError:
+            pass
+        # -- A whole get returns the stored bytes verbatim: the
+        #    decryption is cached by the content's hash, so a put of
+        #    other content re-decrypts, and no re-serialization
+        #    happens on read
+        await store.put('etc/verbat.jw-secret', b'v=1\n', ftype = FT_SECRET)
+        assert (await store.get('etc/verbat.jw-secret')).stdout == b'v=1\n'
+        # -- Non-canonical content is stored and returned as-is
+        content = b'# a comment\nv=  padded  \n'
+        await store.put('etc/verbat.jw-secret', content, ftype = FT_SECRET)
+        assert (await store.get('etc/verbat.jw-secret')).stdout == content
+        # -- A key put merges through the stored content: the comment
+        #    and the padding do not survive the merge, and a whole
+        #    get after the put sees the merged entry
+        await store.put('etc/verbat.jw-secret/k2', b'2', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/verbat.jw-secret')).stdout == \
+            b'v=padded\nk2=2\n'
+        # -- A blob put replaces the blob: a get after the put returns
+        #    the new content
+        await store.put(
+            'etc/vb.pem.jw-secret-file',
+            b'-----BEGIN ONE-----\n',
+            ftype = FT_SECRET_FILE
+        )
+        assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN ONE-----\n'
+        await store.put(
+            'etc/vb.pem.jw-secret-file',
+            b'-----BEGIN TWO-----\n',
+            ftype = FT_SECRET_FILE
+        )
+        assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN TWO-----\n'
+        # -- A put of content sops normalizes on read (blank lines
+        #    dropped, a trailing newline added) is a no-op when
+        #    repeated: the no-op check compares the sops-canonical
+        #    forms, so the repository file is not re-encrypted
+        content = b'n=1\n\nn2=2'
+        await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET)
+        assert (await store.get('etc/nl.jw-secret')).stdout == b'n=1\nn2=2\n'
+        repo_file = os.path.join(repo, 'com/example/host/etc/nl.jw-secret')
+        with open(repo_file, 'rb') as fb:
+            first = fb.read()
+        await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET)
+        with open(repo_file, 'rb') as fb:
+            assert fb.read() == first
+        # -- ...and a changed content still lands
+        await store.put('etc/nl.jw-secret', b'n=1\nn2=3', ftype = FT_SECRET)
+        with open(repo_file, 'rb') as fb:
+            assert fb.read() != first
+        # -- Already-canonical content no-ops on the byte check
+        await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET)
+        repo_file = os.path.join(repo, 'com/example/host/etc/nlc.jw-secret')
+        with open(repo_file, 'rb') as fb:
+            first = fb.read()
+        await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET)
+        with open(repo_file, 'rb') as fb:
+            assert fb.read() == first
+        # -- A blob is byte-exact: dropping the trailing newline is
+        #    a change, not a no-op
+        repo_file = os.path.join(repo, 'com/example/host/etc/nlb.pem.jw-secret-file')
+        await store.put('etc/nlb.pem.jw-secret-file', b'x\n', ftype = FT_SECRET_FILE)
+        with open(repo_file, 'rb') as fb:
+            first = fb.read()
+        await store.put('etc/nlb.pem.jw-secret-file', b'x', ftype = FT_SECRET_FILE)
+        with open(repo_file, 'rb') as fb:
+            assert fb.read() != first
+        await store.close()
+
+        # -- JW_SECRETS_SOPS_ROOT: a base directory within the repository
+        base = os.path.join(tmp, 'repo2', 'my', 'secrets')
+        encrypt(base, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient)
+        os.environ['JW_SECRETS_SOPS_REPO'] = os.path.join(tmp, 'repo2')
+        os.environ['JW_SECRETS_SOPS_ROOT'] = 'my/secrets'
+        store = FileContext.create('sops://')
+        await store.open()
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        await store.close()
+
+        # -- The same store as a git URL is cloned on the fly and
+        #    the checkout is removed by close()
+        hooks = os.path.join(tmp, 'hooks')
+        os.makedirs(hooks, exist_ok = True)
+        env = dict(
+            os.environ,
+            GIT_AUTHOR_NAME = 'test',
+            GIT_AUTHOR_EMAIL = 'test@example.com',
+            GIT_COMMITTER_NAME = 'test',
+            GIT_COMMITTER_EMAIL = 'test@example.com',
+        )
+        subprocess.run(
+            ['git', 'init', '--quiet'],
+            cwd = repo,
+            check = True,
+            env = env,
+            capture_output = True
+        )
+        subprocess.run(
+            ['git', 'add', '-A'],
+            cwd = repo,
+            check = True,
+            env = env,
+            capture_output = True
+        )
+        subprocess.run(
+            [
+                'git',
+                '-c',
+                'commit.gpgsign=false',
+                '-c',
+                f'core.hooksPath={hooks}',
+                'commit',
+                '--quiet',
+                '-m',
+                'test',
+            ],
+            cwd = repo,
+            check = True,
+            env = env,
+            capture_output = True
+        )
+        os.environ['JW_SECRETS_SOPS_REPO'] = 'file://' + repo
+        os.environ['JW_SECRETS_SOPS_ROOT'] = ''
+        checkouts = set(glob('/tmp/jw-pkg-sops-*'))
+        store = FileContext.create('sops://')
+        await store.open()
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        await store.close()
+        assert set(glob('/tmp/jw-pkg-sops-*')) == checkouts
+
+        # -- Key resolution: JW_SECRETS_SOPS_KEY_PATH is the sole
+        #    input; a SOPS_AGE_KEY_FILE in the process environment is
+        #    ignored, since the store sets it per invocation on the
+        #    sops subprocess. This section also switches the
+        #    repository from the git URL to the local directory
+        os.environ['SOPS_AGE_KEY_FILE'] = '/nonexistent/age-key'
+        os.environ['JW_SECRETS_SOPS_REPO'] = repo
+        os.environ['JW_SECRETS_SOPS_ROOT'] = ''
+        store = FileContext.create('sops://')
+        await store.open()
+        assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
+        await store.close()
+        del os.environ['SOPS_AGE_KEY_FILE']
+
+        # -- No key file: the failure names the house variable
+        saved = os.environ['JW_SECRETS_SOPS_KEY_PATH']
+        del os.environ['JW_SECRETS_SOPS_KEY_PATH']
+        store = FileContext.create('sops://')
+        await store.open()
+        try:
+            await store.get('default/etc/a.jw-secret/a')
+            assert False, 'expected a key resolution failure'
+        except SopsKeyError as e:
+            assert 'JW_SECRETS_SOPS_KEY_PATH is not set' in str(e)
+        await store.close()
+        os.environ['JW_SECRETS_SOPS_KEY_PATH'] = saved
+
+def main() -> None:
+    if not shutil.which('sops') or not shutil.which('age-keygen') \
+        or not shutil.which('git'):
+        print(
+            'Skipping secrets sops store tests (sops, age-keygen, and '
+            'git required)'
+        )
+        return
+    asyncio.run(run_tests())
+    print('All secrets sops store tests passed')
+
+main()