lib.ec.secret_store.TarGpg: Add the write path

The TarGpg store is read-only: put() and unlink() raise, so the store can
be copied from, but not into, and a copy that lands entries and blobs below
a host prefix has no tarball target.

This matches the way jw-pkg secrets has worked for a long time. Since
Stores are now FileContext-backed, a write path can be cleanly implemented,
and that's what this commit does.

A put rebuilds the tarball: the existing members minus the removed, plus
the written one; the result is signed with the recipient from
JW_SECRETS_TARGPG_RECIPIENT and written back to the tarball location, which
must be a local path. A missing tarball is an empty store, so a put creates
it.

The put dispatch follows the other stores: a typed entry put replaces the
entry with the given serialized content, a key put merges into its entry,
creating it if it does not exist yet, and an untyped put below an entry
lands a key there, otherwise a blob. unlink() removes the node's member.

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0

src/python/jw/pkg/lib/ec/secret_store/TarGpg.py

@@ -4,14 +4,16 @@ import errno
 import io
 import os
 import stat
+import tempfile
 import time
 
 from tarfile import TarFile, TarInfo
 from typing import TYPE_CHECKING, Iterable, override
 
 from ...file_types import FT_DIR
-from ...kv import parse_kv
+from ...kv import parse_kv, serialize_kv
 from ...Result import Result
+from ...Uri import Uri
 from .Base import Base
 from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
 
@@ -19,7 +21,7 @@ if TYPE_CHECKING:
     from typing import Any
 
     from ...base import StatResult
-    from ...Uri import Uri
+    from ...file_types import FileType
 
 class TarGpg(Base):
     """A store of secrets in a GPG-encrypted tarball in the canonical
@@ -40,6 +42,10 @@ class TarGpg(Base):
     The tarball is fetched and decrypted once, then cached. Fetching,
     decrypting, and filtering happen in a local exec context, so the
     GPG key and the decrypted material stay on this machine.
+
+    A write rebuilds the tarball, signs it, and writes it back to
+    the tarball location, which must be a local path; the recipient
+    is deployment information, JW_SECRETS_TARGPG_RECIPIENT.
     """
 
     def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
@@ -225,3 +231,128 @@ class TarGpg(Base):
     async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
         path = self._unchroot(path)
         return await self._call_with_fallback(path, self.__tar_stat)
+
+    async def __tar_write(self, added: dict[str, bytes], removed: set[str]) -> None:
+        # -- Rebuild the tarball: the existing members minus the
+        #    removed, plus the added; the result is signed and
+        #    written back to the tarball location, which must be a
+        #    local path. A missing tarball is an empty store.
+        try:
+            tf = await self.__tar()
+        except FileNotFoundError:
+            tf = None
+        data: dict[str, tuple[TarInfo, bytes]] = {}
+        if tf is not None:
+            for name, info in {m.name: m for m in tf.getmembers()}.items():
+                if name in removed:
+                    continue
+                f = tf.extractfile(info)
+                if f is None:
+                    raise Exception(f'Can\'t read store member "{name}"')
+                data[name] = (info, f.read())
+        buf = io.BytesIO()
+        with TarFile(fileobj = buf, mode = 'w') as out:
+            for name, (info, content) in data.items():
+                out.addfile(info, io.BytesIO(content))
+            for name, content in added.items():
+                info = TarInfo(name)
+                info.size = len(content)
+                info.mode = 0o400
+                out.addfile(info, io.BytesIO(content))
+        u = Uri(self.__tar_uri)
+        if u.scheme not in ('', 'local'):
+            raise Exception(f'TarGpg: cannot write to "{self.__tar_uri}"')
+        recipient = os.environ.get('JW_SECRETS_TARGPG_RECIPIENT', '')
+        if not recipient:
+            raise Exception('TarGpg: JW_SECRETS_TARGPG_RECIPIENT is not set')
+        fd, tar_tmp = tempfile.mkstemp()
+        with os.fdopen(fd, 'wb') as f:
+            f.write(buf.getvalue())
+        from ..Local import Local
+        await Local().run(
+            [
+                'gpg',
+                '--batch',
+                '--yes',
+                '--quiet',
+                '--no-tty',
+                '--trust-model',
+                'always',
+                '--recipient',
+                recipient,
+                '--encrypt',
+                '--output',
+                u.path or self.__tar_uri,
+                tar_tmp,
+            ],
+            throw = True,
+        )
+        os.unlink(tar_tmp)
+        self.__tf = None
+        self.__kv_cache.clear()
+
+    async def __put_entry(self, entry: str, content: bytes) -> Result:
+        # -- Replace the entry: the content is the serialized
+        #    entry, so a key not in it is gone
+        self.__kv_cache.pop(entry, None)
+        await self.__tar_write({entry: content}, set())
+        return Result()
+
+    async def __put_key(self, entry: str, key: str, content: bytes) -> Result:
+        # -- Merge the key into the entry, creating the entry if it
+        #    does not exist yet
+        self.__kv_cache.pop(entry, None)
+        try:
+            kv = await self.__entry_kv(entry)
+        except FileNotFoundError:
+            kv = {}
+        kv[key] = content.decode()
+        await self.__tar_write({entry: serialize_kv(kv)}, set())
+        return Result()
+
+    async def __put_blob(self, path: str, content: bytes) -> Result:
+        await self.__tar_write({path: content}, set())
+        return Result()
+
+    @override
+    async def _put(
+        self,
+        path: str,
+        content: bytes,
+        wd: str | None,
+        throw: bool,
+        verbose: bool | None,
+        title: str,
+        owner: str | None,
+        group: str | None,
+        mode: str | None,
+        atomic: bool,
+        ftype: FileType,
+    ) -> Result:
+        # -- A write does not take the default fallback
+        path = self._unchroot(path)
+        storage_path = self._path_in_storage(path)
+        if not storage_path:
+            raise ValueError(f'put: "{path}" is not an entry/key or blob path')
+        if ftype.is_subtype(FT_SECRET):
+            return await self.__put_entry(storage_path, content)
+        if ftype.is_subtype(FT_SECRET_KEY):
+            entry, sep, key = storage_path.rpartition('/')
+            if not sep or not key:
+                raise ValueError(f'put: "{path}" is not of the form path/key')
+            return await self.__put_key(entry, key, content)
+        if ftype.is_subtype(FT_SECRET_FILE):
+            return await self.__put_blob(storage_path, content)
+        raise ValueError(f'put: ftype {ftype} is not a store content type')
+
+    @override
+    async def _unlink(self, path: str) -> None:
+        # -- Remove the node's member; the path names the node
+        path = self._unchroot(path)
+        storage_path = self._path_in_storage(path)
+        members = await self.__members()
+        if storage_path in members:
+            self.__kv_cache.pop(storage_path, None)
+            await self.__tar_write({}, {storage_path})
+        else:
+            raise FileNotFoundError(path)

test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py

@@ -7,6 +7,11 @@ import subprocess
 import tarfile
 import tempfile
 
+from typing import override
+
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.ec.secret_store.KvFile import KvFile
 from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
 from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg
 from jw.pkg.lib.file_types import FT_DIR
@@ -225,6 +230,123 @@ async def run_tests() -> None:
         with open(dst) as f:
             assert f.read() == 'a=2\n'
 
+        # -- put(): a key put creates its entry and merges into an
+        #    existing one, an entry put replaces the whole, an
+        #    untyped put below an entry lands a key there, and a
+        #    blob put lands a .jw-secret-file; the tarball is
+        #    amended and re-signed in place
+        os.environ['JW_SECRETS_TARGPG_RECIPIENT'] = 'test@example.com'
+        store = FileContext.create('targpg://host.example.com')
+        await store.open()
+        await store.put('etc/e.jw-secret/e', b'5', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/e.jw-secret/e')).stdout == b'5'
+        await store.put('etc/e.jw-secret/f', b'6', ftype = FT_SECRET_KEY)
+        assert set(await store.readdir('etc/e.jw-secret')) == {'e', 'f'}
+        await store.put('etc/a.jw-secret', b'a=9\n', ftype = FT_SECRET)
+        assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\n'
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
+        await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY)
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
+        await store.put(
+            'etc/new.pem.jw-secret-file',
+            b'-----BEGIN NEW-----\n',
+            ftype = FT_SECRET_FILE
+        )
+        assert (await store.get('etc/new.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN NEW-----\n'
+        await store.unlink('etc/b.jw-secret')
+        assert not await store.file_exists('etc/b.jw-secret')
+        await store.close()
+        # -- The tarball holds the amended members, still signed;
+        #    an untouched member keeps its content
+        data = subprocess.run(
+            ['gpg', '--batch', '--quiet', '--decrypt', enc_path],
+            check = True,
+            capture_output = True,
+            env = env,
+        ).stdout
+        with tarfile.open(fileobj = io.BytesIO(data)) as tf:
+            names = {m.name for m in tf.getmembers()}
+            mem = tf.extractfile('default/etc/a.jw-secret')
+            assert mem is not None
+            assert mem.read() == b'a=1\n'
+        assert 'com/example/host/etc/e.jw-secret' in names
+        assert 'com/example/host/etc/a.jw-secret' in names
+        assert 'com/example/host/etc/new.pem.jw-secret-file' in names
+        assert 'com/example/host/etc/b.jw-secret' not in names
+
+        # -- A put that changes nothing is a no-op: the pre-read is
+        #    fallback-aware, and no write happens, so no recipient
+        #    is needed; a different content still writes
+        del os.environ['JW_SECRETS_TARGPG_RECIPIENT']
+        store = FileContext.create('targpg://host.example.com')
+        await store.open()
+        await store.put('etc/e.jw-secret', b'e=5\nf=6\n', ftype = FT_SECRET)
+        try:
+            await store.put('etc/e.jw-secret', b'e=5\nf=7\n', ftype = FT_SECRET)
+            raise AssertionError('expected Exception')
+        except Exception as e:
+            assert 'JW_SECRETS_TARGPG_RECIPIENT' in str(e)
+        await store.close()
+        os.environ['JW_SECRETS_TARGPG_RECIPIENT'] = 'test@example.com'
+
+        # -- A put creates the tarball if it does not exist yet
+        new_tar = os.path.join(tmp, 'new.tar.gpg')
+        os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + new_tar
+        store = FileContext.create('targpg://host.example.com')
+        await store.open()
+        await store.put('etc/z.jw-secret', b'z=1\n', ftype = FT_SECRET)
+        assert (await store.get('etc/z.jw-secret')).stdout == b'z=1\n'
+        await store.close()
+        assert os.path.isfile(new_tar)
+        os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path
+
+        # -- A store as a copy destination: the walk asks the source
+        #    which node is an entry; entries land as entry members
+        #    below the destination host's prefix, blobs at their
+        #    config path
+        host3 = os.path.join(tmp, 'host3')
+        os.makedirs(os.path.join(host3, 'etc'))
+        with open(os.path.join(host3, 'etc', 'foo.conf.jw-secret'), 'w') as fh:
+            fh.write('f1=1\nf2=2\n')
+        with open(os.path.join(host3, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh:
+            fh.write('-----BEGIN CERT3-----\n')
+
+        class FakeDistro(SuseDistro):
+
+            def __init__(self, files: list[str]) -> None:
+                super().__init__(Local(), id = 'suse')
+                self.files = files
+
+            @override
+            async def _pkg_files(self, names: list[str] | None) -> list[str]:
+                return self.files
+
+        fake = FakeDistro(
+            [
+                f'{host3}/etc/foo.conf.jw-tmpl',
+                f'{host3}/etc/cert.pem.jw-tmpl',
+            ]
+        )
+        src = KvFile(Local(), distro = fake)
+        base = host3.lstrip('/')
+        ret = await copy(src, 'targpg://host3.example.com', recursive = True)
+        assert isinstance(ret, list)
+        # -- The copy transports the on-host file names unchanged
+        assert set(ret) == {
+            f'{base}/etc/foo.conf.jw-secret',
+            f'{base}/etc/cert.pem.jw-secret-file',
+        }
+        tstore = FileContext.create('targpg://host3.example.com')
+        await tstore.open()
+        assert set(await tstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \
+            {'f1', 'f2'}
+        assert (await tstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \
+            b'1'
+        assert (await tstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT3-----\n'
+        await tstore.close()
+
 def main() -> None:
     if not shutil.which('gpg'):
         print('Skipping secrets targpg store tests (gpg required)')