From f70cc48eb693ad27804c17fed658f2d43fe44fbf Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sun, 4 Oct 2026 19:09:20 +0200 Subject: [PATCH] lib.ec.secret_store.TarGpg: Add the write path The TarGpg store is read-only: put() and unlink() raise, so the store can be copied from, but not into, and a copy that lands entries and blobs below a host prefix has no tarball target. This matches the way jw-pkg secrets has worked for a long time. Since Stores are now FileContext-backed, a write path can be cleanly implemented, and that's what this commit does. A put rebuilds the tarball: the existing members minus the removed, plus the written one; the result is signed with the recipient from JW_SECRETS_TARGPG_RECIPIENT and written back to the tarball location, which must be a local path. A missing tarball is an empty store, so a put creates it. The put dispatch follows the other stores: a typed entry put replaces the entry with the given serialized content, a key put merges into its entry, creating it if it does not exist yet, and an untyped put below an entry lands a key there, otherwise a blob. unlink() removes the node's member. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 --- .../jw/pkg/lib/ec/secret_store/TarGpg.py | 135 +++++++++++++++++- .../jw/pkg/lib/ec/secret_store/TarGpg/test.py | 122 ++++++++++++++++ 2 files changed, 255 insertions(+), 2 deletions(-) diff --git a/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py b/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py index 45ac4b68..11245e79 100644 --- a/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py +++ b/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py @@ -4,14 +4,16 @@ import errno import io import os import stat +import tempfile import time from tarfile import TarFile, TarInfo from typing import TYPE_CHECKING, Iterable, override from ...file_types import FT_DIR -from ...kv import parse_kv +from ...kv import parse_kv, serialize_kv from ...Result import Result +from ...Uri import Uri from .Base import Base from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY @@ -19,7 +21,7 @@ if TYPE_CHECKING: from typing import Any from ...base import StatResult - from ...Uri import Uri + from ...file_types import FileType class TarGpg(Base): """A store of secrets in a GPG-encrypted tarball in the canonical @@ -40,6 +42,10 @@ class TarGpg(Base): The tarball is fetched and decrypted once, then cached. Fetching, decrypting, and filtering happen in a local exec context, so the GPG key and the decrypted material stay on this machine. + + A write rebuilds the tarball, signs it, and writes it back to + the tarball location, which must be a local path; the recipient + is deployment information, JW_SECRETS_TARGPG_RECIPIENT. """ def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None: @@ -225,3 +231,128 @@ class TarGpg(Base): async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: path = self._unchroot(path) return await self._call_with_fallback(path, self.__tar_stat) + + async def __tar_write(self, added: dict[str, bytes], removed: set[str]) -> None: + # -- Rebuild the tarball: the existing members minus the + # removed, plus the added; the result is signed and + # written back to the tarball location, which must be a + # local path. A missing tarball is an empty store. + try: + tf = await self.__tar() + except FileNotFoundError: + tf = None + data: dict[str, tuple[TarInfo, bytes]] = {} + if tf is not None: + for name, info in {m.name: m for m in tf.getmembers()}.items(): + if name in removed: + continue + f = tf.extractfile(info) + if f is None: + raise Exception(f'Can\'t read store member "{name}"') + data[name] = (info, f.read()) + buf = io.BytesIO() + with TarFile(fileobj = buf, mode = 'w') as out: + for name, (info, content) in data.items(): + out.addfile(info, io.BytesIO(content)) + for name, content in added.items(): + info = TarInfo(name) + info.size = len(content) + info.mode = 0o400 + out.addfile(info, io.BytesIO(content)) + u = Uri(self.__tar_uri) + if u.scheme not in ('', 'local'): + raise Exception(f'TarGpg: cannot write to "{self.__tar_uri}"') + recipient = os.environ.get('JW_SECRETS_TARGPG_RECIPIENT', '') + if not recipient: + raise Exception('TarGpg: JW_SECRETS_TARGPG_RECIPIENT is not set') + fd, tar_tmp = tempfile.mkstemp() + with os.fdopen(fd, 'wb') as f: + f.write(buf.getvalue()) + from ..Local import Local + await Local().run( + [ + 'gpg', + '--batch', + '--yes', + '--quiet', + '--no-tty', + '--trust-model', + 'always', + '--recipient', + recipient, + '--encrypt', + '--output', + u.path or self.__tar_uri, + tar_tmp, + ], + throw = True, + ) + os.unlink(tar_tmp) + self.__tf = None + self.__kv_cache.clear() + + async def __put_entry(self, entry: str, content: bytes) -> Result: + # -- Replace the entry: the content is the serialized + # entry, so a key not in it is gone + self.__kv_cache.pop(entry, None) + await self.__tar_write({entry: content}, set()) + return Result() + + async def __put_key(self, entry: str, key: str, content: bytes) -> Result: + # -- Merge the key into the entry, creating the entry if it + # does not exist yet + self.__kv_cache.pop(entry, None) + try: + kv = await self.__entry_kv(entry) + except FileNotFoundError: + kv = {} + kv[key] = content.decode() + await self.__tar_write({entry: serialize_kv(kv)}, set()) + return Result() + + async def __put_blob(self, path: str, content: bytes) -> Result: + await self.__tar_write({path: content}, set()) + return Result() + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + # -- A write does not take the default fallback + path = self._unchroot(path) + storage_path = self._path_in_storage(path) + if not storage_path: + raise ValueError(f'put: "{path}" is not an entry/key or blob path') + if ftype.is_subtype(FT_SECRET): + return await self.__put_entry(storage_path, content) + if ftype.is_subtype(FT_SECRET_KEY): + entry, sep, key = storage_path.rpartition('/') + if not sep or not key: + raise ValueError(f'put: "{path}" is not of the form path/key') + return await self.__put_key(entry, key, content) + if ftype.is_subtype(FT_SECRET_FILE): + return await self.__put_blob(storage_path, content) + raise ValueError(f'put: ftype {ftype} is not a store content type') + + @override + async def _unlink(self, path: str) -> None: + # -- Remove the node's member; the path names the node + path = self._unchroot(path) + storage_path = self._path_in_storage(path) + members = await self.__members() + if storage_path in members: + self.__kv_cache.pop(storage_path, None) + await self.__tar_write({}, {storage_path}) + else: + raise FileNotFoundError(path) diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py index 42ca9780..a475cb89 100644 --- a/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py @@ -7,6 +7,11 @@ import subprocess import tarfile import tempfile +from typing import override + +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.ec.secret_store.KvFile import KvFile from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg from jw.pkg.lib.file_types import FT_DIR @@ -225,6 +230,123 @@ async def run_tests() -> None: with open(dst) as f: assert f.read() == 'a=2\n' + # -- put(): a key put creates its entry and merges into an + # existing one, an entry put replaces the whole, an + # untyped put below an entry lands a key there, and a + # blob put lands a .jw-secret-file; the tarball is + # amended and re-signed in place + os.environ['JW_SECRETS_TARGPG_RECIPIENT'] = 'test@example.com' + store = FileContext.create('targpg://host.example.com') + await store.open() + await store.put('etc/e.jw-secret/e', b'5', ftype = FT_SECRET_KEY) + assert (await store.get('etc/e.jw-secret/e')).stdout == b'5' + await store.put('etc/e.jw-secret/f', b'6', ftype = FT_SECRET_KEY) + assert set(await store.readdir('etc/e.jw-secret')) == {'e', 'f'} + await store.put('etc/a.jw-secret', b'a=9\n', ftype = FT_SECRET) + assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\n' + assert (await store.get('etc/a.jw-secret/a')).stdout == b'9' + await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY) + assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'} + await store.put( + 'etc/new.pem.jw-secret-file', + b'-----BEGIN NEW-----\n', + ftype = FT_SECRET_FILE + ) + assert (await store.get('etc/new.pem.jw-secret-file')).stdout == \ + b'-----BEGIN NEW-----\n' + await store.unlink('etc/b.jw-secret') + assert not await store.file_exists('etc/b.jw-secret') + await store.close() + # -- The tarball holds the amended members, still signed; + # an untouched member keeps its content + data = subprocess.run( + ['gpg', '--batch', '--quiet', '--decrypt', enc_path], + check = True, + capture_output = True, + env = env, + ).stdout + with tarfile.open(fileobj = io.BytesIO(data)) as tf: + names = {m.name for m in tf.getmembers()} + mem = tf.extractfile('default/etc/a.jw-secret') + assert mem is not None + assert mem.read() == b'a=1\n' + assert 'com/example/host/etc/e.jw-secret' in names + assert 'com/example/host/etc/a.jw-secret' in names + assert 'com/example/host/etc/new.pem.jw-secret-file' in names + assert 'com/example/host/etc/b.jw-secret' not in names + + # -- A put that changes nothing is a no-op: the pre-read is + # fallback-aware, and no write happens, so no recipient + # is needed; a different content still writes + del os.environ['JW_SECRETS_TARGPG_RECIPIENT'] + store = FileContext.create('targpg://host.example.com') + await store.open() + await store.put('etc/e.jw-secret', b'e=5\nf=6\n', ftype = FT_SECRET) + try: + await store.put('etc/e.jw-secret', b'e=5\nf=7\n', ftype = FT_SECRET) + raise AssertionError('expected Exception') + except Exception as e: + assert 'JW_SECRETS_TARGPG_RECIPIENT' in str(e) + await store.close() + os.environ['JW_SECRETS_TARGPG_RECIPIENT'] = 'test@example.com' + + # -- A put creates the tarball if it does not exist yet + new_tar = os.path.join(tmp, 'new.tar.gpg') + os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + new_tar + store = FileContext.create('targpg://host.example.com') + await store.open() + await store.put('etc/z.jw-secret', b'z=1\n', ftype = FT_SECRET) + assert (await store.get('etc/z.jw-secret')).stdout == b'z=1\n' + await store.close() + assert os.path.isfile(new_tar) + os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path + + # -- A store as a copy destination: the walk asks the source + # which node is an entry; entries land as entry members + # below the destination host's prefix, blobs at their + # config path + host3 = os.path.join(tmp, 'host3') + os.makedirs(os.path.join(host3, 'etc')) + with open(os.path.join(host3, 'etc', 'foo.conf.jw-secret'), 'w') as fh: + fh.write('f1=1\nf2=2\n') + with open(os.path.join(host3, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh: + fh.write('-----BEGIN CERT3-----\n') + + class FakeDistro(SuseDistro): + + def __init__(self, files: list[str]) -> None: + super().__init__(Local(), id = 'suse') + self.files = files + + @override + async def _pkg_files(self, names: list[str] | None) -> list[str]: + return self.files + + fake = FakeDistro( + [ + f'{host3}/etc/foo.conf.jw-tmpl', + f'{host3}/etc/cert.pem.jw-tmpl', + ] + ) + src = KvFile(Local(), distro = fake) + base = host3.lstrip('/') + ret = await copy(src, 'targpg://host3.example.com', recursive = True) + assert isinstance(ret, list) + # -- The copy transports the on-host file names unchanged + assert set(ret) == { + f'{base}/etc/foo.conf.jw-secret', + f'{base}/etc/cert.pem.jw-secret-file', + } + tstore = FileContext.create('targpg://host3.example.com') + await tstore.open() + assert set(await tstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \ + {'f1', 'f2'} + assert (await tstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \ + b'1' + assert (await tstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \ + b'-----BEGIN CERT3-----\n' + await tstore.close() + def main() -> None: if not shutil.which('gpg'): print('Skipping secrets targpg store tests (gpg required)') -- 2.56.0