From 60a91c0692edf45d4371a9268f74017a56a8e2b8 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 3 Oct 2026 21:39:33 +0200 Subject: [PATCH] lib.ec.secret_store.Sops: Add store class This commit adds a new class inheriting from secret_store.Base: Sops. The currently available classes allow to store secrets offline in a GPG-encrypted tarball, but those are not versioned. This commit allows to use a SOPS / AGE-encrypted Git repository as storage backend instead. The additions in detail: - Sops: the store implementation; the repository location comes from JW_SECRETS_SOPS_REPO, get() on an entry returns the decrypted serialized content, and a typed put (FT_SECRET) re-encrypts the entry wholesale - The age key comes from JW_SECRETS_SOPS_KEY_PATH only: the house variable is the sole input to the store, and SOPS_AGE_KEY_FILE is set on the sops subprocess per invocation, never read from the process environment; SopsKeyError is raised when the variable is not set - FileContext.create(): the sops:// scheme - The store's unit tests, covering the node types, the deploy and fetch round trip, the whole-entry get and put, the key resolution, and the default fallback Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 --- src/python/jw/pkg/lib/FileContext.py | 4 + src/python/jw/pkg/lib/ec/secret_store/Sops.py | 410 +++++++++++ .../jw/pkg/lib/ec/secret_store/Sops/Makefile | 7 + .../jw/pkg/lib/ec/secret_store/Sops/test.py | 645 ++++++++++++++++++ 4 files changed, 1066 insertions(+) create mode 100644 src/python/jw/pkg/lib/ec/secret_store/Sops.py create mode 100644 test/unit/python/jw/pkg/lib/ec/secret_store/Sops/Makefile create mode 100644 test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index aec381e5..0bbde74a 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -360,6 +360,10 @@ class FileContext(abc.ABC): from .ec.Curl import Curl return Curl(uri, *args, **kwargs) + case 'sops': + from .ec.secret_store.Sops import Sops + + return Sops(uri, *args, **kwargs) case 'targpg': from .ec.secret_store.TarGpg import TarGpg diff --git a/src/python/jw/pkg/lib/ec/secret_store/Sops.py b/src/python/jw/pkg/lib/ec/secret_store/Sops.py new file mode 100644 index 00000000..6096aa3c --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/Sops.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import errno +import hashlib +import os + +from contextlib import suppress +from typing import TYPE_CHECKING, Iterable, override + +if TYPE_CHECKING: + from typing import Any + + from ...ExecContext import ExecContext + from ...Uri import Uri + from ...base import StatResult + from ...file_types import FileType + +from ...file_types import FT_DIR +from ...kv import parse_kv, serialize_kv +from ...Result import Result +from .Base import Base +from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY + +def _sops_dotenv_norm(content: bytes) -> bytes: + # -- The form sops' dotenv serialization produces: blank + # lines dropped, a single trailing newline, everything + # else verbatim. The identity for content already in + # that form + if not content: + return content + return b'\n'.join(line for line in content.split(b'\n') if line) + b'\n' + +class SopsKeyError(Exception): + """The age key file of the sops store could not be resolved""" + +class Sops(Base): + """A secret store of SOPS-encrypted files in a canonical layout. + + The repository location is deployment information, not part of + the address: JW_SECRETS_SOPS_REPO names a local directory (used + as-is) or a git URL (cloned into a temporary directory until + close()), and JW_SECRETS_SOPS_ROOT an optional base directory + within it. The store URI addresses a subtree of the namespace, + as in the base class. + + A store path names a file, and optionally a key below an entry. + The file of an entry is its address, and its keys are + addressable below it; a blob is a file at its address holding + non-key-value content. The store does not identify content by + file name: a file that decrypts to a key-value file is an entry, + anything else a blob. The repository files are sops files in + dotenv format: an entry's keys are its data lines, a blob is + stored under a single data key. + + put() writes the file at its address, sops-encrypting the + content for its type: an entry wholesale, a key merged into its + entry, a blob as raw bytes. The store accepts entry, key, and + blob content only; the encryption recipient is derived from the + SOPS age key, and committing the repository is the caller's + step. + + Files are decrypted with sops in a local exec context, and the + age key file is resolved from JW_SECRETS_SOPS_KEY_PATH, so the + key and the decrypted material stay on this machine. + """ + + def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None: + super().__init__(uri, *args, **kwargs) + self.__repo = os.environ.get('JW_SECRETS_SOPS_REPO', '.') + self.__base = os.environ.get('JW_SECRETS_SOPS_ROOT', '').strip('/') + self.__kv_cache: dict[str, dict[str, str]] = {} + self.__decrypt_cache: dict[bytes, bytes] = {} + self.___keyfile: str | None = None + self.___recipient: str | None = None + self.___git_checkout: str | None = None + self.___crypt_ec: ExecContext | None = None + + async def __crypt_ec(self) -> ExecContext: + if self.___crypt_ec is None: + from ..Local import Local + + # -- Security: decrypt in a local exec context for now so + # that the SOPS key and the decrypted material never + # leave this machine + self.___crypt_ec = Local() + return self.___crypt_ec + + async def __git_checkout(self) -> str: + if self.___git_checkout is None: + from ...base import InputMode + + ec = await self.__crypt_ec() + if os.path.isdir(self.__repo): + self.___git_checkout = self.__repo + else: + ret = await ec.mktemp('/tmp/jw-pkg-sops-XXXXXX', directory = True) + try: + await ec.run( + ['git', 'clone', '--quiet', self.__repo, ret], + cmd_input = InputMode.NonInteractive, + throw = True, + ) + except Exception: + await ec.erase(ret) + raise + self.___git_checkout = ret + return self.___git_checkout + + async def __git_subdir(self, path: str) -> str: + checkout = await self.__git_checkout() + base = f'{checkout}/{self.__base}' if self.__base else checkout + return f'{base}/{path}' if path else base + + async def __git_get(self, path: str) -> Result: + if path == '' or os.path.isdir(await self.__git_subdir(path)): + raise IsADirectoryError(path) + file_path = await self.__git_subdir(path) + if await self.__is_entry(path): + # -- The entry file is the serialized entry + return Result(stdout = await self.__decrypt(file_path)) + if '/' in path: + entry, key = path.rsplit('/', 1) + if await self.__is_entry(entry): + kv = await self.__decrypt_kv_content(entry) + try: + return Result(stdout = kv[key].encode()) + except KeyError: + raise FileNotFoundError(path) from None + if os.path.isfile(file_path): + from ...base import InputMode + + ec = await self.__crypt_ec() + result = await ec.run( + [ + 'sops', + '--decrypt', + '--input-type', + 'dotenv', + '--output-type', + 'binary', + file_path, + ], + cmd_input = InputMode.NonInteractive, + mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()}, + throw = True, + ) + return Result(stdout = result.stdout) + raise FileNotFoundError(path) + + async def __git_readdir(self, path: str) -> Iterable[str]: + + def __names(d: str) -> set[str]: + # -- No name filter: every file is store content + return set(os.listdir(d)) + + if await self.__is_entry(path): + return (await self.__decrypt_kv_content(path)).keys() + os_path = await self.__git_subdir(path) + if os.path.isfile(os_path): + raise NotADirectoryError(path) + if os.path.isdir(os_path): + names = __names(os_path) + default_path = self._default_path_in_storage(path) + if default_path is not None: + d = await self.__git_subdir(default_path) + if os.path.isdir(d): + names |= __names(d) + return sorted(names) + raise FileNotFoundError(path) + + async def __git_stat( + self, + storage_path: str, + follow_symlinks: bool, + ) -> StatResult: + stat_fn = os.lstat if not follow_symlinks else os.stat + file_path = await self.__git_subdir(storage_path) + if await self.__is_entry(storage_path): + return self._stat_result_from_os(stat_fn(file_path), FT_SECRET) + entry, sep, key = storage_path.rpartition('/') + if sep and key and await self.__is_entry(entry): + if key in await self.__decrypt_kv_content(entry): + # -- A key has no file of its own: the size stays 0 + st = stat_fn(await self.__git_subdir(entry)) + return self._stat_result_from_os(st, FT_SECRET_KEY, size = 0) + if os.path.isfile(file_path): + return self._stat_result_from_os(stat_fn(file_path), FT_SECRET_FILE) + if os.path.isdir(file_path): + return self._stat_result_from_os(stat_fn(file_path), FT_DIR) + raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}') + + async def __keyfile(self) -> str: + # -- The house variable is the sole input; SOPS_AGE_KEY_FILE + # is set per invocation on the sops subprocess, never read + # from the process environment + if self.___keyfile is None: + keyfile = os.environ.get('JW_SECRETS_SOPS_KEY_PATH') + if not keyfile: + raise SopsKeyError('JW_SECRETS_SOPS_KEY_PATH is not set') + self.___keyfile = keyfile + return self.___keyfile + + async def __recipient(self) -> str: + if self.___recipient is None: + from ...base import InputMode + + keyfile = await self.__keyfile() + ec = await self.__crypt_ec() + # -- Pipe the output: in interactive mode the pty relays + # the child's stdout to the console as well + result = await ec.run( + ['age-keygen', '-y', keyfile], + cmd_input = InputMode.NonInteractive, + throw = True, + ) + self.___recipient = result.stdout.decode().strip() + return self.___recipient + + async def __put_key(self, entry: str, key: str, content: bytes) -> Result: + # -- Merge the key into the entry, replacing the entry's + # existing value for the key + self.__kv_cache.pop(entry, None) + try: + kv = await self.__decrypt_kv_content(entry) + except FileNotFoundError: + kv = {} + kv[key] = content.decode() + await self.__encrypt(await self.__git_subdir(entry), serialize_kv(kv), 'dotenv') + return Result() + + async def __put_entry(self, entry: str, content: bytes) -> Result: + # -- Replace the entry: the content is the serialized + # entry, encrypted wholesale, so a key not in it is gone + self.__kv_cache.pop(entry, None) + await self.__encrypt(await self.__git_subdir(entry), content, 'dotenv') + return Result() + + async def __put_blob(self, path: str, content: bytes) -> Result: + await self.__encrypt(await self.__git_subdir(path), content, 'binary') + return Result() + + async def __encrypt(self, path: str, content: bytes, input_type: str) -> None: + # -- Encrypt the content to the repository file. The + # plaintext lives in a temporary file next to it until the + # encryption succeeds, and the recipient is derived from + # the SOPS age key the store decrypts with + from ...base import InputMode + + ec = await self.__crypt_ec() + recipient = await self.__recipient() + os.makedirs(os.path.dirname(path), exist_ok = True) + tmp = f'{path}.tmp' + try: + with open(tmp, 'wb') as f: + f.write(content) + await ec.run( + [ + 'sops', + '--input-type', + input_type, + '--output-type', + 'dotenv', + '--encrypt', + '--age', + recipient, + '--output', + path, + tmp, + ], + cmd_input = InputMode.NonInteractive, + mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()}, + throw = True, + ) + except Exception: + with suppress(FileNotFoundError): + os.unlink(tmp) + raise + os.unlink(tmp) + + async def __decrypt(self, path: str) -> bytes: + # -- The decryption is a pure function of the file's content, + # so the result is cached by the content's hash: the entry + # check and the content fetch of one get() share one + # decryption, a repeated read of an unchanged file is + # free, and a changed file - after a put, or an external + # update of the store - re-decrypts under its new hash + # without any invalidation + if not os.path.isfile(path): + raise FileNotFoundError(path) + with open(path, 'rb') as f: + raw = f.read() + key = hashlib.sha256(raw).digest() + ret = self.__decrypt_cache.get(key) + if ret is None: + ec = await self.__crypt_ec() + ret = ( + await ec.run( + [ + 'sops', + 'decrypt', + '--input-type', + 'dotenv', + '--output-type', + 'dotenv', + ], + cmd_input = raw, + mod_env = {'SOPS_AGE_KEY_FILE': await self.__keyfile()}, + throw = True, + ) + ).stdout or b'' + self.__decrypt_cache[key] = ret + return ret + + async def __decrypt_kv_content(self, entry: str) -> dict[str, str]: + ret = self.__kv_cache.get(entry) + if ret is None: + path = await self.__git_subdir(entry) + ret = parse_kv(await self.__decrypt(path)) + self.__kv_cache[entry] = ret + return ret + + async def __is_entry(self, entry: str) -> bool: + # -- The file at entry is an entry: it decrypts to a kv + # file, and the single data key sops gives a blob is not + # all of its content + path = await self.__git_subdir(entry) + if not os.path.isfile(path): + return False + try: + kv = await self.__decrypt_kv_content(entry) + except SopsKeyError: + raise + except Exception: + return False + return not (len(kv) == 1 and 'data' in kv) + + # -- Override base class methods + + @override + async def _close(self) -> None: + if self.___git_checkout is not None and self.___git_checkout != self.__repo: + ec = await self.__crypt_ec() + await ec.erase(self.___git_checkout) + self.___git_checkout = None + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + path = self._unchroot(path) + return await self._call_with_fallback( + path, + lambda p: self.__git_stat(p, follow_symlinks), + ) + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + path = self._unchroot(path) + return await self._call_with_fallback(path, self.__git_get) + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + # -- Resolve the address to the absolute namespace path in + # the host tree; a write does not take the default + # fallback. + path = self._unchroot(path) + if not path: + raise ValueError(f'put: "{path}" is not an entry/key or blob path') + storage_path = self._path_in_storage(path) + if ftype.is_subtype(FT_SECRET): + return await self.__put_entry(storage_path, content) + if ftype.is_subtype(FT_SECRET_KEY): + entry, sep, key = storage_path.rpartition('/') + if not sep or not key: + raise ValueError(f'put: "{path}" is not of the form path/key') + return await self.__put_key(entry, key, content) + if ftype.is_subtype(FT_SECRET_FILE): + return await self.__put_blob(storage_path, content) + raise ValueError(f'put: ftype {ftype} is not a store content type') + + @override + def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool: + # -- sops re-serializes a dotenv entry on decrypt: it + # drops blank lines and adds a trailing newline, so + # the byte check of Base._put_is_noop() never fires + # for content outside that form, and every put + # re-encrypts the unchanged entry. Compare the + # sops-canonical forms. A blob round-trips + # byte-exact and keeps the byte check + if ftype.is_subtype(FT_SECRET): + return _sops_dotenv_norm(existing) == _sops_dotenv_norm(content) + return existing == content + + @override + async def _readdir(self, path: str) -> Iterable[str]: + path = self._unchroot(path) + return await self._call_with_fallback(path, self.__git_readdir) diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/Makefile b/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/Makefile new file mode 100644 index 00000000..e0959bfb --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py b/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py new file mode 100644 index 00000000..38968f97 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py @@ -0,0 +1,645 @@ +import asyncio +import os +import shutil +import stat +import subprocess +import tempfile + +from glob import glob +from typing import override + +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.ec.secret_store.KvFile import KvFile +from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY +from jw.pkg.lib.ec.secret_store.Sops import Sops, SopsKeyError +from jw.pkg.lib.file_types import FT_DIR +from jw.pkg.lib.FileContext import FileContext +from jw.pkg.lib.kv import parse_kv +from jw.pkg.lib.util import copy + +def encrypt(repo: str, rel: str, text: str, input_type: str, recipient: str) -> None: + path = os.path.join(repo, rel) + os.makedirs(os.path.dirname(path), exist_ok = True) + with open(path, 'w') as f: + f.write(text) + subprocess.run( + [ + 'sops', + '--input-type', + input_type, + '--output-type', + 'dotenv', + '--encrypt', + '--age', + recipient, + '--output', + path, + path + ], + check = True, + capture_output = True, + ) + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + # -- A throw-away age key, resolved via JW_SECRETS_SOPS_KEY_PATH + keyfile = os.path.join(tmp, 'key.txt') + subprocess.run( + ['age-keygen', '-o', keyfile], check = True, capture_output = True + ) + recipient = subprocess.run( + ['age-keygen', '-y', keyfile], check = True, capture_output = True + ).stdout.decode().strip() + os.environ['JW_SECRETS_SOPS_KEY_PATH'] = keyfile + + # -- A store in the canonical layout, SOPS-encrypted + repo = os.path.join(tmp, 'repo') + encrypt(repo, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient) + encrypt(repo, 'default/etc/d.jw-secret', 'd=4\n', 'dotenv', recipient) + encrypt(repo, 'com/example/host/etc/a.jw-secret', 'a=2\n', 'dotenv', recipient) + encrypt(repo, 'com/example/host/etc/b.jw-secret', 'b=3\n', 'dotenv', recipient) + # Non-key-value content is stored as a .jw-secret-file and + # addressable as a blob + encrypt( + repo, + 'com/example/host/etc/c.pem.jw-secret-file', + '-----BEGIN CERT-----\n', + 'binary', + recipient + ) + # A branch with a blob, for the deploy: it must land as a + # real directory, not a phantom entry + encrypt( + repo, + 'com/example/host/etc/new/x.pem.jw-secret-file', + '-----BEGIN NEW BRANCH CERT-----\n', + 'binary', + recipient + ) + # A file without a content suffix is store content too: + # non-key-value content is a blob + encrypt( + repo, + 'com/example/host/etc/README.md', + 'not key-value content\n', + 'binary', + recipient + ) + + # -- The whole store: strict, no fallback + os.environ['JW_SECRETS_SOPS_REPO'] = repo + os.environ['JW_SECRETS_SOPS_ROOT'] = '' + store = FileContext.create('sops://') + await store.open() + assert set(await store.readdir('')) == {'default', 'com'} + # -- The names as stored: the store does not strip or filter + # by file name, a suffixed name being the address of its + # node + assert set(await store.readdir('com/example/host/etc')) == \ + {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', 'new', + 'README.md'} + assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1' + assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \ + b'2' + assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \ + b'3' + + # The entry is gettable as a whole: the decrypted dotenv + assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n' + + # A blob is a file: get() returns its bytes, readdir() refuses + assert (await store.get( + 'com/example/host/etc/c.pem.jw-secret-file')).stdout == \ + b'-----BEGIN CERT-----\n' + try: + await store.readdir('com/example/host/etc/c.pem.jw-secret-file') + raise AssertionError('expected NotADirectoryError') + except NotADirectoryError: + pass + + # A file without a content suffix is store content too: a + # non-key-value file is a blob, listed, statable, gettable + assert 'README.md' in set(await store.readdir('com/example/host/etc')) + st = await store.stat('com/example/host/etc/README.md') + assert st.ftype is FT_SECRET_FILE + assert (await store.get('com/example/host/etc/README.md')).stdout == \ + b'not key-value content\n' + + # A missing entry and a missing key raise FileNotFoundError + for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'): + try: + await store.get(path) + raise AssertionError(f'expected FileNotFoundError for {path}') + except FileNotFoundError: + pass + await store.close() + # A local directory is used as-is; close() must not remove it + assert os.path.isdir(repo) + + # -- A host-rooted store applies the default fallback + store = FileContext.create('sops://host.example.com') + assert isinstance(store, Sops) + await store.open() + assert (await store.get('etc/a.jw-secret/a')).stdout == b'2' + # A directory listing is the union with the default tree: d + # exists only under default/, but is listable as it is + # gettable + assert set(await store.readdir('etc')) == \ + {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', + 'd.jw-secret', 'new', 'README.md'} + # A miss below the host is retried below the default tree + assert (await store.get('etc/d.jw-secret/d')).stdout == b'4' + # A key listing stays host-wins-wholesale + assert set(await store.readdir('etc/d.jw-secret')) == {'d'} + + # -- stat() answers the node's type: an entry is a file that + # enumerates its keys, a key a leaf below it, a blob a + # file, a branch a plain directory; a key has no file of + # its own, and keeps the size 0 + st = await store.stat('etc') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + assert st.size == os.path.getsize(os.path.join(repo, 'com/example/host/etc')) + st = await store.stat('etc/a.jw-secret') + assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode) + assert st.size == os.path.getsize( + os.path.join(repo, 'com/example/host/etc/a.jw-secret') + ) + st = await store.stat('etc/a.jw-secret/a') + assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode) + assert st.size == 0 + st = await store.stat('etc/c.pem.jw-secret-file') + assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode) + assert st.size == os.path.getsize( + os.path.join(repo, 'com/example/host/etc/c.pem.jw-secret-file') + ) + # A miss below the host falls back to the default tree + st = await store.stat('etc/d.jw-secret/d') + assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode) + try: + await store.stat('etc/missing') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + # -- file_exists() works on the store, being built on stat() + assert await store.file_exists('etc/a.jw-secret') + assert not await store.file_exists('etc/missing') + await store.close() + + # -- A store address as a copy source: FileContext.create() + # resolves the scheme + dst = os.path.join(tmp, 'copied', 'a') + os.makedirs(os.path.dirname(dst)) + ret = await copy('sops://host.example.com/etc/a.jw-secret/a', dst) + assert ret == dst + with open(dst) as f: + assert f.read() == '2' + + # A single object that is an entry: the serialized whole + # lands at the destination + dst = os.path.join(tmp, 'copied', 'entry') + os.makedirs(os.path.dirname(dst), exist_ok = True) + ret = await copy('sops://host.example.com/etc/a.jw-secret', dst) + assert ret == dst + with open(dst) as f: + assert f.read() == 'a=2\n' + + # -- A store entry as a recursive copy source: the entry is + # gettable as a whole, and the walk terminates at it + tree = os.path.join(tmp, 'tree') + ret = await copy( + 'sops://host.example.com/etc/a.jw-secret', tree, recursive = True + ) + assert ret == tree + with open(tree) as f: + assert f.read() == 'a=2\n' + + # -- Deploy: a recursive copy from the store to the on-host + # layout; the kvfile scheme wraps the target context, + # chrooted at the root of the on-host tree. The host tree + # is a precondition: etc/ exists + host_tree = os.path.join(tmp, 'host-tree') + os.makedirs(os.path.join(host_tree, 'etc')) + ret = await copy( + 'sops://host.example.com', f'kvfile://{host_tree}', recursive = True + ) + assert isinstance(ret, list) + assert set(ret) == { + 'etc/a.jw-secret', + 'etc/b.jw-secret', + 'etc/c.pem.jw-secret-file', + 'etc/d.jw-secret', + 'etc/new/x.pem.jw-secret-file', + 'etc/README.md', + } + # -- Entries land as entry files, mode 0400; the host + # value wins over default, and the default-only entry d + # lands from the union listing + for entry, kv in ( + ('etc/a', {'a': '2'}), + ('etc/b', {'b': '3'}), + ('etc/d', {'d': '4'}), + ): + path = os.path.join(host_tree, *entry.split('/')) + '.jw-secret' + assert os.path.isfile(path) + assert stat.S_IMODE(os.stat(path).st_mode) == 0o400 + with open(path, 'rb') as fb: + assert parse_kv(fb.read()) == kv + # -- The blob lands as a .jw-secret-file at its config path, + # and no .jw-secret garbage is created for the branch + pem = os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file') + with open(pem) as f: + assert f.read() == '-----BEGIN CERT-----\n' + assert not os.path.exists(os.path.join(host_tree, 'etc.jw-secret')) + # -- The unsuffixed blob lands suffixed at its config path + readme = os.path.join(host_tree, 'etc', 'README.md.jw-secret-file') + with open(readme) as f: + assert f.read() == 'not key-value content\n' + # -- A new branch is a real directory, not a phantom entry: + # its blob lands suffixed below it + assert os.path.isdir(os.path.join(host_tree, 'etc', 'new')) + new_pem = os.path.join(host_tree, 'etc', 'new', 'x.pem.jw-secret-file') + with open(new_pem) as f: + assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n' + assert not os.path.exists(os.path.join(host_tree, 'etc', 'new.jw-secret')) + + # -- A recursive copy onto a plain host tree: an entry + # lands as a single file with its serialized content, a + # branch as a directory, and a blob as a file + host = os.path.join(tmp, 'plain-host') + os.makedirs(host) + ret = await copy('sops://host.example.com', host, recursive = True) + assert isinstance(ret, list) + assert set(ret) == { + f'{host}/etc/a.jw-secret', + f'{host}/etc/b.jw-secret', + f'{host}/etc/c.pem.jw-secret-file', + f'{host}/etc/d.jw-secret', + f'{host}/etc/new/x.pem.jw-secret-file', + f'{host}/etc/README.md', + } + with open(os.path.join(host, 'etc', 'a.jw-secret')) as f: + assert f.read() == 'a=2\n' + assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a')) + with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f: + assert f.read() == '-----BEGIN CERT-----\n' + assert os.path.isdir(os.path.join(host, 'etc', 'new')) + with open(os.path.join(host, 'etc', 'new', 'x.pem.jw-secret-file')) as f: + assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n' + + # -- A single object into the on-host layout: with --parents, + # the object lands at its source address + host_tree = os.path.join(tmp, 'host-tree2') + os.makedirs(os.path.join(host_tree, 'etc')) + ret = await copy( + 'sops://host.example.com/etc/a.jw-secret/a', + f'kvfile://{host_tree}', + parents = True + ) + assert ret == 'etc/a.jw-secret/a' + with open(os.path.join(host_tree, 'etc', 'a.jw-secret'), 'rb') as fb: + assert parse_kv(fb.read()) == {'a': '2'} + ret = await copy( + 'sops://host.example.com/etc/c.pem.jw-secret-file', + f'kvfile://{host_tree}', + parents = True + ) + assert ret == 'etc/c.pem.jw-secret-file' + with open(os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')) as f: + assert f.read() == '-----BEGIN CERT-----\n' + + # -- Fetch: a recursive copy from the on-host layout into + # the store. The kvfile source is handed the host's + # package set, and lists its branches from it; the + # entries and the blob land below the destination host's + # prefix + host_tree = os.path.join(tmp, 'fetch-host') + os.makedirs(os.path.join(host_tree, 'etc')) + with open(os.path.join(host_tree, 'etc', 'foo.conf.jw-secret'), 'w') as fh: + fh.write('f1=1\nf2=2\n') + with open(os.path.join(host_tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh: + fh.write('-----BEGIN FETCHED CERT-----\n') + # -- As on a real host, the deployed config and cert are + # present too: the key paths below the entry have no + # host counterpart + with open(os.path.join(host_tree, 'etc', 'foo.conf'), 'w') as fh: + fh.write('deployed config\n') + with open(os.path.join(host_tree, 'etc', 'cert.pem'), 'w') as fh: + fh.write('deployed cert\n') + + class FakeDistro(SuseDistro): + + def __init__(self, files: list[str]) -> None: + super().__init__(Local(), id = 'suse') + self.files = files + + @override + async def _pkg_files(self, names: list[str] | None) -> list[str]: + return self.files + + fake = FakeDistro( + [ + f'{host_tree}/etc/foo.conf.jw-tmpl', + f'{host_tree}/etc/cert.pem.jw-tmpl', + ] + ) + # -- The package file list reports host-absolute paths, so + # the fixture tree sits at its path below the host root + src = KvFile(Local(), distro = fake) + base = host_tree.lstrip('/') + ret = await copy(src, 'sops://fetch.example.com', recursive = True) + assert isinstance(ret, list) + # -- The copy transports the on-host file names unchanged + assert set(ret) == { + f'{base}/etc/foo.conf.jw-secret', + f'{base}/etc/cert.pem.jw-secret-file', + } + fstore = FileContext.create('sops://fetch.example.com') + await fstore.open() + assert set(await fstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \ + {'f1', 'f2'} + assert (await fstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \ + b'1' + assert (await fstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \ + b'-----BEGIN FETCHED CERT-----\n' + await fstore.close() + + # -- A host subtree: the fallback mirrors below the subtree. + # root stays the URI path; the namespace root is separate + store = FileContext.create('sops://host.example.com/etc') + assert isinstance(store, Sops) + await store.open() + assert store.root == '/etc' + assert (await store.get('a.jw-secret/a')).stdout == b'2' + assert (await store.get('d.jw-secret/d')).stdout == b'4' + await store.close() + + # -- A store is always chrooted: an explicit value is + # overridden, the address space stays relative + store = Sops('sops://host.example.com/etc', chroot = False) + await store.open() + assert store.chroot + assert (await store.get('a.jw-secret/a')).stdout == b'2' + assert (await store.get('d.jw-secret/d')).stdout == b'4' + assert set(await store.readdir('a.jw-secret')) == {'a'} + await store.close() + + # -- The default tree is terminal: no fallback from it + store = FileContext.create('sops://default') + await store.open() + assert (await store.get('etc/a.jw-secret/a')).stdout == b'1' + try: + await store.get('etc/missing') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + await store.close() + + # -- A store as a copy destination: the walk asks the source + # which node is an entry; entries land as entry files + # below the destination host's prefix, blobs at their + # config path, sops-encrypted in the repository + ret = await copy( + 'sops://host.example.com', 'sops://host2.example.com', recursive = True + ) + assert isinstance(ret, list) + assert set(ret) == { + 'etc/a.jw-secret', + 'etc/b.jw-secret', + 'etc/c.pem.jw-secret-file', + 'etc/d.jw-secret', + 'etc/new/x.pem.jw-secret-file', + 'etc/README.md', + } + store = FileContext.create('sops://host2.example.com') + await store.open() + assert (await store.get('etc/a.jw-secret/a')).stdout == b'2' + assert (await store.get('etc/b.jw-secret/b')).stdout == b'3' + assert (await store.get('etc/d.jw-secret/d')).stdout == b'4' + assert (await store.get('etc/c.pem.jw-secret-file')).stdout == \ + b'-----BEGIN CERT-----\n' + await store.close() + # -- The content lands in suffixed files below the host + # prefix, and no entry is created for the branch + assert os.path.isfile(os.path.join(repo, 'com/example/host2/etc/a.jw-secret')) + assert os.path.isfile( + os.path.join(repo, 'com/example/host2/etc/c.pem.jw-secret-file') + ) + assert not os.path.exists(os.path.join(repo, 'com/example/host2/etc.jw-secret')) + + # -- put(): a key put merges into its entry, a leaf put + # writes a blob, a leaf below an existing entry lands as + # a key there, and a top-level object is refused + store = FileContext.create('sops://host.example.com') + await store.open() + await store.put('etc/e.jw-secret/e', b'5', ftype = FT_SECRET_KEY) + assert (await store.get('etc/e.jw-secret/e')).stdout == b'5' + await store.put( + 'etc/f.pem.jw-secret-file', + b'-----BEGIN KEY-----\n', + ftype = FT_SECRET_FILE + ) + assert (await store.get('etc/f.pem.jw-secret-file')).stdout == \ + b'-----BEGIN KEY-----\n' + await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY) + assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'} + await store.put('etc/a.jw-secret/a', b'9', ftype = FT_SECRET_KEY) + assert (await store.get('etc/a.jw-secret/a')).stdout == b'9' + # -- a typed put replaces the entry: the serialized content + # is written wholesale, keys not in it are gone + await store.put('etc/a.jw-secret', b'a=9\nextra=e\n', ftype = FT_SECRET) + assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'} + assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\nextra=e\n' + assert (await store.get('etc/a.jw-secret/a')).stdout == b'9' + # -- A one-component address is a blob at a top-level config + # path, as on the read side + await store.put('top', b'1', ftype = FT_SECRET_FILE) + assert (await store.get('top')).stdout == b'1' + # -- An untyped put is refused: the store content types are + # entry, key, and blob + try: + await store.put('etc/untyped', b'x') + raise AssertionError('expected ValueError') + except ValueError: + pass + # -- The repository files are sops files in dotenv format, + # as in the pre-existing store: the flat sops_version + # data line marks it + for rel in ('com/example/host/etc/e.jw-secret', + 'com/example/host/etc/f.pem.jw-secret-file'): + with open(os.path.join(repo, rel), 'rb') as data: + assert b'sops_version=' in data.read() + # -- The namespace root itself has no store home + try: + await store.put('', b'1') + raise AssertionError('expected ValueError') + except ValueError: + pass + # -- A whole get returns the stored bytes verbatim: the + # decryption is cached by the content's hash, so a put of + # other content re-decrypts, and no re-serialization + # happens on read + await store.put('etc/verbat.jw-secret', b'v=1\n', ftype = FT_SECRET) + assert (await store.get('etc/verbat.jw-secret')).stdout == b'v=1\n' + # -- Non-canonical content is stored and returned as-is + content = b'# a comment\nv= padded \n' + await store.put('etc/verbat.jw-secret', content, ftype = FT_SECRET) + assert (await store.get('etc/verbat.jw-secret')).stdout == content + # -- A key put merges through the stored content: the comment + # and the padding do not survive the merge, and a whole + # get after the put sees the merged entry + await store.put('etc/verbat.jw-secret/k2', b'2', ftype = FT_SECRET_KEY) + assert (await store.get('etc/verbat.jw-secret')).stdout == \ + b'v=padded\nk2=2\n' + # -- A blob put replaces the blob: a get after the put returns + # the new content + await store.put( + 'etc/vb.pem.jw-secret-file', + b'-----BEGIN ONE-----\n', + ftype = FT_SECRET_FILE + ) + assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \ + b'-----BEGIN ONE-----\n' + await store.put( + 'etc/vb.pem.jw-secret-file', + b'-----BEGIN TWO-----\n', + ftype = FT_SECRET_FILE + ) + assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \ + b'-----BEGIN TWO-----\n' + # -- A put of content sops normalizes on read (blank lines + # dropped, a trailing newline added) is a no-op when + # repeated: the no-op check compares the sops-canonical + # forms, so the repository file is not re-encrypted + content = b'n=1\n\nn2=2' + await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET) + assert (await store.get('etc/nl.jw-secret')).stdout == b'n=1\nn2=2\n' + repo_file = os.path.join(repo, 'com/example/host/etc/nl.jw-secret') + with open(repo_file, 'rb') as fb: + first = fb.read() + await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET) + with open(repo_file, 'rb') as fb: + assert fb.read() == first + # -- ...and a changed content still lands + await store.put('etc/nl.jw-secret', b'n=1\nn2=3', ftype = FT_SECRET) + with open(repo_file, 'rb') as fb: + assert fb.read() != first + # -- Already-canonical content no-ops on the byte check + await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET) + repo_file = os.path.join(repo, 'com/example/host/etc/nlc.jw-secret') + with open(repo_file, 'rb') as fb: + first = fb.read() + await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET) + with open(repo_file, 'rb') as fb: + assert fb.read() == first + # -- A blob is byte-exact: dropping the trailing newline is + # a change, not a no-op + repo_file = os.path.join(repo, 'com/example/host/etc/nlb.pem.jw-secret-file') + await store.put('etc/nlb.pem.jw-secret-file', b'x\n', ftype = FT_SECRET_FILE) + with open(repo_file, 'rb') as fb: + first = fb.read() + await store.put('etc/nlb.pem.jw-secret-file', b'x', ftype = FT_SECRET_FILE) + with open(repo_file, 'rb') as fb: + assert fb.read() != first + await store.close() + + # -- JW_SECRETS_SOPS_ROOT: a base directory within the repository + base = os.path.join(tmp, 'repo2', 'my', 'secrets') + encrypt(base, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient) + os.environ['JW_SECRETS_SOPS_REPO'] = os.path.join(tmp, 'repo2') + os.environ['JW_SECRETS_SOPS_ROOT'] = 'my/secrets' + store = FileContext.create('sops://') + await store.open() + assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1' + await store.close() + + # -- The same store as a git URL is cloned on the fly and + # the checkout is removed by close() + hooks = os.path.join(tmp, 'hooks') + os.makedirs(hooks, exist_ok = True) + env = dict( + os.environ, + GIT_AUTHOR_NAME = 'test', + GIT_AUTHOR_EMAIL = 'test@example.com', + GIT_COMMITTER_NAME = 'test', + GIT_COMMITTER_EMAIL = 'test@example.com', + ) + subprocess.run( + ['git', 'init', '--quiet'], + cwd = repo, + check = True, + env = env, + capture_output = True + ) + subprocess.run( + ['git', 'add', '-A'], + cwd = repo, + check = True, + env = env, + capture_output = True + ) + subprocess.run( + [ + 'git', + '-c', + 'commit.gpgsign=false', + '-c', + f'core.hooksPath={hooks}', + 'commit', + '--quiet', + '-m', + 'test', + ], + cwd = repo, + check = True, + env = env, + capture_output = True + ) + os.environ['JW_SECRETS_SOPS_REPO'] = 'file://' + repo + os.environ['JW_SECRETS_SOPS_ROOT'] = '' + checkouts = set(glob('/tmp/jw-pkg-sops-*')) + store = FileContext.create('sops://') + await store.open() + assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1' + await store.close() + assert set(glob('/tmp/jw-pkg-sops-*')) == checkouts + + # -- Key resolution: JW_SECRETS_SOPS_KEY_PATH is the sole + # input; a SOPS_AGE_KEY_FILE in the process environment is + # ignored, since the store sets it per invocation on the + # sops subprocess. This section also switches the + # repository from the git URL to the local directory + os.environ['SOPS_AGE_KEY_FILE'] = '/nonexistent/age-key' + os.environ['JW_SECRETS_SOPS_REPO'] = repo + os.environ['JW_SECRETS_SOPS_ROOT'] = '' + store = FileContext.create('sops://') + await store.open() + assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1' + await store.close() + del os.environ['SOPS_AGE_KEY_FILE'] + + # -- No key file: the failure names the house variable + saved = os.environ['JW_SECRETS_SOPS_KEY_PATH'] + del os.environ['JW_SECRETS_SOPS_KEY_PATH'] + store = FileContext.create('sops://') + await store.open() + try: + await store.get('default/etc/a.jw-secret/a') + assert False, 'expected a key resolution failure' + except SopsKeyError as e: + assert 'JW_SECRETS_SOPS_KEY_PATH is not set' in str(e) + await store.close() + os.environ['JW_SECRETS_SOPS_KEY_PATH'] = saved + +def main() -> None: + if not shutil.which('sops') or not shutil.which('age-keygen') \ + or not shutil.which('git'): + print( + 'Skipping secrets sops store tests (sops, age-keygen, and ' + 'git required)' + ) + return + asyncio.run(run_tests()) + print('All secrets sops store tests passed') + +main() -- 2.56.0