lib.ec.secret_store.Vault: Add the store class

This commit adds the Vault store and the vault:// dispatch in FileContext.

A Vault KV v2 server maps nicely into jw-pkg's secret handling abstraction
as just another secret store backend. Environment variables identify the
storage details, analogous to the way it's handled in the SOPS backend.

- JW_SECRETS_VAULT_ADDR names the server, a unix socket address or an http
  URL; the standard VAULT_ADDR is never read; JW_SECRETS_VAULT_MOUNT the
  mount path, default secret; the token is the standard VAULT_TOKEN
- The transport is one curl per API call, through the Local context: a
  unix:// server is reached through its socket, an http:// server through
  its port
- A node's type is derived at stat time: an entry, a blob, a key below an
  entry, a branch if the path has children, otherwise a miss
- A blob's content is base64 under a single key of its KV secret, the
  marker being a path component, so a KV secret that is not a blob is
  always an entry

The test drives a dev server fixture from the workspace: stat/readdir/get
on the fixture, the default fallback, the storage marker, and the copy
forms - single key, whole entry, recursive entry, deploy to the on-host
layout, and a host subtree.

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0

test/unit/python/jw/pkg/lib/ec/secret_store/Vault/Makefile

@@ -0,0 +1,7 @@
+TOPDIR = ../../../../../../../../../
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-run.mk
+
+all:
+test: run

test/unit/python/jw/pkg/lib/ec/secret_store/Vault/test.py

@@ -0,0 +1,383 @@
+import asyncio
+import base64
+import json
+import os
+import stat
+import subprocess
+import tempfile
+import uuid
+
+from pathlib import Path
+from typing import Any, override
+
+from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
+from jw.pkg.lib.ec.Local import Local
+from jw.pkg.lib.ec.secret_store.KvFile import KvFile
+from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
+from jw.pkg.lib.ec.secret_store.Vault import BLOB_KEY
+from jw.pkg.lib.file_types import FT_DIR
+from jw.pkg.lib.FileContext import FileContext
+from jw.pkg.lib.kv import parse_kv
+from jw.pkg.lib.util import copy
+
+# -- The dev server is a fixture of the workspace, not a dependency
+#    of the tree: test/vault, started by
+#    test/scripts/setup-vault-test-server.sh
+def vault_base() -> Path:
+    return Path(__file__).resolve().parents[10] / 'test' / 'vault'
+
+def vault_sock() -> str:
+    return str(vault_base() / 'vault.sock')
+
+def vault_api(method: str,
+              api_path: str,
+              data: dict[str, Any] | None = None) -> subprocess.CompletedProcess[str]:
+    cmd = [
+        'curl',
+        '--silent',
+        '--show-error',
+        '--fail-with-body',
+        '--request',
+        method,
+        '--unix-socket',
+        vault_sock(),
+        '--header',
+        f'X-Vault-Token: {os.environ["VAULT_TOKEN"]}',
+        # -- Required by Vault 3 to route the request to the API
+        '--header',
+        'X-Vault-Request: true',
+        f'http://localhost/v1/{api_path}',
+    ]
+    if data is not None:
+        cmd += ['--data', json.dumps(data)]
+    return subprocess.run(cmd, capture_output = True, text = True)
+
+def vault_put_kv(rel: str, kv: dict[str, str]) -> None:
+    vault_api('POST', f'secret/data/{rel}', {'data': kv}).check_returncode()
+
+def vault_get_meta_version(rel: str) -> int:
+    ret = vault_api('GET', f'secret/data/{rel}')
+    ret.check_returncode()
+    return int(json.loads(ret.stdout)['data']['metadata']['version'])
+
+def have_vault() -> bool:
+    # -- Skip the test if the dev server is not up
+    if not os.path.exists(vault_sock()) or not (vault_base() / 'token').exists():
+        return False
+    os.environ['JW_SECRETS_VAULT_ADDR'] = f'unix://{vault_sock()}'
+    os.environ['VAULT_TOKEN'] = (vault_base() / 'token').read_text().strip()
+    ret = vault_api('GET', 'secret/metadata?list=true')
+    return ret.returncode == 0
+
+async def run_tests() -> None:
+    if not have_vault():
+        return
+
+    # -- A unique host per run, so the test does not collide with
+    #    whatever else the dev server holds
+    host = f'vault-{uuid.uuid4().hex[:8]}.example.com'
+    rev = '/'.join(reversed(host.split('.')))
+
+    # -- The fixture, seeded through the API: an entry with two
+    #    keys, a key-only default counterpart, a default-only entry,
+    #    a blob, and a branch holding a blob. Entries and blobs
+    #    alike are KV secrets at their store paths
+    vault_put_kv(f'{rev}/etc/a.jw-secret', {'a': '1', 'b': '2'})
+    vault_put_kv('default/etc/a.jw-secret', {'a': 'default-a'})
+    vault_put_kv('default/etc/d.jw-secret', {'d': 'default-d'})
+    vault_put_kv(
+        f'{rev}/etc/c.pem.jw-secret-file',
+        {BLOB_KEY: base64.b64encode(b'-----BEGIN CERT-----\n').decode()},
+    )
+    vault_put_kv(
+        f'{rev}/etc/new/x.pem.jw-secret-file',
+        {BLOB_KEY: base64.b64encode(b'-----BEGIN NEW BRANCH CERT-----\n').decode()},
+    )
+
+    with tempfile.TemporaryDirectory() as tmp:
+        store = FileContext.create(f'vault://{host}')
+        assert store is not None
+        await store.open()
+
+        # -- stat: the node's type is derived; the size is 0, as in
+        #    /proc
+        st = await store.stat('etc/a.jw-secret')
+        assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
+        assert st.size == 0
+        st = await store.stat('etc/a.jw-secret/b')
+        assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
+        st = await store.stat('etc/c.pem.jw-secret-file')
+        assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
+        st = await store.stat('etc/new')
+        assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
+        try:
+            await store.stat('etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        # -- file_exists() works on the store, being built on stat()
+        assert await store.file_exists('etc/a.jw-secret')
+        assert not await store.file_exists('etc/missing')
+
+        # -- The address is the KV secret's path: the blob at
+        #    etc/c.pem.jw-secret-file is the KV secret at that path
+        body = vault_api('GET', f'secret/data/{rev}/etc/c.pem.jw-secret-file')
+        body.check_returncode()
+        assert json.loads(body.stdout)['data']['data'] == {
+            BLOB_KEY: base64.b64encode(b'-----BEGIN CERT-----\n').decode(),
+        }
+
+        # -- readdir: the entry lists its keys
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'b'}
+        # -- the branch lists the names below it as stored, the
+        #    default tree unioned in
+        assert set(await store.readdir('etc')) == \
+            {'a.jw-secret', 'c.pem.jw-secret-file', 'd.jw-secret', 'new'}
+        # -- the root lists the host tree
+        assert set(await store.readdir('')) == {'etc'}
+        # -- a blob is not a directory
+        try:
+            await store.readdir('etc/c.pem.jw-secret-file')
+            raise AssertionError('expected NotADirectoryError')
+        except NotADirectoryError:
+            pass
+        # -- a miss is an error
+        try:
+            await store.readdir('etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+
+        # -- get: the key, the whole entry, the blob
+        assert (await store.get('etc/a.jw-secret/b')).stdout == b'2'
+        assert (await store.get('etc/a.jw-secret')).stdout == b'a=1\nb=2\n'
+        assert (await store.get('etc/c.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN CERT-----\n'
+        # -- a miss is an error
+        try:
+            await store.get('etc/a.jw-secret/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+
+        # -- The default fallback: a miss below the host is retried
+        #    below the default tree
+        assert (await store.get('etc/d.jw-secret')).stdout == b'd=default-d\n'
+        await store.close()
+
+        # -- A store address as a copy source: FileContext.create()
+        #    resolves the scheme
+        dst = os.path.join(tmp, 'copied', 'a')
+        os.makedirs(os.path.dirname(dst))
+        ret = await copy(f'vault://{host}/etc/a.jw-secret/a', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == '1'
+
+        # A single object that is an entry: the serialized whole
+        # lands at the destination
+        dst = os.path.join(tmp, 'copied', 'entry')
+        os.makedirs(os.path.dirname(dst), exist_ok = True)
+        ret = await copy(f'vault://{host}/etc/a.jw-secret', dst)
+        assert ret == dst
+        with open(dst) as f:
+            assert f.read() == 'a=1\nb=2\n'
+
+        # -- A store entry as a recursive copy source: the entry is
+        #    gettable as a whole, and the walk terminates at it
+        tree = os.path.join(tmp, 'tree')
+        ret = await copy(f'vault://{host}/etc/a.jw-secret', tree, recursive = True)
+        assert ret == tree
+        with open(tree) as f:
+            assert f.read() == 'a=1\nb=2\n'
+
+        # -- Deploy: a recursive copy from the store to the on-host
+        #    layout; the kvfile scheme wraps the target context,
+        #    chrooted at the root of the on-host tree. The host tree
+        #    is a precondition: etc/ exists
+        host_tree = os.path.join(tmp, 'host-tree')
+        os.makedirs(os.path.join(host_tree, 'etc'))
+        ret = await copy(f'vault://{host}', f'kvfile://{host_tree}', recursive = True)
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            'etc/a.jw-secret',
+            'etc/c.pem.jw-secret-file',
+            'etc/d.jw-secret',
+            'etc/new/x.pem.jw-secret-file',
+        }
+        # -- Entries land as entry files, mode 0400; the host
+        #    value wins over default, and the default-only entry d
+        #    lands from the union listing
+        for entry, kv in (
+            ('etc/a.jw-secret', {'a': '1', 'b': '2'}),
+            ('etc/d.jw-secret', {'d': 'default-d'}),
+        ):
+            path = os.path.join(host_tree, *entry.split('/'))
+            assert os.path.isfile(path)
+            assert stat.S_IMODE(os.stat(path).st_mode) == 0o400
+            with open(path, 'rb') as fb:
+                assert parse_kv(fb.read()) == kv
+        # -- The blob lands as a .jw-secret-file at its config path,
+        #    and no .jw-secret garbage is created for the branch
+        pem = os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')
+        with open(pem) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert not os.path.exists(os.path.join(host_tree, 'etc.jw-secret'))
+        # -- A branch is a real directory: its blob lands suffixed
+        #    below it
+        assert os.path.isdir(os.path.join(host_tree, 'etc', 'new'))
+        new_pem = os.path.join(host_tree, 'etc', 'new', 'x.pem.jw-secret-file')
+        with open(new_pem) as f:
+            assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
+        assert not os.path.exists(os.path.join(host_tree, 'etc', 'new.jw-secret'))
+
+        # -- A recursive copy onto a plain host tree: an entry
+        #    lands as a single file with its serialized content, a
+        #    branch as a directory, and a blob as a file
+        plain_host = os.path.join(tmp, 'plain-host')
+        os.makedirs(plain_host)
+        ret = await copy(f'vault://{host}', plain_host, recursive = True)
+        assert isinstance(ret, list)
+        assert set(ret) == {
+            f'{plain_host}/etc/a.jw-secret',
+            f'{plain_host}/etc/c.pem.jw-secret-file',
+            f'{plain_host}/etc/d.jw-secret',
+            f'{plain_host}/etc/new/x.pem.jw-secret-file',
+        }
+        with open(os.path.join(plain_host, 'etc', 'a.jw-secret')) as f:
+            assert f.read() == 'a=1\nb=2\n'
+        assert not os.path.exists(os.path.join(plain_host, 'etc', 'a', 'a'))
+        with open(os.path.join(plain_host, 'etc', 'c.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN CERT-----\n'
+        assert os.path.isdir(os.path.join(plain_host, 'etc', 'new'))
+        with open(os.path.join(plain_host, 'etc', 'new', 'x.pem.jw-secret-file')) as f:
+            assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
+
+        # -- A host subtree: the fallback mirrors below the subtree.
+        #    root stays the URI path; the namespace root is separate
+        store = FileContext.create(f'vault://{host}/etc')
+        assert store is not None
+        assert store.root == '/etc'
+        await store.open()
+        assert (await store.get('a.jw-secret/a')).stdout == b'1'
+        assert (await store.get('d.jw-secret/d')).stdout == b'default-d'
+        await store.close()
+
+        # -- put(): a key put merges into its entry, a leaf put
+        #    writes a blob, a leaf below an existing entry lands as
+        #    a key there, and a top-level object is a blob at a
+        #    top-level config path
+        store = FileContext.create(f'vault://{host}')
+        await store.open()
+        await store.put('etc/w1.jw-secret/k1', b'v1', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/w1.jw-secret/k1')).stdout == b'v1'
+        await store.put(
+            'etc/w2.pem.jw-secret-file',
+            b'-----BEGIN KEY-----\n',
+            ftype = FT_SECRET_FILE
+        )
+        assert (await store.get('etc/w2.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN KEY-----\n'
+        await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY)
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'b', 'extra'}
+        await store.put('etc/a.jw-secret/a', b'9', ftype = FT_SECRET_KEY)
+        assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
+        # -- a typed put replaces the entry: the serialized content
+        #    is written wholesale, and a KV v2 write is a full
+        #    replacement, so a key not in it is gone
+        await store.put('etc/a.jw-secret', b'a=9\nextra=e\n', ftype = FT_SECRET)
+        assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
+        assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\nextra=e\n'
+        try:
+            await store.get('etc/a.jw-secret/b')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        await store.put('top', b'1', ftype = FT_SECRET_FILE)
+        assert (await store.get('top')).stdout == b'1'
+        # -- a put that changes nothing leaves the secret version
+        #    untouched
+        version = vault_get_meta_version(f'{rev}/etc/w1.jw-secret')
+        await store.put('etc/w1.jw-secret/k1', b'v1', ftype = FT_SECRET_KEY)
+        assert vault_get_meta_version(f'{rev}/etc/w1.jw-secret') == version
+        # -- The namespace root itself has no store home
+        try:
+            await store.put('', b'1')
+            raise AssertionError('expected ValueError')
+        except ValueError:
+            pass
+        await store.close()
+
+        # -- unlink(): drops the entry, or the blob, and the name
+        #    leaves the listing
+        store = FileContext.create(f'vault://{host}')
+        await store.open()
+        await store.unlink('etc/w1.jw-secret')
+        assert not await store.file_exists('etc/w1.jw-secret')
+        await store.unlink('etc/w2.pem.jw-secret-file')
+        assert not await store.file_exists('etc/w2.pem.jw-secret-file')
+        try:
+            await store.unlink('etc/missing')
+            raise AssertionError('expected FileNotFoundError')
+        except FileNotFoundError:
+            pass
+        await store.close()
+
+        # -- Fetch: a recursive copy from the on-host layout into
+        #    the store. The kvfile source is handed the host's
+        #    package set, and the entries and the blob land below
+        #    the destination host's prefix
+        fetch_host = f'vault-{uuid.uuid4().hex[:8]}.fetch.example.com'
+        host_tree = os.path.join(tmp, 'fetch-host')
+        os.makedirs(os.path.join(host_tree, 'etc'))
+        with open(os.path.join(host_tree, 'etc', 'foo.conf.jw-secret'), 'w') as fh:
+            fh.write('f1=1\nf2=2\n')
+        with open(os.path.join(host_tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh:
+            fh.write('-----BEGIN FETCHED CERT-----\n')
+        # -- As on a real host, the deployed config and cert are
+        #    present too: the key paths below the entry have no
+        #    host counterpart
+        with open(os.path.join(host_tree, 'etc', 'foo.conf'), 'w') as fh:
+            fh.write('deployed config\n')
+        with open(os.path.join(host_tree, 'etc', 'cert.pem'), 'w') as fh:
+            fh.write('deployed cert\n')
+
+        class FakeDistro(SuseDistro):
+
+            def __init__(self, files: list[str]) -> None:
+                super().__init__(Local(), id = 'suse')
+                self.files = files
+
+            @override
+            async def _pkg_files(self, names: list[str] | None) -> list[str]:
+                return self.files
+
+        fake = FakeDistro(
+            [
+                f'{host_tree}/etc/foo.conf.jw-tmpl',
+                f'{host_tree}/etc/cert.pem.jw-tmpl',
+            ]
+        )
+        src = KvFile(Local(), distro = fake)
+        base = host_tree.lstrip('/')
+        ret = await copy(src, f'vault://{fetch_host}', recursive = True)
+        assert isinstance(ret, list)
+        # -- The copy transports the on-host file names unchanged
+        assert set(ret) == {
+            f'{base}/etc/foo.conf.jw-secret',
+            f'{base}/etc/cert.pem.jw-secret-file',
+        }
+        fstore = FileContext.create(f'vault://{fetch_host}')
+        await fstore.open()
+        assert set(await fstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \
+            {'f1', 'f2'}
+        assert (await fstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \
+            b'1'
+        assert (await fstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \
+            b'-----BEGIN FETCHED CERT-----\n'
+        await fstore.close()
+
+def main() -> None:
+    asyncio.run(run_tests())
+
+main()