janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

lib.ec.secret_store.Vault: Add the store class
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 4m37s
Details
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 4m56s
Details
CI / Packaging test (pull_request) Successful in 0s
Details
CI / Packaging - Kali Linux (push) Successful in 5m8s
Details
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 5m0s
Details
CI / Packaging test (push) Successful in 0s
Details

Browse source 
This commit adds the Vault store and the vault:// dispatch in FileContext.

A Vault KV v2 server maps nicely into jw-pkg's secret handling abstraction
as just another secret store backend. Environment variables identify the
storage details, analogous to the way it's handled in the SOPS backend.

- JW_SECRETS_VAULT_ADDR names the server, a unix socket address or an http
  URL; the standard VAULT_ADDR is never read; JW_SECRETS_VAULT_MOUNT the
  mount path, default secret; the token is the standard VAULT_TOKEN
- The transport is one curl per API call, through the Local context: a
  unix:// server is reached through its socket, an http:// server through
  its port
- A node's type is derived at stat time: an entry, a blob, a key below an
  entry, a branch if the path has children, otherwise a miss
- A blob's content is base64 under a single key of its KV secret, the
  marker being a path component, so a KV secret that is not a blob is
  always an entry

The test drives a dev server fixture from the workspace: stat/readdir/get
on the fixture, the default fallback, the storage marker, and the copy
forms - single key, whole entry, recursive entry, deploy to the on-host
layout, and a host subtree.

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.2.0
This commit is contained in:
Jan Lindemann 2026-10-04 22:30:10 +02:00
commit 8f87982490
Signed by: Jan Lindemann
GPG key ID: 3750640C9E25DD61
2 changed files with 744 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/python/jw/pkg/lib/FileContext.py
View file

@ -376,6 +376,10 @@ class FileContext(abc.ABC):
f'{scheme}://{uri.authority}', chroot = False
)
return KvFile(inner, uri = uri)
case 'vault':
from .ec.secret_store.Vault import Vault
return Vault(uri, *args, **kwargs)
case _:
pass
raise Exception(

350
src/python/jw/pkg/lib/ec/secret_store/Vault.py Normal file
View file

@ -0,0 +1,350 @@
from __future__ import annotations
import base64
import json
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Iterable, override
if TYPE_CHECKING:
from ...base import StatResult
from ...file_types import FileType
from ...Uri import Uri
from ...file_types import FT_DIR
from ...kv import parse_kv, serialize_kv
from ...Result import Result
from .Base import Base
from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
# -- The single key of a blob's KV secret: the base64 bytes
BLOB_KEY = 'content'
class Vault(Base):
"""A secret store in a Vault KV v2 backend.
The server and the mount are deployment information, not part of
the address: JW_SECRETS_VAULT_ADDR names the server, a unix://
socket address or an http:// URL, and JW_SECRETS_VAULT_MOUNT the
KV v2 mount path, defaulting to secret. The standard VAULT_ADDR
is never read. The store URI addresses a subtree of the
namespace, as in the base class; the token comes from the
standard VAULT_TOKEN environment variable.
A store path names a KV secret, and optionally a key below an
entry. The KV secret of an entry is its address, the map being
the entry, and its keys are addressable below it; a blob is a KV
secret at its address, holding the base64 bytes under a single
key. The store does not identify content by path: a KV map whose
only key is the blob's content key is a blob, anything else an
entry.
put() writes the secret at its address: a key merged into its
entry, an entry wholesale, or a blob as base64 under the single
content key. The store accepts entry, key, and blob content
only; a KV v2 write is a full replacement, so a key not in the
content is gone.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
super().__init__(uri, *args, **kwargs)
self.__addr = os.environ.get('JW_SECRETS_VAULT_ADDR')
self.__mount = os.environ.get('JW_SECRETS_VAULT_MOUNT', 'secret')
self.__secret_cache: dict[str, tuple[dict[str, str], float]] = {}
@staticmethod
def __mtime(created_time: str) -> float:
if not created_time:
return 0
return datetime.fromisoformat(created_time.replace('Z', '+00:00')).timestamp()
@staticmethod
def __is_blob_map(kv: dict[str, str]) -> bool:
# -- The map is a blob: the base64 bytes under the single
# content key
if len(kv) != 1 or BLOB_KEY not in kv:
return False
try:
base64.b64decode(kv[BLOB_KEY], validate = True)
except (ValueError, TypeError):
return False
return True
async def __api(
self,
method: str,
storage_path: str,
data: dict[str, Any] | None = None,
list_path: bool = False,
metadata: bool = False,
) -> dict[str, Any]:
# -- One Vault API call, <method> /v1/<mount>/<data|metadata>/<path>;
# a 404 is a miss, any other failure an error
from ...base import InputMode
from ..Local import Local
if not self.__addr:
raise Exception('JW_SECRETS_VAULT_ADDR is not set')
token = os.environ.get('VAULT_TOKEN')
if not token:
raise Exception('VAULT_TOKEN is not set')
kind = 'metadata' if metadata or list_path else 'data'
api_path = f'{self.__mount}/{kind}/{storage_path}'
if list_path:
api_path += '?list=true'
if self.__addr.startswith('unix://'):
url = f'http://localhost/v1/{api_path}'
socket_args = ['--unix-socket', self.__addr[len('unix://'):]]
else:
base = self.__addr.rstrip('/')
if not base.startswith('http'):
base = f'http://{base}'
url = f'{base}/v1/{api_path}'
socket_args = []
cmd = [
'curl',
'--silent',
'--show-error',
'--fail-with-body',
'--request',
method,
*socket_args,
'--header',
f'X-Vault-Token: {token}',
# -- Required by Vault 3 to route the request to the API
# rather than the web UI
'--header',
'X-Vault-Request: true',
url,
]
if data is not None:
cmd += ['--data', json.dumps(data)]
ret = await Local().run(
cmd, cmd_input = InputMode.NonInteractive, throw = False
)
body: dict[str, Any] = {}
if ret.stdout:
body = json.loads(ret.stdout)
if ret.status != 0:
if ret.status == 22:
raise FileNotFoundError(storage_path)
raise Exception(
f'vault api: {method} {api_path} failed: {ret.stderr.decode()}'
)
return body
async def __secret(self, storage_path: str) -> tuple[dict[str, str], float]:
# -- The KV secret at the storage path: its data map and the
# time derived from the created_time metadata
ret = self.__secret_cache.get(storage_path)
if ret is None:
body = await self.__api('GET', storage_path)
payload = body.get('data') or {}
raw = payload.get('data') or {}
kv = {str(k): str(v) for k, v in raw.items()}
meta = payload.get('metadata') or {}
ret = (kv, self.__mtime(str(meta.get('created_time', ''))))
self.__secret_cache[storage_path] = ret
return ret
async def __list(self, storage_path: str) -> list[str]:
# -- The children at the storage path, in the address space;
# a subdirectory is marked with a trailing slash
body = await self.__api('GET', storage_path, list_path = True)
keys = (body.get('data') or {}).get('keys') or []
ret = []
for k in keys:
if k.endswith('/'):
ret.append(k[:-1])
else:
ret.append(k)
return ret
async def __vault_stat(
self,
storage_path: str,
follow_symlinks: bool,
) -> StatResult:
try:
kv, mtime = await self.__secret(storage_path)
except FileNotFoundError:
pass
else:
if self.__is_blob_map(kv):
return self._stat_result('0', '0', mtime, FT_SECRET_FILE)
return self._stat_result('0', '0', mtime, FT_SECRET)
entry, sep, key = storage_path.rpartition('/')
if sep and key:
try:
kv, mtime = await self.__secret(entry)
except FileNotFoundError:
pass
else:
if key in kv:
return self._stat_result('0', '0', mtime, FT_SECRET_KEY)
if await self.__list(storage_path):
return self._stat_result('0', '0', 0, FT_DIR)
raise FileNotFoundError(storage_path)
async def __vault_readdir(self, storage_path: str) -> Iterable[str]:
if storage_path:
try:
kv, _ = await self.__secret(storage_path)
except FileNotFoundError:
pass
else:
if self.__is_blob_map(kv):
raise NotADirectoryError(storage_path)
return kv.keys()
names: set[str] = set()
try:
names.update(await self.__list(storage_path))
except FileNotFoundError:
pass
default_path = self._default_path_in_storage(storage_path)
if default_path is not None:
try:
names.update(await self.__list(default_path))
except FileNotFoundError:
pass
if not names:
raise FileNotFoundError(storage_path)
return sorted(names)
async def __vault_get(self, storage_path: str) -> Result:
# -- The entry is the KV map, serialized; the blob is the
# base64 bytes under BLOB_KEY of its KV secret
try:
kv, _ = await self.__secret(storage_path)
except FileNotFoundError:
kv = None
if kv is not None:
if self.__is_blob_map(kv):
return Result(stdout = base64.b64decode(kv[BLOB_KEY]))
return Result(stdout = serialize_kv(kv))
if '/' in storage_path:
entry, key = storage_path.rsplit('/', 1)
try:
kv, _ = await self.__secret(entry)
except FileNotFoundError:
raise FileNotFoundError(storage_path) from None
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(storage_path) from None
raise FileNotFoundError(storage_path)
# -- Override base class methods
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
path = self._unchroot(path)
return await self._call_with_fallback(
path,
lambda p: self.__vault_stat(p, follow_symlinks),
)
@override
async def _get(
self,
path: str,
wd: str | None,
throw: bool,
verbose: bool | None,
title: str,
) -> Result:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__vault_get)
@override
async def _readdir(self, path: str) -> Iterable[str]:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__vault_readdir)
async def __put_entry(self, storage_path: str, content: bytes) -> Result:
# -- Replace the entry: the content is the serialized entry,
# and a KV v2 write is a full replacement, so a key not in
# it is gone
kv = parse_kv(content)
self.__secret_cache.pop(storage_path, None)
await self.__api('POST', storage_path, data = {'data': kv})
return Result()
async def __put_key(self, storage_path: str, key: str, content: bytes) -> Result:
# -- Merge the key into the entry, replacing its existing
# value for the key
try:
kv, _ = await self.__secret(storage_path)
except FileNotFoundError:
kv = {}
kv[key] = content.decode()
self.__secret_cache.pop(storage_path, None)
await self.__api('POST', storage_path, data = {'data': kv})
return Result()
async def __put_blob(self, storage_path: str, content: bytes) -> Result:
self.__secret_cache.pop(storage_path, None)
await self.__api(
'POST',
storage_path,
data = {'data': {
BLOB_KEY: base64.b64encode(content).decode()
}},
)
return Result()
async def __vault_unlink(self, storage_path: str) -> None:
# -- Drop the secret at the storage path: the data and its
# metadata, so the name leaves the listing. A DELETE is a
# blind success on KV v2 - it 204s even for a path that
# never existed - so probe with a GET first
try:
await self.__api('GET', storage_path)
except FileNotFoundError:
raise
await self.__api('DELETE', storage_path)
await self.__api('DELETE', storage_path, metadata = True)
self.__secret_cache.pop(storage_path, None)
@override
async def _put(
self,
path: str,
content: bytes,
wd: str | None,
throw: bool,
verbose: bool | None,
title: str,
owner: str | None,
group: str | None,
mode: str | None,
atomic: bool,
ftype: FileType,
) -> Result:
# -- Resolve the address to the absolute namespace path in
# the host tree; a write does not take the default
# fallback.
path = self._unchroot(path)
if not path:
raise ValueError(f'put: "{path}" is not an entry/key or blob path')
storage_path = self._path_in_storage(path)
if ftype.is_subtype(FT_SECRET):
return await self.__put_entry(storage_path, content)
if ftype.is_subtype(FT_SECRET_KEY):
entry, sep, key = storage_path.rpartition('/')
if not sep or not key:
raise ValueError(f'put: "{path}" is not of the form path/key')
return await self.__put_key(entry, key, content)
if ftype.is_subtype(FT_SECRET_FILE):
return await self.__put_blob(storage_path, content)
raise ValueError(f'put: ftype {ftype} is not a store content type')
@override
async def _unlink(self, path: str) -> None:
# -- Remove the node; a write does not take the default
# fallback
path = self._unchroot(path)
if not path:
raise ValueError(f'unlink: "{path}" is not a node path')
await self.__vault_unlink(self._path_in_storage(path))