janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

lib.ec.secret_store: Add and use framework #146

Merged
Jan Lindemann merged 1 commit from jan/feature/20261017-lib-ec-secret-store-add-and-use-framework into master 2026-10-17 13:02:24 +02:00 AGit
Conversation 0 Commits 1 Files changed 18 +1835 -169
18 changed files with 1835 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

120
src/python/jw/pkg/cmds/secrets/lib/DistroContext.py
View file

@ -3,11 +3,21 @@ from __future__ import annotations
import re
import sys
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from ....lib.ec.secret_store.Base import Base as SecretStore
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import (
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_KEY,
SECRET_SUFFIX,
host_prefix,
)
from ....lib.FileContext import FileContext
from ....lib.log import DEBUG, NOTICE, WARNING, log
from ....lib.ProcFilterGpg import ProcFilterGpg
from .FilesContext import FilesContext
if TYPE_CHECKING:
@ -19,7 +29,7 @@ if TYPE_CHECKING:
class DistroContext(FilesContext):
def __init__(self, distro: Distro) -> None:
super().__init__(distro.ctx)
super().__init__(distro.ctx, distro = distro)
self.__distro = distro
async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@ -40,7 +50,7 @@ class DistroContext(FilesContext):
) -> list[str]:
ret = []
for tmpl in await self.list_template_files(pkg_names):
path = str(Path(tmpl).with_suffix('.jw-secret'))
path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
if ignore_missing and not await self.ctx.file_exists(path):
continue
ret.append(path)
@ -101,64 +111,60 @@ class DistroContext(FilesContext):
verbose: bool = False,
) -> None:
async def _read_secret_tar_blob(src_uri: str) -> bytes:
ec = self.ctx
from ....lib.ec.Local import Local
from ....lib.util import get
if not isinstance(ec, Local):
# Security: Use a local exec context for decrypting and
# filtering secrets
ec = Local()
return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
def _is_needed_secret(path: str) -> bool:
return path in secret_paths
from .tar import extract as tar_extract
from .tar import filter as tar_filter
from .tar import merge as tar_merge
from .tar import rewrite as tar_rewrite
if only_missing:
raise NotImplementedError('--only-missing is not yet implemented')
secret_paths = await self.list_secret_paths(pkg_names)
fc = FileContext.create(src_uri, chroot = True)
await fc.open()
try:
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in tar file')
host_root_in_tar = '/'.join(reversed(hostname.split('.')))
host_rx = re.compile(r'^' + host_root_in_tar)
default_rx = re.compile(r'^default')
filtered_paths: list[str] = []
extracted_paths: list[str] = []
blob_all = await _read_secret_tar_blob(src_uri)
if blob_all is None:
raise Exception(f'Tar blob {src_uri} is empty')
blob_host_filtered = tar_filter(
blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
raise Exception('Have no hostname to find secrets in the store')
if isinstance(fc, SecretStore):
if not fc.whole:
raise Exception(
f'src {src_uri} must name the whole store, not a '
f'rooted subtree'
)
blob_host_transformed = tar_rewrite(
blob_host_filtered, lambda p: re.sub(host_rx, '', p)
)
blob_default_filtered = tar_filter(
blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
)
blob_default_transformed = tar_rewrite(
blob_default_filtered, lambda p: re.sub(default_rx, '', p)
)
blob_secret_material = tar_merge(
[blob_host_transformed, blob_default_transformed], overwrite = False
)
blob_needed = tar_filter(
blob_secret_material, _is_needed_secret, extracted_paths
)
await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
for path in secret_paths:
if path not in extracted_paths:
log(NOTICE, f'not extracted: {path}')
src: FileContext = fc
# -- Store candidates: the host tree, then the default
# tree
prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
else:
log(NOTICE, f'extracted: {path}')
# -- A file tree in the on-host layout: wrap it in the kvfile store
src = KvFile(fc, distro = self.__distro)
prefixes = ('', )
# -- The store and the kvfile source are alike addressed
# by the on-host file name
suffix = SECRET_SUFFIX
for entry in await self.list_compilation_targets(pkg_names):
# -- A present entry, even an empty one, wins over the
# fallback
entry_rel = f'{entry.lstrip("/")}{suffix}'
addr = entry_rel
keys: list[str] | None = None
for pfx in prefixes:
addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
try:
keys = list(await src.readdir(addr))
break
except FileNotFoundError:
continue
if keys is None:
log(NOTICE, f'not extracted: {entry}')
continue
log(NOTICE, f'extracted: {entry}')
pairs: list[tuple[str, bytes]] = []
for key in keys:
val = await src.get(f'{addr}/{key}')
pairs.append((key, val.stdout))
dst = self.store
with suppress(FileNotFoundError):
await dst.unlink(entry_rel)
await dst.mkdir(entry_rel, ftype = FT_SECRET)
for key, content in pairs:
await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
finally:
await fc.close()

59
src/python/jw/pkg/cmds/secrets/lib/FilesContext.py
View file

@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
from ....lib.Distro import Distro
from ....lib.FileContext import FileContext
from ....lib.ec.secret_store.Base import Base
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import SECRET_SUFFIX
from ....lib.log import DEBUG, NOTICE, WARNING, log
from .base import Attrs
class FilesContext:
def __init__(self, ctx: FileContext) -> None:
def __init__(
self,
ctx: FileContext,
store: Base | None = None,
distro: Distro | None = None,
) -> None:
self.__ctx = ctx
if store is None:
store = KvFile(ctx, distro = distro)
self.__store = store
@property
def ctx(self) -> FileContext:
return self.__ctx
async def _read_key_value_file(
self,
path: str,
throw: bool = False,
) -> dict[str, str]:
ret: dict[str, str] = {}
try:
result = await self.ctx.get(path)
for line in result.stdout_str.splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
except FileNotFoundError:
log(DEBUG, f'File not found {path}')
return ret
@property
def store(self) -> Base:
return self.__store
def _parse_attributes(self, content: str) -> Attrs | None:
@ -143,6 +137,10 @@ class FilesContext:
for key, val in replace.items():
new_content = new_content.replace(key, val)
unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
if unresolved:
raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
tmp_path: str | None = None
try:
@ -200,12 +198,21 @@ class FilesContext:
) -> bool:
path_tmpl = target_path + '.jw-tmpl'
path_secret_file = target_path + '.jw-secret-file'
path_secret = target_path + '.jw-secret'
path_secret = target_path + SECRET_SUFFIX
attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
attrs.update(
await self._read_attributes([path_tmpl, path_secret_file, path_secret])
)
replace = await self._read_key_value_file(path_secret)
entry = f'{target_path}{SECRET_SUFFIX}'
try:
keys = await self.__store.readdir(entry)
except FileNotFoundError:
log(DEBUG, f'No secrets for target {target_path}')
keys = []
replace: dict[str, str] = {}
for key in keys:
val = await self.__store.get(f'{entry}/{key}')
replace[key] = val.stdout.decode()
for src in [path_secret_file, path_tmpl]:
try:
await self._compile_one_template_file(

83
src/python/jw/pkg/cmds/secrets/lib/tar.py
View file

@ -1,83 +0,0 @@
from __future__ import annotations
import io
import tarfile
from tarfile import TarFile
from typing import TYPE_CHECKING, Callable
from ....lib.ExecContext import ExecContext
from ....lib.log import DEBUG, log
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
def filter(
blob: bytes,
path_filter: Callable[[str], bool] | None,
matched: list[str] | None = None,
) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
if path_filter is not None and not path_filter(info.name):
continue
log(DEBUG, f'Adding {info.name}')
if matched is not None:
matched.append(info.name)
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
new_name = rewrite_filter(info.name)
log(DEBUG, f'Rewriting {info.name} -> {new_name}')
info.name = new_name
# -- A pax "path" supplement from the input would override
# the rewritten name on re-serialization
info.pax_headers = {
k: v
for k, v in info.pax_headers.items() if k != 'path'
}
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
for blob in blobs:
tf_in = TarFile(fileobj = io.BytesIO(blob))
existing_names = tf_out.getnames()
for info in tf_in.getmembers():
if not overwrite and info.name in existing_names:
continue
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
async def extract(
dst: FileContext,
blob: bytes,
root: str | None = None,
verbose: bool = False
) -> None:
cmd = ['tar']
if root is not None:
cmd += ['-C', root]
if verbose:
cmd += '-v'
cmd += ['-x', '-f', '-']
if not isinstance(dst, ExecContext):
raise NotImplementedError(
'Extracting tar files to a non-executable '
f'context is not yet implemented: {dst}'
)
await dst.run(cmd, verbose = verbose, cmd_input = blob)

12
src/python/jw/pkg/lib/FileContext.py
View file

@ -360,6 +360,18 @@ class FileContext(abc.ABC):
from .ec.Curl import Curl
return Curl(uri, *args, **kwargs)
case 'targpg':
from .ec.secret_store.TarGpg import TarGpg
return TarGpg(uri, *args, **kwargs)
case 'kvfile':
from .ec.secret_store.KvFile import KvFile
scheme = 'ssh' if uri.hostname else 'local'
inner = FileContext.create(
f'{scheme}://{uri.authority}', chroot = False
)
return KvFile(inner, uri = uri)
case _:
pass
raise Exception(

252
src/python/jw/pkg/lib/ec/secret_store/Base.py Normal file
View file

@ -0,0 +1,252 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override
from ...base import Result, StatResult
from ...file_types import FT_DIR, FT_REG
from ...FileContext import FileContext
from .layout import DEFAULT_PREFIX, host_prefix
if TYPE_CHECKING:
import os
from typing import Any, Iterable
from ...file_types import FileType
from ...Uri import Uri
T = TypeVar('T')
class Base(FileContext):
"""A store of secret entries in the canonical store layout.
A store is a FileContext over the store namespace. The URI
addresses a subtree of that namespace: no authority means the
whole store, a host authority means that host's tree (its
reversed domain prefix), `default` names the default tree, and a
path narrows further. The subtree is the store's namespace
root; addresses passed to the methods are relative to it.
A store is a typed file store: a store path names a file, and
put() persists the content at the address as-is. The store does
not identify content by file name; what a file holds - an entry,
or a blob - is a property of its content, resolved at stat
time. A node is what its content makes it:
- an entry carries key-value secrets and behaves like a
directory: readdir() on the entry lists its keys, get() on
<entry>/<key> returns the value of the key, and get() on the
entry itself returns the serialized entry
- a branch is a path prefix: readdir() lists its children
- a blob is non-key-value content: get() returns its bytes
A store rooted at a host applies the default fallback: an
address that misses below the host is retried below the default
tree. A whole-store or default-rooted store is strict; callers
apply the fallback themselves.
The URI-addressable stores are created by FileContext.create()
from their scheme. The kvfile store is the store of the on-host
layout: its addresses are config paths of the host it is rooted
at, and it may be constructed directly over a host context.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
# -- A store is always chrooted. The flag states the
# fact, and the chroot it arms is undone in the
# protected layer by _unchroot(); an explicit value
# is overridden.
kwargs['chroot'] = True
super().__init__(uri, *args, **kwargs)
@property
def __host_root(self) -> str:
# -- The root of the host tree this store is rooted at, '' if
# the store is whole-store or rooted at the default tree:
# then there is no host tree to fall back from, and the
# caller applies the default fallback itself.
host = self.uri.hostname or ''
if host and host != 'default':
return host_prefix(host)
return ''
@property
def __root_in_storage(self) -> str:
# -- The namespace node this store is rooted at: the
# reversed host prefix of the URI authority, if any
# (`default` is a reserved word), followed by the URI
# path.
host = self.uri.hostname or ''
path = self.uri.path.lstrip('/')
if host and host != 'default':
host = host_prefix(host)
return '/'.join(p for p in (host, path) if p)
def _path_in_storage(self, path: str) -> str:
path = path.lstrip('/')
if not self.__root_in_storage:
return path
if not path:
return self.__root_in_storage
return f'{self.__root_in_storage}/{path}'
def _default_path_in_storage(self, path: str) -> str | None:
host_root = self.__host_root
ns = self.__root_in_storage
if not host_root or not (path == ns or path.startswith(f'{ns}/')):
return None
return f'{DEFAULT_PREFIX}{path[len(host_root):]}'
def _unchroot(self, path: str) -> str:
# -- The inverse of _chroot(): the public wrappers hand the
# protected layer addresses prefixed with the root; store
# addresses are relative to it, and the root itself is the
# empty address. Already-relative addresses pass through.
if not self.root:
return path.lstrip('/')
if path == self.root:
return ''
prefix = f'{self.root}/'
if path.startswith(prefix):
return path[len(prefix):]
return path
async def _call_with_fallback(
self,
path: str,
func: Callable[[str], Awaitable[T]],
) -> T:
storage_path = self._path_in_storage(path)
try:
return await func(storage_path)
except FileNotFoundError:
default_storage_path = self._default_path_in_storage(storage_path)
if default_storage_path is None:
raise
return await func(default_storage_path)
@override
async def put(
self,
path: str,
content: bytes,
wd: str | None = None,
throw: bool = True,
verbose: bool | None = None,
title: str | None = None,
owner: str | None = None,
group: str | None = None,
mode: int | None = None,
atomic: bool = False,
ftype: FileType = FT_REG,
) -> Result:
# -- A put that changes nothing is a no-op: the pre-read is
# fallback-aware, so a value already represented, below
# the host tree or below the default tree, does not
# trigger a store write. Metadata arguments opt out: they
# may be the only change
if owner is None and group is None and mode is None:
try:
existing = await self.get(path)
except Exception:
pass
else:
if self._put_is_noop(existing.stdout, content, ftype):
return Result()
return await super().put(
path,
content,
wd,
throw,
verbose,
title,
owner,
group,
mode,
atomic,
ftype,
)
def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
# -- A put that leaves the stored content byte-identical
# changes nothing. A store that normalizes content on
# read overrides this to compare the normalized forms
return existing == content
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a file on the backing store, yet it lists
# its keys: answer from the node's type, not the
# object's mode
try:
return (await self._stat(path, follow_symlinks)).is_dir
except FileNotFoundError:
return False
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- A store is listable: a directory of keys, entries, or
# blobs, whatever its nodes hold.
raise NotImplementedError()
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The base store materializes nothing: branches are
# implicit namespace prefixes, and entries materialize on
# their first write. A store whose nodes have a host home
# overrides this
pass
def _stat_result(
self,
owner: str,
group: str,
mtime: float,
ftype: FileType,
mode: int | None = None,
size: int = 0,
) -> StatResult:
# -- The metadata of a node: the mode and size, if given, are
# the ones of the file, directory, or member behind the
# node; a node without a backing object keeps the
# synthesized type bits and a size of 0. The access and
# change times follow the modification time
import stat
if mode is None:
mode = (
(stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else
(stat.S_IFREG | 0o400)
)
return StatResult(
mode = mode,
owner = owner,
group = group,
size = size,
atime = mtime,
mtime = mtime,
ctime = mtime,
ftype = ftype,
)
def _stat_result_from_os(
self,
st: os.stat_result,
ftype: FileType,
size: int | None = None,
) -> StatResult:
return self._stat_result(
owner = str(st.st_uid),
group = str(st.st_gid),
mtime = st.st_mtime,
ftype = ftype,
mode = st.st_mode,
size = st.st_size if size is None else size,
)
@property
def whole(self) -> bool:
# -- True, if the store is rooted at the whole store namespace:
# no authority, no path
return not self.__root_in_storage

439
src/python/jw/pkg/lib/ec/secret_store/KvFile.py Normal file
View file

@ -0,0 +1,439 @@
from __future__ import annotations
import errno
import os
from typing import TYPE_CHECKING, Iterable, override
if TYPE_CHECKING:
from ...Distro import Distro
from ...FileContext import FileContext
from ...Uri import Uri
from ...base import StatResult
from ...file_types import FileType
from ...FcCache import FcCache
from ...file_types import FT_COMPOUND, FT_DIR
from ...kv import parse_kv, serialize_kv
from ...Result import Result
from .Base import Base
from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX
class KvFile(Base):
"""The on-host store of a host's package secrets.
A config path X is either an entry, the file X.jw-secret with
KEY=VALUE lines, whose children in the address space are its
keys, or a blob, the file X.jw-secret-file, whose content is the
file itself, a leaf. Any other path is a branch: a real
directory on the host, with no file of its own.
Rooted at the host: addresses are config paths relative to the
host root, absolute on the host with the leading slash. A URI
may root the store at a subtree of the host tree; its path is
composed into every address, and the URI is the store's
identity. A suffixed address names the on-host file of its
config path, the suffix picking the kind: .jw-secret the
entry, .jw-secret-file the blob.
get reads a key below an entry or the blob at its config path;
put writes a key into its entry file, or a blob as a
.jw-secret-file at its config path, and accepts entry, key, and
blob content only; unlink drops the file the address names. No
default fallback.
An entry lists its keys. A branch lists the config paths below
it, the root included: the host's package set is the bounded
base, the .jw-tmpl files of the installed packages, presence-
checked on the host. The distro is autodetected from the host
on first use, or given in the constructor; the autodetected
distro applies the JW_DEFAULT_PKG_FILTER package filter, if
set, so a URI-built context honors the app's documented
filter fallback. is_dir() answers from the entry file and the
host tree, so the write path never pays for the enumeration.
The on-host tree is a precondition: mkdir materializes a missing
entry file below an existing directory, and mkdir and put fail
with EROFS below a missing one. KvFile never creates
directories. Entry and blob files are mode 0400;
owner and group follow the put request, if given.
"""
def __init__(
self,
ctx: FileContext,
uri: str | Uri | None = None,
distro: Distro | None = None,
) -> None:
# -- The URI roots the namespace at the host tree's root, or
# at a subtree of it: its path is composed into every
# address, and it is the store's identity. A direct
# construction names the context's host
if uri is None:
uri = f'kvfile://{ctx.uri.hostname or "target"}'
super().__init__(uri = uri)
self.__raw = ctx
self.__distro: Distro | None = distro
self.___config_paths: list[str] | None = None
self.__subtree = self.uri.path.lstrip('/')
# -- The context is read through a cache: the read path
# probes the same handful of paths again and again, and
# the cache pays the transport once per path
self.__ctx = FcCache(ctx)
@override
async def _open(self) -> None:
# -- The lifecycle propagates to the host context: an
# open() pin on the store, the CopyContext of a copy
# walk, keeps the host's transport open for its
# duration, and the run()s within it share it
await self.__raw.open()
@override
async def _close(self) -> None:
await self.__raw.close()
def __addr(self, path: str) -> str:
# -- The host-relative address: the subtree the URI roots
# the store at, then the address relative to it
path = self._unchroot(path)
if not path:
return self.__subtree
if not self.__subtree:
return path
return f'{self.__subtree}/{path}'
@staticmethod
def __resolve(path: str) -> tuple[str, bool]:
# -- The config path of the address, and whether the address
# names a blob file: a suffixed address is the on-host
# file name of its config path, the suffix picking the
# kind
if path.endswith(SECRET_SUFFIX):
return path[:-len(SECRET_SUFFIX)], False
if path.endswith(BLOB_SUFFIX):
return path[:-len(BLOB_SUFFIX)], True
return path, False
async def __config_paths(self) -> list[str]:
# -- The config paths of the host's package secrets: the
# templates of the installed packages, relative to the
# host root. The package database is the bounded base, a
# filesystem scan is not. Computed once: the package set
# is stable for the lifetime of the session
if self.___config_paths is None:
if self.__distro is None:
from ...Distro import Distro
from ...ExecContext import ExecContext
# -- The distro probes the host through the raw
# context: the cache is a plain file context, no
# exec of its own
if not isinstance(self.__raw, ExecContext):
raise ValueError(
'KvFile: a distro must be given when the host'
' context is not an exec context'
)
# -- A URI-built context has no app distro to
# inherit the package filter from: apply the
# app's documented fallback
from ...PackageFilter import PackageFilterString
pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER')
self.__distro = await Distro.instantiate(
ec = self.__raw,
default_pkg_filter = (
PackageFilterString(pkg_filter_str) if pkg_filter_str else None
),
)
# -- The packages' file paths, filtered to the templates
ret: set[str] = set()
for f in await self.__distro.pkg_files():
if not f.endswith('.jw-tmpl'):
continue
ret.add(f.lstrip('/')[:-len('.jw-tmpl')])
self.___config_paths = sorted(ret)
# -- Warm the stat cache with one host command, so the
# per-level presence checks and the walk's node stats
# hit memory: the root, the candidate files of both
# kinds, the directories they live in, and the file
# names a suffixed address would probe those
# directories by
warm = ['/']
warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths]
warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths]
dirs: set[str] = set()
for p in self.___config_paths:
parts = p.split('/')
for i in range(1, len(parts)):
dirs.add('/'.join(parts[:i]))
for d in sorted(dirs):
warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}']
await self.__ctx.warm(warm)
assert self.___config_paths is not None
return self.___config_paths
async def __entry_kv(self, entry: str) -> dict[str, str]:
path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}'
if not await self.__ctx.file_exists(path):
raise FileNotFoundError(entry)
ret = await self.__ctx.get(path)
return parse_kv(ret.stdout)
async def __is_entry(self, path: str) -> bool:
# -- The entry file on the host; KvFile has no default tree.
# The path is host-relative, as from __addr()
if not path:
return False
return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}')
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
# -- The metadata of the host file behind the node, the type
# overridden: an entry is a file that enumerates its
# keys, a key a leaf below it; a branch is a plain
# directory. A key has no file of its own, so it keeps
# the size 0
path = self.__addr(path)
config, is_blob = self.__resolve(path)
async def __stat_of(real: str, ftype: FileType) -> StatResult:
st = await self.__ctx.stat(real, follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
ftype,
mode = st.mode,
size = st.size,
)
if not config:
return await __stat_of('/', FT_DIR)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
return await __stat_of(real, FT_SECRET_FILE)
if await self.__is_entry(config):
return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET)
parent, sep, key = config.rpartition('/')
if sep and key:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent) and key in await self.__entry_kv(parent):
# -- The type is synthesized, the mode the one of
# the entry file
st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
FT_SECRET_KEY,
mode = st.mode,
)
blob = f'/{config}{BLOB_SUFFIX}'
if await self.__ctx.file_exists(blob):
return await __stat_of(blob, FT_SECRET_FILE)
if await self.__ctx.is_dir(f'/{config}'):
return await __stat_of(f'/{config}', FT_DIR)
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a directory of its keys, a key below an
# entry is a leaf: the entry is a file on the host, so
# there is no host path below it to stat. A branch is a
# real directory on the host. The package set is not
# consulted: the write path never pays for the
# enumeration
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if not config:
return True
parent, sep, _ = config.rpartition('/')
if sep:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent):
return False
if await self.__is_entry(config):
return True
if is_blob:
return False
return await self.__ctx.is_dir(f'/{config}')
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- An entry lists its keys; a branch, the root included,
# lists the config paths below it, presence-checked on
# the host, under the on-host file name of the file that
# backs it
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if config and await self.__is_entry(config):
return (await self.__entry_kv(config)).keys()
if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'):
raise NotADirectoryError(path)
children: set[str] = set()
prefix = f'{config}/' if config else ''
for cfg in await self.__config_paths():
if not cfg.startswith(prefix):
continue
rest = cfg[len(prefix):]
label, sep, _ = rest.partition('/')
if sep:
# -- A deeper candidate lists its branch label, if a
# secret is present below it: the label names a
# directory on the host, which a present secret
# below implies
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \
await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(label)
continue
# -- A direct child lists under the on-host file name of
# the file that backs it, entry and blob alike
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'):
children.add(f'{label}{SECRET_SUFFIX}')
if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(f'{label}{BLOB_SUFFIX}')
if config and not children:
# -- A branch the host lacks is absent; a real directory
# with no secret children lists empty
if not await self.__ctx.is_dir(f'/{config}'):
raise FileNotFoundError(path)
return children
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(path)
return await self.__ctx.get(real)
real = f'/{config}'
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
# -- The entry file is the serialized entry
return await self.__ctx.get(real + SECRET_SUFFIX)
if await self.__ctx.file_exists(real + BLOB_SUFFIX):
return await self.__ctx.get(real + BLOB_SUFFIX)
if '/' not in config:
raise FileNotFoundError(path)
entry, key = config.rsplit('/', 1)
entry, _ = self.__resolve(entry)
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The address space has no directories of its own: a node
# is a branch (a real directory) or an entry (a suffixed
# file); the type says which to build. Below a missing
# branch, mkdir fails with EROFS: the caller creates the
# levels
path = self.__addr(path)
config, _ = self.__resolve(path)
real = f'/{config}'
if ftype.is_subtype(FT_COMPOUND):
# -- An entry: materialize the empty entry file
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True)
elif ftype.is_subtype(FT_DIR):
# -- A branch: a real directory
if await self.__ctx.is_dir(real):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.mkdir(real, mode)
else:
raise ValueError(f'mkdir: "{path}" is not a directory type')
@override
async def _put(
self,
path: str,
content: bytes,
wd: str | None,
throw: bool,
verbose: bool | None,
title: str,
owner: str | None,
group: str | None,
mode: str | None,
atomic: bool,
ftype: FileType,
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if ftype.is_subtype(FT_SECRET):
# -- The whole entry: the content is the serialized
# entry, written over the entry file; the branch is
# a precondition, as for a blob
if is_blob:
raise ValueError(
f'put: "{path}" names a blob file, but the ftype is an entry'
)
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{SECRET_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
if ftype.is_subtype(FT_SECRET_FILE):
# -- A blob: the content is the file itself, at its
# on-host file name
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{BLOB_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
entry, sep, key = config.rpartition('/')
if not sep or not entry or not key or '/' in key:
raise ValueError(f'put: "{path}" is not an entry/key path')
entry, _ = self.__resolve(entry)
real_entry = f'/{entry}'
if ftype.is_subtype(FT_SECRET_KEY):
# -- A key: merge into the entry file, which must exist;
# mkdir() is the materialization. The put is atomic,
# so it also overwrites an existing entry file, which
# is mode 0400
if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'):
raise OSError(errno.EROFS, f'put: "{entry}" has no entry file')
kv = await self.__entry_kv(entry)
kv[key] = content.decode()
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'{real_entry}{SECRET_SUFFIX}',
serialize_kv(kv),
owner = owner,
group = group,
mode = m,
atomic = True,
)
raise ValueError(f'put: ftype {ftype} is not a store content type')
@override
async def _unlink(self, path: str) -> None:
# -- Unlink the file the address names: the address is the
# on-host file name below the subtree, so the host path
# is the composed address itself
await self.__ctx.unlink(f'/{self.__addr(path)}')

4
src/python/jw/pkg/lib/ec/secret_store/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../../../../../../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-mod.mk

227
src/python/jw/pkg/lib/ec/secret_store/TarGpg.py Normal file
View file

@ -0,0 +1,227 @@
from __future__ import annotations
import errno
import io
import os
import stat
import time
from tarfile import TarFile, TarInfo
from typing import TYPE_CHECKING, Iterable, override
from ...file_types import FT_DIR
from ...kv import parse_kv
from ...Result import Result
from .Base import Base
from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
if TYPE_CHECKING:
from typing import Any
from ...base import StatResult
from ...Uri import Uri
class TarGpg(Base):
"""A store of secrets in a GPG-encrypted tarball in the canonical
layout.
The tarball location is deployment information, not part of the
address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The
store URI addresses a subtree of the namespace, as in the base
class.
A store path names a member, and optionally a key below an
entry. The member of an entry is its address, and its keys are
addressable below it; a blob is a member at its address holding
non-key-value content. The store does not identify content by
member name: a member whose content is a key-value file is an
entry, anything else a blob.
The tarball is fetched and decrypted once, then cached. Fetching,
decrypting, and filtering happen in a local exec context, so the
GPG key and the decrypted material stay on this machine.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
super().__init__(uri, *args, **kwargs)
self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '')
self.__tf: TarFile | None = None
self.__kv_cache: dict[str, dict[str, str]] = {}
async def __tar(self) -> TarFile:
if self.__tf is None:
from ...ProcFilterGpg import ProcFilterGpg
from ...util import get
from ..Local import Local
data = (
await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local()))
).stdout
self.__tf = TarFile(fileobj = io.BytesIO(data))
return self.__tf
async def __members(self) -> dict[str, TarInfo]:
tf = await self.__tar()
return {m.name: m for m in tf.getmembers()}
async def __entry_kv(self, entry: str) -> dict[str, str]:
if entry not in self.__kv_cache:
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
raise FileNotFoundError(entry) from None
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store entry "{entry}"')
self.__kv_cache[entry] = parse_kv(buf.read())
return self.__kv_cache[entry]
async def __is_entry(self, entry: str) -> bool:
# -- The member at entry is an entry: its content is a
# non-empty kv file
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
return False
if not info.isfile():
return False
buf = tf.extractfile(info)
if buf is None:
return False
return len(parse_kv(buf.read())) != 0
async def __tar_stat(self, storage_path: str) -> StatResult:
# -- The metadata of the member that stands behind the node;
# a branch has no member of its own and is synthesized, a
# key keeps the size 0
members = await self.__members()
def __meta(name: str) -> tuple[str, str, float, int, int]:
# -- A member's mode carries the permission bits only, so
# the type bits are synthesized; a store member is
# always a regular file
mi = members[name]
return (
mi.uname or str(mi.uid),
mi.gname or str(mi.gid),
mi.mtime,
stat.S_IFREG | (mi.mode & 0o777),
mi.size,
)
if await self.__is_entry(storage_path):
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET,
mode = mode,
size = size,
)
par, sep, key = storage_path.rpartition('/')
if sep and key and await self.__is_entry(par):
if key in await self.__entry_kv(par):
# -- The type is synthesized, the mode the one of the
# entry member, the size stays 0
owner, group, mtime, mode, _ = __meta(par)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_KEY,
mode = mode,
)
if storage_path in members:
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_FILE,
mode = mode,
size = size,
)
if not storage_path or \
any(name.startswith(f'{storage_path}/') for name in members):
return self._stat_result(
str(os.getuid()),
str(os.getgid()),
time.time(),
FT_DIR,
)
raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
async def __tar_get(self, path: str) -> Result:
members = await self.__members()
if path == '':
raise IsADirectoryError(path)
if await self.__is_entry(path):
# -- The entry member is the serialized entry
tf = await self.__tar()
buf = tf.extractfile(members[path])
if buf is None:
raise Exception(f'Can\'t read store entry "{path}"')
return Result(stdout = buf.read())
if any(name.startswith(f'{path}/') for name in members):
raise IsADirectoryError(path)
if '/' in path:
entry, key = path.rsplit('/', 1)
if await self.__is_entry(entry):
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
info = members.get(path)
if info is not None and info.isfile():
tf = await self.__tar()
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store blob "{path}"')
return Result(stdout = buf.read())
raise FileNotFoundError(path)
async def __tar_readdir(self, path: str) -> Iterable[str]:
members = await self.__members()
if await self.__is_entry(path):
return (await self.__entry_kv(path)).keys()
if path in members:
raise NotADirectoryError(path)
def __names(prefix: str) -> set[str]:
# -- No name filter: every member is store content
ret: set[str] = set()
for name in members:
if not name.startswith(prefix):
continue
rest = name[len(prefix):]
ret.add(rest.partition('/')[0])
return ret
ret = __names(f'{path}/' if path else '')
mirror = self._default_path_in_storage(path)
if mirror is not None:
ret |= __names(f'{mirror}/' if mirror else '')
if not ret:
raise FileNotFoundError(path)
return sorted(ret)
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_get)
@override
async def _readdir(self, path: str) -> Iterable[str]:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_readdir)
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_stat)

52
src/python/jw/pkg/lib/ec/secret_store/layout.py Normal file
View file

@ -0,0 +1,52 @@
from __future__ import annotations
from ...file_types import FT_COMPOUND, FT_REG, file_types
# -- The canonical store layout, shared by every store implementation and
# the install pipeline that consumes it:
#
# default/<path> entries applying to all hosts
# <host-root>/<path> entries applying to a single host, where
# <host-root> is the host's FQDN, reversed and split
# on dots (com/example/host for host.example.com)
#
# Host-specific entries take precedence over default ones.
# Suffix of the key-value secret file accompanying a compiled target
SECRET_SUFFIX = '.jw-secret'
# Suffix of a non-key-value secret file (certificate, key, ...)
BLOB_SUFFIX = '.jw-secret-file'
DEFAULT_PREFIX = 'default'
def content_name(name: str) -> str | None:
# -- The config path of an on-host secret file name: the name
# with its content suffix removed, or None if the name
# carries neither suffix
for suffix in (SECRET_SUFFIX, BLOB_SUFFIX):
if name.endswith(suffix):
return name.removesuffix(suffix)
return None
def host_prefix(hostname: str) -> str:
return '/'.join(reversed(hostname.split('.')))
# -- The file types of the store content, registered with the global
# file type registry: an entry is a compound of its keys, a key a
# leaf below it, a blob a file
FT_NAME_SECRET = 'secret'
FT_NAME_SECRET_KEY = 'secret-key'
FT_NAME_SECRET_FILE = 'secret-file'
file_types.register_types(
{
FT_NAME_SECRET: FT_COMPOUND,
FT_NAME_SECRET_KEY: FT_REG,
FT_NAME_SECRET_FILE: FT_REG,
}
)
FT_SECRET = file_types.get(FT_NAME_SECRET)
FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY)
FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE)

21
src/python/jw/pkg/lib/kv.py Normal file
View file

@ -0,0 +1,21 @@
from __future__ import annotations
def parse_kv(data: bytes) -> dict[str, str]:
"""Parse a dotenv-style key-value entry into a dict"""
ret: dict[str, str] = {}
for line in data.decode().splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
return ret
def serialize_kv(kv: dict[str, str]) -> bytes:
"""Serialize a dict back into a dotenv-style key-value entry"""
return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode()

46
test/integration/jw_pkg/secrets/test.py
View file

@ -45,6 +45,7 @@ from jw.pkg.lib.Package import Package
HOST = 'jannet.example.com'
HOST_PREFIX = '/'.join(reversed(HOST.split('.')))
PKG = 'fixture-pkg'
PKG2 = 'fixture-pkg2'
USER = pwd.getpwuid(os.getuid()).pw_name
GROUP = grp.getgrgid(os.getgid()).gr_name
@ -189,12 +190,21 @@ def main() -> None:
# extract it, and must not leave an empty file behind
with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f:
f.write('x=X\n')
# -- qux references a macro that has no store entry: the
# compile must fail rather than emit a literal macro
with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f:
f.write(
f'# conf: owner={USER}, group={GROUP}, mode=640\n'
'present=PRESENT\n'
'missing=__UNRESOLVED__\n'
)
with open(target + '/etc/readme.txt', 'w') as f:
f.write('plain package file\n')
foo_tmpl = target + '/etc/foo.conf.jw-tmpl'
bar_tmpl = target + '/etc/bar.conf.jw-tmpl'
baz_tmpl = target + '/etc/baz.conf.jw-tmpl'
qux_tmpl = target + '/etc/qux.conf.jw-tmpl'
foo_secret = target + '/etc/foo.conf.jw-secret'
bar_secret = target + '/etc/bar.conf.jw-secret'
baz_secret = target + '/etc/baz.conf.jw-secret'
@ -231,7 +241,10 @@ def main() -> None:
distro = FakeDistro(
Local('local://' + HOST),
{PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt']},
{
PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'],
PKG2: [qux_tmpl],
},
)
# -- list-templates: only the .jw-tmpl files of the package
@ -356,6 +369,37 @@ def main() -> None:
with open(bar_out) as f:
assert f.read() == 'plain=plain-value\n'
# -- compile-templates fails on an unresolved macro: qux
# resolves PRESENT from its secret file, but the template
# also references __UNRESOLVED__, which is not in the
# store. The failure must name the macro, and no compiled
# file may be written
qux_secret = target + '/etc/qux.conf.jw-secret'
qux_out = target + '/etc/qux.conf'
with open(qux_secret, 'w') as f:
f.write('PRESENT=there\n')
try:
run_cli(
distro,
[
'secrets',
'compile-templates',
'--owner',
USER,
'--group',
GROUP,
'--mode',
'400',
PKG2,
],
)
except RuntimeError as e:
assert 'Unresolved macros' in str(e), e
assert '__UNRESOLVED__' in str(e), e
else:
raise AssertionError('unresolved macro must fail the compile')
assert not os.path.exists(qux_out)
out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
assert out == [foo_out, bar_out, baz_out], out

7
test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

384
test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py Normal file
View file

@ -0,0 +1,384 @@
import asyncio
import errno
import os
import stat
import tempfile
from typing import Any, override
from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
from jw.pkg.lib.ec.Local import Local
from jw.pkg.lib.ec.secret_store.KvFile import KvFile
from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
from jw.pkg.lib.file_types import FT_DIR
from jw.pkg.lib.FileContext import FileContext
from jw.pkg.lib.Package import Package
async def run_tests() -> None:
with tempfile.TemporaryDirectory() as tmp:
target = os.path.join(tmp, 'etc', 'foo.conf')
os.makedirs(os.path.dirname(target))
with open(target + '.jw-secret', 'w') as f:
f.write(
'# a comment\n\n key1 = value1 \nkey2=value2\n'
'=novalue\nkey3=\n'
)
# FileContext.create() resolves the kvfile scheme, keeping
# the URI as the store's identity
created = FileContext.create('kvfile://')
assert isinstance(created, KvFile)
assert created.uri.scheme == 'kvfile'
store = KvFile(Local())
assert set(await store.readdir(target)) == {'key1', 'key2', 'key3'}
assert (await store.get(f'{target}/key1')).stdout == b'value1'
assert (await store.get(f'{target}/key3')).stdout == b''
# The entry is gettable as a whole: the serialized file
assert (await store.get(target)).stdout == \
b'# a comment\n\n key1 = value1 \nkey2=value2\n=novalue\nkey3=\n'
# -- stat() answers the node's type, the metadata of the
# host file behind it; a key has no file of its own, and
# keeps the size 0
blob = os.path.join(tmp, 'etc', 'bar.pem')
with open(blob + '.jw-secret-file', 'w') as f:
f.write('a blob\n')
st = await store.stat(target)
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == os.path.getsize(target + '.jw-secret')
st = await store.stat(f'{target}/key1')
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == 0
st = await store.stat(blob)
assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == os.path.getsize(blob + '.jw-secret-file')
st = await store.stat(os.path.dirname(target))
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert not st.is_gettable
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert not st.is_gettable
try:
await store.stat(f'{target}/missing')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- file_exists() works on the store, being built on stat()
assert await store.file_exists(target)
assert not await store.file_exists(f'{target}/missing')
# A missing key and a missing entry raise FileNotFoundError
for path in (f'{target}/missing', os.path.join(tmp, 'etc', 'missing.conf')):
try:
await store.get(path)
raise AssertionError(f'expected FileNotFoundError for {path}')
except FileNotFoundError:
pass
# put writes a key into the entry file, merging with the
# existing keys
await store.put(f'{target}/key4', b'value4', ftype = FT_SECRET_KEY)
assert (await store.get(f'{target}/key4')).stdout == b'value4'
assert (await store.get(f'{target}/key1')).stdout == b'value1'
# -- a second put overwrites the entry file, which is mode 0400
await store.put(f'{target}/key1', b'value1-new', ftype = FT_SECRET_KEY)
assert (await store.get(f'{target}/key1')).stdout == b'value1-new'
assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
new_entry = os.path.join(tmp, 'etc', 'new.conf')
# -- a key put does not create: below a missing entry file
# it fails with EROFS; mkdir() is the materialization
try:
await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
raise AssertionError('expected EROFS')
except OSError as e:
assert e.errno == errno.EROFS
await store.mkdir(new_entry, ftype = FT_SECRET)
await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
with open(new_entry + '.jw-secret') as f:
assert f.read() == 'k=v\n'
# -- a typed put replaces the entry: the serialized content
# is written over the entry file, mode 0400
await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
assert set(await store.readdir(target)) == {'new1', 'new2'}
assert (await store.get(target)).stdout == b'new1=n1\nnew2=n2\n'
with open(target + '.jw-secret') as f:
assert f.read() == 'new1=n1\nnew2=n2\n'
assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
# -- a put that changes nothing does not rewrite the file
mtime = os.stat(target + '.jw-secret').st_mtime_ns
await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
assert os.stat(target + '.jw-secret').st_mtime_ns == mtime
# -- An untyped put is refused: the store content types are
# entry, key, and blob
try:
await store.put('HOSTNAME', b'x')
raise AssertionError('expected ValueError')
except ValueError:
pass
# -- put: a blob below a branch lands as a .jw-secret-file at
# its config path, mode 0400
pem = os.path.join(tmp, 'etc', 'cert.pem.jw-secret-file')
await store.put(
os.path.join(tmp, 'etc', 'cert.pem'),
b'-----BEGIN CERT-----\n',
ftype = FT_SECRET_FILE,
)
with open(pem) as f:
assert f.read() == '-----BEGIN CERT-----\n'
assert stat.S_IMODE(os.stat(pem).st_mode) == 0o400
# -- mkdir: the type says which node to build, the default
# a branch; a branch is a real directory, an entry an
# empty entry file, mode 0400
await store.mkdir(os.path.join(tmp, 'etc'))
assert not os.path.exists(os.path.join(tmp, 'etc.jw-secret'))
baz = os.path.join(tmp, 'etc', 'baz')
await store.mkdir(baz)
assert os.path.isdir(baz)
bar = os.path.join(tmp, 'etc', 'bar.conf')
await store.mkdir(bar, ftype = FT_SECRET)
assert os.path.isfile(bar + '.jw-secret')
assert open(bar + '.jw-secret').read() == ''
assert stat.S_IMODE(os.stat(bar + '.jw-secret').st_mode) == 0o400
# -- mkdir and put fail with EROFS below a missing directory
for fn in (
lambda: store.mkdir(os.path.join(tmp, 'no', 'such', 'entry')),
lambda: store.put(
os.path.join(tmp, 'no', 'such', 'entry', 'key'),
b'x',
ftype = FT_SECRET_KEY, ),
):
try:
await fn()
raise AssertionError('expected EROFS')
except OSError as e:
assert e.errno == errno.EROFS
# -- put forwards owner and group to the host context: an
# unknown group fails the put
try:
await store.put(
f'{target}/key7', b'v7', ftype = FT_SECRET_KEY, group = 'no-such-group'
)
except AssertionError:
raise
except Exception as e:
assert 'no-such-group' in str(e)
else:
raise AssertionError('expected the put to fail')
# unlink drops the file the address names, and a missing
# file raises
await store.unlink(target + '.jw-secret')
assert not os.path.exists(target + '.jw-secret')
try:
await store.unlink(target + '.jw-secret')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- Branch listing: the config paths below a branch, the
# root included, come from the host's package set, the
# templates of the installed packages, presence-checked
# on the host. A fake distro supplies the list
class FakeDistro(SuseDistro):
def __init__(self, files: list[str]) -> None:
super().__init__(Local(), id = 'suse')
self.files = files
self.queries = 0
@override
async def _pkg_files(self, names: list[str] | None) -> list[str]:
self.queries += 1
return self.files
tree = os.path.join(tmp, 'tree')
os.makedirs(os.path.join(tree, 'etc'))
with open(os.path.join(tree, 'etc', 'foo.conf.jw-secret'), 'w') as f:
f.write('k1=v1\nk2=v2\n')
with open(os.path.join(tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as f:
f.write('-----BEGIN CERT-----\n')
with open(os.path.join(tree, 'etc', 'other.conf.jw-secret'), 'w') as f:
f.write('o1=v1\n')
# -- A template without a secret file is not a config path;
# a package file that is not a template is not one
# either
fake = FakeDistro(
[
f'{tree}/etc/foo.conf.jw-tmpl',
f'{tree}/etc/cert.pem.jw-tmpl',
f'{tree}/etc/ghost.conf.jw-tmpl',
f'{tree}/etc/other.conf',
]
)
# -- The address space is rooted at the host root; the
# package file list reports host-absolute paths, so the
# fixture tree sits at its path below the root
bstore = KvFile(Local(), distro = fake)
base = tree.lstrip('/')
# -- is_dir answers from the host state, without consulting
# the package set
assert await bstore.is_dir('')
assert await bstore.is_dir(base)
assert await bstore.is_dir(f'{base}/etc/foo.conf')
assert not await bstore.is_dir(f'{base}/etc/cert.pem')
assert fake.queries == 0
# -- is_gettable answers from the node's type: an entry,
# a key, a blob; not a branch, the root included
assert await bstore.is_gettable(f'{base}/etc/foo.conf')
assert await bstore.is_gettable(f'{base}/etc/foo.conf/k1')
assert await bstore.is_gettable(f'{base}/etc/cert.pem')
assert not await bstore.is_gettable(f'{base}/etc')
assert not await bstore.is_gettable('')
assert fake.queries == 0
# -- Listing consults it, once: each branch lists the secret
# files below it, under their on-host file names,
# presence-checked
assert set(await bstore.readdir('')) == {'tmp'}
assert fake.queries == 1
assert set(await bstore.readdir(f'{base}/etc')) == \
{'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
assert fake.queries == 1
# -- An entry still lists its keys, without the package set
assert set(await bstore.readdir(f'{base}/etc/foo.conf')) == {'k1', 'k2'}
assert fake.queries == 1
# -- A branch without config paths raises FileNotFoundError
try:
await bstore.readdir('no/such')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- A real directory with no secret children lists empty
os.makedirs(os.path.join(tree, 'var', 'empty'))
assert set(await bstore.readdir(f'{base}/var/empty')) == set()
# -- The blob reads back through the store
assert (await bstore.get(f'{base}/etc/cert.pem')).stdout == \
b'-----BEGIN CERT-----\n'
# -- The entry's config path is also a deployed file on the
# host, as on a real host: a key below it has no host
# path, and is_dir() must not stat below the file
with open(os.path.join(tree, 'etc', 'foo.conf'), 'w') as f:
f.write('deployed\n')
assert not (await bstore.is_dir(f'{base}/etc/foo.conf/k1'))
assert (await bstore.get(f'{base}/etc/foo.conf/k1')).stdout == b'v1'
# -- A URI may root the store at a subtree of the host tree:
# create() keeps the URI as the store's identity, and its
# path is composed into every address. The inner context
# is host-rooted, so the on-host files are reached
# without a double path
created = FileContext.create(f'kvfile://{tree}/etc')
assert created.uri.path == f'{tree}/etc'
sstore = KvFile(Local(), uri = f'kvfile://{tree}/etc', distro = fake)
await sstore.open()
assert set(await sstore.readdir('')) == \
{'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
# -- An entry lists its keys, suffixed and unsuffixed alike
assert set(await sstore.readdir('foo.conf.jw-secret')) == {'k1', 'k2'}
assert set(await sstore.readdir('foo.conf')) == {'k1', 'k2'}
assert (await sstore.get('foo.conf/k1')).stdout == b'v1'
assert (await sstore.get('cert.pem')).stdout == \
b'-----BEGIN CERT-----\n'
# -- A suffixed address names the on-host file of its config
# path, the suffix picking the kind
assert (await sstore.get('foo.conf.jw-secret')).stdout == \
b'k1=v1\nk2=v2\n'
assert (await sstore.get('cert.pem.jw-secret-file')).stdout == \
b'-----BEGIN CERT-----\n'
st = await sstore.stat('foo.conf.jw-secret')
assert st.ftype is FT_SECRET
st = await sstore.stat('foo.conf.jw-secret/k1')
assert st.ftype is FT_SECRET_KEY
st = await sstore.stat('cert.pem.jw-secret-file')
assert st.ftype is FT_SECRET_FILE
# -- A blob address refuses to list
try:
await sstore.readdir('cert.pem.jw-secret-file')
raise AssertionError('expected NotADirectoryError')
except NotADirectoryError:
pass
# -- A write through the subtree root lands in the subtree
await sstore.mkdir('sub')
await sstore.mkdir('sub/conf', ftype = FT_SECRET)
await sstore.put('sub/conf/k', b'v', ftype = FT_SECRET_KEY)
with open(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) as f:
assert f.read() == 'k=v\n'
await sstore.unlink('sub/conf.jw-secret')
assert not os.path.exists(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret'))
await sstore.close()
# -- The autodetected distro applies the app's documented
# filter fallback: JW_DEFAULT_PKG_FILTER reaches
# Distro.instantiate() as the default package filter
from unittest import mock
from jw.pkg.lib.Distro import Distro
captured: dict[str, Any] = {}
async def __instantiate(*args: Any, **kwargs: Any) -> FakeDistro:
captured['kwargs'] = kwargs
return fake
os.environ['JW_DEFAULT_PKG_FILTER'] = 'url=~jw.dev'
try:
with mock.patch.object(Distro,
'instantiate',
new = classmethod(__instantiate)):
astore = KvFile(Local())
assert set(await astore.readdir('')) == {'tmp'}
flt = captured['kwargs'].get('default_pkg_filter')
assert flt is not None
assert flt.match(Package('fixture', url = 'https://jw.dev/fixture.rpm'))
assert not flt.match(
Package('other', url = 'https://elsewhere/other.rpm')
)
finally:
del os.environ['JW_DEFAULT_PKG_FILTER']
# -- The lifecycle propagates to the host context: an
# open() pin on the store keeps the host's transport
# open for its duration, and the run()s within it share
# it
class CountingLocal(Local):
def __init__(self) -> None:
super().__init__()
self.opens = 0
self.closes = 0
@override
async def _open(self) -> None:
self.opens += 1
await super()._open()
@override
async def _close(self) -> None:
self.closes += 1
await super()._close()
cl = CountingLocal()
pstore = KvFile(cl)
await pstore.open()
assert cl.opens == 1
await cl.run(['true'])
await cl.run(['true'])
assert cl.opens == 1
assert cl.closes == 0
await pstore.close()
assert cl.closes == 1
asyncio.run(run_tests())
print('All secrets kvfile store tests passed')

4
test/unit/python/jw/pkg/lib/ec/secret_store/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/dirs.mk

7
test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

235
test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py Normal file
View file

@ -0,0 +1,235 @@
import asyncio
import io
import os
import shutil
import stat
import subprocess
import tarfile
import tempfile
from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg
from jw.pkg.lib.file_types import FT_DIR
from jw.pkg.lib.FileContext import FileContext
from jw.pkg.lib.util import copy
def make_tar(path: str, entries: dict[str, str]) -> None:
buf = io.BytesIO()
with tarfile.open(fileobj = buf, mode = 'w') as tf:
for name, text in entries.items():
info = tarfile.TarInfo(name)
data = text.encode()
info.size = len(data)
info.mode = 0o400
tf.addfile(info, io.BytesIO(data))
with open(path, 'wb') as f:
f.write(buf.getvalue())
async def run_tests() -> None:
with tempfile.TemporaryDirectory() as tmp:
# -- A throw-away GPG home with a key for the test
gpghome = os.path.join(tmp, 'gnupg')
os.makedirs(gpghome, mode = 0o700)
env = dict(os.environ, GNUPGHOME = gpghome)
subprocess.run(
[
'gpg',
'--batch',
'--pinentry-mode',
'loopback',
'--passphrase',
'',
'--quick-gen-key',
'test@example.com',
],
check = True,
capture_output = True,
env = env,
)
# -- A store in the canonical layout, GPG-encrypted tarball
tar_path = os.path.join(tmp, 'secrets.tar')
make_tar(
tar_path,
{
'default/etc/a.jw-secret': 'a=1\n',
'default/etc/d.jw-secret': 'd=4\n',
'com/example/host/etc/a.jw-secret': 'a=2\n',
'com/example/host/etc/b.jw-secret': 'b=3\n',
# Non-key-value content is stored as a .jw-secret-file
# and addressable as a blob
'com/example/host/etc/c.pem.jw-secret-file': \
'-----BEGIN CERT-----\n',
# A member without a content suffix is store
# content too: non-key-value content is a blob
'com/example/host/etc/README.md': 'not key-value content\n',
}
)
enc_path = tar_path + '.gpg'
subprocess.run(
[
'gpg',
'--batch',
'--yes',
'--quiet',
'--no-tty',
'--trust-model',
'always',
'--recipient',
'test@example.com',
'--encrypt',
'--output',
enc_path,
tar_path,
],
check = True,
capture_output = True,
env = env,
)
# The store decrypts in a local exec context, which inherits
# the test's GNUPGHOME; the tarball location is deployment
# information, passed via the environment
os.environ['GNUPGHOME'] = gpghome
os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path
# -- The whole store: strict, no fallback
store = FileContext.create('targpg://')
await store.open()
assert isinstance(store, TarGpg)
assert set(await store.readdir('')) == {'default', 'com'}
# -- The names as stored: the store does not strip or filter
# by member name, a suffixed name being the address of
# its node
assert set(await store.readdir('com/example/host/etc')) == \
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
'README.md'}
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
b'2'
assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
b'3'
# The entry is gettable as a whole: the member's content
assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
# A blob is a file: get() returns its bytes, readdir() refuses
pem = 'com/example/host/etc/c.pem.jw-secret-file'
assert (await store.get(pem)).stdout == b'-----BEGIN CERT-----\n'
try:
await store.readdir(pem)
raise AssertionError('expected NotADirectoryError')
except NotADirectoryError:
pass
# A member without a content suffix is store content too: a
# non-key-value member is a blob, listed, statable, gettable
assert 'README.md' in set(await store.readdir('com/example/host/etc'))
st = await store.stat('com/example/host/etc/README.md')
assert st.ftype is FT_SECRET_FILE
assert (await store.get('com/example/host/etc/README.md')).stdout == \
b'not key-value content\n'
# A missing entry and a missing key raise FileNotFoundError
for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
try:
await store.get(path)
raise AssertionError(f'expected FileNotFoundError for {path}')
except FileNotFoundError:
pass
# -- stat() answers the node's type, the metadata of the
# member behind it; a key has no member of its own, and a
# branch none at all, so they keep the size 0
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
st = await store.stat('default/etc/a.jw-secret')
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
assert st.size == len((await store.get('default/etc/a.jw-secret')).stdout)
st = await store.stat('default/etc/a.jw-secret/a')
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
assert st.size == 0
st = await store.stat('com/example/host/etc/c.pem.jw-secret-file')
assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
assert st.size == len(
(await store.get('com/example/host/etc/c.pem.jw-secret-file')).stdout
)
st = await store.stat('default/etc')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert st.size == 0
# -- is_dir answers the node's listability: an entry lists
# its keys, a branch its children; a key is a leaf
assert await store.is_dir('default/etc/a.jw-secret')
assert await store.is_dir('default/etc')
assert not await store.is_dir('default/etc/a.jw-secret/a')
try:
await store.stat('default/etc/missing')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- file_exists() works on the store, being built on stat()
assert await store.file_exists('default/etc/a.jw-secret')
assert not await store.file_exists('default/etc/missing')
await store.close()
# -- A host-rooted store applies the default fallback
store = FileContext.create('targpg://host.example.com')
assert isinstance(store, TarGpg)
await store.open()
assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
# A directory listing is the union with the default tree: d
# exists only under default/, but is listable as it is
# gettable
assert set(await store.readdir('etc')) == \
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
'd.jw-secret', 'README.md'}
# A miss below the host is retried below the default tree
assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
# A key listing stays host-wins-wholesale
assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
# -- stat() distinguishes an entry from a branch and the root
st = await store.stat('etc/a.jw-secret')
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
st = await store.stat('etc')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
await store.close()
# -- A store as a copy source: an entry lands as a single
# file with its serialized content, a branch as a
# directory, and a blob as a file; the default-only entry
# d lands from the union listing
host = os.path.join(tmp, 'host-tree')
os.makedirs(host)
ret = await copy('targpg://host.example.com', host, recursive = True)
assert isinstance(ret, list)
assert set(ret) == {
f'{host}/etc/a.jw-secret',
f'{host}/etc/b.jw-secret',
f'{host}/etc/c.pem.jw-secret-file',
f'{host}/etc/d.jw-secret',
f'{host}/etc/README.md',
}
with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
assert f.read() == 'a=2\n'
assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
assert f.read() == '-----BEGIN CERT-----\n'
# -- A single object that is an entry: the serialized whole
# lands at the destination
dst = os.path.join(tmp, 'copied', 'entry')
os.makedirs(os.path.dirname(dst))
ret = await copy('targpg://host.example.com/etc/a.jw-secret', dst)
assert ret == dst
with open(dst) as f:
assert f.read() == 'a=2\n'
def main() -> None:
if not shutil.which('gpg'):
print('Skipping secrets targpg store tests (gpg required)')
return
asyncio.run(run_tests())
print('All secrets targpg store tests passed')
main()

7
test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

41
test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py Normal file
View file

@ -0,0 +1,41 @@
from jw.pkg.lib.ec.secret_store.layout import (
BLOB_SUFFIX,
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_FILE,
FT_SECRET_KEY,
SECRET_SUFFIX,
content_name,
host_prefix,
)
from jw.pkg.lib.file_types import FT_COMPOUND, FT_DIR, FT_REG
# The layout constants are the store contract
assert DEFAULT_PREFIX == 'default'
assert SECRET_SUFFIX == '.jw-secret'
assert BLOB_SUFFIX == '.jw-secret-file'
# content_name strips the content suffix, and names without a
# content suffix are not store content
assert content_name('etc/a.jw-secret') == 'etc/a'
assert content_name('etc/c.pem.jw-secret-file') == 'etc/c.pem'
assert content_name('etc/README.md') is None
# An FQDN maps to its reversed, dot-split host root
assert host_prefix('host.example.com') == 'com/example/host'
assert host_prefix('a.b.c.d') == 'd/c/b/a'
# A single-label hostname maps to itself
assert host_prefix('localhost') == 'localhost'
# The store content types are registered with the file types: an
# entry is a kind of compound, a key and a blob a kind of file
assert FT_SECRET.is_subtype(FT_COMPOUND)
assert FT_SECRET.is_subtype(FT_REG)
assert not FT_SECRET.is_subtype(FT_DIR)
assert FT_SECRET_KEY.is_subtype(FT_REG)
assert not FT_SECRET_KEY.is_subtype(FT_DIR)
assert FT_SECRET_FILE.is_subtype(FT_REG)
assert not FT_SECRET_FILE.is_subtype(FT_DIR)
print('All secrets layout tests passed')