janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

lib.ec.secret_store: Add and use framework
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 4m48s
Details
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 5m12s
Details
CI / Packaging test (pull_request) Successful in 0s
Details
CI / Packaging - Kali Linux (push) Successful in 4m35s
Details
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 4m49s
Details
CI / Packaging test (push) Successful in 0s
Details

Browse source 
"jw-pkg secrets install" copies secret material from a GPG-encrypted tar
archive to dotenv .jw-secret files on a target host. This commit refactors
the underlying code and adds hooks for the inverse copy direction, and for
more secret sources and targets.

The peers between which secrets are copied are represented by the new
abstract secret_store.Base class, and the existing tar blob and host
storages, inheriting from that class, are TarGpg and KvFile. Moreover,
secret_store.Base is inheriting from FileContext, so all of jw-pkg's
file-like operations are available as hooks.

Secrets inside instances of two file contexts are addressable as
(targpg|kvfile)://target.host.com/path/to/file.conf.jw-secret. The on-host
KvFile will look either locally or via SSH for that file, and TarGpg will
look inside a tar archive specified in the JW_SECRETS_TARGPG_TAR
environment variable.

Uris with these schemas can be passed to FileContext.create(), which will
return a chrooted file context of the correct type.

KvFile.stat("/path/to/file.conf.jw-secret").ftype is FT_SECRET which
inherits from FT_COMPOUND - which means that readdir() works on such
animals. Not entirely sure if that's a good idea, but it feels pretty
generic. Let's see where that takes us.

DistroContext.install() after this refactor reads its source through the
store interface and writes the landing through the KvFile store.

Both secret stores are exercised through the existing unit test of jw-pkg
secrets install, and unit tests for both stores are also added by this
commit.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-10-03 21:39:32 +02:00
commit f08f658d33
Signed by: Jan Lindemann
GPG key ID: 3750640C9E25DD61
10 changed files with 1835 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

126
src/python/jw/pkg/cmds/secrets/lib/DistroContext.py
View file

@ -3,11 +3,21 @@ from __future__ import annotations
import re
import sys
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from ....lib.ec.secret_store.Base import Base as SecretStore
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import (
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_KEY,
SECRET_SUFFIX,
host_prefix,
)
from ....lib.FileContext import FileContext
from ....lib.log import DEBUG, NOTICE, WARNING, log
from ....lib.ProcFilterGpg import ProcFilterGpg
from .FilesContext import FilesContext
if TYPE_CHECKING:
@ -19,7 +29,7 @@ if TYPE_CHECKING:
class DistroContext(FilesContext):
def __init__(self, distro: Distro) -> None:
super().__init__(distro.ctx)
super().__init__(distro.ctx, distro = distro)
self.__distro = distro
async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@ -40,7 +50,7 @@ class DistroContext(FilesContext):
) -> list[str]:
ret = []
for tmpl in await self.list_template_files(pkg_names):
path = str(Path(tmpl).with_suffix('.jw-secret'))
path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
if ignore_missing and not await self.ctx.file_exists(path):
continue
ret.append(path)
@ -101,64 +111,60 @@ class DistroContext(FilesContext):
verbose: bool = False,
) -> None:
async def _read_secret_tar_blob(src_uri: str) -> bytes:
ec = self.ctx
from ....lib.ec.Local import Local
from ....lib.util import get
if not isinstance(ec, Local):
# Security: Use a local exec context for decrypting and
# filtering secrets
ec = Local()
return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
def _is_needed_secret(path: str) -> bool:
return path in secret_paths
from .tar import extract as tar_extract
from .tar import filter as tar_filter
from .tar import merge as tar_merge
from .tar import rewrite as tar_rewrite
if only_missing:
raise NotImplementedError('--only-missing is not yet implemented')
secret_paths = await self.list_secret_paths(pkg_names)
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in tar file')
host_root_in_tar = '/'.join(reversed(hostname.split('.')))
host_rx = re.compile(r'^' + host_root_in_tar)
default_rx = re.compile(r'^default')
filtered_paths: list[str] = []
extracted_paths: list[str] = []
blob_all = await _read_secret_tar_blob(src_uri)
if blob_all is None:
raise Exception(f'Tar blob {src_uri} is empty')
blob_host_filtered = tar_filter(
blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
)
blob_host_transformed = tar_rewrite(
blob_host_filtered, lambda p: re.sub(host_rx, '', p)
)
blob_default_filtered = tar_filter(
blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
)
blob_default_transformed = tar_rewrite(
blob_default_filtered, lambda p: re.sub(default_rx, '', p)
)
blob_secret_material = tar_merge(
[blob_host_transformed, blob_default_transformed], overwrite = False
)
blob_needed = tar_filter(
blob_secret_material, _is_needed_secret, extracted_paths
)
await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
for path in secret_paths:
if path not in extracted_paths:
log(NOTICE, f'not extracted: {path}')
fc = FileContext.create(src_uri, chroot = True)
await fc.open()
try:
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in the store')
if isinstance(fc, SecretStore):
if not fc.whole:
raise Exception(
f'src {src_uri} must name the whole store, not a '
f'rooted subtree'
)
src: FileContext = fc
# -- Store candidates: the host tree, then the default
# tree
prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
else:
log(NOTICE, f'extracted: {path}')
# -- A file tree in the on-host layout: wrap it in the kvfile store
src = KvFile(fc, distro = self.__distro)
prefixes = ('', )
# -- The store and the kvfile source are alike addressed
# by the on-host file name
suffix = SECRET_SUFFIX
for entry in await self.list_compilation_targets(pkg_names):
# -- A present entry, even an empty one, wins over the
# fallback
entry_rel = f'{entry.lstrip("/")}{suffix}'
addr = entry_rel
keys: list[str] | None = None
for pfx in prefixes:
addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
try:
keys = list(await src.readdir(addr))
break
except FileNotFoundError:
continue
if keys is None:
log(NOTICE, f'not extracted: {entry}')
continue
log(NOTICE, f'extracted: {entry}')
pairs: list[tuple[str, bytes]] = []
for key in keys:
val = await src.get(f'{addr}/{key}')
pairs.append((key, val.stdout))
dst = self.store
with suppress(FileNotFoundError):
await dst.unlink(entry_rel)
await dst.mkdir(entry_rel, ftype = FT_SECRET)
for key, content in pairs:
await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
finally:
await fc.close()

59
src/python/jw/pkg/cmds/secrets/lib/FilesContext.py
View file

@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
from ....lib.Distro import Distro
from ....lib.FileContext import FileContext
from ....lib.ec.secret_store.Base import Base
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import SECRET_SUFFIX
from ....lib.log import DEBUG, NOTICE, WARNING, log
from .base import Attrs
class FilesContext:
def __init__(self, ctx: FileContext) -> None:
def __init__(
self,
ctx: FileContext,
store: Base | None = None,
distro: Distro | None = None,
) -> None:
self.__ctx = ctx
if store is None:
store = KvFile(ctx, distro = distro)
self.__store = store
@property
def ctx(self) -> FileContext:
return self.__ctx
async def _read_key_value_file(
self,
path: str,
throw: bool = False,
) -> dict[str, str]:
ret: dict[str, str] = {}
try:
result = await self.ctx.get(path)
for line in result.stdout_str.splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
except FileNotFoundError:
log(DEBUG, f'File not found {path}')
return ret
@property
def store(self) -> Base:
return self.__store
def _parse_attributes(self, content: str) -> Attrs | None:
@ -143,6 +137,10 @@ class FilesContext:
for key, val in replace.items():
new_content = new_content.replace(key, val)
unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
if unresolved:
raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
tmp_path: str | None = None
try:
@ -200,12 +198,21 @@ class FilesContext:
) -> bool:
path_tmpl = target_path + '.jw-tmpl'
path_secret_file = target_path + '.jw-secret-file'
path_secret = target_path + '.jw-secret'
path_secret = target_path + SECRET_SUFFIX
attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
attrs.update(
await self._read_attributes([path_tmpl, path_secret_file, path_secret])
)
replace = await self._read_key_value_file(path_secret)
entry = f'{target_path}{SECRET_SUFFIX}'
try:
keys = await self.__store.readdir(entry)
except FileNotFoundError:
log(DEBUG, f'No secrets for target {target_path}')
keys = []
replace: dict[str, str] = {}
for key in keys:
val = await self.__store.get(f'{entry}/{key}')
replace[key] = val.stdout.decode()
for src in [path_secret_file, path_tmpl]:
try:
await self._compile_one_template_file(

83
src/python/jw/pkg/cmds/secrets/lib/tar.py
View file

@ -1,83 +0,0 @@
from __future__ import annotations
import io
import tarfile
from tarfile import TarFile
from typing import TYPE_CHECKING, Callable
from ....lib.ExecContext import ExecContext
from ....lib.log import DEBUG, log
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
def filter(
blob: bytes,
path_filter: Callable[[str], bool] | None,
matched: list[str] | None = None,
) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
if path_filter is not None and not path_filter(info.name):
continue
log(DEBUG, f'Adding {info.name}')
if matched is not None:
matched.append(info.name)
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
new_name = rewrite_filter(info.name)
log(DEBUG, f'Rewriting {info.name} -> {new_name}')
info.name = new_name
# -- A pax "path" supplement from the input would override
# the rewritten name on re-serialization
info.pax_headers = {
k: v
for k, v in info.pax_headers.items() if k != 'path'
}
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
for blob in blobs:
tf_in = TarFile(fileobj = io.BytesIO(blob))
existing_names = tf_out.getnames()
for info in tf_in.getmembers():
if not overwrite and info.name in existing_names:
continue
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
async def extract(
dst: FileContext,
blob: bytes,
root: str | None = None,
verbose: bool = False
) -> None:
cmd = ['tar']
if root is not None:
cmd += ['-C', root]
if verbose:
cmd += '-v'
cmd += ['-x', '-f', '-']
if not isinstance(dst, ExecContext):
raise NotImplementedError(
'Extracting tar files to a non-executable '
f'context is not yet implemented: {dst}'
)
await dst.run(cmd, verbose = verbose, cmd_input = blob)

12
src/python/jw/pkg/lib/FileContext.py
View file

@ -360,6 +360,18 @@ class FileContext(abc.ABC):
from .ec.Curl import Curl
return Curl(uri, *args, **kwargs)
case 'targpg':
from .ec.secret_store.TarGpg import TarGpg
return TarGpg(uri, *args, **kwargs)
case 'kvfile':
from .ec.secret_store.KvFile import KvFile
scheme = 'ssh' if uri.hostname else 'local'
inner = FileContext.create(
f'{scheme}://{uri.authority}', chroot = False
)
return KvFile(inner, uri = uri)
case _:
pass
raise Exception(

252
src/python/jw/pkg/lib/ec/secret_store/Base.py Normal file
View file

@ -0,0 +1,252 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override
from ...base import Result, StatResult
from ...file_types import FT_DIR, FT_REG
from ...FileContext import FileContext
from .layout import DEFAULT_PREFIX, host_prefix
if TYPE_CHECKING:
import os
from typing import Any, Iterable
from ...file_types import FileType
from ...Uri import Uri
T = TypeVar('T')
class Base(FileContext):
"""A store of secret entries in the canonical store layout.
A store is a FileContext over the store namespace. The URI
addresses a subtree of that namespace: no authority means the
whole store, a host authority means that host's tree (its
reversed domain prefix), `default` names the default tree, and a
path narrows further. The subtree is the store's namespace
root; addresses passed to the methods are relative to it.
A store is a typed file store: a store path names a file, and
put() persists the content at the address as-is. The store does
not identify content by file name; what a file holds - an entry,
or a blob - is a property of its content, resolved at stat
time. A node is what its content makes it:
- an entry carries key-value secrets and behaves like a
directory: readdir() on the entry lists its keys, get() on
<entry>/<key> returns the value of the key, and get() on the
entry itself returns the serialized entry
- a branch is a path prefix: readdir() lists its children
- a blob is non-key-value content: get() returns its bytes
A store rooted at a host applies the default fallback: an
address that misses below the host is retried below the default
tree. A whole-store or default-rooted store is strict; callers
apply the fallback themselves.
The URI-addressable stores are created by FileContext.create()
from their scheme. The kvfile store is the store of the on-host
layout: its addresses are config paths of the host it is rooted
at, and it may be constructed directly over a host context.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
# -- A store is always chrooted. The flag states the
# fact, and the chroot it arms is undone in the
# protected layer by _unchroot(); an explicit value
# is overridden.
kwargs['chroot'] = True
super().__init__(uri, *args, **kwargs)
@property
def __host_root(self) -> str:
# -- The root of the host tree this store is rooted at, '' if
# the store is whole-store or rooted at the default tree:
# then there is no host tree to fall back from, and the
# caller applies the default fallback itself.
host = self.uri.hostname or ''
if host and host != 'default':
return host_prefix(host)
return ''
@property
def __root_in_storage(self) -> str:
# -- The namespace node this store is rooted at: the
# reversed host prefix of the URI authority, if any
# (`default` is a reserved word), followed by the URI
# path.
host = self.uri.hostname or ''
path = self.uri.path.lstrip('/')
if host and host != 'default':
host = host_prefix(host)
return '/'.join(p for p in (host, path) if p)
def _path_in_storage(self, path: str) -> str:
path = path.lstrip('/')
if not self.__root_in_storage:
return path
if not path:
return self.__root_in_storage
return f'{self.__root_in_storage}/{path}'
def _default_path_in_storage(self, path: str) -> str | None:
host_root = self.__host_root
ns = self.__root_in_storage
if not host_root or not (path == ns or path.startswith(f'{ns}/')):
return None
return f'{DEFAULT_PREFIX}{path[len(host_root):]}'
def _unchroot(self, path: str) -> str:
# -- The inverse of _chroot(): the public wrappers hand the
# protected layer addresses prefixed with the root; store
# addresses are relative to it, and the root itself is the
# empty address. Already-relative addresses pass through.
if not self.root:
return path.lstrip('/')
if path == self.root:
return ''
prefix = f'{self.root}/'
if path.startswith(prefix):
return path[len(prefix):]
return path
async def _call_with_fallback(
self,
path: str,
func: Callable[[str], Awaitable[T]],
) -> T:
storage_path = self._path_in_storage(path)
try:
return await func(storage_path)
except FileNotFoundError:
default_storage_path = self._default_path_in_storage(storage_path)
if default_storage_path is None:
raise
return await func(default_storage_path)
@override
async def put(
self,
path: str,
content: bytes,
wd: str | None = None,
throw: bool = True,
verbose: bool | None = None,
title: str | None = None,
owner: str | None = None,
group: str | None = None,
mode: int | None = None,
atomic: bool = False,
ftype: FileType = FT_REG,
) -> Result:
# -- A put that changes nothing is a no-op: the pre-read is
# fallback-aware, so a value already represented, below
# the host tree or below the default tree, does not
# trigger a store write. Metadata arguments opt out: they
# may be the only change
if owner is None and group is None and mode is None:
try:
existing = await self.get(path)
except Exception:
pass
else:
if self._put_is_noop(existing.stdout, content, ftype):
return Result()
return await super().put(
path,
content,
wd,
throw,
verbose,
title,
owner,
group,
mode,
atomic,
ftype,
)
def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
# -- A put that leaves the stored content byte-identical
# changes nothing. A store that normalizes content on
# read overrides this to compare the normalized forms
return existing == content
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a file on the backing store, yet it lists
# its keys: answer from the node's type, not the
# object's mode
try:
return (await self._stat(path, follow_symlinks)).is_dir
except FileNotFoundError:
return False
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- A store is listable: a directory of keys, entries, or
# blobs, whatever its nodes hold.
raise NotImplementedError()
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The base store materializes nothing: branches are
# implicit namespace prefixes, and entries materialize on
# their first write. A store whose nodes have a host home
# overrides this
pass
def _stat_result(
self,
owner: str,
group: str,
mtime: float,
ftype: FileType,
mode: int | None = None,
size: int = 0,
) -> StatResult:
# -- The metadata of a node: the mode and size, if given, are
# the ones of the file, directory, or member behind the
# node; a node without a backing object keeps the
# synthesized type bits and a size of 0. The access and
# change times follow the modification time
import stat
if mode is None:
mode = (
(stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else
(stat.S_IFREG | 0o400)
)
return StatResult(
mode = mode,
owner = owner,
group = group,
size = size,
atime = mtime,
mtime = mtime,
ctime = mtime,
ftype = ftype,
)
def _stat_result_from_os(
self,
st: os.stat_result,
ftype: FileType,
size: int | None = None,
) -> StatResult:
return self._stat_result(
owner = str(st.st_uid),
group = str(st.st_gid),
mtime = st.st_mtime,
ftype = ftype,
mode = st.st_mode,
size = st.st_size if size is None else size,
)
@property
def whole(self) -> bool:
# -- True, if the store is rooted at the whole store namespace:
# no authority, no path
return not self.__root_in_storage

439
src/python/jw/pkg/lib/ec/secret_store/KvFile.py Normal file
View file

@ -0,0 +1,439 @@
from __future__ import annotations
import errno
import os
from typing import TYPE_CHECKING, Iterable, override
if TYPE_CHECKING:
from ...Distro import Distro
from ...FileContext import FileContext
from ...Uri import Uri
from ...base import StatResult
from ...file_types import FileType
from ...FcCache import FcCache
from ...file_types import FT_COMPOUND, FT_DIR
from ...kv import parse_kv, serialize_kv
from ...Result import Result
from .Base import Base
from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX
class KvFile(Base):
"""The on-host store of a host's package secrets.
A config path X is either an entry, the file X.jw-secret with
KEY=VALUE lines, whose children in the address space are its
keys, or a blob, the file X.jw-secret-file, whose content is the
file itself, a leaf. Any other path is a branch: a real
directory on the host, with no file of its own.
Rooted at the host: addresses are config paths relative to the
host root, absolute on the host with the leading slash. A URI
may root the store at a subtree of the host tree; its path is
composed into every address, and the URI is the store's
identity. A suffixed address names the on-host file of its
config path, the suffix picking the kind: .jw-secret the
entry, .jw-secret-file the blob.
get reads a key below an entry or the blob at its config path;
put writes a key into its entry file, or a blob as a
.jw-secret-file at its config path, and accepts entry, key, and
blob content only; unlink drops the file the address names. No
default fallback.
An entry lists its keys. A branch lists the config paths below
it, the root included: the host's package set is the bounded
base, the .jw-tmpl files of the installed packages, presence-
checked on the host. The distro is autodetected from the host
on first use, or given in the constructor; the autodetected
distro applies the JW_DEFAULT_PKG_FILTER package filter, if
set, so a URI-built context honors the app's documented
filter fallback. is_dir() answers from the entry file and the
host tree, so the write path never pays for the enumeration.
The on-host tree is a precondition: mkdir materializes a missing
entry file below an existing directory, and mkdir and put fail
with EROFS below a missing one. KvFile never creates
directories. Entry and blob files are mode 0400;
owner and group follow the put request, if given.
"""
def __init__(
self,
ctx: FileContext,
uri: str | Uri | None = None,
distro: Distro | None = None,
) -> None:
# -- The URI roots the namespace at the host tree's root, or
# at a subtree of it: its path is composed into every
# address, and it is the store's identity. A direct
# construction names the context's host
if uri is None:
uri = f'kvfile://{ctx.uri.hostname or "target"}'
super().__init__(uri = uri)
self.__raw = ctx
self.__distro: Distro | None = distro
self.___config_paths: list[str] | None = None
self.__subtree = self.uri.path.lstrip('/')
# -- The context is read through a cache: the read path
# probes the same handful of paths again and again, and
# the cache pays the transport once per path
self.__ctx = FcCache(ctx)
@override
async def _open(self) -> None:
# -- The lifecycle propagates to the host context: an
# open() pin on the store, the CopyContext of a copy
# walk, keeps the host's transport open for its
# duration, and the run()s within it share it
await self.__raw.open()
@override
async def _close(self) -> None:
await self.__raw.close()
def __addr(self, path: str) -> str:
# -- The host-relative address: the subtree the URI roots
# the store at, then the address relative to it
path = self._unchroot(path)
if not path:
return self.__subtree
if not self.__subtree:
return path
return f'{self.__subtree}/{path}'
@staticmethod
def __resolve(path: str) -> tuple[str, bool]:
# -- The config path of the address, and whether the address
# names a blob file: a suffixed address is the on-host
# file name of its config path, the suffix picking the
# kind
if path.endswith(SECRET_SUFFIX):
return path[:-len(SECRET_SUFFIX)], False
if path.endswith(BLOB_SUFFIX):
return path[:-len(BLOB_SUFFIX)], True
return path, False
async def __config_paths(self) -> list[str]:
# -- The config paths of the host's package secrets: the
# templates of the installed packages, relative to the
# host root. The package database is the bounded base, a
# filesystem scan is not. Computed once: the package set
# is stable for the lifetime of the session
if self.___config_paths is None:
if self.__distro is None:
from ...Distro import Distro
from ...ExecContext import ExecContext
# -- The distro probes the host through the raw
# context: the cache is a plain file context, no
# exec of its own
if not isinstance(self.__raw, ExecContext):
raise ValueError(
'KvFile: a distro must be given when the host'
' context is not an exec context'
)
# -- A URI-built context has no app distro to
# inherit the package filter from: apply the
# app's documented fallback
from ...PackageFilter import PackageFilterString
pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER')
self.__distro = await Distro.instantiate(
ec = self.__raw,
default_pkg_filter = (
PackageFilterString(pkg_filter_str) if pkg_filter_str else None
),
)
# -- The packages' file paths, filtered to the templates
ret: set[str] = set()
for f in await self.__distro.pkg_files():
if not f.endswith('.jw-tmpl'):
continue
ret.add(f.lstrip('/')[:-len('.jw-tmpl')])
self.___config_paths = sorted(ret)
# -- Warm the stat cache with one host command, so the
# per-level presence checks and the walk's node stats
# hit memory: the root, the candidate files of both
# kinds, the directories they live in, and the file
# names a suffixed address would probe those
# directories by
warm = ['/']
warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths]
warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths]
dirs: set[str] = set()
for p in self.___config_paths:
parts = p.split('/')
for i in range(1, len(parts)):
dirs.add('/'.join(parts[:i]))
for d in sorted(dirs):
warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}']
await self.__ctx.warm(warm)
assert self.___config_paths is not None
return self.___config_paths
async def __entry_kv(self, entry: str) -> dict[str, str]:
path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}'
if not await self.__ctx.file_exists(path):
raise FileNotFoundError(entry)
ret = await self.__ctx.get(path)
return parse_kv(ret.stdout)
async def __is_entry(self, path: str) -> bool:
# -- The entry file on the host; KvFile has no default tree.
# The path is host-relative, as from __addr()
if not path:
return False
return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}')
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
# -- The metadata of the host file behind the node, the type
# overridden: an entry is a file that enumerates its
# keys, a key a leaf below it; a branch is a plain
# directory. A key has no file of its own, so it keeps
# the size 0
path = self.__addr(path)
config, is_blob = self.__resolve(path)
async def __stat_of(real: str, ftype: FileType) -> StatResult:
st = await self.__ctx.stat(real, follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
ftype,
mode = st.mode,
size = st.size,
)
if not config:
return await __stat_of('/', FT_DIR)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
return await __stat_of(real, FT_SECRET_FILE)
if await self.__is_entry(config):
return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET)
parent, sep, key = config.rpartition('/')
if sep and key:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent) and key in await self.__entry_kv(parent):
# -- The type is synthesized, the mode the one of
# the entry file
st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
FT_SECRET_KEY,
mode = st.mode,
)
blob = f'/{config}{BLOB_SUFFIX}'
if await self.__ctx.file_exists(blob):
return await __stat_of(blob, FT_SECRET_FILE)
if await self.__ctx.is_dir(f'/{config}'):
return await __stat_of(f'/{config}', FT_DIR)
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a directory of its keys, a key below an
# entry is a leaf: the entry is a file on the host, so
# there is no host path below it to stat. A branch is a
# real directory on the host. The package set is not
# consulted: the write path never pays for the
# enumeration
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if not config:
return True
parent, sep, _ = config.rpartition('/')
if sep:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent):
return False
if await self.__is_entry(config):
return True
if is_blob:
return False
return await self.__ctx.is_dir(f'/{config}')
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- An entry lists its keys; a branch, the root included,
# lists the config paths below it, presence-checked on
# the host, under the on-host file name of the file that
# backs it
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if config and await self.__is_entry(config):
return (await self.__entry_kv(config)).keys()
if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'):
raise NotADirectoryError(path)
children: set[str] = set()
prefix = f'{config}/' if config else ''
for cfg in await self.__config_paths():
if not cfg.startswith(prefix):
continue
rest = cfg[len(prefix):]
label, sep, _ = rest.partition('/')
if sep:
# -- A deeper candidate lists its branch label, if a
# secret is present below it: the label names a
# directory on the host, which a present secret
# below implies
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \
await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(label)
continue
# -- A direct child lists under the on-host file name of
# the file that backs it, entry and blob alike
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'):
children.add(f'{label}{SECRET_SUFFIX}')
if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(f'{label}{BLOB_SUFFIX}')
if config and not children:
# -- A branch the host lacks is absent; a real directory
# with no secret children lists empty
if not await self.__ctx.is_dir(f'/{config}'):
raise FileNotFoundError(path)
return children
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(path)
return await self.__ctx.get(real)
real = f'/{config}'
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
# -- The entry file is the serialized entry
return await self.__ctx.get(real + SECRET_SUFFIX)
if await self.__ctx.file_exists(real + BLOB_SUFFIX):
return await self.__ctx.get(real + BLOB_SUFFIX)
if '/' not in config:
raise FileNotFoundError(path)
entry, key = config.rsplit('/', 1)
entry, _ = self.__resolve(entry)
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The address space has no directories of its own: a node
# is a branch (a real directory) or an entry (a suffixed
# file); the type says which to build. Below a missing
# branch, mkdir fails with EROFS: the caller creates the
# levels
path = self.__addr(path)
config, _ = self.__resolve(path)
real = f'/{config}'
if ftype.is_subtype(FT_COMPOUND):
# -- An entry: materialize the empty entry file
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True)
elif ftype.is_subtype(FT_DIR):
# -- A branch: a real directory
if await self.__ctx.is_dir(real):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.mkdir(real, mode)
else:
raise ValueError(f'mkdir: "{path}" is not a directory type')
@override
async def _put(
self,
path: str,
content: bytes,
wd: str | None,
throw: bool,
verbose: bool | None,
title: str,
owner: str | None,
group: str | None,
mode: str | None,
atomic: bool,
ftype: FileType,
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if ftype.is_subtype(FT_SECRET):
# -- The whole entry: the content is the serialized
# entry, written over the entry file; the branch is
# a precondition, as for a blob
if is_blob:
raise ValueError(
f'put: "{path}" names a blob file, but the ftype is an entry'
)
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{SECRET_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
if ftype.is_subtype(FT_SECRET_FILE):
# -- A blob: the content is the file itself, at its
# on-host file name
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{BLOB_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
entry, sep, key = config.rpartition('/')
if not sep or not entry or not key or '/' in key:
raise ValueError(f'put: "{path}" is not an entry/key path')
entry, _ = self.__resolve(entry)
real_entry = f'/{entry}'
if ftype.is_subtype(FT_SECRET_KEY):
# -- A key: merge into the entry file, which must exist;
# mkdir() is the materialization. The put is atomic,
# so it also overwrites an existing entry file, which
# is mode 0400
if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'):
raise OSError(errno.EROFS, f'put: "{entry}" has no entry file')
kv = await self.__entry_kv(entry)
kv[key] = content.decode()
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'{real_entry}{SECRET_SUFFIX}',
serialize_kv(kv),
owner = owner,
group = group,
mode = m,
atomic = True,
)
raise ValueError(f'put: ftype {ftype} is not a store content type')
@override
async def _unlink(self, path: str) -> None:
# -- Unlink the file the address names: the address is the
# on-host file name below the subtree, so the host path
# is the composed address itself
await self.__ctx.unlink(f'/{self.__addr(path)}')

4
src/python/jw/pkg/lib/ec/secret_store/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../../../../../../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-mod.mk

227
src/python/jw/pkg/lib/ec/secret_store/TarGpg.py Normal file
View file

@ -0,0 +1,227 @@
from __future__ import annotations
import errno
import io
import os
import stat
import time
from tarfile import TarFile, TarInfo
from typing import TYPE_CHECKING, Iterable, override
from ...file_types import FT_DIR
from ...kv import parse_kv
from ...Result import Result
from .Base import Base
from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
if TYPE_CHECKING:
from typing import Any
from ...base import StatResult
from ...Uri import Uri
class TarGpg(Base):
"""A store of secrets in a GPG-encrypted tarball in the canonical
layout.
The tarball location is deployment information, not part of the
address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The
store URI addresses a subtree of the namespace, as in the base
class.
A store path names a member, and optionally a key below an
entry. The member of an entry is its address, and its keys are
addressable below it; a blob is a member at its address holding
non-key-value content. The store does not identify content by
member name: a member whose content is a key-value file is an
entry, anything else a blob.
The tarball is fetched and decrypted once, then cached. Fetching,
decrypting, and filtering happen in a local exec context, so the
GPG key and the decrypted material stay on this machine.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
super().__init__(uri, *args, **kwargs)
self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '')
self.__tf: TarFile | None = None
self.__kv_cache: dict[str, dict[str, str]] = {}
async def __tar(self) -> TarFile:
if self.__tf is None:
from ...ProcFilterGpg import ProcFilterGpg
from ...util import get
from ..Local import Local
data = (
await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local()))
).stdout
self.__tf = TarFile(fileobj = io.BytesIO(data))
return self.__tf
async def __members(self) -> dict[str, TarInfo]:
tf = await self.__tar()
return {m.name: m for m in tf.getmembers()}
async def __entry_kv(self, entry: str) -> dict[str, str]:
if entry not in self.__kv_cache:
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
raise FileNotFoundError(entry) from None
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store entry "{entry}"')
self.__kv_cache[entry] = parse_kv(buf.read())
return self.__kv_cache[entry]
async def __is_entry(self, entry: str) -> bool:
# -- The member at entry is an entry: its content is a
# non-empty kv file
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
return False
if not info.isfile():
return False
buf = tf.extractfile(info)
if buf is None:
return False
return len(parse_kv(buf.read())) != 0
async def __tar_stat(self, storage_path: str) -> StatResult:
# -- The metadata of the member that stands behind the node;
# a branch has no member of its own and is synthesized, a
# key keeps the size 0
members = await self.__members()
def __meta(name: str) -> tuple[str, str, float, int, int]:
# -- A member's mode carries the permission bits only, so
# the type bits are synthesized; a store member is
# always a regular file
mi = members[name]
return (
mi.uname or str(mi.uid),
mi.gname or str(mi.gid),
mi.mtime,
stat.S_IFREG | (mi.mode & 0o777),
mi.size,
)
if await self.__is_entry(storage_path):
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET,
mode = mode,
size = size,
)
par, sep, key = storage_path.rpartition('/')
if sep and key and await self.__is_entry(par):
if key in await self.__entry_kv(par):
# -- The type is synthesized, the mode the one of the
# entry member, the size stays 0
owner, group, mtime, mode, _ = __meta(par)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_KEY,
mode = mode,
)
if storage_path in members:
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_FILE,
mode = mode,
size = size,
)
if not storage_path or \
any(name.startswith(f'{storage_path}/') for name in members):
return self._stat_result(
str(os.getuid()),
str(os.getgid()),
time.time(),
FT_DIR,
)
raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
async def __tar_get(self, path: str) -> Result:
members = await self.__members()
if path == '':
raise IsADirectoryError(path)
if await self.__is_entry(path):
# -- The entry member is the serialized entry
tf = await self.__tar()
buf = tf.extractfile(members[path])
if buf is None:
raise Exception(f'Can\'t read store entry "{path}"')
return Result(stdout = buf.read())
if any(name.startswith(f'{path}/') for name in members):
raise IsADirectoryError(path)
if '/' in path:
entry, key = path.rsplit('/', 1)
if await self.__is_entry(entry):
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
info = members.get(path)
if info is not None and info.isfile():
tf = await self.__tar()
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store blob "{path}"')
return Result(stdout = buf.read())
raise FileNotFoundError(path)
async def __tar_readdir(self, path: str) -> Iterable[str]:
members = await self.__members()
if await self.__is_entry(path):
return (await self.__entry_kv(path)).keys()
if path in members:
raise NotADirectoryError(path)
def __names(prefix: str) -> set[str]:
# -- No name filter: every member is store content
ret: set[str] = set()
for name in members:
if not name.startswith(prefix):
continue
rest = name[len(prefix):]
ret.add(rest.partition('/')[0])
return ret
ret = __names(f'{path}/' if path else '')
mirror = self._default_path_in_storage(path)
if mirror is not None:
ret |= __names(f'{mirror}/' if mirror else '')
if not ret:
raise FileNotFoundError(path)
return sorted(ret)
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_get)
@override
async def _readdir(self, path: str) -> Iterable[str]:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_readdir)
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_stat)

52
src/python/jw/pkg/lib/ec/secret_store/layout.py Normal file
View file

@ -0,0 +1,52 @@
from __future__ import annotations
from ...file_types import FT_COMPOUND, FT_REG, file_types
# -- The canonical store layout, shared by every store implementation and
# the install pipeline that consumes it:
#
# default/<path> entries applying to all hosts
# <host-root>/<path> entries applying to a single host, where
# <host-root> is the host's FQDN, reversed and split
# on dots (com/example/host for host.example.com)
#
# Host-specific entries take precedence over default ones.
# Suffix of the key-value secret file accompanying a compiled target
SECRET_SUFFIX = '.jw-secret'
# Suffix of a non-key-value secret file (certificate, key, ...)
BLOB_SUFFIX = '.jw-secret-file'
DEFAULT_PREFIX = 'default'
def content_name(name: str) -> str | None:
# -- The config path of an on-host secret file name: the name
# with its content suffix removed, or None if the name
# carries neither suffix
for suffix in (SECRET_SUFFIX, BLOB_SUFFIX):
if name.endswith(suffix):
return name.removesuffix(suffix)
return None
def host_prefix(hostname: str) -> str:
return '/'.join(reversed(hostname.split('.')))
# -- The file types of the store content, registered with the global
# file type registry: an entry is a compound of its keys, a key a
# leaf below it, a blob a file
FT_NAME_SECRET = 'secret'
FT_NAME_SECRET_KEY = 'secret-key'
FT_NAME_SECRET_FILE = 'secret-file'
file_types.register_types(
{
FT_NAME_SECRET: FT_COMPOUND,
FT_NAME_SECRET_KEY: FT_REG,
FT_NAME_SECRET_FILE: FT_REG,
}
)
FT_SECRET = file_types.get(FT_NAME_SECRET)
FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY)
FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE)

21
src/python/jw/pkg/lib/kv.py Normal file
View file

@ -0,0 +1,21 @@
from __future__ import annotations
def parse_kv(data: bytes) -> dict[str, str]:
"""Parse a dotenv-style key-value entry into a dict"""
ret: dict[str, str] = {}
for line in data.decode().splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
return ret
def serialize_kv(kv: dict[str, str]) -> bytes:
"""Serialize a dict back into a dotenv-style key-value entry"""
return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode()