janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

lib.ec.secret_store: Add and use framework
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 4m48s
Details
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 5m12s
Details
CI / Packaging test (pull_request) Successful in 0s
Details
CI / Packaging - Kali Linux (push) Successful in 4m35s
Details
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 4m49s
Details
CI / Packaging test (push) Successful in 0s
Details

Browse source 
"jw-pkg secrets install" copies secret material from a GPG-encrypted tar
archive to dotenv .jw-secret files on a target host. This commit refactors
the underlying code and adds hooks for the inverse copy direction, and for
more secret sources and targets.

The peers between which secrets are copied are represented by the new
abstract secret_store.Base class, and the existing tar blob and host
storages, inheriting from that class, are TarGpg and KvFile. Moreover,
secret_store.Base is inheriting from FileContext, so all of jw-pkg's
file-like operations are available as hooks.

Secrets inside instances of two file contexts are addressable as
(targpg|kvfile)://target.host.com/path/to/file.conf.jw-secret. The on-host
KvFile will look either locally or via SSH for that file, and TarGpg will
look inside a tar archive specified in the JW_SECRETS_TARGPG_TAR
environment variable.

Uris with these schemas can be passed to FileContext.create(), which will
return a chrooted file context of the correct type.

KvFile.stat("/path/to/file.conf.jw-secret").ftype is FT_SECRET which
inherits from FT_COMPOUND - which means that readdir() works on such
animals. Not entirely sure if that's a good idea, but it feels pretty
generic. Let's see where that takes us.

DistroContext.install() after this refactor reads its source through the
store interface and writes the landing through the KvFile store.

Both secret stores are exercised through the existing unit test of jw-pkg
secrets install, and unit tests for both stores are also added by this
commit.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-10-03 21:39:32 +02:00
commit f08f658d33
Signed by: Jan Lindemann
GPG key ID: 3750640C9E25DD61
3 changed files with 1835 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

126
src/python/jw/pkg/cmds/secrets/lib/DistroContext.py
View file

@ -3,11 +3,21 @@ from __future__ import annotations
import re
import sys
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from ....lib.ec.secret_store.Base import Base as SecretStore
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import (
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_KEY,
SECRET_SUFFIX,
host_prefix,
)
from ....lib.FileContext import FileContext
from ....lib.log import DEBUG, NOTICE, WARNING, log
from ....lib.ProcFilterGpg import ProcFilterGpg
from .FilesContext import FilesContext
if TYPE_CHECKING:
@ -19,7 +29,7 @@ if TYPE_CHECKING:
class DistroContext(FilesContext):
def __init__(self, distro: Distro) -> None:
super().__init__(distro.ctx)
super().__init__(distro.ctx, distro = distro)
self.__distro = distro
async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@ -40,7 +50,7 @@ class DistroContext(FilesContext):
) -> list[str]:
ret = []
for tmpl in await self.list_template_files(pkg_names):
path = str(Path(tmpl).with_suffix('.jw-secret'))
path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
if ignore_missing and not await self.ctx.file_exists(path):
continue
ret.append(path)
@ -101,64 +111,60 @@ class DistroContext(FilesContext):
verbose: bool = False,
) -> None:
async def _read_secret_tar_blob(src_uri: str) -> bytes:
ec = self.ctx
from ....lib.ec.Local import Local
from ....lib.util import get
if not isinstance(ec, Local):
# Security: Use a local exec context for decrypting and
# filtering secrets
ec = Local()
return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
def _is_needed_secret(path: str) -> bool:
return path in secret_paths
from .tar import extract as tar_extract
from .tar import filter as tar_filter
from .tar import merge as tar_merge
from .tar import rewrite as tar_rewrite
if only_missing:
raise NotImplementedError('--only-missing is not yet implemented')
secret_paths = await self.list_secret_paths(pkg_names)
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in tar file')
host_root_in_tar = '/'.join(reversed(hostname.split('.')))
host_rx = re.compile(r'^' + host_root_in_tar)
default_rx = re.compile(r'^default')
filtered_paths: list[str] = []
extracted_paths: list[str] = []
blob_all = await _read_secret_tar_blob(src_uri)
if blob_all is None:
raise Exception(f'Tar blob {src_uri} is empty')
blob_host_filtered = tar_filter(
blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
)
blob_host_transformed = tar_rewrite(
blob_host_filtered, lambda p: re.sub(host_rx, '', p)
)
blob_default_filtered = tar_filter(
blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
)
blob_default_transformed = tar_rewrite(
blob_default_filtered, lambda p: re.sub(default_rx, '', p)
)
blob_secret_material = tar_merge(
[blob_host_transformed, blob_default_transformed], overwrite = False
)
blob_needed = tar_filter(
blob_secret_material, _is_needed_secret, extracted_paths
)
await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
for path in secret_paths:
if path not in extracted_paths:
log(NOTICE, f'not extracted: {path}')
fc = FileContext.create(src_uri, chroot = True)
await fc.open()
try:
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in the store')
if isinstance(fc, SecretStore):
if not fc.whole:
raise Exception(
f'src {src_uri} must name the whole store, not a '
f'rooted subtree'
)
src: FileContext = fc
# -- Store candidates: the host tree, then the default
# tree
prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
else:
log(NOTICE, f'extracted: {path}')
# -- A file tree in the on-host layout: wrap it in the kvfile store
src = KvFile(fc, distro = self.__distro)
prefixes = ('', )
# -- The store and the kvfile source are alike addressed
# by the on-host file name
suffix = SECRET_SUFFIX
for entry in await self.list_compilation_targets(pkg_names):
# -- A present entry, even an empty one, wins over the
# fallback
entry_rel = f'{entry.lstrip("/")}{suffix}'
addr = entry_rel
keys: list[str] | None = None
for pfx in prefixes:
addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
try:
keys = list(await src.readdir(addr))
break
except FileNotFoundError:
continue
if keys is None:
log(NOTICE, f'not extracted: {entry}')
continue
log(NOTICE, f'extracted: {entry}')
pairs: list[tuple[str, bytes]] = []
for key in keys:
val = await src.get(f'{addr}/{key}')
pairs.append((key, val.stdout))
dst = self.store
with suppress(FileNotFoundError):
await dst.unlink(entry_rel)
await dst.mkdir(entry_rel, ftype = FT_SECRET)
for key, content in pairs:
await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
finally:
await fc.close()

59
src/python/jw/pkg/cmds/secrets/lib/FilesContext.py
View file

@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
from ....lib.Distro import Distro
from ....lib.FileContext import FileContext
from ....lib.ec.secret_store.Base import Base
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import SECRET_SUFFIX
from ....lib.log import DEBUG, NOTICE, WARNING, log
from .base import Attrs
class FilesContext:
def __init__(self, ctx: FileContext) -> None:
def __init__(
self,
ctx: FileContext,
store: Base | None = None,
distro: Distro | None = None,
) -> None:
self.__ctx = ctx
if store is None:
store = KvFile(ctx, distro = distro)
self.__store = store
@property
def ctx(self) -> FileContext:
return self.__ctx
async def _read_key_value_file(
self,
path: str,
throw: bool = False,
) -> dict[str, str]:
ret: dict[str, str] = {}
try:
result = await self.ctx.get(path)
for line in result.stdout_str.splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
except FileNotFoundError:
log(DEBUG, f'File not found {path}')
return ret
@property
def store(self) -> Base:
return self.__store
def _parse_attributes(self, content: str) -> Attrs | None:
@ -143,6 +137,10 @@ class FilesContext:
for key, val in replace.items():
new_content = new_content.replace(key, val)
unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
if unresolved:
raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
tmp_path: str | None = None
try:
@ -200,12 +198,21 @@ class FilesContext:
) -> bool:
path_tmpl = target_path + '.jw-tmpl'
path_secret_file = target_path + '.jw-secret-file'
path_secret = target_path + '.jw-secret'
path_secret = target_path + SECRET_SUFFIX
attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
attrs.update(
await self._read_attributes([path_tmpl, path_secret_file, path_secret])
)
replace = await self._read_key_value_file(path_secret)
entry = f'{target_path}{SECRET_SUFFIX}'
try:
keys = await self.__store.readdir(entry)
except FileNotFoundError:
log(DEBUG, f'No secrets for target {target_path}')
keys = []
replace: dict[str, str] = {}
for key in keys:
val = await self.__store.get(f'{entry}/{key}')
replace[key] = val.stdout.decode()
for src in [path_secret_file, path_tmpl]:
try:
await self._compile_one_template_file(

83
src/python/jw/pkg/cmds/secrets/lib/tar.py
View file

@ -1,83 +0,0 @@
from __future__ import annotations
import io
import tarfile
from tarfile import TarFile
from typing import TYPE_CHECKING, Callable
from ....lib.ExecContext import ExecContext
from ....lib.log import DEBUG, log
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
def filter(
blob: bytes,
path_filter: Callable[[str], bool] | None,
matched: list[str] | None = None,
) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
if path_filter is not None and not path_filter(info.name):
continue
log(DEBUG, f'Adding {info.name}')
if matched is not None:
matched.append(info.name)
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
new_name = rewrite_filter(info.name)
log(DEBUG, f'Rewriting {info.name} -> {new_name}')
info.name = new_name
# -- A pax "path" supplement from the input would override
# the rewritten name on re-serialization
info.pax_headers = {
k: v
for k, v in info.pax_headers.items() if k != 'path'
}
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
for blob in blobs:
tf_in = TarFile(fileobj = io.BytesIO(blob))
existing_names = tf_out.getnames()
for info in tf_in.getmembers():
if not overwrite and info.name in existing_names:
continue
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
async def extract(
dst: FileContext,
blob: bytes,
root: str | None = None,
verbose: bool = False
) -> None:
cmd = ['tar']
if root is not None:
cmd += ['-C', root]
if verbose:
cmd += '-v'
cmd += ['-x', '-f', '-']
if not isinstance(dst, ExecContext):
raise NotImplementedError(
'Extracting tar files to a non-executable '
f'context is not yet implemented: {dst}'
)
await dst.run(cmd, verbose = verbose, cmd_input = blob)