janware/jw-pkg
Watch
1
Code Pull requests Activity Actions

lib.ec.secret_store: Add and use framework
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 4m48s
Details
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 5m12s
Details
CI / Packaging test (pull_request) Successful in 0s
Details
CI / Packaging - Kali Linux (push) Successful in 4m35s
Details
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 4m49s
Details
CI / Packaging test (push) Successful in 0s
Details

Browse source 
"jw-pkg secrets install" copies secret material from a GPG-encrypted tar
archive to dotenv .jw-secret files on a target host. This commit refactors
the underlying code and adds hooks for the inverse copy direction, and for
more secret sources and targets.

The peers between which secrets are copied are represented by the new
abstract secret_store.Base class, and the existing tar blob and host
storages, inheriting from that class, are TarGpg and KvFile. Moreover,
secret_store.Base is inheriting from FileContext, so all of jw-pkg's
file-like operations are available as hooks.

Secrets inside instances of two file contexts are addressable as
(targpg|kvfile)://target.host.com/path/to/file.conf.jw-secret. The on-host
KvFile will look either locally or via SSH for that file, and TarGpg will
look inside a tar archive specified in the JW_SECRETS_TARGPG_TAR
environment variable.

Uris with these schemas can be passed to FileContext.create(), which will
return a chrooted file context of the correct type.

KvFile.stat("/path/to/file.conf.jw-secret").ftype is FT_SECRET which
inherits from FT_COMPOUND - which means that readdir() works on such
animals. Not entirely sure if that's a good idea, but it feels pretty
generic. Let's see where that takes us.

DistroContext.install() after this refactor reads its source through the
store interface and writes the landing through the KvFile store.

Both secret stores are exercised through the existing unit test of jw-pkg
secrets install, and unit tests for both stores are also added by this
commit.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2026-10-03 21:39:32 +02:00
commit f08f658d33
Signed by: Jan Lindemann
GPG key ID: 3750640C9E25DD61
18 changed files with 1835 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

126
src/python/jw/pkg/cmds/secrets/lib/DistroContext.py
View file

@ -3,11 +3,21 @@ from __future__ import annotations
import re
import sys
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from ....lib.ec.secret_store.Base import Base as SecretStore
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import (
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_KEY,
SECRET_SUFFIX,
host_prefix,
)
from ....lib.FileContext import FileContext
from ....lib.log import DEBUG, NOTICE, WARNING, log
from ....lib.ProcFilterGpg import ProcFilterGpg
from .FilesContext import FilesContext
if TYPE_CHECKING:
@ -19,7 +29,7 @@ if TYPE_CHECKING:
class DistroContext(FilesContext):
def __init__(self, distro: Distro) -> None:
super().__init__(distro.ctx)
super().__init__(distro.ctx, distro = distro)
self.__distro = distro
async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@ -40,7 +50,7 @@ class DistroContext(FilesContext):
) -> list[str]:
ret = []
for tmpl in await self.list_template_files(pkg_names):
path = str(Path(tmpl).with_suffix('.jw-secret'))
path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
if ignore_missing and not await self.ctx.file_exists(path):
continue
ret.append(path)
@ -101,64 +111,60 @@ class DistroContext(FilesContext):
verbose: bool = False,
) -> None:
async def _read_secret_tar_blob(src_uri: str) -> bytes:
ec = self.ctx
from ....lib.ec.Local import Local
from ....lib.util import get
if not isinstance(ec, Local):
# Security: Use a local exec context for decrypting and
# filtering secrets
ec = Local()
return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
def _is_needed_secret(path: str) -> bool:
return path in secret_paths
from .tar import extract as tar_extract
from .tar import filter as tar_filter
from .tar import merge as tar_merge
from .tar import rewrite as tar_rewrite
if only_missing:
raise NotImplementedError('--only-missing is not yet implemented')
secret_paths = await self.list_secret_paths(pkg_names)
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in tar file')
host_root_in_tar = '/'.join(reversed(hostname.split('.')))
host_rx = re.compile(r'^' + host_root_in_tar)
default_rx = re.compile(r'^default')
filtered_paths: list[str] = []
extracted_paths: list[str] = []
blob_all = await _read_secret_tar_blob(src_uri)
if blob_all is None:
raise Exception(f'Tar blob {src_uri} is empty')
blob_host_filtered = tar_filter(
blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
)
blob_host_transformed = tar_rewrite(
blob_host_filtered, lambda p: re.sub(host_rx, '', p)
)
blob_default_filtered = tar_filter(
blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
)
blob_default_transformed = tar_rewrite(
blob_default_filtered, lambda p: re.sub(default_rx, '', p)
)
blob_secret_material = tar_merge(
[blob_host_transformed, blob_default_transformed], overwrite = False
)
blob_needed = tar_filter(
blob_secret_material, _is_needed_secret, extracted_paths
)
await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
for path in secret_paths:
if path not in extracted_paths:
log(NOTICE, f'not extracted: {path}')
fc = FileContext.create(src_uri, chroot = True)
await fc.open()
try:
hostname = self.ctx.uri.hostname
if not hostname:
raise Exception('Have no hostname to find secrets in the store')
if isinstance(fc, SecretStore):
if not fc.whole:
raise Exception(
f'src {src_uri} must name the whole store, not a '
f'rooted subtree'
)
src: FileContext = fc
# -- Store candidates: the host tree, then the default
# tree
prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
else:
log(NOTICE, f'extracted: {path}')
# -- A file tree in the on-host layout: wrap it in the kvfile store
src = KvFile(fc, distro = self.__distro)
prefixes = ('', )
# -- The store and the kvfile source are alike addressed
# by the on-host file name
suffix = SECRET_SUFFIX
for entry in await self.list_compilation_targets(pkg_names):
# -- A present entry, even an empty one, wins over the
# fallback
entry_rel = f'{entry.lstrip("/")}{suffix}'
addr = entry_rel
keys: list[str] | None = None
for pfx in prefixes:
addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
try:
keys = list(await src.readdir(addr))
break
except FileNotFoundError:
continue
if keys is None:
log(NOTICE, f'not extracted: {entry}')
continue
log(NOTICE, f'extracted: {entry}')
pairs: list[tuple[str, bytes]] = []
for key in keys:
val = await src.get(f'{addr}/{key}')
pairs.append((key, val.stdout))
dst = self.store
with suppress(FileNotFoundError):
await dst.unlink(entry_rel)
await dst.mkdir(entry_rel, ftype = FT_SECRET)
for key, content in pairs:
await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
finally:
await fc.close()

59
src/python/jw/pkg/cmds/secrets/lib/FilesContext.py
View file

@ -9,42 +9,36 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
from ....lib.Distro import Distro
from ....lib.FileContext import FileContext
from ....lib.ec.secret_store.Base import Base
from ....lib.ec.secret_store.KvFile import KvFile
from ....lib.ec.secret_store.layout import SECRET_SUFFIX
from ....lib.log import DEBUG, NOTICE, WARNING, log
from .base import Attrs
class FilesContext:
def __init__(self, ctx: FileContext) -> None:
def __init__(
self,
ctx: FileContext,
store: Base | None = None,
distro: Distro | None = None,
) -> None:
self.__ctx = ctx
if store is None:
store = KvFile(ctx, distro = distro)
self.__store = store
@property
def ctx(self) -> FileContext:
return self.__ctx
async def _read_key_value_file(
self,
path: str,
throw: bool = False,
) -> dict[str, str]:
ret: dict[str, str] = {}
try:
result = await self.ctx.get(path)
for line in result.stdout_str.splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
except FileNotFoundError:
log(DEBUG, f'File not found {path}')
return ret
@property
def store(self) -> Base:
return self.__store
def _parse_attributes(self, content: str) -> Attrs | None:
@ -143,6 +137,10 @@ class FilesContext:
for key, val in replace.items():
new_content = new_content.replace(key, val)
unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
if unresolved:
raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
tmp_path: str | None = None
try:
@ -200,12 +198,21 @@ class FilesContext:
) -> bool:
path_tmpl = target_path + '.jw-tmpl'
path_secret_file = target_path + '.jw-secret-file'
path_secret = target_path + '.jw-secret'
path_secret = target_path + SECRET_SUFFIX
attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs())
attrs.update(
await self._read_attributes([path_tmpl, path_secret_file, path_secret])
)
replace = await self._read_key_value_file(path_secret)
entry = f'{target_path}{SECRET_SUFFIX}'
try:
keys = await self.__store.readdir(entry)
except FileNotFoundError:
log(DEBUG, f'No secrets for target {target_path}')
keys = []
replace: dict[str, str] = {}
for key in keys:
val = await self.__store.get(f'{entry}/{key}')
replace[key] = val.stdout.decode()
for src in [path_secret_file, path_tmpl]:
try:
await self._compile_one_template_file(

83
src/python/jw/pkg/cmds/secrets/lib/tar.py
View file

@ -1,83 +0,0 @@
from __future__ import annotations
import io
import tarfile
from tarfile import TarFile
from typing import TYPE_CHECKING, Callable
from ....lib.ExecContext import ExecContext
from ....lib.log import DEBUG, log
if TYPE_CHECKING:
from typing import Iterable
from ....lib.FileContext import FileContext
def filter(
blob: bytes,
path_filter: Callable[[str], bool] | None,
matched: list[str] | None = None,
) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
if path_filter is not None and not path_filter(info.name):
continue
log(DEBUG, f'Adding {info.name}')
if matched is not None:
matched.append(info.name)
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
tf_in = TarFile(fileobj = io.BytesIO(blob))
for info in tf_in.getmembers():
new_name = rewrite_filter(info.name)
log(DEBUG, f'Rewriting {info.name} -> {new_name}')
info.name = new_name
# -- A pax "path" supplement from the input would override
# the rewritten name on re-serialization
info.pax_headers = {
k: v
for k, v in info.pax_headers.items() if k != 'path'
}
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes:
ret = io.BytesIO()
with tarfile.open(fileobj = ret, mode = 'w') as tf_out:
for blob in blobs:
tf_in = TarFile(fileobj = io.BytesIO(blob))
existing_names = tf_out.getnames()
for info in tf_in.getmembers():
if not overwrite and info.name in existing_names:
continue
buf = tf_in.extractfile(info)
tf_out.addfile(info, buf)
return ret.getvalue()
async def extract(
dst: FileContext,
blob: bytes,
root: str | None = None,
verbose: bool = False
) -> None:
cmd = ['tar']
if root is not None:
cmd += ['-C', root]
if verbose:
cmd += '-v'
cmd += ['-x', '-f', '-']
if not isinstance(dst, ExecContext):
raise NotImplementedError(
'Extracting tar files to a non-executable '
f'context is not yet implemented: {dst}'
)
await dst.run(cmd, verbose = verbose, cmd_input = blob)

12
src/python/jw/pkg/lib/FileContext.py
View file

@ -360,6 +360,18 @@ class FileContext(abc.ABC):
from .ec.Curl import Curl
return Curl(uri, *args, **kwargs)
case 'targpg':
from .ec.secret_store.TarGpg import TarGpg
return TarGpg(uri, *args, **kwargs)
case 'kvfile':
from .ec.secret_store.KvFile import KvFile
scheme = 'ssh' if uri.hostname else 'local'
inner = FileContext.create(
f'{scheme}://{uri.authority}', chroot = False
)
return KvFile(inner, uri = uri)
case _:
pass
raise Exception(

252
src/python/jw/pkg/lib/ec/secret_store/Base.py Normal file
View file

@ -0,0 +1,252 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override
from ...base import Result, StatResult
from ...file_types import FT_DIR, FT_REG
from ...FileContext import FileContext
from .layout import DEFAULT_PREFIX, host_prefix
if TYPE_CHECKING:
import os
from typing import Any, Iterable
from ...file_types import FileType
from ...Uri import Uri
T = TypeVar('T')
class Base(FileContext):
"""A store of secret entries in the canonical store layout.
A store is a FileContext over the store namespace. The URI
addresses a subtree of that namespace: no authority means the
whole store, a host authority means that host's tree (its
reversed domain prefix), `default` names the default tree, and a
path narrows further. The subtree is the store's namespace
root; addresses passed to the methods are relative to it.
A store is a typed file store: a store path names a file, and
put() persists the content at the address as-is. The store does
not identify content by file name; what a file holds - an entry,
or a blob - is a property of its content, resolved at stat
time. A node is what its content makes it:
- an entry carries key-value secrets and behaves like a
directory: readdir() on the entry lists its keys, get() on
<entry>/<key> returns the value of the key, and get() on the
entry itself returns the serialized entry
- a branch is a path prefix: readdir() lists its children
- a blob is non-key-value content: get() returns its bytes
A store rooted at a host applies the default fallback: an
address that misses below the host is retried below the default
tree. A whole-store or default-rooted store is strict; callers
apply the fallback themselves.
The URI-addressable stores are created by FileContext.create()
from their scheme. The kvfile store is the store of the on-host
layout: its addresses are config paths of the host it is rooted
at, and it may be constructed directly over a host context.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
# -- A store is always chrooted. The flag states the
# fact, and the chroot it arms is undone in the
# protected layer by _unchroot(); an explicit value
# is overridden.
kwargs['chroot'] = True
super().__init__(uri, *args, **kwargs)
@property
def __host_root(self) -> str:
# -- The root of the host tree this store is rooted at, '' if
# the store is whole-store or rooted at the default tree:
# then there is no host tree to fall back from, and the
# caller applies the default fallback itself.
host = self.uri.hostname or ''
if host and host != 'default':
return host_prefix(host)
return ''
@property
def __root_in_storage(self) -> str:
# -- The namespace node this store is rooted at: the
# reversed host prefix of the URI authority, if any
# (`default` is a reserved word), followed by the URI
# path.
host = self.uri.hostname or ''
path = self.uri.path.lstrip('/')
if host and host != 'default':
host = host_prefix(host)
return '/'.join(p for p in (host, path) if p)
def _path_in_storage(self, path: str) -> str:
path = path.lstrip('/')
if not self.__root_in_storage:
return path
if not path:
return self.__root_in_storage
return f'{self.__root_in_storage}/{path}'
def _default_path_in_storage(self, path: str) -> str | None:
host_root = self.__host_root
ns = self.__root_in_storage
if not host_root or not (path == ns or path.startswith(f'{ns}/')):
return None
return f'{DEFAULT_PREFIX}{path[len(host_root):]}'
def _unchroot(self, path: str) -> str:
# -- The inverse of _chroot(): the public wrappers hand the
# protected layer addresses prefixed with the root; store
# addresses are relative to it, and the root itself is the
# empty address. Already-relative addresses pass through.
if not self.root:
return path.lstrip('/')
if path == self.root:
return ''
prefix = f'{self.root}/'
if path.startswith(prefix):
return path[len(prefix):]
return path
async def _call_with_fallback(
self,
path: str,
func: Callable[[str], Awaitable[T]],
) -> T:
storage_path = self._path_in_storage(path)
try:
return await func(storage_path)
except FileNotFoundError:
default_storage_path = self._default_path_in_storage(storage_path)
if default_storage_path is None:
raise
return await func(default_storage_path)
@override
async def put(
self,
path: str,
content: bytes,
wd: str | None = None,
throw: bool = True,
verbose: bool | None = None,
title: str | None = None,
owner: str | None = None,
group: str | None = None,
mode: int | None = None,
atomic: bool = False,
ftype: FileType = FT_REG,
) -> Result:
# -- A put that changes nothing is a no-op: the pre-read is
# fallback-aware, so a value already represented, below
# the host tree or below the default tree, does not
# trigger a store write. Metadata arguments opt out: they
# may be the only change
if owner is None and group is None and mode is None:
try:
existing = await self.get(path)
except Exception:
pass
else:
if self._put_is_noop(existing.stdout, content, ftype):
return Result()
return await super().put(
path,
content,
wd,
throw,
verbose,
title,
owner,
group,
mode,
atomic,
ftype,
)
def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool:
# -- A put that leaves the stored content byte-identical
# changes nothing. A store that normalizes content on
# read overrides this to compare the normalized forms
return existing == content
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a file on the backing store, yet it lists
# its keys: answer from the node's type, not the
# object's mode
try:
return (await self._stat(path, follow_symlinks)).is_dir
except FileNotFoundError:
return False
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- A store is listable: a directory of keys, entries, or
# blobs, whatever its nodes hold.
raise NotImplementedError()
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The base store materializes nothing: branches are
# implicit namespace prefixes, and entries materialize on
# their first write. A store whose nodes have a host home
# overrides this
pass
def _stat_result(
self,
owner: str,
group: str,
mtime: float,
ftype: FileType,
mode: int | None = None,
size: int = 0,
) -> StatResult:
# -- The metadata of a node: the mode and size, if given, are
# the ones of the file, directory, or member behind the
# node; a node without a backing object keeps the
# synthesized type bits and a size of 0. The access and
# change times follow the modification time
import stat
if mode is None:
mode = (
(stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else
(stat.S_IFREG | 0o400)
)
return StatResult(
mode = mode,
owner = owner,
group = group,
size = size,
atime = mtime,
mtime = mtime,
ctime = mtime,
ftype = ftype,
)
def _stat_result_from_os(
self,
st: os.stat_result,
ftype: FileType,
size: int | None = None,
) -> StatResult:
return self._stat_result(
owner = str(st.st_uid),
group = str(st.st_gid),
mtime = st.st_mtime,
ftype = ftype,
mode = st.st_mode,
size = st.st_size if size is None else size,
)
@property
def whole(self) -> bool:
# -- True, if the store is rooted at the whole store namespace:
# no authority, no path
return not self.__root_in_storage

439
src/python/jw/pkg/lib/ec/secret_store/KvFile.py Normal file
View file

@ -0,0 +1,439 @@
from __future__ import annotations
import errno
import os
from typing import TYPE_CHECKING, Iterable, override
if TYPE_CHECKING:
from ...Distro import Distro
from ...FileContext import FileContext
from ...Uri import Uri
from ...base import StatResult
from ...file_types import FileType
from ...FcCache import FcCache
from ...file_types import FT_COMPOUND, FT_DIR
from ...kv import parse_kv, serialize_kv
from ...Result import Result
from .Base import Base
from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX
class KvFile(Base):
"""The on-host store of a host's package secrets.
A config path X is either an entry, the file X.jw-secret with
KEY=VALUE lines, whose children in the address space are its
keys, or a blob, the file X.jw-secret-file, whose content is the
file itself, a leaf. Any other path is a branch: a real
directory on the host, with no file of its own.
Rooted at the host: addresses are config paths relative to the
host root, absolute on the host with the leading slash. A URI
may root the store at a subtree of the host tree; its path is
composed into every address, and the URI is the store's
identity. A suffixed address names the on-host file of its
config path, the suffix picking the kind: .jw-secret the
entry, .jw-secret-file the blob.
get reads a key below an entry or the blob at its config path;
put writes a key into its entry file, or a blob as a
.jw-secret-file at its config path, and accepts entry, key, and
blob content only; unlink drops the file the address names. No
default fallback.
An entry lists its keys. A branch lists the config paths below
it, the root included: the host's package set is the bounded
base, the .jw-tmpl files of the installed packages, presence-
checked on the host. The distro is autodetected from the host
on first use, or given in the constructor; the autodetected
distro applies the JW_DEFAULT_PKG_FILTER package filter, if
set, so a URI-built context honors the app's documented
filter fallback. is_dir() answers from the entry file and the
host tree, so the write path never pays for the enumeration.
The on-host tree is a precondition: mkdir materializes a missing
entry file below an existing directory, and mkdir and put fail
with EROFS below a missing one. KvFile never creates
directories. Entry and blob files are mode 0400;
owner and group follow the put request, if given.
"""
def __init__(
self,
ctx: FileContext,
uri: str | Uri | None = None,
distro: Distro | None = None,
) -> None:
# -- The URI roots the namespace at the host tree's root, or
# at a subtree of it: its path is composed into every
# address, and it is the store's identity. A direct
# construction names the context's host
if uri is None:
uri = f'kvfile://{ctx.uri.hostname or "target"}'
super().__init__(uri = uri)
self.__raw = ctx
self.__distro: Distro | None = distro
self.___config_paths: list[str] | None = None
self.__subtree = self.uri.path.lstrip('/')
# -- The context is read through a cache: the read path
# probes the same handful of paths again and again, and
# the cache pays the transport once per path
self.__ctx = FcCache(ctx)
@override
async def _open(self) -> None:
# -- The lifecycle propagates to the host context: an
# open() pin on the store, the CopyContext of a copy
# walk, keeps the host's transport open for its
# duration, and the run()s within it share it
await self.__raw.open()
@override
async def _close(self) -> None:
await self.__raw.close()
def __addr(self, path: str) -> str:
# -- The host-relative address: the subtree the URI roots
# the store at, then the address relative to it
path = self._unchroot(path)
if not path:
return self.__subtree
if not self.__subtree:
return path
return f'{self.__subtree}/{path}'
@staticmethod
def __resolve(path: str) -> tuple[str, bool]:
# -- The config path of the address, and whether the address
# names a blob file: a suffixed address is the on-host
# file name of its config path, the suffix picking the
# kind
if path.endswith(SECRET_SUFFIX):
return path[:-len(SECRET_SUFFIX)], False
if path.endswith(BLOB_SUFFIX):
return path[:-len(BLOB_SUFFIX)], True
return path, False
async def __config_paths(self) -> list[str]:
# -- The config paths of the host's package secrets: the
# templates of the installed packages, relative to the
# host root. The package database is the bounded base, a
# filesystem scan is not. Computed once: the package set
# is stable for the lifetime of the session
if self.___config_paths is None:
if self.__distro is None:
from ...Distro import Distro
from ...ExecContext import ExecContext
# -- The distro probes the host through the raw
# context: the cache is a plain file context, no
# exec of its own
if not isinstance(self.__raw, ExecContext):
raise ValueError(
'KvFile: a distro must be given when the host'
' context is not an exec context'
)
# -- A URI-built context has no app distro to
# inherit the package filter from: apply the
# app's documented fallback
from ...PackageFilter import PackageFilterString
pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER')
self.__distro = await Distro.instantiate(
ec = self.__raw,
default_pkg_filter = (
PackageFilterString(pkg_filter_str) if pkg_filter_str else None
),
)
# -- The packages' file paths, filtered to the templates
ret: set[str] = set()
for f in await self.__distro.pkg_files():
if not f.endswith('.jw-tmpl'):
continue
ret.add(f.lstrip('/')[:-len('.jw-tmpl')])
self.___config_paths = sorted(ret)
# -- Warm the stat cache with one host command, so the
# per-level presence checks and the walk's node stats
# hit memory: the root, the candidate files of both
# kinds, the directories they live in, and the file
# names a suffixed address would probe those
# directories by
warm = ['/']
warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths]
warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths]
dirs: set[str] = set()
for p in self.___config_paths:
parts = p.split('/')
for i in range(1, len(parts)):
dirs.add('/'.join(parts[:i]))
for d in sorted(dirs):
warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}']
await self.__ctx.warm(warm)
assert self.___config_paths is not None
return self.___config_paths
async def __entry_kv(self, entry: str) -> dict[str, str]:
path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}'
if not await self.__ctx.file_exists(path):
raise FileNotFoundError(entry)
ret = await self.__ctx.get(path)
return parse_kv(ret.stdout)
async def __is_entry(self, path: str) -> bool:
# -- The entry file on the host; KvFile has no default tree.
# The path is host-relative, as from __addr()
if not path:
return False
return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}')
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
# -- The metadata of the host file behind the node, the type
# overridden: an entry is a file that enumerates its
# keys, a key a leaf below it; a branch is a plain
# directory. A key has no file of its own, so it keeps
# the size 0
path = self.__addr(path)
config, is_blob = self.__resolve(path)
async def __stat_of(real: str, ftype: FileType) -> StatResult:
st = await self.__ctx.stat(real, follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
ftype,
mode = st.mode,
size = st.size,
)
if not config:
return await __stat_of('/', FT_DIR)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
return await __stat_of(real, FT_SECRET_FILE)
if await self.__is_entry(config):
return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET)
parent, sep, key = config.rpartition('/')
if sep and key:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent) and key in await self.__entry_kv(parent):
# -- The type is synthesized, the mode the one of
# the entry file
st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks)
return self._stat_result(
st.owner,
st.group,
st.mtime,
FT_SECRET_KEY,
mode = st.mode,
)
blob = f'/{config}{BLOB_SUFFIX}'
if await self.__ctx.file_exists(blob):
return await __stat_of(blob, FT_SECRET_FILE)
if await self.__ctx.is_dir(f'/{config}'):
return await __stat_of(f'/{config}', FT_DIR)
raise FileNotFoundError(errno.ENOENT, f'stat: {path}')
@override
async def _is_dir(self, path: str, follow_symlinks: bool) -> bool:
# -- An entry is a directory of its keys, a key below an
# entry is a leaf: the entry is a file on the host, so
# there is no host path below it to stat. A branch is a
# real directory on the host. The package set is not
# consulted: the write path never pays for the
# enumeration
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if not config:
return True
parent, sep, _ = config.rpartition('/')
if sep:
parent, _ = self.__resolve(parent)
if await self.__is_entry(parent):
return False
if await self.__is_entry(config):
return True
if is_blob:
return False
return await self.__ctx.is_dir(f'/{config}')
@override
async def _readdir(self, path: str) -> Iterable[str]:
# -- An entry lists its keys; a branch, the root included,
# lists the config paths below it, presence-checked on
# the host, under the on-host file name of the file that
# backs it
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if config and await self.__is_entry(config):
return (await self.__entry_kv(config)).keys()
if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'):
raise NotADirectoryError(path)
children: set[str] = set()
prefix = f'{config}/' if config else ''
for cfg in await self.__config_paths():
if not cfg.startswith(prefix):
continue
rest = cfg[len(prefix):]
label, sep, _ = rest.partition('/')
if sep:
# -- A deeper candidate lists its branch label, if a
# secret is present below it: the label names a
# directory on the host, which a present secret
# below implies
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \
await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(label)
continue
# -- A direct child lists under the on-host file name of
# the file that backs it, entry and blob alike
if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'):
children.add(f'{label}{SECRET_SUFFIX}')
if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'):
children.add(f'{label}{BLOB_SUFFIX}')
if config and not children:
# -- A branch the host lacks is absent; a real directory
# with no secret children lists empty
if not await self.__ctx.is_dir(f'/{config}'):
raise FileNotFoundError(path)
return children
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if is_blob:
real = f'/{config}{BLOB_SUFFIX}'
if not await self.__ctx.file_exists(real):
raise FileNotFoundError(path)
return await self.__ctx.get(real)
real = f'/{config}'
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
# -- The entry file is the serialized entry
return await self.__ctx.get(real + SECRET_SUFFIX)
if await self.__ctx.file_exists(real + BLOB_SUFFIX):
return await self.__ctx.get(real + BLOB_SUFFIX)
if '/' not in config:
raise FileNotFoundError(path)
entry, key = config.rsplit('/', 1)
entry, _ = self.__resolve(entry)
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
@override
async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None:
# -- The address space has no directories of its own: a node
# is a branch (a real directory) or an entry (a suffixed
# file); the type says which to build. Below a missing
# branch, mkdir fails with EROFS: the caller creates the
# levels
path = self.__addr(path)
config, _ = self.__resolve(path)
real = f'/{config}'
if ftype.is_subtype(FT_COMPOUND):
# -- An entry: materialize the empty entry file
if await self.__ctx.file_exists(real + SECRET_SUFFIX):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True)
elif ftype.is_subtype(FT_DIR):
# -- A branch: a real directory
if await self.__ctx.is_dir(real):
return
if not await self.__ctx.is_dir(os.path.dirname(real)):
raise OSError(
errno.EROFS, f'mkdir: "{path}" is below a missing directory'
)
await self.__ctx.mkdir(real, mode)
else:
raise ValueError(f'mkdir: "{path}" is not a directory type')
@override
async def _put(
self,
path: str,
content: bytes,
wd: str | None,
throw: bool,
verbose: bool | None,
title: str,
owner: str | None,
group: str | None,
mode: str | None,
atomic: bool,
ftype: FileType,
) -> Result:
path = self.__addr(path)
config, is_blob = self.__resolve(path)
if ftype.is_subtype(FT_SECRET):
# -- The whole entry: the content is the serialized
# entry, written over the entry file; the branch is
# a precondition, as for a blob
if is_blob:
raise ValueError(
f'put: "{path}" names a blob file, but the ftype is an entry'
)
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{SECRET_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
if ftype.is_subtype(FT_SECRET_FILE):
# -- A blob: the content is the file itself, at its
# on-host file name
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'/{config}{BLOB_SUFFIX}',
content,
owner = owner,
group = group,
mode = m,
atomic = True,
)
entry, sep, key = config.rpartition('/')
if not sep or not entry or not key or '/' in key:
raise ValueError(f'put: "{path}" is not an entry/key path')
entry, _ = self.__resolve(entry)
real_entry = f'/{entry}'
if ftype.is_subtype(FT_SECRET_KEY):
# -- A key: merge into the entry file, which must exist;
# mkdir() is the materialization. The put is atomic,
# so it also overwrites an existing entry file, which
# is mode 0400
if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'):
raise OSError(errno.EROFS, f'put: "{entry}" has no entry file')
kv = await self.__entry_kv(entry)
kv[key] = content.decode()
m = 0o400 if mode is None else int(mode, 8)
return await self.__ctx.put(
f'{real_entry}{SECRET_SUFFIX}',
serialize_kv(kv),
owner = owner,
group = group,
mode = m,
atomic = True,
)
raise ValueError(f'put: ftype {ftype} is not a store content type')
@override
async def _unlink(self, path: str) -> None:
# -- Unlink the file the address names: the address is the
# on-host file name below the subtree, so the host path
# is the composed address itself
await self.__ctx.unlink(f'/{self.__addr(path)}')

4
src/python/jw/pkg/lib/ec/secret_store/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../../../../../../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-mod.mk

227
src/python/jw/pkg/lib/ec/secret_store/TarGpg.py Normal file
View file

@ -0,0 +1,227 @@
from __future__ import annotations
import errno
import io
import os
import stat
import time
from tarfile import TarFile, TarInfo
from typing import TYPE_CHECKING, Iterable, override
from ...file_types import FT_DIR
from ...kv import parse_kv
from ...Result import Result
from .Base import Base
from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
if TYPE_CHECKING:
from typing import Any
from ...base import StatResult
from ...Uri import Uri
class TarGpg(Base):
"""A store of secrets in a GPG-encrypted tarball in the canonical
layout.
The tarball location is deployment information, not part of the
address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The
store URI addresses a subtree of the namespace, as in the base
class.
A store path names a member, and optionally a key below an
entry. The member of an entry is its address, and its keys are
addressable below it; a blob is a member at its address holding
non-key-value content. The store does not identify content by
member name: a member whose content is a key-value file is an
entry, anything else a blob.
The tarball is fetched and decrypted once, then cached. Fetching,
decrypting, and filtering happen in a local exec context, so the
GPG key and the decrypted material stay on this machine.
"""
def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None:
super().__init__(uri, *args, **kwargs)
self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '')
self.__tf: TarFile | None = None
self.__kv_cache: dict[str, dict[str, str]] = {}
async def __tar(self) -> TarFile:
if self.__tf is None:
from ...ProcFilterGpg import ProcFilterGpg
from ...util import get
from ..Local import Local
data = (
await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local()))
).stdout
self.__tf = TarFile(fileobj = io.BytesIO(data))
return self.__tf
async def __members(self) -> dict[str, TarInfo]:
tf = await self.__tar()
return {m.name: m for m in tf.getmembers()}
async def __entry_kv(self, entry: str) -> dict[str, str]:
if entry not in self.__kv_cache:
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
raise FileNotFoundError(entry) from None
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store entry "{entry}"')
self.__kv_cache[entry] = parse_kv(buf.read())
return self.__kv_cache[entry]
async def __is_entry(self, entry: str) -> bool:
# -- The member at entry is an entry: its content is a
# non-empty kv file
tf = await self.__tar()
try:
info = tf.getmember(entry)
except KeyError:
return False
if not info.isfile():
return False
buf = tf.extractfile(info)
if buf is None:
return False
return len(parse_kv(buf.read())) != 0
async def __tar_stat(self, storage_path: str) -> StatResult:
# -- The metadata of the member that stands behind the node;
# a branch has no member of its own and is synthesized, a
# key keeps the size 0
members = await self.__members()
def __meta(name: str) -> tuple[str, str, float, int, int]:
# -- A member's mode carries the permission bits only, so
# the type bits are synthesized; a store member is
# always a regular file
mi = members[name]
return (
mi.uname or str(mi.uid),
mi.gname or str(mi.gid),
mi.mtime,
stat.S_IFREG | (mi.mode & 0o777),
mi.size,
)
if await self.__is_entry(storage_path):
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET,
mode = mode,
size = size,
)
par, sep, key = storage_path.rpartition('/')
if sep and key and await self.__is_entry(par):
if key in await self.__entry_kv(par):
# -- The type is synthesized, the mode the one of the
# entry member, the size stays 0
owner, group, mtime, mode, _ = __meta(par)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_KEY,
mode = mode,
)
if storage_path in members:
owner, group, mtime, mode, size = __meta(storage_path)
return self._stat_result(
owner,
group,
mtime,
FT_SECRET_FILE,
mode = mode,
size = size,
)
if not storage_path or \
any(name.startswith(f'{storage_path}/') for name in members):
return self._stat_result(
str(os.getuid()),
str(os.getgid()),
time.time(),
FT_DIR,
)
raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}')
async def __tar_get(self, path: str) -> Result:
members = await self.__members()
if path == '':
raise IsADirectoryError(path)
if await self.__is_entry(path):
# -- The entry member is the serialized entry
tf = await self.__tar()
buf = tf.extractfile(members[path])
if buf is None:
raise Exception(f'Can\'t read store entry "{path}"')
return Result(stdout = buf.read())
if any(name.startswith(f'{path}/') for name in members):
raise IsADirectoryError(path)
if '/' in path:
entry, key = path.rsplit('/', 1)
if await self.__is_entry(entry):
kv = await self.__entry_kv(entry)
try:
return Result(stdout = kv[key].encode())
except KeyError:
raise FileNotFoundError(path) from None
info = members.get(path)
if info is not None and info.isfile():
tf = await self.__tar()
buf = tf.extractfile(info)
if buf is None:
raise Exception(f'Can\'t read store blob "{path}"')
return Result(stdout = buf.read())
raise FileNotFoundError(path)
async def __tar_readdir(self, path: str) -> Iterable[str]:
members = await self.__members()
if await self.__is_entry(path):
return (await self.__entry_kv(path)).keys()
if path in members:
raise NotADirectoryError(path)
def __names(prefix: str) -> set[str]:
# -- No name filter: every member is store content
ret: set[str] = set()
for name in members:
if not name.startswith(prefix):
continue
rest = name[len(prefix):]
ret.add(rest.partition('/')[0])
return ret
ret = __names(f'{path}/' if path else '')
mirror = self._default_path_in_storage(path)
if mirror is not None:
ret |= __names(f'{mirror}/' if mirror else '')
if not ret:
raise FileNotFoundError(path)
return sorted(ret)
@override
async def _get(
self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str
) -> Result:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_get)
@override
async def _readdir(self, path: str) -> Iterable[str]:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_readdir)
@override
async def _stat(self, path: str, follow_symlinks: bool) -> StatResult:
path = self._unchroot(path)
return await self._call_with_fallback(path, self.__tar_stat)

52
src/python/jw/pkg/lib/ec/secret_store/layout.py Normal file
View file

@ -0,0 +1,52 @@
from __future__ import annotations
from ...file_types import FT_COMPOUND, FT_REG, file_types
# -- The canonical store layout, shared by every store implementation and
# the install pipeline that consumes it:
#
# default/<path> entries applying to all hosts
# <host-root>/<path> entries applying to a single host, where
# <host-root> is the host's FQDN, reversed and split
# on dots (com/example/host for host.example.com)
#
# Host-specific entries take precedence over default ones.
# Suffix of the key-value secret file accompanying a compiled target
SECRET_SUFFIX = '.jw-secret'
# Suffix of a non-key-value secret file (certificate, key, ...)
BLOB_SUFFIX = '.jw-secret-file'
DEFAULT_PREFIX = 'default'
def content_name(name: str) -> str | None:
# -- The config path of an on-host secret file name: the name
# with its content suffix removed, or None if the name
# carries neither suffix
for suffix in (SECRET_SUFFIX, BLOB_SUFFIX):
if name.endswith(suffix):
return name.removesuffix(suffix)
return None
def host_prefix(hostname: str) -> str:
return '/'.join(reversed(hostname.split('.')))
# -- The file types of the store content, registered with the global
# file type registry: an entry is a compound of its keys, a key a
# leaf below it, a blob a file
FT_NAME_SECRET = 'secret'
FT_NAME_SECRET_KEY = 'secret-key'
FT_NAME_SECRET_FILE = 'secret-file'
file_types.register_types(
{
FT_NAME_SECRET: FT_COMPOUND,
FT_NAME_SECRET_KEY: FT_REG,
FT_NAME_SECRET_FILE: FT_REG,
}
)
FT_SECRET = file_types.get(FT_NAME_SECRET)
FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY)
FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE)

21
src/python/jw/pkg/lib/kv.py Normal file
View file

@ -0,0 +1,21 @@
from __future__ import annotations
def parse_kv(data: bytes) -> dict[str, str]:
"""Parse a dotenv-style key-value entry into a dict"""
ret: dict[str, str] = {}
for line in data.decode().splitlines():
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' not in line:
continue
key, val = line.split('=', 1)
key = key.strip()
val = val.strip()
if key:
ret[key] = val
return ret
def serialize_kv(kv: dict[str, str]) -> bytes:
"""Serialize a dict back into a dotenv-style key-value entry"""
return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode()

46
test/integration/jw_pkg/secrets/test.py
View file

@ -45,6 +45,7 @@ from jw.pkg.lib.Package import Package
HOST = 'jannet.example.com'
HOST_PREFIX = '/'.join(reversed(HOST.split('.')))
PKG = 'fixture-pkg'
PKG2 = 'fixture-pkg2'
USER = pwd.getpwuid(os.getuid()).pw_name
GROUP = grp.getgrgid(os.getgid()).gr_name
@ -189,12 +190,21 @@ def main() -> None:
# extract it, and must not leave an empty file behind
with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f:
f.write('x=X\n')
# -- qux references a macro that has no store entry: the
# compile must fail rather than emit a literal macro
with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f:
f.write(
f'# conf: owner={USER}, group={GROUP}, mode=640\n'
'present=PRESENT\n'
'missing=__UNRESOLVED__\n'
)
with open(target + '/etc/readme.txt', 'w') as f:
f.write('plain package file\n')
foo_tmpl = target + '/etc/foo.conf.jw-tmpl'
bar_tmpl = target + '/etc/bar.conf.jw-tmpl'
baz_tmpl = target + '/etc/baz.conf.jw-tmpl'
qux_tmpl = target + '/etc/qux.conf.jw-tmpl'
foo_secret = target + '/etc/foo.conf.jw-secret'
bar_secret = target + '/etc/bar.conf.jw-secret'
baz_secret = target + '/etc/baz.conf.jw-secret'
@ -231,7 +241,10 @@ def main() -> None:
distro = FakeDistro(
Local('local://' + HOST),
{PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt']},
{
PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'],
PKG2: [qux_tmpl],
},
)
# -- list-templates: only the .jw-tmpl files of the package
@ -356,6 +369,37 @@ def main() -> None:
with open(bar_out) as f:
assert f.read() == 'plain=plain-value\n'
# -- compile-templates fails on an unresolved macro: qux
# resolves PRESENT from its secret file, but the template
# also references __UNRESOLVED__, which is not in the
# store. The failure must name the macro, and no compiled
# file may be written
qux_secret = target + '/etc/qux.conf.jw-secret'
qux_out = target + '/etc/qux.conf'
with open(qux_secret, 'w') as f:
f.write('PRESENT=there\n')
try:
run_cli(
distro,
[
'secrets',
'compile-templates',
'--owner',
USER,
'--group',
GROUP,
'--mode',
'400',
PKG2,
],
)
except RuntimeError as e:
assert 'Unresolved macros' in str(e), e
assert '__UNRESOLVED__' in str(e), e
else:
raise AssertionError('unresolved macro must fail the compile')
assert not os.path.exists(qux_out)
out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
assert out == [foo_out, bar_out, baz_out], out

7
test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

384
test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py Normal file
View file

@ -0,0 +1,384 @@
import asyncio
import errno
import os
import stat
import tempfile
from typing import Any, override
from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
from jw.pkg.lib.ec.Local import Local
from jw.pkg.lib.ec.secret_store.KvFile import KvFile
from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
from jw.pkg.lib.file_types import FT_DIR
from jw.pkg.lib.FileContext import FileContext
from jw.pkg.lib.Package import Package
async def run_tests() -> None:
with tempfile.TemporaryDirectory() as tmp:
target = os.path.join(tmp, 'etc', 'foo.conf')
os.makedirs(os.path.dirname(target))
with open(target + '.jw-secret', 'w') as f:
f.write(
'# a comment\n\n key1 = value1 \nkey2=value2\n'
'=novalue\nkey3=\n'
)
# FileContext.create() resolves the kvfile scheme, keeping
# the URI as the store's identity
created = FileContext.create('kvfile://')
assert isinstance(created, KvFile)
assert created.uri.scheme == 'kvfile'
store = KvFile(Local())
assert set(await store.readdir(target)) == {'key1', 'key2', 'key3'}
assert (await store.get(f'{target}/key1')).stdout == b'value1'
assert (await store.get(f'{target}/key3')).stdout == b''
# The entry is gettable as a whole: the serialized file
assert (await store.get(target)).stdout == \
b'# a comment\n\n key1 = value1 \nkey2=value2\n=novalue\nkey3=\n'
# -- stat() answers the node's type, the metadata of the
# host file behind it; a key has no file of its own, and
# keeps the size 0
blob = os.path.join(tmp, 'etc', 'bar.pem')
with open(blob + '.jw-secret-file', 'w') as f:
f.write('a blob\n')
st = await store.stat(target)
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == os.path.getsize(target + '.jw-secret')
st = await store.stat(f'{target}/key1')
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == 0
st = await store.stat(blob)
assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
assert st.is_gettable
assert st.size == os.path.getsize(blob + '.jw-secret-file')
st = await store.stat(os.path.dirname(target))
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert not st.is_gettable
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert not st.is_gettable
try:
await store.stat(f'{target}/missing')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- file_exists() works on the store, being built on stat()
assert await store.file_exists(target)
assert not await store.file_exists(f'{target}/missing')
# A missing key and a missing entry raise FileNotFoundError
for path in (f'{target}/missing', os.path.join(tmp, 'etc', 'missing.conf')):
try:
await store.get(path)
raise AssertionError(f'expected FileNotFoundError for {path}')
except FileNotFoundError:
pass
# put writes a key into the entry file, merging with the
# existing keys
await store.put(f'{target}/key4', b'value4', ftype = FT_SECRET_KEY)
assert (await store.get(f'{target}/key4')).stdout == b'value4'
assert (await store.get(f'{target}/key1')).stdout == b'value1'
# -- a second put overwrites the entry file, which is mode 0400
await store.put(f'{target}/key1', b'value1-new', ftype = FT_SECRET_KEY)
assert (await store.get(f'{target}/key1')).stdout == b'value1-new'
assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
new_entry = os.path.join(tmp, 'etc', 'new.conf')
# -- a key put does not create: below a missing entry file
# it fails with EROFS; mkdir() is the materialization
try:
await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
raise AssertionError('expected EROFS')
except OSError as e:
assert e.errno == errno.EROFS
await store.mkdir(new_entry, ftype = FT_SECRET)
await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY)
with open(new_entry + '.jw-secret') as f:
assert f.read() == 'k=v\n'
# -- a typed put replaces the entry: the serialized content
# is written over the entry file, mode 0400
await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
assert set(await store.readdir(target)) == {'new1', 'new2'}
assert (await store.get(target)).stdout == b'new1=n1\nnew2=n2\n'
with open(target + '.jw-secret') as f:
assert f.read() == 'new1=n1\nnew2=n2\n'
assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400
# -- a put that changes nothing does not rewrite the file
mtime = os.stat(target + '.jw-secret').st_mtime_ns
await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET)
assert os.stat(target + '.jw-secret').st_mtime_ns == mtime
# -- An untyped put is refused: the store content types are
# entry, key, and blob
try:
await store.put('HOSTNAME', b'x')
raise AssertionError('expected ValueError')
except ValueError:
pass
# -- put: a blob below a branch lands as a .jw-secret-file at
# its config path, mode 0400
pem = os.path.join(tmp, 'etc', 'cert.pem.jw-secret-file')
await store.put(
os.path.join(tmp, 'etc', 'cert.pem'),
b'-----BEGIN CERT-----\n',
ftype = FT_SECRET_FILE,
)
with open(pem) as f:
assert f.read() == '-----BEGIN CERT-----\n'
assert stat.S_IMODE(os.stat(pem).st_mode) == 0o400
# -- mkdir: the type says which node to build, the default
# a branch; a branch is a real directory, an entry an
# empty entry file, mode 0400
await store.mkdir(os.path.join(tmp, 'etc'))
assert not os.path.exists(os.path.join(tmp, 'etc.jw-secret'))
baz = os.path.join(tmp, 'etc', 'baz')
await store.mkdir(baz)
assert os.path.isdir(baz)
bar = os.path.join(tmp, 'etc', 'bar.conf')
await store.mkdir(bar, ftype = FT_SECRET)
assert os.path.isfile(bar + '.jw-secret')
assert open(bar + '.jw-secret').read() == ''
assert stat.S_IMODE(os.stat(bar + '.jw-secret').st_mode) == 0o400
# -- mkdir and put fail with EROFS below a missing directory
for fn in (
lambda: store.mkdir(os.path.join(tmp, 'no', 'such', 'entry')),
lambda: store.put(
os.path.join(tmp, 'no', 'such', 'entry', 'key'),
b'x',
ftype = FT_SECRET_KEY, ),
):
try:
await fn()
raise AssertionError('expected EROFS')
except OSError as e:
assert e.errno == errno.EROFS
# -- put forwards owner and group to the host context: an
# unknown group fails the put
try:
await store.put(
f'{target}/key7', b'v7', ftype = FT_SECRET_KEY, group = 'no-such-group'
)
except AssertionError:
raise
except Exception as e:
assert 'no-such-group' in str(e)
else:
raise AssertionError('expected the put to fail')
# unlink drops the file the address names, and a missing
# file raises
await store.unlink(target + '.jw-secret')
assert not os.path.exists(target + '.jw-secret')
try:
await store.unlink(target + '.jw-secret')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- Branch listing: the config paths below a branch, the
# root included, come from the host's package set, the
# templates of the installed packages, presence-checked
# on the host. A fake distro supplies the list
class FakeDistro(SuseDistro):
def __init__(self, files: list[str]) -> None:
super().__init__(Local(), id = 'suse')
self.files = files
self.queries = 0
@override
async def _pkg_files(self, names: list[str] | None) -> list[str]:
self.queries += 1
return self.files
tree = os.path.join(tmp, 'tree')
os.makedirs(os.path.join(tree, 'etc'))
with open(os.path.join(tree, 'etc', 'foo.conf.jw-secret'), 'w') as f:
f.write('k1=v1\nk2=v2\n')
with open(os.path.join(tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as f:
f.write('-----BEGIN CERT-----\n')
with open(os.path.join(tree, 'etc', 'other.conf.jw-secret'), 'w') as f:
f.write('o1=v1\n')
# -- A template without a secret file is not a config path;
# a package file that is not a template is not one
# either
fake = FakeDistro(
[
f'{tree}/etc/foo.conf.jw-tmpl',
f'{tree}/etc/cert.pem.jw-tmpl',
f'{tree}/etc/ghost.conf.jw-tmpl',
f'{tree}/etc/other.conf',
]
)
# -- The address space is rooted at the host root; the
# package file list reports host-absolute paths, so the
# fixture tree sits at its path below the root
bstore = KvFile(Local(), distro = fake)
base = tree.lstrip('/')
# -- is_dir answers from the host state, without consulting
# the package set
assert await bstore.is_dir('')
assert await bstore.is_dir(base)
assert await bstore.is_dir(f'{base}/etc/foo.conf')
assert not await bstore.is_dir(f'{base}/etc/cert.pem')
assert fake.queries == 0
# -- is_gettable answers from the node's type: an entry,
# a key, a blob; not a branch, the root included
assert await bstore.is_gettable(f'{base}/etc/foo.conf')
assert await bstore.is_gettable(f'{base}/etc/foo.conf/k1')
assert await bstore.is_gettable(f'{base}/etc/cert.pem')
assert not await bstore.is_gettable(f'{base}/etc')
assert not await bstore.is_gettable('')
assert fake.queries == 0
# -- Listing consults it, once: each branch lists the secret
# files below it, under their on-host file names,
# presence-checked
assert set(await bstore.readdir('')) == {'tmp'}
assert fake.queries == 1
assert set(await bstore.readdir(f'{base}/etc')) == \
{'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
assert fake.queries == 1
# -- An entry still lists its keys, without the package set
assert set(await bstore.readdir(f'{base}/etc/foo.conf')) == {'k1', 'k2'}
assert fake.queries == 1
# -- A branch without config paths raises FileNotFoundError
try:
await bstore.readdir('no/such')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- A real directory with no secret children lists empty
os.makedirs(os.path.join(tree, 'var', 'empty'))
assert set(await bstore.readdir(f'{base}/var/empty')) == set()
# -- The blob reads back through the store
assert (await bstore.get(f'{base}/etc/cert.pem')).stdout == \
b'-----BEGIN CERT-----\n'
# -- The entry's config path is also a deployed file on the
# host, as on a real host: a key below it has no host
# path, and is_dir() must not stat below the file
with open(os.path.join(tree, 'etc', 'foo.conf'), 'w') as f:
f.write('deployed\n')
assert not (await bstore.is_dir(f'{base}/etc/foo.conf/k1'))
assert (await bstore.get(f'{base}/etc/foo.conf/k1')).stdout == b'v1'
# -- A URI may root the store at a subtree of the host tree:
# create() keeps the URI as the store's identity, and its
# path is composed into every address. The inner context
# is host-rooted, so the on-host files are reached
# without a double path
created = FileContext.create(f'kvfile://{tree}/etc')
assert created.uri.path == f'{tree}/etc'
sstore = KvFile(Local(), uri = f'kvfile://{tree}/etc', distro = fake)
await sstore.open()
assert set(await sstore.readdir('')) == \
{'cert.pem.jw-secret-file', 'foo.conf.jw-secret'}
# -- An entry lists its keys, suffixed and unsuffixed alike
assert set(await sstore.readdir('foo.conf.jw-secret')) == {'k1', 'k2'}
assert set(await sstore.readdir('foo.conf')) == {'k1', 'k2'}
assert (await sstore.get('foo.conf/k1')).stdout == b'v1'
assert (await sstore.get('cert.pem')).stdout == \
b'-----BEGIN CERT-----\n'
# -- A suffixed address names the on-host file of its config
# path, the suffix picking the kind
assert (await sstore.get('foo.conf.jw-secret')).stdout == \
b'k1=v1\nk2=v2\n'
assert (await sstore.get('cert.pem.jw-secret-file')).stdout == \
b'-----BEGIN CERT-----\n'
st = await sstore.stat('foo.conf.jw-secret')
assert st.ftype is FT_SECRET
st = await sstore.stat('foo.conf.jw-secret/k1')
assert st.ftype is FT_SECRET_KEY
st = await sstore.stat('cert.pem.jw-secret-file')
assert st.ftype is FT_SECRET_FILE
# -- A blob address refuses to list
try:
await sstore.readdir('cert.pem.jw-secret-file')
raise AssertionError('expected NotADirectoryError')
except NotADirectoryError:
pass
# -- A write through the subtree root lands in the subtree
await sstore.mkdir('sub')
await sstore.mkdir('sub/conf', ftype = FT_SECRET)
await sstore.put('sub/conf/k', b'v', ftype = FT_SECRET_KEY)
with open(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) as f:
assert f.read() == 'k=v\n'
await sstore.unlink('sub/conf.jw-secret')
assert not os.path.exists(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret'))
await sstore.close()
# -- The autodetected distro applies the app's documented
# filter fallback: JW_DEFAULT_PKG_FILTER reaches
# Distro.instantiate() as the default package filter
from unittest import mock
from jw.pkg.lib.Distro import Distro
captured: dict[str, Any] = {}
async def __instantiate(*args: Any, **kwargs: Any) -> FakeDistro:
captured['kwargs'] = kwargs
return fake
os.environ['JW_DEFAULT_PKG_FILTER'] = 'url=~jw.dev'
try:
with mock.patch.object(Distro,
'instantiate',
new = classmethod(__instantiate)):
astore = KvFile(Local())
assert set(await astore.readdir('')) == {'tmp'}
flt = captured['kwargs'].get('default_pkg_filter')
assert flt is not None
assert flt.match(Package('fixture', url = 'https://jw.dev/fixture.rpm'))
assert not flt.match(
Package('other', url = 'https://elsewhere/other.rpm')
)
finally:
del os.environ['JW_DEFAULT_PKG_FILTER']
# -- The lifecycle propagates to the host context: an
# open() pin on the store keeps the host's transport
# open for its duration, and the run()s within it share
# it
class CountingLocal(Local):
def __init__(self) -> None:
super().__init__()
self.opens = 0
self.closes = 0
@override
async def _open(self) -> None:
self.opens += 1
await super()._open()
@override
async def _close(self) -> None:
self.closes += 1
await super()._close()
cl = CountingLocal()
pstore = KvFile(cl)
await pstore.open()
assert cl.opens == 1
await cl.run(['true'])
await cl.run(['true'])
assert cl.opens == 1
assert cl.closes == 0
await pstore.close()
assert cl.closes == 1
asyncio.run(run_tests())
print('All secrets kvfile store tests passed')

4
test/unit/python/jw/pkg/lib/ec/secret_store/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/dirs.mk

7
test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../../
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

235
test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py Normal file
View file

@ -0,0 +1,235 @@
import asyncio
import io
import os
import shutil
import stat
import subprocess
import tarfile
import tempfile
from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg
from jw.pkg.lib.file_types import FT_DIR
from jw.pkg.lib.FileContext import FileContext
from jw.pkg.lib.util import copy
def make_tar(path: str, entries: dict[str, str]) -> None:
buf = io.BytesIO()
with tarfile.open(fileobj = buf, mode = 'w') as tf:
for name, text in entries.items():
info = tarfile.TarInfo(name)
data = text.encode()
info.size = len(data)
info.mode = 0o400
tf.addfile(info, io.BytesIO(data))
with open(path, 'wb') as f:
f.write(buf.getvalue())
async def run_tests() -> None:
with tempfile.TemporaryDirectory() as tmp:
# -- A throw-away GPG home with a key for the test
gpghome = os.path.join(tmp, 'gnupg')
os.makedirs(gpghome, mode = 0o700)
env = dict(os.environ, GNUPGHOME = gpghome)
subprocess.run(
[
'gpg',
'--batch',
'--pinentry-mode',
'loopback',
'--passphrase',
'',
'--quick-gen-key',
'test@example.com',
],
check = True,
capture_output = True,
env = env,
)
# -- A store in the canonical layout, GPG-encrypted tarball
tar_path = os.path.join(tmp, 'secrets.tar')
make_tar(
tar_path,
{
'default/etc/a.jw-secret': 'a=1\n',
'default/etc/d.jw-secret': 'd=4\n',
'com/example/host/etc/a.jw-secret': 'a=2\n',
'com/example/host/etc/b.jw-secret': 'b=3\n',
# Non-key-value content is stored as a .jw-secret-file
# and addressable as a blob
'com/example/host/etc/c.pem.jw-secret-file': \
'-----BEGIN CERT-----\n',
# A member without a content suffix is store
# content too: non-key-value content is a blob
'com/example/host/etc/README.md': 'not key-value content\n',
}
)
enc_path = tar_path + '.gpg'
subprocess.run(
[
'gpg',
'--batch',
'--yes',
'--quiet',
'--no-tty',
'--trust-model',
'always',
'--recipient',
'test@example.com',
'--encrypt',
'--output',
enc_path,
tar_path,
],
check = True,
capture_output = True,
env = env,
)
# The store decrypts in a local exec context, which inherits
# the test's GNUPGHOME; the tarball location is deployment
# information, passed via the environment
os.environ['GNUPGHOME'] = gpghome
os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path
# -- The whole store: strict, no fallback
store = FileContext.create('targpg://')
await store.open()
assert isinstance(store, TarGpg)
assert set(await store.readdir('')) == {'default', 'com'}
# -- The names as stored: the store does not strip or filter
# by member name, a suffixed name being the address of
# its node
assert set(await store.readdir('com/example/host/etc')) == \
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
'README.md'}
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
b'2'
assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
b'3'
# The entry is gettable as a whole: the member's content
assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
# A blob is a file: get() returns its bytes, readdir() refuses
pem = 'com/example/host/etc/c.pem.jw-secret-file'
assert (await store.get(pem)).stdout == b'-----BEGIN CERT-----\n'
try:
await store.readdir(pem)
raise AssertionError('expected NotADirectoryError')
except NotADirectoryError:
pass
# A member without a content suffix is store content too: a
# non-key-value member is a blob, listed, statable, gettable
assert 'README.md' in set(await store.readdir('com/example/host/etc'))
st = await store.stat('com/example/host/etc/README.md')
assert st.ftype is FT_SECRET_FILE
assert (await store.get('com/example/host/etc/README.md')).stdout == \
b'not key-value content\n'
# A missing entry and a missing key raise FileNotFoundError
for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
try:
await store.get(path)
raise AssertionError(f'expected FileNotFoundError for {path}')
except FileNotFoundError:
pass
# -- stat() answers the node's type, the metadata of the
# member behind it; a key has no member of its own, and a
# branch none at all, so they keep the size 0
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
st = await store.stat('default/etc/a.jw-secret')
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
assert st.size == len((await store.get('default/etc/a.jw-secret')).stdout)
st = await store.stat('default/etc/a.jw-secret/a')
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
assert st.size == 0
st = await store.stat('com/example/host/etc/c.pem.jw-secret-file')
assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
assert st.size == len(
(await store.get('com/example/host/etc/c.pem.jw-secret-file')).stdout
)
st = await store.stat('default/etc')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
assert st.size == 0
# -- is_dir answers the node's listability: an entry lists
# its keys, a branch its children; a key is a leaf
assert await store.is_dir('default/etc/a.jw-secret')
assert await store.is_dir('default/etc')
assert not await store.is_dir('default/etc/a.jw-secret/a')
try:
await store.stat('default/etc/missing')
raise AssertionError('expected FileNotFoundError')
except FileNotFoundError:
pass
# -- file_exists() works on the store, being built on stat()
assert await store.file_exists('default/etc/a.jw-secret')
assert not await store.file_exists('default/etc/missing')
await store.close()
# -- A host-rooted store applies the default fallback
store = FileContext.create('targpg://host.example.com')
assert isinstance(store, TarGpg)
await store.open()
assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
# A directory listing is the union with the default tree: d
# exists only under default/, but is listable as it is
# gettable
assert set(await store.readdir('etc')) == \
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
'd.jw-secret', 'README.md'}
# A miss below the host is retried below the default tree
assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
# A key listing stays host-wins-wholesale
assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
# -- stat() distinguishes an entry from a branch and the root
st = await store.stat('etc/a.jw-secret')
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
st = await store.stat('etc')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
st = await store.stat('')
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
await store.close()
# -- A store as a copy source: an entry lands as a single
# file with its serialized content, a branch as a
# directory, and a blob as a file; the default-only entry
# d lands from the union listing
host = os.path.join(tmp, 'host-tree')
os.makedirs(host)
ret = await copy('targpg://host.example.com', host, recursive = True)
assert isinstance(ret, list)
assert set(ret) == {
f'{host}/etc/a.jw-secret',
f'{host}/etc/b.jw-secret',
f'{host}/etc/c.pem.jw-secret-file',
f'{host}/etc/d.jw-secret',
f'{host}/etc/README.md',
}
with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
assert f.read() == 'a=2\n'
assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
assert f.read() == '-----BEGIN CERT-----\n'
# -- A single object that is an entry: the serialized whole
# lands at the destination
dst = os.path.join(tmp, 'copied', 'entry')
os.makedirs(os.path.dirname(dst))
ret = await copy('targpg://host.example.com/etc/a.jw-secret', dst)
assert ret == dst
with open(dst) as f:
assert f.read() == 'a=2\n'
def main() -> None:
if not shutil.which('gpg'):
print('Skipping secrets targpg store tests (gpg required)')
return
asyncio.run(run_tests())
print('All secrets targpg store tests passed')
main()

7
test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile Normal file
View file

@ -0,0 +1,7 @@
TOPDIR = ../../../../../../../../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/py-run.mk
all:
test: run

41
test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py Normal file
View file

@ -0,0 +1,41 @@
from jw.pkg.lib.ec.secret_store.layout import (
BLOB_SUFFIX,
DEFAULT_PREFIX,
FT_SECRET,
FT_SECRET_FILE,
FT_SECRET_KEY,
SECRET_SUFFIX,
content_name,
host_prefix,
)
from jw.pkg.lib.file_types import FT_COMPOUND, FT_DIR, FT_REG
# The layout constants are the store contract
assert DEFAULT_PREFIX == 'default'
assert SECRET_SUFFIX == '.jw-secret'
assert BLOB_SUFFIX == '.jw-secret-file'
# content_name strips the content suffix, and names without a
# content suffix are not store content
assert content_name('etc/a.jw-secret') == 'etc/a'
assert content_name('etc/c.pem.jw-secret-file') == 'etc/c.pem'
assert content_name('etc/README.md') is None
# An FQDN maps to its reversed, dot-split host root
assert host_prefix('host.example.com') == 'com/example/host'
assert host_prefix('a.b.c.d') == 'd/c/b/a'
# A single-label hostname maps to itself
assert host_prefix('localhost') == 'localhost'
# The store content types are registered with the file types: an
# entry is a kind of compound, a key and a blob a kind of file
assert FT_SECRET.is_subtype(FT_COMPOUND)
assert FT_SECRET.is_subtype(FT_REG)
assert not FT_SECRET.is_subtype(FT_DIR)
assert FT_SECRET_KEY.is_subtype(FT_REG)
assert not FT_SECRET_KEY.is_subtype(FT_DIR)
assert FT_SECRET_FILE.is_subtype(FT_REG)
assert not FT_SECRET_FILE.is_subtype(FT_DIR)
print('All secrets layout tests passed')