diff --git a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py index 67fe0648..610e4976 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py @@ -3,11 +3,21 @@ from __future__ import annotations import re import sys +from contextlib import suppress from pathlib import Path from typing import TYPE_CHECKING +from ....lib.ec.secret_store.Base import Base as SecretStore +from ....lib.ec.secret_store.KvFile import KvFile +from ....lib.ec.secret_store.layout import ( + DEFAULT_PREFIX, + FT_SECRET, + FT_SECRET_KEY, + SECRET_SUFFIX, + host_prefix, +) +from ....lib.FileContext import FileContext from ....lib.log import DEBUG, NOTICE, WARNING, log -from ....lib.ProcFilterGpg import ProcFilterGpg from .FilesContext import FilesContext if TYPE_CHECKING: @@ -19,7 +29,7 @@ if TYPE_CHECKING: class DistroContext(FilesContext): def __init__(self, distro: Distro) -> None: - super().__init__(distro.ctx) + super().__init__(distro.ctx, distro = distro) self.__distro = distro async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]: @@ -40,7 +50,7 @@ class DistroContext(FilesContext): ) -> list[str]: ret = [] for tmpl in await self.list_template_files(pkg_names): - path = str(Path(tmpl).with_suffix('.jw-secret')) + path = str(Path(tmpl).with_suffix(SECRET_SUFFIX)) if ignore_missing and not await self.ctx.file_exists(path): continue ret.append(path) @@ -101,64 +111,60 @@ class DistroContext(FilesContext): verbose: bool = False, ) -> None: - async def _read_secret_tar_blob(src_uri: str) -> bytes: - ec = self.ctx - from ....lib.ec.Local import Local - from ....lib.util import get - - if not isinstance(ec, Local): - # Security: Use a local exec context for decrypting and - # filtering secrets - ec = Local() - return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout - - def _is_needed_secret(path: str) -> bool: - return path in secret_paths - - from .tar import extract as tar_extract - from .tar import filter as tar_filter - from .tar import merge as tar_merge - from .tar import rewrite as tar_rewrite - if only_missing: raise NotImplementedError('--only-missing is not yet implemented') - secret_paths = await self.list_secret_paths(pkg_names) - hostname = self.ctx.uri.hostname - if not hostname: - raise Exception('Have no hostname to find secrets in tar file') - host_root_in_tar = '/'.join(reversed(hostname.split('.'))) - host_rx = re.compile(r'^' + host_root_in_tar) - default_rx = re.compile(r'^default') - - filtered_paths: list[str] = [] - extracted_paths: list[str] = [] - - blob_all = await _read_secret_tar_blob(src_uri) - if blob_all is None: - raise Exception(f'Tar blob {src_uri} is empty') - blob_host_filtered = tar_filter( - blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths - ) - blob_host_transformed = tar_rewrite( - blob_host_filtered, lambda p: re.sub(host_rx, '', p) - ) - blob_default_filtered = tar_filter( - blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths - ) - blob_default_transformed = tar_rewrite( - blob_default_filtered, lambda p: re.sub(default_rx, '', p) - ) - blob_secret_material = tar_merge( - [blob_host_transformed, blob_default_transformed], overwrite = False - ) - blob_needed = tar_filter( - blob_secret_material, _is_needed_secret, extracted_paths - ) - - await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose) - for path in secret_paths: - if path not in extracted_paths: - log(NOTICE, f'not extracted: {path}') + fc = FileContext.create(src_uri, chroot = True) + await fc.open() + try: + hostname = self.ctx.uri.hostname + if not hostname: + raise Exception('Have no hostname to find secrets in the store') + if isinstance(fc, SecretStore): + if not fc.whole: + raise Exception( + f'src {src_uri} must name the whole store, not a ' + f'rooted subtree' + ) + src: FileContext = fc + # -- Store candidates: the host tree, then the default + # tree + prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX) else: - log(NOTICE, f'extracted: {path}') + # -- A file tree in the on-host layout: wrap it in the kvfile store + src = KvFile(fc, distro = self.__distro) + prefixes = ('', ) + + # -- The store and the kvfile source are alike addressed + # by the on-host file name + suffix = SECRET_SUFFIX + + for entry in await self.list_compilation_targets(pkg_names): + # -- A present entry, even an empty one, wins over the + # fallback + entry_rel = f'{entry.lstrip("/")}{suffix}' + addr = entry_rel + keys: list[str] | None = None + for pfx in prefixes: + addr = f'{pfx}/{entry_rel}' if pfx else entry_rel + try: + keys = list(await src.readdir(addr)) + break + except FileNotFoundError: + continue + if keys is None: + log(NOTICE, f'not extracted: {entry}') + continue + log(NOTICE, f'extracted: {entry}') + pairs: list[tuple[str, bytes]] = [] + for key in keys: + val = await src.get(f'{addr}/{key}') + pairs.append((key, val.stdout)) + dst = self.store + with suppress(FileNotFoundError): + await dst.unlink(entry_rel) + await dst.mkdir(entry_rel, ftype = FT_SECRET) + for key, content in pairs: + await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY) + finally: + await fc.close() diff --git a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py index 065934f6..d657b5e7 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py @@ -9,42 +9,36 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Iterable - from ....lib.FileContext import FileContext + from ....lib.Distro import Distro + from ....lib.FileContext import FileContext + from ....lib.ec.secret_store.Base import Base + +from ....lib.ec.secret_store.KvFile import KvFile +from ....lib.ec.secret_store.layout import SECRET_SUFFIX from ....lib.log import DEBUG, NOTICE, WARNING, log from .base import Attrs class FilesContext: - def __init__(self, ctx: FileContext) -> None: + def __init__( + self, + ctx: FileContext, + store: Base | None = None, + distro: Distro | None = None, + ) -> None: self.__ctx = ctx + if store is None: + store = KvFile(ctx, distro = distro) + self.__store = store @property def ctx(self) -> FileContext: return self.__ctx - async def _read_key_value_file( - self, - path: str, - throw: bool = False, - ) -> dict[str, str]: - ret: dict[str, str] = {} - try: - result = await self.ctx.get(path) - for line in result.stdout_str.splitlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' not in line: - continue - key, val = line.split('=', 1) - key = key.strip() - val = val.strip() - if key: - ret[key] = val - except FileNotFoundError: - log(DEBUG, f'File not found {path}') - return ret + @property + def store(self) -> Base: + return self.__store def _parse_attributes(self, content: str) -> Attrs | None: @@ -143,6 +137,10 @@ class FilesContext: for key, val in replace.items(): new_content = new_content.replace(key, val) + unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content))) + if unresolved: + raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}') + tmp_path: str | None = None try: @@ -200,12 +198,21 @@ class FilesContext: ) -> bool: path_tmpl = target_path + '.jw-tmpl' path_secret_file = target_path + '.jw-secret-file' - path_secret = target_path + '.jw-secret' + path_secret = target_path + SECRET_SUFFIX attrs = copy.deepcopy(default_attrs if default_attrs is not None else Attrs()) attrs.update( await self._read_attributes([path_tmpl, path_secret_file, path_secret]) ) - replace = await self._read_key_value_file(path_secret) + entry = f'{target_path}{SECRET_SUFFIX}' + try: + keys = await self.__store.readdir(entry) + except FileNotFoundError: + log(DEBUG, f'No secrets for target {target_path}') + keys = [] + replace: dict[str, str] = {} + for key in keys: + val = await self.__store.get(f'{entry}/{key}') + replace[key] = val.stdout.decode() for src in [path_secret_file, path_tmpl]: try: await self._compile_one_template_file( diff --git a/src/python/jw/pkg/cmds/secrets/lib/tar.py b/src/python/jw/pkg/cmds/secrets/lib/tar.py deleted file mode 100644 index 80009148..00000000 --- a/src/python/jw/pkg/cmds/secrets/lib/tar.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import io -import tarfile - -from tarfile import TarFile -from typing import TYPE_CHECKING, Callable - -from ....lib.ExecContext import ExecContext -from ....lib.log import DEBUG, log - -if TYPE_CHECKING: - from typing import Iterable - - from ....lib.FileContext import FileContext - -def filter( - blob: bytes, - path_filter: Callable[[str], bool] | None, - matched: list[str] | None = None, -) -> bytes: - ret = io.BytesIO() - with tarfile.open(fileobj = ret, mode = 'w') as tf_out: - tf_in = TarFile(fileobj = io.BytesIO(blob)) - for info in tf_in.getmembers(): - if path_filter is not None and not path_filter(info.name): - continue - log(DEBUG, f'Adding {info.name}') - if matched is not None: - matched.append(info.name) - buf = tf_in.extractfile(info) - tf_out.addfile(info, buf) - return ret.getvalue() - -def rewrite(blob: bytes, rewrite_filter: Callable[[str], str]) -> bytes: - ret = io.BytesIO() - with tarfile.open(fileobj = ret, mode = 'w') as tf_out: - tf_in = TarFile(fileobj = io.BytesIO(blob)) - for info in tf_in.getmembers(): - new_name = rewrite_filter(info.name) - log(DEBUG, f'Rewriting {info.name} -> {new_name}') - info.name = new_name - # -- A pax "path" supplement from the input would override - # the rewritten name on re-serialization - info.pax_headers = { - k: v - for k, v in info.pax_headers.items() if k != 'path' - } - buf = tf_in.extractfile(info) - tf_out.addfile(info, buf) - return ret.getvalue() - -def merge(blobs: Iterable[bytes], overwrite: bool = False) -> bytes: - ret = io.BytesIO() - with tarfile.open(fileobj = ret, mode = 'w') as tf_out: - for blob in blobs: - tf_in = TarFile(fileobj = io.BytesIO(blob)) - existing_names = tf_out.getnames() - for info in tf_in.getmembers(): - if not overwrite and info.name in existing_names: - continue - buf = tf_in.extractfile(info) - tf_out.addfile(info, buf) - return ret.getvalue() - -async def extract( - dst: FileContext, - blob: bytes, - root: str | None = None, - verbose: bool = False -) -> None: - cmd = ['tar'] - if root is not None: - cmd += ['-C', root] - if verbose: - cmd += '-v' - cmd += ['-x', '-f', '-'] - if not isinstance(dst, ExecContext): - raise NotImplementedError( - 'Extracting tar files to a non-executable ' - f'context is not yet implemented: {dst}' - ) - await dst.run(cmd, verbose = verbose, cmd_input = blob) diff --git a/src/python/jw/pkg/lib/FileContext.py b/src/python/jw/pkg/lib/FileContext.py index 736caa97..aec381e5 100644 --- a/src/python/jw/pkg/lib/FileContext.py +++ b/src/python/jw/pkg/lib/FileContext.py @@ -360,6 +360,18 @@ class FileContext(abc.ABC): from .ec.Curl import Curl return Curl(uri, *args, **kwargs) + case 'targpg': + from .ec.secret_store.TarGpg import TarGpg + + return TarGpg(uri, *args, **kwargs) + case 'kvfile': + from .ec.secret_store.KvFile import KvFile + + scheme = 'ssh' if uri.hostname else 'local' + inner = FileContext.create( + f'{scheme}://{uri.authority}', chroot = False + ) + return KvFile(inner, uri = uri) case _: pass raise Exception( diff --git a/src/python/jw/pkg/lib/ec/secret_store/Base.py b/src/python/jw/pkg/lib/ec/secret_store/Base.py new file mode 100644 index 00000000..f09fa340 --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/Base.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar, override + +from ...base import Result, StatResult +from ...file_types import FT_DIR, FT_REG +from ...FileContext import FileContext +from .layout import DEFAULT_PREFIX, host_prefix + +if TYPE_CHECKING: + import os + + from typing import Any, Iterable + + from ...file_types import FileType + from ...Uri import Uri + +T = TypeVar('T') + +class Base(FileContext): + """A store of secret entries in the canonical store layout. + + A store is a FileContext over the store namespace. The URI + addresses a subtree of that namespace: no authority means the + whole store, a host authority means that host's tree (its + reversed domain prefix), `default` names the default tree, and a + path narrows further. The subtree is the store's namespace + root; addresses passed to the methods are relative to it. + + A store is a typed file store: a store path names a file, and + put() persists the content at the address as-is. The store does + not identify content by file name; what a file holds - an entry, + or a blob - is a property of its content, resolved at stat + time. A node is what its content makes it: + + - an entry carries key-value secrets and behaves like a + directory: readdir() on the entry lists its keys, get() on + / returns the value of the key, and get() on the + entry itself returns the serialized entry + - a branch is a path prefix: readdir() lists its children + - a blob is non-key-value content: get() returns its bytes + + A store rooted at a host applies the default fallback: an + address that misses below the host is retried below the default + tree. A whole-store or default-rooted store is strict; callers + apply the fallback themselves. + + The URI-addressable stores are created by FileContext.create() + from their scheme. The kvfile store is the store of the on-host + layout: its addresses are config paths of the host it is rooted + at, and it may be constructed directly over a host context. + """ + + def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None: + # -- A store is always chrooted. The flag states the + # fact, and the chroot it arms is undone in the + # protected layer by _unchroot(); an explicit value + # is overridden. + kwargs['chroot'] = True + super().__init__(uri, *args, **kwargs) + + @property + def __host_root(self) -> str: + # -- The root of the host tree this store is rooted at, '' if + # the store is whole-store or rooted at the default tree: + # then there is no host tree to fall back from, and the + # caller applies the default fallback itself. + host = self.uri.hostname or '' + if host and host != 'default': + return host_prefix(host) + return '' + + @property + def __root_in_storage(self) -> str: + + # -- The namespace node this store is rooted at: the + # reversed host prefix of the URI authority, if any + # (`default` is a reserved word), followed by the URI + # path. + host = self.uri.hostname or '' + path = self.uri.path.lstrip('/') + if host and host != 'default': + host = host_prefix(host) + return '/'.join(p for p in (host, path) if p) + + def _path_in_storage(self, path: str) -> str: + path = path.lstrip('/') + if not self.__root_in_storage: + return path + if not path: + return self.__root_in_storage + return f'{self.__root_in_storage}/{path}' + + def _default_path_in_storage(self, path: str) -> str | None: + host_root = self.__host_root + ns = self.__root_in_storage + if not host_root or not (path == ns or path.startswith(f'{ns}/')): + return None + return f'{DEFAULT_PREFIX}{path[len(host_root):]}' + + def _unchroot(self, path: str) -> str: + # -- The inverse of _chroot(): the public wrappers hand the + # protected layer addresses prefixed with the root; store + # addresses are relative to it, and the root itself is the + # empty address. Already-relative addresses pass through. + if not self.root: + return path.lstrip('/') + if path == self.root: + return '' + prefix = f'{self.root}/' + if path.startswith(prefix): + return path[len(prefix):] + return path + + async def _call_with_fallback( + self, + path: str, + func: Callable[[str], Awaitable[T]], + ) -> T: + storage_path = self._path_in_storage(path) + try: + return await func(storage_path) + except FileNotFoundError: + default_storage_path = self._default_path_in_storage(storage_path) + if default_storage_path is None: + raise + return await func(default_storage_path) + + @override + async def put( + self, + path: str, + content: bytes, + wd: str | None = None, + throw: bool = True, + verbose: bool | None = None, + title: str | None = None, + owner: str | None = None, + group: str | None = None, + mode: int | None = None, + atomic: bool = False, + ftype: FileType = FT_REG, + ) -> Result: + # -- A put that changes nothing is a no-op: the pre-read is + # fallback-aware, so a value already represented, below + # the host tree or below the default tree, does not + # trigger a store write. Metadata arguments opt out: they + # may be the only change + if owner is None and group is None and mode is None: + try: + existing = await self.get(path) + except Exception: + pass + else: + if self._put_is_noop(existing.stdout, content, ftype): + return Result() + return await super().put( + path, + content, + wd, + throw, + verbose, + title, + owner, + group, + mode, + atomic, + ftype, + ) + + def _put_is_noop(self, existing: bytes, content: bytes, ftype: FileType) -> bool: + # -- A put that leaves the stored content byte-identical + # changes nothing. A store that normalizes content on + # read overrides this to compare the normalized forms + return existing == content + + @override + async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: + # -- An entry is a file on the backing store, yet it lists + # its keys: answer from the node's type, not the + # object's mode + try: + return (await self._stat(path, follow_symlinks)).is_dir + except FileNotFoundError: + return False + + @override + async def _readdir(self, path: str) -> Iterable[str]: + # -- A store is listable: a directory of keys, entries, or + # blobs, whatever its nodes hold. + raise NotImplementedError() + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + # -- The base store materializes nothing: branches are + # implicit namespace prefixes, and entries materialize on + # their first write. A store whose nodes have a host home + # overrides this + pass + + def _stat_result( + self, + owner: str, + group: str, + mtime: float, + ftype: FileType, + mode: int | None = None, + size: int = 0, + ) -> StatResult: + # -- The metadata of a node: the mode and size, if given, are + # the ones of the file, directory, or member behind the + # node; a node without a backing object keeps the + # synthesized type bits and a size of 0. The access and + # change times follow the modification time + import stat + + if mode is None: + mode = ( + (stat.S_IFDIR | 0o755) if ftype.is_subtype(FT_DIR) else + (stat.S_IFREG | 0o400) + ) + return StatResult( + mode = mode, + owner = owner, + group = group, + size = size, + atime = mtime, + mtime = mtime, + ctime = mtime, + ftype = ftype, + ) + + def _stat_result_from_os( + self, + st: os.stat_result, + ftype: FileType, + size: int | None = None, + ) -> StatResult: + return self._stat_result( + owner = str(st.st_uid), + group = str(st.st_gid), + mtime = st.st_mtime, + ftype = ftype, + mode = st.st_mode, + size = st.st_size if size is None else size, + ) + + @property + def whole(self) -> bool: + # -- True, if the store is rooted at the whole store namespace: + # no authority, no path + return not self.__root_in_storage diff --git a/src/python/jw/pkg/lib/ec/secret_store/KvFile.py b/src/python/jw/pkg/lib/ec/secret_store/KvFile.py new file mode 100644 index 00000000..2edea5d3 --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/KvFile.py @@ -0,0 +1,439 @@ +from __future__ import annotations + +import errno +import os + +from typing import TYPE_CHECKING, Iterable, override + +if TYPE_CHECKING: + from ...Distro import Distro + from ...FileContext import FileContext + from ...Uri import Uri + from ...base import StatResult + from ...file_types import FileType + +from ...FcCache import FcCache +from ...file_types import FT_COMPOUND, FT_DIR +from ...kv import parse_kv, serialize_kv +from ...Result import Result +from .Base import Base +from .layout import BLOB_SUFFIX, FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY, SECRET_SUFFIX + +class KvFile(Base): + """The on-host store of a host's package secrets. + + A config path X is either an entry, the file X.jw-secret with + KEY=VALUE lines, whose children in the address space are its + keys, or a blob, the file X.jw-secret-file, whose content is the + file itself, a leaf. Any other path is a branch: a real + directory on the host, with no file of its own. + + Rooted at the host: addresses are config paths relative to the + host root, absolute on the host with the leading slash. A URI + may root the store at a subtree of the host tree; its path is + composed into every address, and the URI is the store's + identity. A suffixed address names the on-host file of its + config path, the suffix picking the kind: .jw-secret the + entry, .jw-secret-file the blob. + + get reads a key below an entry or the blob at its config path; + put writes a key into its entry file, or a blob as a + .jw-secret-file at its config path, and accepts entry, key, and + blob content only; unlink drops the file the address names. No + default fallback. + + An entry lists its keys. A branch lists the config paths below + it, the root included: the host's package set is the bounded + base, the .jw-tmpl files of the installed packages, presence- + checked on the host. The distro is autodetected from the host + on first use, or given in the constructor; the autodetected + distro applies the JW_DEFAULT_PKG_FILTER package filter, if + set, so a URI-built context honors the app's documented + filter fallback. is_dir() answers from the entry file and the + host tree, so the write path never pays for the enumeration. + + The on-host tree is a precondition: mkdir materializes a missing + entry file below an existing directory, and mkdir and put fail + with EROFS below a missing one. KvFile never creates + directories. Entry and blob files are mode 0400; + owner and group follow the put request, if given. + """ + + def __init__( + self, + ctx: FileContext, + uri: str | Uri | None = None, + distro: Distro | None = None, + ) -> None: + # -- The URI roots the namespace at the host tree's root, or + # at a subtree of it: its path is composed into every + # address, and it is the store's identity. A direct + # construction names the context's host + if uri is None: + uri = f'kvfile://{ctx.uri.hostname or "target"}' + super().__init__(uri = uri) + self.__raw = ctx + self.__distro: Distro | None = distro + self.___config_paths: list[str] | None = None + self.__subtree = self.uri.path.lstrip('/') + # -- The context is read through a cache: the read path + # probes the same handful of paths again and again, and + # the cache pays the transport once per path + self.__ctx = FcCache(ctx) + + @override + async def _open(self) -> None: + # -- The lifecycle propagates to the host context: an + # open() pin on the store, the CopyContext of a copy + # walk, keeps the host's transport open for its + # duration, and the run()s within it share it + await self.__raw.open() + + @override + async def _close(self) -> None: + await self.__raw.close() + + def __addr(self, path: str) -> str: + # -- The host-relative address: the subtree the URI roots + # the store at, then the address relative to it + path = self._unchroot(path) + if not path: + return self.__subtree + if not self.__subtree: + return path + return f'{self.__subtree}/{path}' + + @staticmethod + def __resolve(path: str) -> tuple[str, bool]: + # -- The config path of the address, and whether the address + # names a blob file: a suffixed address is the on-host + # file name of its config path, the suffix picking the + # kind + if path.endswith(SECRET_SUFFIX): + return path[:-len(SECRET_SUFFIX)], False + if path.endswith(BLOB_SUFFIX): + return path[:-len(BLOB_SUFFIX)], True + return path, False + + async def __config_paths(self) -> list[str]: + # -- The config paths of the host's package secrets: the + # templates of the installed packages, relative to the + # host root. The package database is the bounded base, a + # filesystem scan is not. Computed once: the package set + # is stable for the lifetime of the session + if self.___config_paths is None: + if self.__distro is None: + from ...Distro import Distro + from ...ExecContext import ExecContext + + # -- The distro probes the host through the raw + # context: the cache is a plain file context, no + # exec of its own + if not isinstance(self.__raw, ExecContext): + raise ValueError( + 'KvFile: a distro must be given when the host' + ' context is not an exec context' + ) + # -- A URI-built context has no app distro to + # inherit the package filter from: apply the + # app's documented fallback + from ...PackageFilter import PackageFilterString + + pkg_filter_str = os.environ.get('JW_DEFAULT_PKG_FILTER') + self.__distro = await Distro.instantiate( + ec = self.__raw, + default_pkg_filter = ( + PackageFilterString(pkg_filter_str) if pkg_filter_str else None + ), + ) + # -- The packages' file paths, filtered to the templates + ret: set[str] = set() + for f in await self.__distro.pkg_files(): + if not f.endswith('.jw-tmpl'): + continue + ret.add(f.lstrip('/')[:-len('.jw-tmpl')]) + self.___config_paths = sorted(ret) + # -- Warm the stat cache with one host command, so the + # per-level presence checks and the walk's node stats + # hit memory: the root, the candidate files of both + # kinds, the directories they live in, and the file + # names a suffixed address would probe those + # directories by + warm = ['/'] + warm += [f'/{p}{SECRET_SUFFIX}' for p in self.___config_paths] + warm += [f'/{p}{BLOB_SUFFIX}' for p in self.___config_paths] + dirs: set[str] = set() + for p in self.___config_paths: + parts = p.split('/') + for i in range(1, len(parts)): + dirs.add('/'.join(parts[:i])) + for d in sorted(dirs): + warm += [f'/{d}', f'/{d}{SECRET_SUFFIX}', f'/{d}{BLOB_SUFFIX}'] + await self.__ctx.warm(warm) + assert self.___config_paths is not None + return self.___config_paths + + async def __entry_kv(self, entry: str) -> dict[str, str]: + path = f'/{entry.lstrip("/")}{SECRET_SUFFIX}' + if not await self.__ctx.file_exists(path): + raise FileNotFoundError(entry) + ret = await self.__ctx.get(path) + return parse_kv(ret.stdout) + + async def __is_entry(self, path: str) -> bool: + # -- The entry file on the host; KvFile has no default tree. + # The path is host-relative, as from __addr() + if not path: + return False + return await self.__ctx.file_exists(f'/{path}{SECRET_SUFFIX}') + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + # -- The metadata of the host file behind the node, the type + # overridden: an entry is a file that enumerates its + # keys, a key a leaf below it; a branch is a plain + # directory. A key has no file of its own, so it keeps + # the size 0 + path = self.__addr(path) + config, is_blob = self.__resolve(path) + + async def __stat_of(real: str, ftype: FileType) -> StatResult: + st = await self.__ctx.stat(real, follow_symlinks) + return self._stat_result( + st.owner, + st.group, + st.mtime, + ftype, + mode = st.mode, + size = st.size, + ) + + if not config: + return await __stat_of('/', FT_DIR) + if is_blob: + real = f'/{config}{BLOB_SUFFIX}' + if not await self.__ctx.file_exists(real): + raise FileNotFoundError(errno.ENOENT, f'stat: {path}') + return await __stat_of(real, FT_SECRET_FILE) + if await self.__is_entry(config): + return await __stat_of(f'/{config}{SECRET_SUFFIX}', FT_SECRET) + parent, sep, key = config.rpartition('/') + if sep and key: + parent, _ = self.__resolve(parent) + if await self.__is_entry(parent) and key in await self.__entry_kv(parent): + # -- The type is synthesized, the mode the one of + # the entry file + st = await self.__ctx.stat(f'/{parent}{SECRET_SUFFIX}', follow_symlinks) + return self._stat_result( + st.owner, + st.group, + st.mtime, + FT_SECRET_KEY, + mode = st.mode, + ) + blob = f'/{config}{BLOB_SUFFIX}' + if await self.__ctx.file_exists(blob): + return await __stat_of(blob, FT_SECRET_FILE) + if await self.__ctx.is_dir(f'/{config}'): + return await __stat_of(f'/{config}', FT_DIR) + raise FileNotFoundError(errno.ENOENT, f'stat: {path}') + + @override + async def _is_dir(self, path: str, follow_symlinks: bool) -> bool: + # -- An entry is a directory of its keys, a key below an + # entry is a leaf: the entry is a file on the host, so + # there is no host path below it to stat. A branch is a + # real directory on the host. The package set is not + # consulted: the write path never pays for the + # enumeration + path = self.__addr(path) + config, is_blob = self.__resolve(path) + if not config: + return True + parent, sep, _ = config.rpartition('/') + if sep: + parent, _ = self.__resolve(parent) + if await self.__is_entry(parent): + return False + if await self.__is_entry(config): + return True + if is_blob: + return False + return await self.__ctx.is_dir(f'/{config}') + + @override + async def _readdir(self, path: str) -> Iterable[str]: + # -- An entry lists its keys; a branch, the root included, + # lists the config paths below it, presence-checked on + # the host, under the on-host file name of the file that + # backs it + path = self.__addr(path) + config, is_blob = self.__resolve(path) + if config and await self.__is_entry(config): + return (await self.__entry_kv(config)).keys() + if is_blob and await self.__ctx.file_exists(f'/{config}{BLOB_SUFFIX}'): + raise NotADirectoryError(path) + children: set[str] = set() + prefix = f'{config}/' if config else '' + for cfg in await self.__config_paths(): + if not cfg.startswith(prefix): + continue + rest = cfg[len(prefix):] + label, sep, _ = rest.partition('/') + if sep: + # -- A deeper candidate lists its branch label, if a + # secret is present below it: the label names a + # directory on the host, which a present secret + # below implies + if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}') or \ + await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'): + children.add(label) + continue + # -- A direct child lists under the on-host file name of + # the file that backs it, entry and blob alike + if await self.__ctx.file_exists(f'/{cfg}{SECRET_SUFFIX}'): + children.add(f'{label}{SECRET_SUFFIX}') + if await self.__ctx.file_exists(f'/{cfg}{BLOB_SUFFIX}'): + children.add(f'{label}{BLOB_SUFFIX}') + if config and not children: + # -- A branch the host lacks is absent; a real directory + # with no secret children lists empty + if not await self.__ctx.is_dir(f'/{config}'): + raise FileNotFoundError(path) + return children + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + path = self.__addr(path) + config, is_blob = self.__resolve(path) + if is_blob: + real = f'/{config}{BLOB_SUFFIX}' + if not await self.__ctx.file_exists(real): + raise FileNotFoundError(path) + return await self.__ctx.get(real) + real = f'/{config}' + if await self.__ctx.file_exists(real + SECRET_SUFFIX): + # -- The entry file is the serialized entry + return await self.__ctx.get(real + SECRET_SUFFIX) + if await self.__ctx.file_exists(real + BLOB_SUFFIX): + return await self.__ctx.get(real + BLOB_SUFFIX) + if '/' not in config: + raise FileNotFoundError(path) + entry, key = config.rsplit('/', 1) + entry, _ = self.__resolve(entry) + kv = await self.__entry_kv(entry) + try: + return Result(stdout = kv[key].encode()) + except KeyError: + raise FileNotFoundError(path) from None + + @override + async def _mkdir(self, path: str, mode: int, ftype: FileType) -> None: + # -- The address space has no directories of its own: a node + # is a branch (a real directory) or an entry (a suffixed + # file); the type says which to build. Below a missing + # branch, mkdir fails with EROFS: the caller creates the + # levels + path = self.__addr(path) + config, _ = self.__resolve(path) + real = f'/{config}' + if ftype.is_subtype(FT_COMPOUND): + # -- An entry: materialize the empty entry file + if await self.__ctx.file_exists(real + SECRET_SUFFIX): + return + if not await self.__ctx.is_dir(os.path.dirname(real)): + raise OSError( + errno.EROFS, f'mkdir: "{path}" is below a missing directory' + ) + await self.__ctx.put(real + SECRET_SUFFIX, b'', mode = 0o400, atomic = True) + elif ftype.is_subtype(FT_DIR): + # -- A branch: a real directory + if await self.__ctx.is_dir(real): + return + if not await self.__ctx.is_dir(os.path.dirname(real)): + raise OSError( + errno.EROFS, f'mkdir: "{path}" is below a missing directory' + ) + await self.__ctx.mkdir(real, mode) + else: + raise ValueError(f'mkdir: "{path}" is not a directory type') + + @override + async def _put( + self, + path: str, + content: bytes, + wd: str | None, + throw: bool, + verbose: bool | None, + title: str, + owner: str | None, + group: str | None, + mode: str | None, + atomic: bool, + ftype: FileType, + ) -> Result: + path = self.__addr(path) + config, is_blob = self.__resolve(path) + if ftype.is_subtype(FT_SECRET): + # -- The whole entry: the content is the serialized + # entry, written over the entry file; the branch is + # a precondition, as for a blob + if is_blob: + raise ValueError( + f'put: "{path}" names a blob file, but the ftype is an entry' + ) + m = 0o400 if mode is None else int(mode, 8) + return await self.__ctx.put( + f'/{config}{SECRET_SUFFIX}', + content, + owner = owner, + group = group, + mode = m, + atomic = True, + ) + if ftype.is_subtype(FT_SECRET_FILE): + # -- A blob: the content is the file itself, at its + # on-host file name + m = 0o400 if mode is None else int(mode, 8) + return await self.__ctx.put( + f'/{config}{BLOB_SUFFIX}', + content, + owner = owner, + group = group, + mode = m, + atomic = True, + ) + entry, sep, key = config.rpartition('/') + if not sep or not entry or not key or '/' in key: + raise ValueError(f'put: "{path}" is not an entry/key path') + entry, _ = self.__resolve(entry) + real_entry = f'/{entry}' + if ftype.is_subtype(FT_SECRET_KEY): + # -- A key: merge into the entry file, which must exist; + # mkdir() is the materialization. The put is atomic, + # so it also overwrites an existing entry file, which + # is mode 0400 + if not await self.__ctx.file_exists(f'{real_entry}{SECRET_SUFFIX}'): + raise OSError(errno.EROFS, f'put: "{entry}" has no entry file') + kv = await self.__entry_kv(entry) + kv[key] = content.decode() + m = 0o400 if mode is None else int(mode, 8) + return await self.__ctx.put( + f'{real_entry}{SECRET_SUFFIX}', + serialize_kv(kv), + owner = owner, + group = group, + mode = m, + atomic = True, + ) + raise ValueError(f'put: ftype {ftype} is not a store content type') + + @override + async def _unlink(self, path: str) -> None: + # -- Unlink the file the address names: the address is the + # on-host file name below the subtree, so the host path + # is the composed address itself + await self.__ctx.unlink(f'/{self.__addr(path)}') diff --git a/src/python/jw/pkg/lib/ec/secret_store/Makefile b/src/python/jw/pkg/lib/ec/secret_store/Makefile new file mode 100644 index 00000000..19fedac9 --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-mod.mk diff --git a/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py b/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py new file mode 100644 index 00000000..45ac4b68 --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/TarGpg.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import errno +import io +import os +import stat +import time + +from tarfile import TarFile, TarInfo +from typing import TYPE_CHECKING, Iterable, override + +from ...file_types import FT_DIR +from ...kv import parse_kv +from ...Result import Result +from .Base import Base +from .layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY + +if TYPE_CHECKING: + from typing import Any + + from ...base import StatResult + from ...Uri import Uri + +class TarGpg(Base): + """A store of secrets in a GPG-encrypted tarball in the canonical + layout. + + The tarball location is deployment information, not part of the + address: JW_SECRETS_TARGPG_TAR names a URL or a local path. The + store URI addresses a subtree of the namespace, as in the base + class. + + A store path names a member, and optionally a key below an + entry. The member of an entry is its address, and its keys are + addressable below it; a blob is a member at its address holding + non-key-value content. The store does not identify content by + member name: a member whose content is a key-value file is an + entry, anything else a blob. + + The tarball is fetched and decrypted once, then cached. Fetching, + decrypting, and filtering happen in a local exec context, so the + GPG key and the decrypted material stay on this machine. + """ + + def __init__(self, uri: str | Uri, *args: Any, **kwargs: Any) -> None: + super().__init__(uri, *args, **kwargs) + self.__tar_uri = os.environ.get('JW_SECRETS_TARGPG_TAR', '') + self.__tf: TarFile | None = None + self.__kv_cache: dict[str, dict[str, str]] = {} + + async def __tar(self) -> TarFile: + if self.__tf is None: + from ...ProcFilterGpg import ProcFilterGpg + from ...util import get + from ..Local import Local + + data = ( + await get(self.__tar_uri, content_filter = ProcFilterGpg(ec = Local())) + ).stdout + self.__tf = TarFile(fileobj = io.BytesIO(data)) + return self.__tf + + async def __members(self) -> dict[str, TarInfo]: + tf = await self.__tar() + return {m.name: m for m in tf.getmembers()} + + async def __entry_kv(self, entry: str) -> dict[str, str]: + if entry not in self.__kv_cache: + tf = await self.__tar() + try: + info = tf.getmember(entry) + except KeyError: + raise FileNotFoundError(entry) from None + buf = tf.extractfile(info) + if buf is None: + raise Exception(f'Can\'t read store entry "{entry}"') + self.__kv_cache[entry] = parse_kv(buf.read()) + return self.__kv_cache[entry] + + async def __is_entry(self, entry: str) -> bool: + # -- The member at entry is an entry: its content is a + # non-empty kv file + tf = await self.__tar() + try: + info = tf.getmember(entry) + except KeyError: + return False + if not info.isfile(): + return False + buf = tf.extractfile(info) + if buf is None: + return False + return len(parse_kv(buf.read())) != 0 + + async def __tar_stat(self, storage_path: str) -> StatResult: + # -- The metadata of the member that stands behind the node; + # a branch has no member of its own and is synthesized, a + # key keeps the size 0 + members = await self.__members() + + def __meta(name: str) -> tuple[str, str, float, int, int]: + # -- A member's mode carries the permission bits only, so + # the type bits are synthesized; a store member is + # always a regular file + mi = members[name] + return ( + mi.uname or str(mi.uid), + mi.gname or str(mi.gid), + mi.mtime, + stat.S_IFREG | (mi.mode & 0o777), + mi.size, + ) + + if await self.__is_entry(storage_path): + owner, group, mtime, mode, size = __meta(storage_path) + return self._stat_result( + owner, + group, + mtime, + FT_SECRET, + mode = mode, + size = size, + ) + par, sep, key = storage_path.rpartition('/') + if sep and key and await self.__is_entry(par): + if key in await self.__entry_kv(par): + # -- The type is synthesized, the mode the one of the + # entry member, the size stays 0 + owner, group, mtime, mode, _ = __meta(par) + return self._stat_result( + owner, + group, + mtime, + FT_SECRET_KEY, + mode = mode, + ) + if storage_path in members: + owner, group, mtime, mode, size = __meta(storage_path) + return self._stat_result( + owner, + group, + mtime, + FT_SECRET_FILE, + mode = mode, + size = size, + ) + if not storage_path or \ + any(name.startswith(f'{storage_path}/') for name in members): + return self._stat_result( + str(os.getuid()), + str(os.getgid()), + time.time(), + FT_DIR, + ) + raise FileNotFoundError(errno.ENOENT, f'stat: {storage_path}') + + async def __tar_get(self, path: str) -> Result: + members = await self.__members() + if path == '': + raise IsADirectoryError(path) + if await self.__is_entry(path): + # -- The entry member is the serialized entry + tf = await self.__tar() + buf = tf.extractfile(members[path]) + if buf is None: + raise Exception(f'Can\'t read store entry "{path}"') + return Result(stdout = buf.read()) + if any(name.startswith(f'{path}/') for name in members): + raise IsADirectoryError(path) + if '/' in path: + entry, key = path.rsplit('/', 1) + if await self.__is_entry(entry): + kv = await self.__entry_kv(entry) + try: + return Result(stdout = kv[key].encode()) + except KeyError: + raise FileNotFoundError(path) from None + info = members.get(path) + if info is not None and info.isfile(): + tf = await self.__tar() + buf = tf.extractfile(info) + if buf is None: + raise Exception(f'Can\'t read store blob "{path}"') + return Result(stdout = buf.read()) + raise FileNotFoundError(path) + + async def __tar_readdir(self, path: str) -> Iterable[str]: + members = await self.__members() + if await self.__is_entry(path): + return (await self.__entry_kv(path)).keys() + if path in members: + raise NotADirectoryError(path) + + def __names(prefix: str) -> set[str]: + # -- No name filter: every member is store content + ret: set[str] = set() + for name in members: + if not name.startswith(prefix): + continue + rest = name[len(prefix):] + ret.add(rest.partition('/')[0]) + return ret + + ret = __names(f'{path}/' if path else '') + mirror = self._default_path_in_storage(path) + if mirror is not None: + ret |= __names(f'{mirror}/' if mirror else '') + if not ret: + raise FileNotFoundError(path) + return sorted(ret) + + @override + async def _get( + self, path: str, wd: str | None, throw: bool, verbose: bool | None, title: str + ) -> Result: + path = self._unchroot(path) + return await self._call_with_fallback(path, self.__tar_get) + + @override + async def _readdir(self, path: str) -> Iterable[str]: + path = self._unchroot(path) + return await self._call_with_fallback(path, self.__tar_readdir) + + @override + async def _stat(self, path: str, follow_symlinks: bool) -> StatResult: + path = self._unchroot(path) + return await self._call_with_fallback(path, self.__tar_stat) diff --git a/src/python/jw/pkg/lib/ec/secret_store/layout.py b/src/python/jw/pkg/lib/ec/secret_store/layout.py new file mode 100644 index 00000000..67956350 --- /dev/null +++ b/src/python/jw/pkg/lib/ec/secret_store/layout.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from ...file_types import FT_COMPOUND, FT_REG, file_types + +# -- The canonical store layout, shared by every store implementation and +# the install pipeline that consumes it: +# +# default/ entries applying to all hosts +# / entries applying to a single host, where +# is the host's FQDN, reversed and split +# on dots (com/example/host for host.example.com) +# +# Host-specific entries take precedence over default ones. + +# Suffix of the key-value secret file accompanying a compiled target +SECRET_SUFFIX = '.jw-secret' + +# Suffix of a non-key-value secret file (certificate, key, ...) +BLOB_SUFFIX = '.jw-secret-file' + +DEFAULT_PREFIX = 'default' + +def content_name(name: str) -> str | None: + # -- The config path of an on-host secret file name: the name + # with its content suffix removed, or None if the name + # carries neither suffix + for suffix in (SECRET_SUFFIX, BLOB_SUFFIX): + if name.endswith(suffix): + return name.removesuffix(suffix) + return None + +def host_prefix(hostname: str) -> str: + return '/'.join(reversed(hostname.split('.'))) + +# -- The file types of the store content, registered with the global +# file type registry: an entry is a compound of its keys, a key a +# leaf below it, a blob a file +FT_NAME_SECRET = 'secret' +FT_NAME_SECRET_KEY = 'secret-key' +FT_NAME_SECRET_FILE = 'secret-file' + +file_types.register_types( + { + FT_NAME_SECRET: FT_COMPOUND, + FT_NAME_SECRET_KEY: FT_REG, + FT_NAME_SECRET_FILE: FT_REG, + } +) + +FT_SECRET = file_types.get(FT_NAME_SECRET) +FT_SECRET_KEY = file_types.get(FT_NAME_SECRET_KEY) +FT_SECRET_FILE = file_types.get(FT_NAME_SECRET_FILE) diff --git a/src/python/jw/pkg/lib/kv.py b/src/python/jw/pkg/lib/kv.py new file mode 100644 index 00000000..46fcf8aa --- /dev/null +++ b/src/python/jw/pkg/lib/kv.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +def parse_kv(data: bytes) -> dict[str, str]: + """Parse a dotenv-style key-value entry into a dict""" + ret: dict[str, str] = {} + for line in data.decode().splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' not in line: + continue + key, val = line.split('=', 1) + key = key.strip() + val = val.strip() + if key: + ret[key] = val + return ret + +def serialize_kv(kv: dict[str, str]) -> bytes: + """Serialize a dict back into a dotenv-style key-value entry""" + return ''.join(f'{key}={val}\n' for key, val in kv.items()).encode() diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index e23ae6b6..3d0aa313 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -45,6 +45,7 @@ from jw.pkg.lib.Package import Package HOST = 'jannet.example.com' HOST_PREFIX = '/'.join(reversed(HOST.split('.'))) PKG = 'fixture-pkg' +PKG2 = 'fixture-pkg2' USER = pwd.getpwuid(os.getuid()).pw_name GROUP = grp.getgrgid(os.getgid()).gr_name @@ -189,12 +190,21 @@ def main() -> None: # extract it, and must not leave an empty file behind with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f: f.write('x=X\n') + # -- qux references a macro that has no store entry: the + # compile must fail rather than emit a literal macro + with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f: + f.write( + f'# conf: owner={USER}, group={GROUP}, mode=640\n' + 'present=PRESENT\n' + 'missing=__UNRESOLVED__\n' + ) with open(target + '/etc/readme.txt', 'w') as f: f.write('plain package file\n') foo_tmpl = target + '/etc/foo.conf.jw-tmpl' bar_tmpl = target + '/etc/bar.conf.jw-tmpl' baz_tmpl = target + '/etc/baz.conf.jw-tmpl' + qux_tmpl = target + '/etc/qux.conf.jw-tmpl' foo_secret = target + '/etc/foo.conf.jw-secret' bar_secret = target + '/etc/bar.conf.jw-secret' baz_secret = target + '/etc/baz.conf.jw-secret' @@ -231,7 +241,10 @@ def main() -> None: distro = FakeDistro( Local('local://' + HOST), - {PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt']}, + { + PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'], + PKG2: [qux_tmpl], + }, ) # -- list-templates: only the .jw-tmpl files of the package @@ -356,6 +369,37 @@ def main() -> None: with open(bar_out) as f: assert f.read() == 'plain=plain-value\n' + # -- compile-templates fails on an unresolved macro: qux + # resolves PRESENT from its secret file, but the template + # also references __UNRESOLVED__, which is not in the + # store. The failure must name the macro, and no compiled + # file may be written + qux_secret = target + '/etc/qux.conf.jw-secret' + qux_out = target + '/etc/qux.conf' + with open(qux_secret, 'w') as f: + f.write('PRESENT=there\n') + try: + run_cli( + distro, + [ + 'secrets', + 'compile-templates', + '--owner', + USER, + '--group', + GROUP, + '--mode', + '400', + PKG2, + ], + ) + except RuntimeError as e: + assert 'Unresolved macros' in str(e), e + assert '__UNRESOLVED__' in str(e), e + else: + raise AssertionError('unresolved macro must fail the compile') + assert not os.path.exists(qux_out) + out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG]) assert out == [foo_out, bar_out, baz_out], out diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile b/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile new file mode 100644 index 00000000..e0959bfb --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py b/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py new file mode 100644 index 00000000..8e084cd2 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/KvFile/test.py @@ -0,0 +1,384 @@ +import asyncio +import errno +import os +import stat +import tempfile + +from typing import Any, override + +from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro +from jw.pkg.lib.ec.Local import Local +from jw.pkg.lib.ec.secret_store.KvFile import KvFile +from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY +from jw.pkg.lib.file_types import FT_DIR +from jw.pkg.lib.FileContext import FileContext +from jw.pkg.lib.Package import Package + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + target = os.path.join(tmp, 'etc', 'foo.conf') + os.makedirs(os.path.dirname(target)) + with open(target + '.jw-secret', 'w') as f: + f.write( + '# a comment\n\n key1 = value1 \nkey2=value2\n' + '=novalue\nkey3=\n' + ) + + # FileContext.create() resolves the kvfile scheme, keeping + # the URI as the store's identity + created = FileContext.create('kvfile://') + assert isinstance(created, KvFile) + assert created.uri.scheme == 'kvfile' + + store = KvFile(Local()) + assert set(await store.readdir(target)) == {'key1', 'key2', 'key3'} + assert (await store.get(f'{target}/key1')).stdout == b'value1' + assert (await store.get(f'{target}/key3')).stdout == b'' + + # The entry is gettable as a whole: the serialized file + assert (await store.get(target)).stdout == \ + b'# a comment\n\n key1 = value1 \nkey2=value2\n=novalue\nkey3=\n' + + # -- stat() answers the node's type, the metadata of the + # host file behind it; a key has no file of its own, and + # keeps the size 0 + blob = os.path.join(tmp, 'etc', 'bar.pem') + with open(blob + '.jw-secret-file', 'w') as f: + f.write('a blob\n') + st = await store.stat(target) + assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode) + assert st.is_gettable + assert st.size == os.path.getsize(target + '.jw-secret') + st = await store.stat(f'{target}/key1') + assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode) + assert st.is_gettable + assert st.size == 0 + st = await store.stat(blob) + assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode) + assert st.is_gettable + assert st.size == os.path.getsize(blob + '.jw-secret-file') + st = await store.stat(os.path.dirname(target)) + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + assert not st.is_gettable + st = await store.stat('') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + assert not st.is_gettable + try: + await store.stat(f'{target}/missing') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + # -- file_exists() works on the store, being built on stat() + assert await store.file_exists(target) + assert not await store.file_exists(f'{target}/missing') + + # A missing key and a missing entry raise FileNotFoundError + for path in (f'{target}/missing', os.path.join(tmp, 'etc', 'missing.conf')): + try: + await store.get(path) + raise AssertionError(f'expected FileNotFoundError for {path}') + except FileNotFoundError: + pass + + # put writes a key into the entry file, merging with the + # existing keys + await store.put(f'{target}/key4', b'value4', ftype = FT_SECRET_KEY) + assert (await store.get(f'{target}/key4')).stdout == b'value4' + assert (await store.get(f'{target}/key1')).stdout == b'value1' + # -- a second put overwrites the entry file, which is mode 0400 + await store.put(f'{target}/key1', b'value1-new', ftype = FT_SECRET_KEY) + assert (await store.get(f'{target}/key1')).stdout == b'value1-new' + assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400 + new_entry = os.path.join(tmp, 'etc', 'new.conf') + # -- a key put does not create: below a missing entry file + # it fails with EROFS; mkdir() is the materialization + try: + await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY) + raise AssertionError('expected EROFS') + except OSError as e: + assert e.errno == errno.EROFS + await store.mkdir(new_entry, ftype = FT_SECRET) + await store.put(f'{new_entry}/k', b'v', ftype = FT_SECRET_KEY) + with open(new_entry + '.jw-secret') as f: + assert f.read() == 'k=v\n' + + # -- a typed put replaces the entry: the serialized content + # is written over the entry file, mode 0400 + await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET) + assert set(await store.readdir(target)) == {'new1', 'new2'} + assert (await store.get(target)).stdout == b'new1=n1\nnew2=n2\n' + with open(target + '.jw-secret') as f: + assert f.read() == 'new1=n1\nnew2=n2\n' + assert stat.S_IMODE(os.stat(target + '.jw-secret').st_mode) == 0o400 + + # -- a put that changes nothing does not rewrite the file + mtime = os.stat(target + '.jw-secret').st_mtime_ns + await store.put(target, b'new1=n1\nnew2=n2\n', ftype = FT_SECRET) + assert os.stat(target + '.jw-secret').st_mtime_ns == mtime + + # -- An untyped put is refused: the store content types are + # entry, key, and blob + try: + await store.put('HOSTNAME', b'x') + raise AssertionError('expected ValueError') + except ValueError: + pass + + # -- put: a blob below a branch lands as a .jw-secret-file at + # its config path, mode 0400 + pem = os.path.join(tmp, 'etc', 'cert.pem.jw-secret-file') + await store.put( + os.path.join(tmp, 'etc', 'cert.pem'), + b'-----BEGIN CERT-----\n', + ftype = FT_SECRET_FILE, + ) + with open(pem) as f: + assert f.read() == '-----BEGIN CERT-----\n' + assert stat.S_IMODE(os.stat(pem).st_mode) == 0o400 + + # -- mkdir: the type says which node to build, the default + # a branch; a branch is a real directory, an entry an + # empty entry file, mode 0400 + await store.mkdir(os.path.join(tmp, 'etc')) + assert not os.path.exists(os.path.join(tmp, 'etc.jw-secret')) + baz = os.path.join(tmp, 'etc', 'baz') + await store.mkdir(baz) + assert os.path.isdir(baz) + bar = os.path.join(tmp, 'etc', 'bar.conf') + await store.mkdir(bar, ftype = FT_SECRET) + assert os.path.isfile(bar + '.jw-secret') + assert open(bar + '.jw-secret').read() == '' + assert stat.S_IMODE(os.stat(bar + '.jw-secret').st_mode) == 0o400 + + # -- mkdir and put fail with EROFS below a missing directory + for fn in ( + lambda: store.mkdir(os.path.join(tmp, 'no', 'such', 'entry')), + lambda: store.put( + os.path.join(tmp, 'no', 'such', 'entry', 'key'), + b'x', + ftype = FT_SECRET_KEY, ), + ): + try: + await fn() + raise AssertionError('expected EROFS') + except OSError as e: + assert e.errno == errno.EROFS + + # -- put forwards owner and group to the host context: an + # unknown group fails the put + try: + await store.put( + f'{target}/key7', b'v7', ftype = FT_SECRET_KEY, group = 'no-such-group' + ) + except AssertionError: + raise + except Exception as e: + assert 'no-such-group' in str(e) + else: + raise AssertionError('expected the put to fail') + + # unlink drops the file the address names, and a missing + # file raises + await store.unlink(target + '.jw-secret') + assert not os.path.exists(target + '.jw-secret') + try: + await store.unlink(target + '.jw-secret') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + + # -- Branch listing: the config paths below a branch, the + # root included, come from the host's package set, the + # templates of the installed packages, presence-checked + # on the host. A fake distro supplies the list + class FakeDistro(SuseDistro): + + def __init__(self, files: list[str]) -> None: + super().__init__(Local(), id = 'suse') + self.files = files + self.queries = 0 + + @override + async def _pkg_files(self, names: list[str] | None) -> list[str]: + self.queries += 1 + return self.files + + tree = os.path.join(tmp, 'tree') + os.makedirs(os.path.join(tree, 'etc')) + with open(os.path.join(tree, 'etc', 'foo.conf.jw-secret'), 'w') as f: + f.write('k1=v1\nk2=v2\n') + with open(os.path.join(tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as f: + f.write('-----BEGIN CERT-----\n') + with open(os.path.join(tree, 'etc', 'other.conf.jw-secret'), 'w') as f: + f.write('o1=v1\n') + # -- A template without a secret file is not a config path; + # a package file that is not a template is not one + # either + fake = FakeDistro( + [ + f'{tree}/etc/foo.conf.jw-tmpl', + f'{tree}/etc/cert.pem.jw-tmpl', + f'{tree}/etc/ghost.conf.jw-tmpl', + f'{tree}/etc/other.conf', + ] + ) + # -- The address space is rooted at the host root; the + # package file list reports host-absolute paths, so the + # fixture tree sits at its path below the root + bstore = KvFile(Local(), distro = fake) + base = tree.lstrip('/') + # -- is_dir answers from the host state, without consulting + # the package set + assert await bstore.is_dir('') + assert await bstore.is_dir(base) + assert await bstore.is_dir(f'{base}/etc/foo.conf') + assert not await bstore.is_dir(f'{base}/etc/cert.pem') + assert fake.queries == 0 + # -- is_gettable answers from the node's type: an entry, + # a key, a blob; not a branch, the root included + assert await bstore.is_gettable(f'{base}/etc/foo.conf') + assert await bstore.is_gettable(f'{base}/etc/foo.conf/k1') + assert await bstore.is_gettable(f'{base}/etc/cert.pem') + assert not await bstore.is_gettable(f'{base}/etc') + assert not await bstore.is_gettable('') + assert fake.queries == 0 + # -- Listing consults it, once: each branch lists the secret + # files below it, under their on-host file names, + # presence-checked + assert set(await bstore.readdir('')) == {'tmp'} + assert fake.queries == 1 + assert set(await bstore.readdir(f'{base}/etc')) == \ + {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'} + assert fake.queries == 1 + # -- An entry still lists its keys, without the package set + assert set(await bstore.readdir(f'{base}/etc/foo.conf')) == {'k1', 'k2'} + assert fake.queries == 1 + # -- A branch without config paths raises FileNotFoundError + try: + await bstore.readdir('no/such') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + # -- A real directory with no secret children lists empty + os.makedirs(os.path.join(tree, 'var', 'empty')) + assert set(await bstore.readdir(f'{base}/var/empty')) == set() + # -- The blob reads back through the store + assert (await bstore.get(f'{base}/etc/cert.pem')).stdout == \ + b'-----BEGIN CERT-----\n' + + # -- The entry's config path is also a deployed file on the + # host, as on a real host: a key below it has no host + # path, and is_dir() must not stat below the file + with open(os.path.join(tree, 'etc', 'foo.conf'), 'w') as f: + f.write('deployed\n') + assert not (await bstore.is_dir(f'{base}/etc/foo.conf/k1')) + assert (await bstore.get(f'{base}/etc/foo.conf/k1')).stdout == b'v1' + + # -- A URI may root the store at a subtree of the host tree: + # create() keeps the URI as the store's identity, and its + # path is composed into every address. The inner context + # is host-rooted, so the on-host files are reached + # without a double path + created = FileContext.create(f'kvfile://{tree}/etc') + assert created.uri.path == f'{tree}/etc' + sstore = KvFile(Local(), uri = f'kvfile://{tree}/etc', distro = fake) + await sstore.open() + assert set(await sstore.readdir('')) == \ + {'cert.pem.jw-secret-file', 'foo.conf.jw-secret'} + # -- An entry lists its keys, suffixed and unsuffixed alike + assert set(await sstore.readdir('foo.conf.jw-secret')) == {'k1', 'k2'} + assert set(await sstore.readdir('foo.conf')) == {'k1', 'k2'} + assert (await sstore.get('foo.conf/k1')).stdout == b'v1' + assert (await sstore.get('cert.pem')).stdout == \ + b'-----BEGIN CERT-----\n' + # -- A suffixed address names the on-host file of its config + # path, the suffix picking the kind + assert (await sstore.get('foo.conf.jw-secret')).stdout == \ + b'k1=v1\nk2=v2\n' + assert (await sstore.get('cert.pem.jw-secret-file')).stdout == \ + b'-----BEGIN CERT-----\n' + st = await sstore.stat('foo.conf.jw-secret') + assert st.ftype is FT_SECRET + st = await sstore.stat('foo.conf.jw-secret/k1') + assert st.ftype is FT_SECRET_KEY + st = await sstore.stat('cert.pem.jw-secret-file') + assert st.ftype is FT_SECRET_FILE + # -- A blob address refuses to list + try: + await sstore.readdir('cert.pem.jw-secret-file') + raise AssertionError('expected NotADirectoryError') + except NotADirectoryError: + pass + # -- A write through the subtree root lands in the subtree + await sstore.mkdir('sub') + await sstore.mkdir('sub/conf', ftype = FT_SECRET) + await sstore.put('sub/conf/k', b'v', ftype = FT_SECRET_KEY) + with open(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) as f: + assert f.read() == 'k=v\n' + await sstore.unlink('sub/conf.jw-secret') + assert not os.path.exists(os.path.join(tree, 'etc', 'sub', 'conf.jw-secret')) + await sstore.close() + + # -- The autodetected distro applies the app's documented + # filter fallback: JW_DEFAULT_PKG_FILTER reaches + # Distro.instantiate() as the default package filter + from unittest import mock + + from jw.pkg.lib.Distro import Distro + + captured: dict[str, Any] = {} + + async def __instantiate(*args: Any, **kwargs: Any) -> FakeDistro: + captured['kwargs'] = kwargs + return fake + + os.environ['JW_DEFAULT_PKG_FILTER'] = 'url=~jw.dev' + try: + with mock.patch.object(Distro, + 'instantiate', + new = classmethod(__instantiate)): + astore = KvFile(Local()) + assert set(await astore.readdir('')) == {'tmp'} + flt = captured['kwargs'].get('default_pkg_filter') + assert flt is not None + assert flt.match(Package('fixture', url = 'https://jw.dev/fixture.rpm')) + assert not flt.match( + Package('other', url = 'https://elsewhere/other.rpm') + ) + finally: + del os.environ['JW_DEFAULT_PKG_FILTER'] + + # -- The lifecycle propagates to the host context: an + # open() pin on the store keeps the host's transport + # open for its duration, and the run()s within it share + # it + class CountingLocal(Local): + + def __init__(self) -> None: + super().__init__() + self.opens = 0 + self.closes = 0 + + @override + async def _open(self) -> None: + self.opens += 1 + await super()._open() + + @override + async def _close(self) -> None: + self.closes += 1 + await super()._close() + + cl = CountingLocal() + pstore = KvFile(cl) + await pstore.open() + assert cl.opens == 1 + await cl.run(['true']) + await cl.run(['true']) + assert cl.opens == 1 + assert cl.closes == 0 + await pstore.close() + assert cl.closes == 1 + +asyncio.run(run_tests()) +print('All secrets kvfile store tests passed') diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/Makefile b/test/unit/python/jw/pkg/lib/ec/secret_store/Makefile new file mode 100644 index 00000000..c5ddf8d5 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile new file mode 100644 index 00000000..e0959bfb --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py new file mode 100644 index 00000000..42ca9780 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/TarGpg/test.py @@ -0,0 +1,235 @@ +import asyncio +import io +import os +import shutil +import stat +import subprocess +import tarfile +import tempfile + +from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY +from jw.pkg.lib.ec.secret_store.TarGpg import TarGpg +from jw.pkg.lib.file_types import FT_DIR +from jw.pkg.lib.FileContext import FileContext +from jw.pkg.lib.util import copy + +def make_tar(path: str, entries: dict[str, str]) -> None: + buf = io.BytesIO() + with tarfile.open(fileobj = buf, mode = 'w') as tf: + for name, text in entries.items(): + info = tarfile.TarInfo(name) + data = text.encode() + info.size = len(data) + info.mode = 0o400 + tf.addfile(info, io.BytesIO(data)) + with open(path, 'wb') as f: + f.write(buf.getvalue()) + +async def run_tests() -> None: + with tempfile.TemporaryDirectory() as tmp: + # -- A throw-away GPG home with a key for the test + gpghome = os.path.join(tmp, 'gnupg') + os.makedirs(gpghome, mode = 0o700) + env = dict(os.environ, GNUPGHOME = gpghome) + subprocess.run( + [ + 'gpg', + '--batch', + '--pinentry-mode', + 'loopback', + '--passphrase', + '', + '--quick-gen-key', + 'test@example.com', + ], + check = True, + capture_output = True, + env = env, + ) + + # -- A store in the canonical layout, GPG-encrypted tarball + tar_path = os.path.join(tmp, 'secrets.tar') + make_tar( + tar_path, + { + 'default/etc/a.jw-secret': 'a=1\n', + 'default/etc/d.jw-secret': 'd=4\n', + 'com/example/host/etc/a.jw-secret': 'a=2\n', + 'com/example/host/etc/b.jw-secret': 'b=3\n', + # Non-key-value content is stored as a .jw-secret-file + # and addressable as a blob + 'com/example/host/etc/c.pem.jw-secret-file': \ + '-----BEGIN CERT-----\n', + # A member without a content suffix is store + # content too: non-key-value content is a blob + 'com/example/host/etc/README.md': 'not key-value content\n', + } + ) + enc_path = tar_path + '.gpg' + subprocess.run( + [ + 'gpg', + '--batch', + '--yes', + '--quiet', + '--no-tty', + '--trust-model', + 'always', + '--recipient', + 'test@example.com', + '--encrypt', + '--output', + enc_path, + tar_path, + ], + check = True, + capture_output = True, + env = env, + ) + + # The store decrypts in a local exec context, which inherits + # the test's GNUPGHOME; the tarball location is deployment + # information, passed via the environment + os.environ['GNUPGHOME'] = gpghome + os.environ['JW_SECRETS_TARGPG_TAR'] = 'local://' + enc_path + + # -- The whole store: strict, no fallback + store = FileContext.create('targpg://') + await store.open() + assert isinstance(store, TarGpg) + assert set(await store.readdir('')) == {'default', 'com'} + # -- The names as stored: the store does not strip or filter + # by member name, a suffixed name being the address of + # its node + assert set(await store.readdir('com/example/host/etc')) == \ + {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', + 'README.md'} + assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1' + assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \ + b'2' + assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \ + b'3' + + # The entry is gettable as a whole: the member's content + assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n' + + # A blob is a file: get() returns its bytes, readdir() refuses + pem = 'com/example/host/etc/c.pem.jw-secret-file' + assert (await store.get(pem)).stdout == b'-----BEGIN CERT-----\n' + try: + await store.readdir(pem) + raise AssertionError('expected NotADirectoryError') + except NotADirectoryError: + pass + + # A member without a content suffix is store content too: a + # non-key-value member is a blob, listed, statable, gettable + assert 'README.md' in set(await store.readdir('com/example/host/etc')) + st = await store.stat('com/example/host/etc/README.md') + assert st.ftype is FT_SECRET_FILE + assert (await store.get('com/example/host/etc/README.md')).stdout == \ + b'not key-value content\n' + + # A missing entry and a missing key raise FileNotFoundError + for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'): + try: + await store.get(path) + raise AssertionError(f'expected FileNotFoundError for {path}') + except FileNotFoundError: + pass + + # -- stat() answers the node's type, the metadata of the + # member behind it; a key has no member of its own, and a + # branch none at all, so they keep the size 0 + st = await store.stat('') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + st = await store.stat('default/etc/a.jw-secret') + assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode) + assert st.size == len((await store.get('default/etc/a.jw-secret')).stdout) + st = await store.stat('default/etc/a.jw-secret/a') + assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode) + assert st.size == 0 + st = await store.stat('com/example/host/etc/c.pem.jw-secret-file') + assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode) + assert st.size == len( + (await store.get('com/example/host/etc/c.pem.jw-secret-file')).stdout + ) + st = await store.stat('default/etc') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + assert st.size == 0 + # -- is_dir answers the node's listability: an entry lists + # its keys, a branch its children; a key is a leaf + assert await store.is_dir('default/etc/a.jw-secret') + assert await store.is_dir('default/etc') + assert not await store.is_dir('default/etc/a.jw-secret/a') + try: + await store.stat('default/etc/missing') + raise AssertionError('expected FileNotFoundError') + except FileNotFoundError: + pass + # -- file_exists() works on the store, being built on stat() + assert await store.file_exists('default/etc/a.jw-secret') + assert not await store.file_exists('default/etc/missing') + await store.close() + + # -- A host-rooted store applies the default fallback + store = FileContext.create('targpg://host.example.com') + assert isinstance(store, TarGpg) + await store.open() + assert (await store.get('etc/a.jw-secret/a')).stdout == b'2' + # A directory listing is the union with the default tree: d + # exists only under default/, but is listable as it is + # gettable + assert set(await store.readdir('etc')) == \ + {'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', + 'd.jw-secret', 'README.md'} + # A miss below the host is retried below the default tree + assert (await store.get('etc/d.jw-secret/d')).stdout == b'4' + # A key listing stays host-wins-wholesale + assert set(await store.readdir('etc/d.jw-secret')) == {'d'} + # -- stat() distinguishes an entry from a branch and the root + st = await store.stat('etc/a.jw-secret') + assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode) + st = await store.stat('etc') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + st = await store.stat('') + assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode) + await store.close() + + # -- A store as a copy source: an entry lands as a single + # file with its serialized content, a branch as a + # directory, and a blob as a file; the default-only entry + # d lands from the union listing + host = os.path.join(tmp, 'host-tree') + os.makedirs(host) + ret = await copy('targpg://host.example.com', host, recursive = True) + assert isinstance(ret, list) + assert set(ret) == { + f'{host}/etc/a.jw-secret', + f'{host}/etc/b.jw-secret', + f'{host}/etc/c.pem.jw-secret-file', + f'{host}/etc/d.jw-secret', + f'{host}/etc/README.md', + } + with open(os.path.join(host, 'etc', 'a.jw-secret')) as f: + assert f.read() == 'a=2\n' + assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a')) + with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f: + assert f.read() == '-----BEGIN CERT-----\n' + # -- A single object that is an entry: the serialized whole + # lands at the destination + dst = os.path.join(tmp, 'copied', 'entry') + os.makedirs(os.path.dirname(dst)) + ret = await copy('targpg://host.example.com/etc/a.jw-secret', dst) + assert ret == dst + with open(dst) as f: + assert f.read() == 'a=2\n' + +def main() -> None: + if not shutil.which('gpg'): + print('Skipping secrets targpg store tests (gpg required)') + return + asyncio.run(run_tests()) + print('All secrets targpg store tests passed') + +main() diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile b/test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile new file mode 100644 index 00000000..d949ac97 --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/layout/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py b/test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py new file mode 100644 index 00000000..17b8d90e --- /dev/null +++ b/test/unit/python/jw/pkg/lib/ec/secret_store/layout/test.py @@ -0,0 +1,41 @@ +from jw.pkg.lib.ec.secret_store.layout import ( + BLOB_SUFFIX, + DEFAULT_PREFIX, + FT_SECRET, + FT_SECRET_FILE, + FT_SECRET_KEY, + SECRET_SUFFIX, + content_name, + host_prefix, +) +from jw.pkg.lib.file_types import FT_COMPOUND, FT_DIR, FT_REG + +# The layout constants are the store contract +assert DEFAULT_PREFIX == 'default' +assert SECRET_SUFFIX == '.jw-secret' +assert BLOB_SUFFIX == '.jw-secret-file' + +# content_name strips the content suffix, and names without a +# content suffix are not store content +assert content_name('etc/a.jw-secret') == 'etc/a' +assert content_name('etc/c.pem.jw-secret-file') == 'etc/c.pem' +assert content_name('etc/README.md') is None + +# An FQDN maps to its reversed, dot-split host root +assert host_prefix('host.example.com') == 'com/example/host' +assert host_prefix('a.b.c.d') == 'd/c/b/a' + +# A single-label hostname maps to itself +assert host_prefix('localhost') == 'localhost' + +# The store content types are registered with the file types: an +# entry is a kind of compound, a key and a blob a kind of file +assert FT_SECRET.is_subtype(FT_COMPOUND) +assert FT_SECRET.is_subtype(FT_REG) +assert not FT_SECRET.is_subtype(FT_DIR) +assert FT_SECRET_KEY.is_subtype(FT_REG) +assert not FT_SECRET_KEY.is_subtype(FT_DIR) +assert FT_SECRET_FILE.is_subtype(FT_REG) +assert not FT_SECRET_FILE.is_subtype(FT_DIR) + +print('All secrets layout tests passed')