lib.ec.secret_store: Add and use framework

"jw-pkg secrets install" copies secret material from a GPG-encrypted tar
archive to dotenv .jw-secret files on a target host. This commit refactors
the underlying code and adds hooks for the inverse copy direction, and for
more secret sources and targets.

The peers between which secrets are copied are represented by the new
abstract secret_store.Base class, and the existing tar blob and host
storages, inheriting from that class, are TarGpg and KvFile. Moreover,
secret_store.Base is inheriting from FileContext, so all of jw-pkg's
file-like operations are available as hooks.

Secrets inside instances of two file contexts are addressable as
(targpg|kvfile)://target.host.com/path/to/file.conf.jw-secret. The on-host
KvFile will look either locally or via SSH for that file, and TarGpg will
look inside a tar archive specified in the JW_SECRETS_TARGPG_TAR
environment variable.

Uris with these schemas can be passed to FileContext.create(), which will
return a chrooted file context of the correct type.

KvFile.stat("/path/to/file.conf.jw-secret").ftype is FT_SECRET which
inherits from FT_COMPOUND - which means that readdir() works on such
animals. Not entirely sure if that's a good idea, but it feels pretty
generic. Let's see where that takes us.

DistroContext.install() after this refactor reads its source through the
store interface and writes the landing through the KvFile store.

Both secret stores are exercised through the existing unit test of jw-pkg
secrets install, and unit tests for both stores are also added by this
commit.

Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/jw/pkg/cmds/secrets/lib/DistroContext.py

@@ -3,11 +3,21 @@ from __future__ import annotations
 import re
 import sys
 
+from contextlib import suppress
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from ....lib.ec.secret_store.Base import Base as SecretStore
+from ....lib.ec.secret_store.KvFile import KvFile
+from ....lib.ec.secret_store.layout import (
+    DEFAULT_PREFIX,
+    FT_SECRET,
+    FT_SECRET_KEY,
+    SECRET_SUFFIX,
+    host_prefix,
+)
+from ....lib.FileContext import FileContext
 from ....lib.log import DEBUG, NOTICE, WARNING, log
-from ....lib.ProcFilterGpg import ProcFilterGpg
 from .FilesContext import FilesContext
 
 if TYPE_CHECKING:
@@ -19,7 +29,7 @@ if TYPE_CHECKING:
 class DistroContext(FilesContext):
 
     def __init__(self, distro: Distro) -> None:
-        super().__init__(distro.ctx)
+        super().__init__(distro.ctx, distro = distro)
         self.__distro = distro
 
     async def match_files(self, pkg_names: Collection[str], pattern: str) -> list[str]:
@@ -40,7 +50,7 @@ class DistroContext(FilesContext):
     ) -> list[str]:
         ret = []
         for tmpl in await self.list_template_files(pkg_names):
-            path = str(Path(tmpl).with_suffix('.jw-secret'))
+            path = str(Path(tmpl).with_suffix(SECRET_SUFFIX))
             if ignore_missing and not await self.ctx.file_exists(path):
                 continue
             ret.append(path)
@@ -101,64 +111,60 @@ class DistroContext(FilesContext):
         verbose: bool = False,
     ) -> None:
 
-        async def _read_secret_tar_blob(src_uri: str) -> bytes:
-            ec = self.ctx
-            from ....lib.ec.Local import Local
-            from ....lib.util import get
-
-            if not isinstance(ec, Local):
-                # Security: Use a local exec context for decrypting and
-                # filtering secrets
-                ec = Local()
-            return (await get(src_uri, content_filter = ProcFilterGpg(ec = ec))).stdout
-
-        def _is_needed_secret(path: str) -> bool:
-            return path in secret_paths
-
-        from .tar import extract as tar_extract
-        from .tar import filter as tar_filter
-        from .tar import merge as tar_merge
-        from .tar import rewrite as tar_rewrite
-
         if only_missing:
             raise NotImplementedError('--only-missing is not yet implemented')
 
-        secret_paths = await self.list_secret_paths(pkg_names)
-        hostname = self.ctx.uri.hostname
-        if not hostname:
-            raise Exception('Have no hostname to find secrets in tar file')
-        host_root_in_tar = '/'.join(reversed(hostname.split('.')))
-        host_rx = re.compile(r'^' + host_root_in_tar)
-        default_rx = re.compile(r'^default')
-
-        filtered_paths: list[str] = []
-        extracted_paths: list[str] = []
-
-        blob_all = await _read_secret_tar_blob(src_uri)
-        if blob_all is None:
-            raise Exception(f'Tar blob {src_uri} is empty')
-        blob_host_filtered = tar_filter(
-            blob_all, lambda p: re.match(host_rx, p) is not None, filtered_paths
-        )
-        blob_host_transformed = tar_rewrite(
-            blob_host_filtered, lambda p: re.sub(host_rx, '', p)
-        )
-        blob_default_filtered = tar_filter(
-            blob_all, lambda p: re.match(default_rx, p) is not None, filtered_paths
-        )
-        blob_default_transformed = tar_rewrite(
-            blob_default_filtered, lambda p: re.sub(default_rx, '', p)
-        )
-        blob_secret_material = tar_merge(
-            [blob_host_transformed, blob_default_transformed], overwrite = False
-        )
-        blob_needed = tar_filter(
-            blob_secret_material, _is_needed_secret, extracted_paths
-        )
-
-        await tar_extract(self.ctx, blob_needed, root = '/', verbose = verbose)
-        for path in secret_paths:
-            if path not in extracted_paths:
-                log(NOTICE, f'not extracted: {path}')
+        fc = FileContext.create(src_uri, chroot = True)
+        await fc.open()
+        try:
+            hostname = self.ctx.uri.hostname
+            if not hostname:
+                raise Exception('Have no hostname to find secrets in the store')
+            if isinstance(fc, SecretStore):
+                if not fc.whole:
+                    raise Exception(
+                        f'src {src_uri} must name the whole store, not a '
+                        f'rooted subtree'
+                    )
+                src: FileContext = fc
+                # -- Store candidates: the host tree, then the default
+                #    tree
+                prefixes: tuple[str, ...] = (host_prefix(hostname), DEFAULT_PREFIX)
             else:
-                log(NOTICE, f'extracted:     {path}')
+                # -- A file tree in the on-host layout: wrap it in the kvfile store
+                src = KvFile(fc, distro = self.__distro)
+                prefixes = ('', )
+
+            # -- The store and the kvfile source are alike addressed
+            #    by the on-host file name
+            suffix = SECRET_SUFFIX
+
+            for entry in await self.list_compilation_targets(pkg_names):
+                # -- A present entry, even an empty one, wins over the
+                #    fallback
+                entry_rel = f'{entry.lstrip("/")}{suffix}'
+                addr = entry_rel
+                keys: list[str] | None = None
+                for pfx in prefixes:
+                    addr = f'{pfx}/{entry_rel}' if pfx else entry_rel
+                    try:
+                        keys = list(await src.readdir(addr))
+                        break
+                    except FileNotFoundError:
+                        continue
+                if keys is None:
+                    log(NOTICE, f'not extracted: {entry}')
+                    continue
+                log(NOTICE, f'extracted:     {entry}')
+                pairs: list[tuple[str, bytes]] = []
+                for key in keys:
+                    val = await src.get(f'{addr}/{key}')
+                    pairs.append((key, val.stdout))
+                dst = self.store
+                with suppress(FileNotFoundError):
+                    await dst.unlink(entry_rel)
+                await dst.mkdir(entry_rel, ftype = FT_SECRET)
+                for key, content in pairs:
+                    await dst.put(f'{entry_rel}/{key}', content, ftype = FT_SECRET_KEY)
+        finally:
+            await fc.close()