cmds.secrets.CmdCompileTemplates: Support -k / --keep-going

jw-pkg secrets compile-templates iterates over all selected templates and
compiles secrets from it - unless it encounters one that can't be compiled
for whatever reason, in which case the iteration errors out without
compiling the remaining secrets.

Add a -k / --keep-going option that makes the iteration not abort over
failed compilations but continue through the remaining templates, then
report an error status at the end.

The integration test gains a good sibling of the failing fixture template:
a fail-fast run must not reach it, and a keep-going run must compile it and
still fail the command.

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0

src/python/jw/pkg/cmds/secrets/Cmd.py

@@ -39,9 +39,14 @@ class Cmd(Base):  # export
         return await self.ctx.remove_compilation_targets(packages)
 
     async def _compile_template_files(
-        self, packages: Collection[str], default_attrs: Attrs
+        self,
+        packages: Collection[str],
+        default_attrs: Attrs,
+        keep_going: bool = False,
     ) -> list[str]:
-        return await self.ctx.compile_template_files(packages, default_attrs)
+        return await self.ctx.compile_template_files(
+            packages, default_attrs, keep_going = keep_going
+        )
 
     def __init__(self, parent: Parent, name: str, help: str) -> None:
         super().__init__(parent, name, help)

src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py

@@ -22,11 +22,17 @@ class CmdCompileTemplates(Cmd):  # export
     @override
     async def _run(self, args: Namespace) -> None:
         attrs = Attrs(args.mode, args.owner, args.group, None)
-        await self._compile_template_files(args.packages, attrs)
+        await self._compile_template_files(args.packages, attrs, args.keep_going)
 
     @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         super().add_arguments(parser)
+        parser.add_argument(
+            '--keep-going',
+            '-k',
+            action = 'store_true',
+            help = 'Compile all templates, reporting the failures at the end',
+        )
         parser.add_argument(
             '--owner', '-o', default = None, help = 'Default output file owner'
         )

src/python/jw/pkg/cmds/secrets/lib/DistroContext.py

@@ -17,7 +17,7 @@ from ....lib.ec.secret_store.layout import (
     host_prefix,
 )
 from ....lib.FileContext import FileContext
-from ....lib.log import DEBUG, NOTICE, WARNING, log
+from ....lib.log import DEBUG, ERR, NOTICE, WARNING, log
 from .FilesContext import FilesContext
 
 if TYPE_CHECKING:
@@ -81,12 +81,25 @@ class DistroContext(FilesContext):
         return ret
 
     async def compile_template_files(
-        self, pkg_names: Collection[str], default_attrs: Attrs
+        self,
+        pkg_names: Collection[str],
+        default_attrs: Attrs,
+        keep_going: bool = False,
     ) -> list[str]:
         ret: list[str] = []
+        failed: list[str] = []
         missing = 0
-        for target in await self.list_compilation_targets(pkg_names):
+        targets = await self.list_compilation_targets(pkg_names)
-            if await self.compile_template_file(target, default_attrs):
+        for target in targets:
+            try:
+                ok = await self.compile_template_file(target, default_attrs)
+            except Exception as e:
+                if not keep_going:
+                    raise
+                log(ERR, f'{target}: {e}')
+                failed.append(target)
+                continue
+            if ok:
                 ret.append(target)
             else:
                 missing += 1
@@ -101,6 +114,11 @@ class DistroContext(FilesContext):
                     f'run sudo {cmdline} again'
                 ),
             )
+        if failed:
+            raise Exception(
+                f'{len(failed)} of {len(targets)} templates failed to '
+                f'compile: {", ".join(failed)}'
+            )
         return ret
 
     async def install(

test/integration/jw_pkg/help/test-expected.txt

@@ -848,7 +848,7 @@ Available subcommands of secrets:
     rm-compilation-output
                         Remove package compilation output files
 ============= Running: jw-pkg.py -t ../../../.. --log-level info secrets compile-templates --help
-usage: jw-pkg.py secrets compile-templates [-h] [--owner OWNER]
+usage: jw-pkg.py secrets compile-templates [-h] [--keep-going] [--owner OWNER]
                                            [--group GROUP] [--mode MODE]
                                            [packages ...]
 
@@ -859,6 +859,8 @@ positional arguments:
 
 options:
   -h, --help         show this help message and exit
+  --keep-going, -k   Compile all templates, reporting the failures at the end
+                     (default: False)
   --owner, -o OWNER  Default output file owner (default: None)
   --group, -g GROUP  Default output file group (default: None)
   --mode, -m MODE    Default output file mode (octal) (default: None)

test/integration/jw_pkg/secrets/test.py

@@ -211,6 +211,12 @@ def main() -> None:
                 'present=PRESENT\n'
                 'missing=__UNRESOLVED__\n'
             )
+        # -- grault is the good sibling of qux in the same package:
+        #    it must not be compiled by a fail-fast run, and must be
+        #    compiled by a keep-going run
+        with open(target + '/etc/grault.conf.jw-tmpl', 'w') as f:
+            f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n'
+                    'g=GAULT\n')
         with open(target + '/etc/readme.txt', 'w') as f:
             f.write('plain package file\n')
 
@@ -271,7 +277,7 @@ def main() -> None:
                     target + '/etc/quux.conf.jw-tmpl',
                     target + '/etc/readme.txt',
                 ],
-                PKG2: [qux_tmpl],
+                PKG2: [qux_tmpl, target + '/etc/grault.conf.jw-tmpl'],
             },
         )
 
@@ -423,8 +429,12 @@ def main() -> None:
         #    file may be written
         qux_secret = target + '/etc/qux.conf.jw-secret'
         qux_out = target + '/etc/qux.conf'
+        grault_secret = target + '/etc/grault.conf.jw-secret'
+        grault_out = target + '/etc/grault.conf'
         with open(qux_secret, 'w') as f:
             f.write('PRESENT=there\n')
+        with open(grault_secret, 'w') as f:
+            f.write('GAULT=grault-value\n')
         try:
             run_cli(
                 distro,
@@ -446,6 +456,37 @@ def main() -> None:
         else:
             raise AssertionError('unresolved macro must fail the compile')
         assert not os.path.exists(qux_out)
+        # -- fail-fast: the sweep stopped at qux and never reached its
+        #    good sibling
+        assert not os.path.exists(grault_out)
+
+        # -- keep-going: the failing sibling does not stop the sweep,
+        #    grault is compiled, and the command still fails, naming
+        #    the failing target
+        try:
+            run_cli(
+                distro,
+                [
+                    'secrets',
+                    'compile-templates',
+                    '--keep-going',
+                    '--owner',
+                    USER,
+                    '--group',
+                    GROUP,
+                    '--mode',
+                    '400',
+                    PKG2,
+                ],
+            )
+        except RuntimeError as e:
+            assert 'failed to compile' in str(e), e
+            assert 'qux.conf' in str(e), e
+        else:
+            raise AssertionError('keep-going must still fail the command')
+        with open(grault_out) as f:
+            assert f.read() == 'g=grault-value\n'
+        assert not os.path.exists(qux_out)
 
         out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
         assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out