From a29af550c442447f9b6a8508c86ff619b654dd9a Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 29 Oct 2026 10:10:03 +0100 Subject: [PATCH 1/2] cmds.secrets.CmdCompileTemplates: Support -k / --keep-going jw-pkg secrets compile-templates iterates over all selected templates and compiles secrets from it - unless it encounters one that can't be compiled for whatever reason, in which case the iteration errors out without compiling the remaining secrets. Add a -k / --keep-going option that makes the iteration not abort over failed compilations but continue through the remaining templates, then report an error status at the end. The integration test gains a good sibling of the failing fixture template: a fail-fast run must not reach it, and a keep-going run must compile it and still fail the command. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- src/python/jw/pkg/cmds/secrets/Cmd.py | 9 +++- .../pkg/cmds/secrets/CmdCompileTemplates.py | 8 +++- .../jw/pkg/cmds/secrets/lib/DistroContext.py | 26 +++++++++-- test/integration/jw_pkg/secrets/test.py | 43 ++++++++++++++++++- 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/python/jw/pkg/cmds/secrets/Cmd.py b/src/python/jw/pkg/cmds/secrets/Cmd.py index 103b12b7..7b0291a4 100644 --- a/src/python/jw/pkg/cmds/secrets/Cmd.py +++ b/src/python/jw/pkg/cmds/secrets/Cmd.py @@ -39,9 +39,14 @@ class Cmd(Base): # export return await self.ctx.remove_compilation_targets(packages) async def _compile_template_files( - self, packages: Collection[str], default_attrs: Attrs + self, + packages: Collection[str], + default_attrs: Attrs, + keep_going: bool = False, ) -> list[str]: - return await self.ctx.compile_template_files(packages, default_attrs) + return await self.ctx.compile_template_files( + packages, default_attrs, keep_going = keep_going + ) def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) diff --git a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py index 6bd8a4fb..3cfbf225 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py +++ b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py @@ -22,11 +22,17 @@ class CmdCompileTemplates(Cmd): # export @override async def _run(self, args: Namespace) -> None: attrs = Attrs(args.mode, args.owner, args.group, None) - await self._compile_template_files(args.packages, attrs) + await self._compile_template_files(args.packages, attrs, args.keep_going) @override def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) + parser.add_argument( + '--keep-going', + '-k', + action = 'store_true', + help = 'Compile all templates, reporting the failures at the end', + ) parser.add_argument( '--owner', '-o', default = None, help = 'Default output file owner' ) diff --git a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py index 610e4976..8b5147f2 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py @@ -17,7 +17,7 @@ from ....lib.ec.secret_store.layout import ( host_prefix, ) from ....lib.FileContext import FileContext -from ....lib.log import DEBUG, NOTICE, WARNING, log +from ....lib.log import DEBUG, ERR, NOTICE, WARNING, log from .FilesContext import FilesContext if TYPE_CHECKING: @@ -81,12 +81,25 @@ class DistroContext(FilesContext): return ret async def compile_template_files( - self, pkg_names: Collection[str], default_attrs: Attrs + self, + pkg_names: Collection[str], + default_attrs: Attrs, + keep_going: bool = False, ) -> list[str]: ret: list[str] = [] + failed: list[str] = [] missing = 0 - for target in await self.list_compilation_targets(pkg_names): - if await self.compile_template_file(target, default_attrs): + targets = await self.list_compilation_targets(pkg_names) + for target in targets: + try: + ok = await self.compile_template_file(target, default_attrs) + except Exception as e: + if not keep_going: + raise + log(ERR, f'{target}: {e}') + failed.append(target) + continue + if ok: ret.append(target) else: missing += 1 @@ -101,6 +114,11 @@ class DistroContext(FilesContext): f'run sudo {cmdline} again' ), ) + if failed: + raise Exception( + f'{len(failed)} of {len(targets)} templates failed to ' + f'compile: {", ".join(failed)}' + ) return ret async def install( diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index 97c431e2..3daed322 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -211,6 +211,12 @@ def main() -> None: 'present=PRESENT\n' 'missing=__UNRESOLVED__\n' ) + # -- grault is the good sibling of qux in the same package: + # it must not be compiled by a fail-fast run, and must be + # compiled by a keep-going run + with open(target + '/etc/grault.conf.jw-tmpl', 'w') as f: + f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n' + 'g=GAULT\n') with open(target + '/etc/readme.txt', 'w') as f: f.write('plain package file\n') @@ -271,7 +277,7 @@ def main() -> None: target + '/etc/quux.conf.jw-tmpl', target + '/etc/readme.txt', ], - PKG2: [qux_tmpl], + PKG2: [qux_tmpl, target + '/etc/grault.conf.jw-tmpl'], }, ) @@ -423,8 +429,12 @@ def main() -> None: # file may be written qux_secret = target + '/etc/qux.conf.jw-secret' qux_out = target + '/etc/qux.conf' + grault_secret = target + '/etc/grault.conf.jw-secret' + grault_out = target + '/etc/grault.conf' with open(qux_secret, 'w') as f: f.write('PRESENT=there\n') + with open(grault_secret, 'w') as f: + f.write('GAULT=grault-value\n') try: run_cli( distro, @@ -446,6 +456,37 @@ def main() -> None: else: raise AssertionError('unresolved macro must fail the compile') assert not os.path.exists(qux_out) + # -- fail-fast: the sweep stopped at qux and never reached its + # good sibling + assert not os.path.exists(grault_out) + + # -- keep-going: the failing sibling does not stop the sweep, + # grault is compiled, and the command still fails, naming + # the failing target + try: + run_cli( + distro, + [ + 'secrets', + 'compile-templates', + '--keep-going', + '--owner', + USER, + '--group', + GROUP, + '--mode', + '400', + PKG2, + ], + ) + except RuntimeError as e: + assert 'failed to compile' in str(e), e + assert 'qux.conf' in str(e), e + else: + raise AssertionError('keep-going must still fail the command') + with open(grault_out) as f: + assert f.read() == 'g=grault-value\n' + assert not os.path.exists(qux_out) out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG]) assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out From 4691f38eecffb1b4980240fb4a6c5dba3b5073dd Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 29 Oct 2026 10:10:03 +0100 Subject: [PATCH 2/2] cmds.secrets.CmdCompileTemplates: Support -k / --keep-going jw-pkg secrets compile-templates iterates over all selected templates and compiles secrets from it - unless it encounters one that can't be compiled for whatever reason, in which case the iteration errors out without compiling the remaining secrets. Add a -k / --keep-going option that makes the iteration not abort over failed compilations but continue through the remaining templates, then report an error status at the end. The integration test gains a good sibling of the failing fixture template: a fail-fast run must not reach it, and a keep-going run must compile it and still fail the command. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 --- src/python/jw/pkg/cmds/secrets/Cmd.py | 9 +++- .../pkg/cmds/secrets/CmdCompileTemplates.py | 8 +++- .../jw/pkg/cmds/secrets/lib/DistroContext.py | 26 +++++++++-- .../integration/jw_pkg/help/test-expected.txt | 4 +- test/integration/jw_pkg/secrets/test.py | 43 ++++++++++++++++++- 5 files changed, 81 insertions(+), 9 deletions(-) diff --git a/src/python/jw/pkg/cmds/secrets/Cmd.py b/src/python/jw/pkg/cmds/secrets/Cmd.py index 103b12b7..7b0291a4 100644 --- a/src/python/jw/pkg/cmds/secrets/Cmd.py +++ b/src/python/jw/pkg/cmds/secrets/Cmd.py @@ -39,9 +39,14 @@ class Cmd(Base): # export return await self.ctx.remove_compilation_targets(packages) async def _compile_template_files( - self, packages: Collection[str], default_attrs: Attrs + self, + packages: Collection[str], + default_attrs: Attrs, + keep_going: bool = False, ) -> list[str]: - return await self.ctx.compile_template_files(packages, default_attrs) + return await self.ctx.compile_template_files( + packages, default_attrs, keep_going = keep_going + ) def __init__(self, parent: Parent, name: str, help: str) -> None: super().__init__(parent, name, help) diff --git a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py index 6bd8a4fb..3cfbf225 100644 --- a/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py +++ b/src/python/jw/pkg/cmds/secrets/CmdCompileTemplates.py @@ -22,11 +22,17 @@ class CmdCompileTemplates(Cmd): # export @override async def _run(self, args: Namespace) -> None: attrs = Attrs(args.mode, args.owner, args.group, None) - await self._compile_template_files(args.packages, attrs) + await self._compile_template_files(args.packages, attrs, args.keep_going) @override def add_arguments(self, parser: ArgumentParser) -> None: super().add_arguments(parser) + parser.add_argument( + '--keep-going', + '-k', + action = 'store_true', + help = 'Compile all templates, reporting the failures at the end', + ) parser.add_argument( '--owner', '-o', default = None, help = 'Default output file owner' ) diff --git a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py index 610e4976..8b5147f2 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/DistroContext.py @@ -17,7 +17,7 @@ from ....lib.ec.secret_store.layout import ( host_prefix, ) from ....lib.FileContext import FileContext -from ....lib.log import DEBUG, NOTICE, WARNING, log +from ....lib.log import DEBUG, ERR, NOTICE, WARNING, log from .FilesContext import FilesContext if TYPE_CHECKING: @@ -81,12 +81,25 @@ class DistroContext(FilesContext): return ret async def compile_template_files( - self, pkg_names: Collection[str], default_attrs: Attrs + self, + pkg_names: Collection[str], + default_attrs: Attrs, + keep_going: bool = False, ) -> list[str]: ret: list[str] = [] + failed: list[str] = [] missing = 0 - for target in await self.list_compilation_targets(pkg_names): - if await self.compile_template_file(target, default_attrs): + targets = await self.list_compilation_targets(pkg_names) + for target in targets: + try: + ok = await self.compile_template_file(target, default_attrs) + except Exception as e: + if not keep_going: + raise + log(ERR, f'{target}: {e}') + failed.append(target) + continue + if ok: ret.append(target) else: missing += 1 @@ -101,6 +114,11 @@ class DistroContext(FilesContext): f'run sudo {cmdline} again' ), ) + if failed: + raise Exception( + f'{len(failed)} of {len(targets)} templates failed to ' + f'compile: {", ".join(failed)}' + ) return ret async def install( diff --git a/test/integration/jw_pkg/help/test-expected.txt b/test/integration/jw_pkg/help/test-expected.txt index fe76fc8d..eac36b1a 100644 --- a/test/integration/jw_pkg/help/test-expected.txt +++ b/test/integration/jw_pkg/help/test-expected.txt @@ -848,7 +848,7 @@ Available subcommands of secrets: rm-compilation-output Remove package compilation output files ============= Running: jw-pkg.py -t ../../../.. --log-level info secrets compile-templates --help -usage: jw-pkg.py secrets compile-templates [-h] [--owner OWNER] +usage: jw-pkg.py secrets compile-templates [-h] [--keep-going] [--owner OWNER] [--group GROUP] [--mode MODE] [packages ...] @@ -859,6 +859,8 @@ positional arguments: options: -h, --help show this help message and exit + --keep-going, -k Compile all templates, reporting the failures at the end + (default: False) --owner, -o OWNER Default output file owner (default: None) --group, -g GROUP Default output file group (default: None) --mode, -m MODE Default output file mode (octal) (default: None) diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index 97c431e2..3daed322 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -211,6 +211,12 @@ def main() -> None: 'present=PRESENT\n' 'missing=__UNRESOLVED__\n' ) + # -- grault is the good sibling of qux in the same package: + # it must not be compiled by a fail-fast run, and must be + # compiled by a keep-going run + with open(target + '/etc/grault.conf.jw-tmpl', 'w') as f: + f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n' + 'g=GAULT\n') with open(target + '/etc/readme.txt', 'w') as f: f.write('plain package file\n') @@ -271,7 +277,7 @@ def main() -> None: target + '/etc/quux.conf.jw-tmpl', target + '/etc/readme.txt', ], - PKG2: [qux_tmpl], + PKG2: [qux_tmpl, target + '/etc/grault.conf.jw-tmpl'], }, ) @@ -423,8 +429,12 @@ def main() -> None: # file may be written qux_secret = target + '/etc/qux.conf.jw-secret' qux_out = target + '/etc/qux.conf' + grault_secret = target + '/etc/grault.conf.jw-secret' + grault_out = target + '/etc/grault.conf' with open(qux_secret, 'w') as f: f.write('PRESENT=there\n') + with open(grault_secret, 'w') as f: + f.write('GAULT=grault-value\n') try: run_cli( distro, @@ -446,6 +456,37 @@ def main() -> None: else: raise AssertionError('unresolved macro must fail the compile') assert not os.path.exists(qux_out) + # -- fail-fast: the sweep stopped at qux and never reached its + # good sibling + assert not os.path.exists(grault_out) + + # -- keep-going: the failing sibling does not stop the sweep, + # grault is compiled, and the command still fails, naming + # the failing target + try: + run_cli( + distro, + [ + 'secrets', + 'compile-templates', + '--keep-going', + '--owner', + USER, + '--group', + GROUP, + '--mode', + '400', + PKG2, + ], + ) + except RuntimeError as e: + assert 'failed to compile' in str(e), e + assert 'qux.conf' in str(e), e + else: + raise AssertionError('keep-going must still fail the command') + with open(grault_out) as f: + assert f.read() == 'g=grault-value\n' + assert not os.path.exists(qux_out) out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG]) assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out