From ea45cb1fed4f16d4a2bc0f9e4b405c6d6c9276c0 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 24 Oct 2026 07:22:47 +0200 Subject: [PATCH 1/2] cmds.secrets.FilesContext: Re-shuffle attrs precedence _compile_one_template_file() re-parses the template's own conf line and, when present, discards the attrs that compile_template_file() merged from the command-line defaults, the template, the .jw-secret-file, and the .jw-secret. The merge already orders the sources so that later non-empty fields override earlier ones; discarding it inverts the order and lets the template's line override the one the operator placed in the .jw-secret. Strip the template's conf line from the content and apply the merged attrs instead, so the .jw-secret conf line takes the last word again, as the compiler did before the rewrite. The integration test gains a target whose template and secret both carry a conf line. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- .../jw/pkg/cmds/secrets/lib/FilesContext.py | 11 ++--- test/integration/jw_pkg/secrets/test.py | 42 +++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py index d657b5e7..7f79afce 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py @@ -115,15 +115,16 @@ class FilesContext: result = await self.ctx.get(src) new_content = result.stdout_str - attrs = self._parse_attributes(new_content) - if attrs is None: - attrs = default_attrs - else: + if self._parse_attributes(new_content) is not None: # -- A conf line is the attribute spec of the output - # file, not part of its content + # file, not part of its content. The merged attrs + # contain this line, and a conf line in the secret + # takes precedence over it. lines = new_content.splitlines(keepends = True) new_content = ''.join(lines[1:]) + attrs = default_attrs + if attrs is not None: if attrs.owner is not None: owner = attrs.owner diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index 3d0aa313..a8706bd2 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -190,6 +190,11 @@ def main() -> None: # extract it, and must not leave an empty file behind with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f: f.write('x=X\n') + # -- corge has a conf line in the template and one in the + # secret: the secret's line takes precedence + with open(target + '/etc/corge.conf.jw-tmpl', 'w') as f: + f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n' + 'key=KEY\n') # -- qux references a macro that has no store entry: the # compile must fail rather than emit a literal macro with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f: @@ -208,9 +213,11 @@ def main() -> None: foo_secret = target + '/etc/foo.conf.jw-secret' bar_secret = target + '/etc/bar.conf.jw-secret' baz_secret = target + '/etc/baz.conf.jw-secret' + corge_secret = target + '/etc/corge.conf.jw-secret' foo_out = target + '/etc/foo.conf' bar_out = target + '/etc/bar.conf' baz_out = target + '/etc/baz.conf' + corge_out = target + '/etc/corge.conf' # -- The GPG-encrypted tar store in canonical layout. Member # names are / @@ -230,6 +237,8 @@ def main() -> None: 'HOSTNAME=jannet\nTOKEN=host-token\n', f'default/{trel}/etc/bar.conf.jw-secret': \ 'PLAIN=plain-value\n', + f'default/{trel}/etc/corge.conf.jw-secret': \ + 'KEY=corge-value\n', # A member that is not a secret of the fixture # package: install must not extract it f'default/{trel}/etc/cert.pem': 'decoy certificate\n', @@ -242,7 +251,13 @@ def main() -> None: distro = FakeDistro( Local('local://' + HOST), { - PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'], + PKG: [ + foo_tmpl, + bar_tmpl, + baz_tmpl, + target + '/etc/corge.conf.jw-tmpl', + target + '/etc/readme.txt', + ], PKG2: [qux_tmpl], }, ) @@ -250,7 +265,8 @@ def main() -> None: # -- list-templates: only the .jw-tmpl files of the package out, _ = run_cli(distro, ['secrets', 'list-templates', PKG]) - assert out == [foo_tmpl, bar_tmpl, baz_tmpl], out + assert out == [foo_tmpl, bar_tmpl, baz_tmpl, + target + '/etc/corge.conf.jw-tmpl'], out # -- list-secrets: nothing installed yet @@ -258,7 +274,7 @@ def main() -> None: assert out == [], out out, _ = run_cli(distro, ['secrets', 'list-secrets', '--all', PKG]) - assert out == [foo_secret, bar_secret, baz_secret], out + assert out == [foo_secret, bar_secret, baz_secret, corge_secret], out # -- list-compilation-output: nothing compiled yet @@ -266,7 +282,7 @@ def main() -> None: assert out == [], out out, _ = run_cli(distro, ['secrets', 'list-compilation-output', '--all', PKG]) - assert out == [foo_out, bar_out, baz_out], out + assert out == [foo_out, bar_out, baz_out, corge_out], out # -- install: decrypt the tarball, keep the host-specific entry # over the default one, extract only the needed secrets @@ -286,7 +302,7 @@ def main() -> None: assert stat.S_IMODE(os.stat(foo_secret).st_mode) == 0o400 out, _ = run_cli(distro, ['secrets', 'list-secrets', PKG]) - assert out == [foo_secret, bar_secret], out + assert out == [foo_secret, bar_secret, corge_secret], out if HAVE_TARGPG: # -- a rooted targpg:// URI is rejected: install addresses @@ -315,11 +331,16 @@ def main() -> None: assert stat.S_IMODE(os.stat(foo_secret).st_mode) == 0o400 # -- The file attributes can also be specified atop the - # .jw-secret; bar takes its mode from there + # .jw-secret; bar takes its mode from there. corge gets a + # line as well: it takes precedence over the template's os.unlink(bar_secret) with open(bar_secret, 'w') as f: f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n') f.write('PLAIN=plain-value\n') + os.unlink(corge_secret) + with open(corge_secret, 'w') as f: + f.write('# conf: mode=400\n') + f.write('KEY=corge-value\n') # -- compile-templates: substitute the keys, apply the # metadata; the conf lines win over the --mode default @@ -346,6 +367,12 @@ def main() -> None: assert (st.st_uid, stat.S_IMODE(st.st_mode)) == (os.getuid(), 0o640), st with open(bar_out) as f: assert f.read() == 'plain=plain-value\n' + # -- the secret's conf line takes precedence over the + # template's: corge comes out 0400, not the template's 640 + st = os.stat(corge_out) + assert (st.st_uid, stat.S_IMODE(st.st_mode)) == (os.getuid(), 0o400), st + with open(corge_out) as f: + assert f.read() == 'key=corge-value\n' # -- compile-templates is idempotent @@ -401,7 +428,7 @@ def main() -> None: assert not os.path.exists(qux_out) out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG]) - assert out == [foo_out, bar_out, baz_out], out + assert out == [foo_out, bar_out, baz_out, corge_out], out # -- rm-compilation-output: drop the compiled files @@ -410,6 +437,7 @@ def main() -> None: assert not os.path.exists(foo_out) assert not os.path.exists(bar_out) assert not os.path.exists(baz_out) + assert not os.path.exists(corge_out) print('All secrets command tests passed') finally: -- 2.56.0 From 6398406a5fdd38ac956a2587fb20db3b987e5b0e Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 24 Oct 2026 07:37:59 +0200 Subject: [PATCH 2/2] cmds.secrets.FilesContext: Ignore commented-out macros The unresolved-macro check scans the substituted content for __MACRO__ names, and a commented-out reference fails the compile like a live one. A disabled config section keeps its macros in the comments as documentation, and the matching store keys are retired along with it, so compiling the template fails on lines that no longer take effect. Skip comment lines when scanning for unresolved macros. The integration test gains a target whose template carries a commented-out macro without a store entry. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.4.0 Signed-off-by: Jan Lindemann --- .../jw/pkg/cmds/secrets/lib/FilesContext.py | 8 ++++- test/integration/jw_pkg/secrets/test.py | 31 ++++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py index 7f79afce..0be62b5d 100644 --- a/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py +++ b/src/python/jw/pkg/cmds/secrets/lib/FilesContext.py @@ -138,7 +138,13 @@ class FilesContext: for key, val in replace.items(): new_content = new_content.replace(key, val) - unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content))) + # -- A commented-out macro is documentation, not a reference; + # it must not fail the compile + live = '\n'.join( + line for line in new_content.splitlines() + if not line.lstrip().startswith('#') + ) + unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', live))) if unresolved: raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}') diff --git a/test/integration/jw_pkg/secrets/test.py b/test/integration/jw_pkg/secrets/test.py index a8706bd2..97c431e2 100644 --- a/test/integration/jw_pkg/secrets/test.py +++ b/test/integration/jw_pkg/secrets/test.py @@ -195,6 +195,14 @@ def main() -> None: with open(target + '/etc/corge.conf.jw-tmpl', 'w') as f: f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n' 'key=KEY\n') + # -- quux carries a commented-out macro without a store entry: + # the compile must succeed, the comment stays verbatim + with open(target + '/etc/quux.conf.jw-tmpl', 'w') as f: + f.write( + f'# conf: owner={USER}, group={GROUP}, mode=640\n' + '# retired=__RETIRED__\n' + 'live=LIVE\n' + ) # -- qux references a macro that has no store entry: the # compile must fail rather than emit a literal macro with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f: @@ -214,10 +222,12 @@ def main() -> None: bar_secret = target + '/etc/bar.conf.jw-secret' baz_secret = target + '/etc/baz.conf.jw-secret' corge_secret = target + '/etc/corge.conf.jw-secret' + quux_secret = target + '/etc/quux.conf.jw-secret' foo_out = target + '/etc/foo.conf' bar_out = target + '/etc/bar.conf' baz_out = target + '/etc/baz.conf' corge_out = target + '/etc/corge.conf' + quux_out = target + '/etc/quux.conf' # -- The GPG-encrypted tar store in canonical layout. Member # names are / @@ -239,6 +249,8 @@ def main() -> None: 'PLAIN=plain-value\n', f'default/{trel}/etc/corge.conf.jw-secret': \ 'KEY=corge-value\n', + f'default/{trel}/etc/quux.conf.jw-secret': \ + 'LIVE=live-value\n', # A member that is not a secret of the fixture # package: install must not extract it f'default/{trel}/etc/cert.pem': 'decoy certificate\n', @@ -256,6 +268,7 @@ def main() -> None: bar_tmpl, baz_tmpl, target + '/etc/corge.conf.jw-tmpl', + target + '/etc/quux.conf.jw-tmpl', target + '/etc/readme.txt', ], PKG2: [qux_tmpl], @@ -266,7 +279,8 @@ def main() -> None: out, _ = run_cli(distro, ['secrets', 'list-templates', PKG]) assert out == [foo_tmpl, bar_tmpl, baz_tmpl, - target + '/etc/corge.conf.jw-tmpl'], out + target + '/etc/corge.conf.jw-tmpl', + target + '/etc/quux.conf.jw-tmpl'], out # -- list-secrets: nothing installed yet @@ -274,7 +288,8 @@ def main() -> None: assert out == [], out out, _ = run_cli(distro, ['secrets', 'list-secrets', '--all', PKG]) - assert out == [foo_secret, bar_secret, baz_secret, corge_secret], out + assert out == [foo_secret, bar_secret, baz_secret, + corge_secret, quux_secret], out # -- list-compilation-output: nothing compiled yet @@ -282,7 +297,7 @@ def main() -> None: assert out == [], out out, _ = run_cli(distro, ['secrets', 'list-compilation-output', '--all', PKG]) - assert out == [foo_out, bar_out, baz_out, corge_out], out + assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out # -- install: decrypt the tarball, keep the host-specific entry # over the default one, extract only the needed secrets @@ -302,7 +317,7 @@ def main() -> None: assert stat.S_IMODE(os.stat(foo_secret).st_mode) == 0o400 out, _ = run_cli(distro, ['secrets', 'list-secrets', PKG]) - assert out == [foo_secret, bar_secret, corge_secret], out + assert out == [foo_secret, bar_secret, corge_secret, quux_secret], out if HAVE_TARGPG: # -- a rooted targpg:// URI is rejected: install addresses @@ -373,6 +388,11 @@ def main() -> None: assert (st.st_uid, stat.S_IMODE(st.st_mode)) == (os.getuid(), 0o400), st with open(corge_out) as f: assert f.read() == 'key=corge-value\n' + # -- the commented-out macro does not fail the compile and + # stays in the output verbatim + with open(quux_out) as f: + assert f.read() == ('# retired=__RETIRED__\n' + 'live=live-value\n') # -- compile-templates is idempotent @@ -428,7 +448,7 @@ def main() -> None: assert not os.path.exists(qux_out) out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG]) - assert out == [foo_out, bar_out, baz_out, corge_out], out + assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out # -- rm-compilation-output: drop the compiled files @@ -438,6 +458,7 @@ def main() -> None: assert not os.path.exists(bar_out) assert not os.path.exists(baz_out) assert not os.path.exists(corge_out) + assert not os.path.exists(quux_out) print('All secrets command tests passed') finally: -- 2.56.0