src/python/jw/pkg/cmds/secrets/lib/FilesContext.py

@@ -115,15 +115,16 @@ class FilesContext:
         result = await self.ctx.get(src)
         new_content = result.stdout_str
 
-        attrs = self._parse_attributes(new_content)
-        if attrs is None:
-            attrs = default_attrs
-        else:
+        if self._parse_attributes(new_content) is not None:
             # -- A conf line is the attribute spec of the output
-            #    file, not part of its content
+            #    file, not part of its content. The merged attrs
+            #    contain this line, and a conf line in the secret
+            #    takes precedence over it.
             lines = new_content.splitlines(keepends = True)
             new_content = ''.join(lines[1:])
 
+        attrs = default_attrs
+
         if attrs is not None:
             if attrs.owner is not None:
                 owner = attrs.owner
@@ -137,7 +138,13 @@ class FilesContext:
         for key, val in replace.items():
             new_content = new_content.replace(key, val)
 
-        unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', new_content)))
+        # -- A commented-out macro is documentation, not a reference;
+        #    it must not fail the compile
+        live = '\n'.join(
+            line for line in new_content.splitlines()
+            if not line.lstrip().startswith('#')
+        )
+        unresolved = sorted(set(re.findall(r'__[A-Z0-9_]+__', live)))
         if unresolved:
             raise Exception(f'Unresolved macros in {src}: {", ".join(unresolved)}')
 

test/integration/jw_pkg/secrets/test.py

@@ -190,6 +190,19 @@ def main() -> None:
         #    extract it, and must not leave an empty file behind
         with open(target + '/etc/baz.conf.jw-tmpl', 'w') as f:
             f.write('x=X\n')
+        # -- corge has a conf line in the template and one in the
+        #    secret: the secret's line takes precedence
+        with open(target + '/etc/corge.conf.jw-tmpl', 'w') as f:
+            f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n'
+                    'key=KEY\n')
+        # -- quux carries a commented-out macro without a store entry:
+        #    the compile must succeed, the comment stays verbatim
+        with open(target + '/etc/quux.conf.jw-tmpl', 'w') as f:
+            f.write(
+                f'# conf: owner={USER}, group={GROUP}, mode=640\n'
+                '# retired=__RETIRED__\n'
+                'live=LIVE\n'
+            )
         # -- qux references a macro that has no store entry: the
         #    compile must fail rather than emit a literal macro
         with open(target + '/etc/qux.conf.jw-tmpl', 'w') as f:
@@ -208,9 +221,13 @@ def main() -> None:
         foo_secret = target + '/etc/foo.conf.jw-secret'
         bar_secret = target + '/etc/bar.conf.jw-secret'
         baz_secret = target + '/etc/baz.conf.jw-secret'
+        corge_secret = target + '/etc/corge.conf.jw-secret'
+        quux_secret = target + '/etc/quux.conf.jw-secret'
         foo_out = target + '/etc/foo.conf'
         bar_out = target + '/etc/bar.conf'
         baz_out = target + '/etc/baz.conf'
+        corge_out = target + '/etc/corge.conf'
+        quux_out = target + '/etc/quux.conf'
 
         # -- The GPG-encrypted tar store in canonical layout. Member
         #    names are <prefix>/<target path without leading slash>
@@ -230,6 +247,10 @@ def main() -> None:
                     'HOSTNAME=jannet\nTOKEN=host-token\n',
                 f'default/{trel}/etc/bar.conf.jw-secret': \
                     'PLAIN=plain-value\n',
+                f'default/{trel}/etc/corge.conf.jw-secret': \
+                    'KEY=corge-value\n',
+                f'default/{trel}/etc/quux.conf.jw-secret': \
+                    'LIVE=live-value\n',
                 # A member that is not a secret of the fixture
                 # package: install must not extract it
                 f'default/{trel}/etc/cert.pem': 'decoy certificate\n',
@@ -242,7 +263,14 @@ def main() -> None:
         distro = FakeDistro(
             Local('local://' + HOST),
             {
-                PKG: [foo_tmpl, bar_tmpl, baz_tmpl, target + '/etc/readme.txt'],
+                PKG: [
+                    foo_tmpl,
+                    bar_tmpl,
+                    baz_tmpl,
+                    target + '/etc/corge.conf.jw-tmpl',
+                    target + '/etc/quux.conf.jw-tmpl',
+                    target + '/etc/readme.txt',
+                ],
                 PKG2: [qux_tmpl],
             },
         )
@@ -250,7 +278,9 @@ def main() -> None:
         # -- list-templates: only the .jw-tmpl files of the package
 
         out, _ = run_cli(distro, ['secrets', 'list-templates', PKG])
-        assert out == [foo_tmpl, bar_tmpl, baz_tmpl], out
+        assert out == [foo_tmpl, bar_tmpl, baz_tmpl,
+                       target + '/etc/corge.conf.jw-tmpl',
+                       target + '/etc/quux.conf.jw-tmpl'], out
 
         # -- list-secrets: nothing installed yet
 
@@ -258,7 +288,8 @@ def main() -> None:
         assert out == [], out
 
         out, _ = run_cli(distro, ['secrets', 'list-secrets', '--all', PKG])
-        assert out == [foo_secret, bar_secret, baz_secret], out
+        assert out == [foo_secret, bar_secret, baz_secret,
+                       corge_secret, quux_secret], out
 
         # -- list-compilation-output: nothing compiled yet
 
@@ -266,7 +297,7 @@ def main() -> None:
         assert out == [], out
 
         out, _ = run_cli(distro, ['secrets', 'list-compilation-output', '--all', PKG])
-        assert out == [foo_out, bar_out, baz_out], out
+        assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out
 
         # -- install: decrypt the tarball, keep the host-specific entry
         #    over the default one, extract only the needed secrets
@@ -286,7 +317,7 @@ def main() -> None:
         assert stat.S_IMODE(os.stat(foo_secret).st_mode) == 0o400
 
         out, _ = run_cli(distro, ['secrets', 'list-secrets', PKG])
-        assert out == [foo_secret, bar_secret], out
+        assert out == [foo_secret, bar_secret, corge_secret, quux_secret], out
 
         if HAVE_TARGPG:
             # -- a rooted targpg:// URI is rejected: install addresses
@@ -315,11 +346,16 @@ def main() -> None:
             assert stat.S_IMODE(os.stat(foo_secret).st_mode) == 0o400
 
         # -- The file attributes can also be specified atop the
-        #    .jw-secret; bar takes its mode from there
+        #    .jw-secret; bar takes its mode from there. corge gets a
+        #    line as well: it takes precedence over the template's
         os.unlink(bar_secret)
         with open(bar_secret, 'w') as f:
             f.write(f'# conf: owner={USER}, group={GROUP}, mode=640\n')
             f.write('PLAIN=plain-value\n')
+        os.unlink(corge_secret)
+        with open(corge_secret, 'w') as f:
+            f.write('# conf: mode=400\n')
+            f.write('KEY=corge-value\n')
 
         # -- compile-templates: substitute the keys, apply the
         #    metadata; the conf lines win over the --mode default
@@ -346,6 +382,17 @@ def main() -> None:
         assert (st.st_uid, stat.S_IMODE(st.st_mode)) == (os.getuid(), 0o640), st
         with open(bar_out) as f:
             assert f.read() == 'plain=plain-value\n'
+        # -- the secret's conf line takes precedence over the
+        #    template's: corge comes out 0400, not the template's 640
+        st = os.stat(corge_out)
+        assert (st.st_uid, stat.S_IMODE(st.st_mode)) == (os.getuid(), 0o400), st
+        with open(corge_out) as f:
+            assert f.read() == 'key=corge-value\n'
+        # -- the commented-out macro does not fail the compile and
+        #    stays in the output verbatim
+        with open(quux_out) as f:
+            assert f.read() == ('# retired=__RETIRED__\n'
+                                'live=live-value\n')
 
         # -- compile-templates is idempotent
 
@@ -401,7 +448,7 @@ def main() -> None:
         assert not os.path.exists(qux_out)
 
         out, _ = run_cli(distro, ['secrets', 'list-compilation-output', PKG])
-        assert out == [foo_out, bar_out, baz_out], out
+        assert out == [foo_out, bar_out, baz_out, corge_out, quux_out], out
 
         # -- rm-compilation-output: drop the compiled files
 
@@ -410,6 +457,8 @@ def main() -> None:
         assert not os.path.exists(foo_out)
         assert not os.path.exists(bar_out)
         assert not os.path.exists(baz_out)
+        assert not os.path.exists(corge_out)
+        assert not os.path.exists(quux_out)
 
         print('All secrets command tests passed')
     finally: