From 7f9f65153fa2d17845976224b84e55db204c0b14 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 12 Sep 2026 18:28:22 +0200 Subject: [PATCH 1/3] cmds.projects.CmdCreateFile: Add --field-keys option The --field option accepts arbitrary KEY=VALUE pairs, inserting them into the rendered template output, but there is no way to declare which keys a template actually supports. Passing a key the template does not use goes unnoticed, and a template cannot offer optional fields that drop out of the output when not supplied. Add a --field-keys option, a comma-separated list of the keys a template accepts. Passing a key via --field that is not in that list is an error, and keys from the list that are not passed are added with an empty value, so the respective field renders as empty and effectively drops out of the output. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.85.1 Signed-off-by: Jan Lindemann --- .../jw/pkg/cmds/projects/CmdCreateFile.py | 29 ++++++++++++++++++- .../integration/jw-pkg/help/test-expected.txt | 8 +++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py index 754f42cb..007d9734 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py +++ b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py @@ -132,11 +132,38 @@ class CmdCreateFile(Cmd): # export metavar = 'KEY=VALUE', help = 'Additional fields to insert into the output file', ) + parser.add_argument( + '--field-keys', + help = ( + 'Possible keys for the --field option, comma-separated. Defaults to ' + 'all passed via the --field option. If this field is specified, ' + 'passing a key in --field that\'s not specified here is an error, ' + 'and not passing one makes the respective field go away from the ' + 'rendering output' + ) + ) parser.add_argument('module', help = 'The module to generate the file for') @override async def _run(self, args: Namespace) -> None: + + # -- Honour --field-keys option + fields = args.field + if args.field_keys: + passed_keys = set([field[0] for field in fields]) + field_keys = set(args.field_keys.split(',')) + if passed_keys - field_keys: + raise Exception( + 'The following keys in --field are not in --field-keys: ' + + ', '.join(sorted(passed_keys - field_keys)) + ) + for key in field_keys - passed_keys: + fields.append((key, '')) + + # -- Find method for --format method = getattr(self, 'render_' + args.format, None) if method is None: # choices already restricts this; keeps the linter happy raise Exception(f'Unsupported output format {args.format}') - sys.stdout.write(method(args.module, args.field)) + + # -- Render + sys.stdout.write(method(args.module, fields)) diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 4ac25287..ae035045 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -336,6 +336,7 @@ usage: jw-pkg.py projects create-file [-h] --format {pyright,tmpl} [--search-path SEARCH_PATH] [--template-name TEMPLATE_NAME] [--quote] [-f KEY=VALUE] + [--field-keys FIELD_KEYS] module Generate a file from project metadata @@ -357,6 +358,13 @@ options: -f, --field KEY=VALUE Additional fields to insert into the output file (default: []) + --field-keys FIELD_KEYS + Possible keys for the --field option, comma-separated. + Defaults to all passed via the --field option. If this + field is specified, passing a key in --field that's + not specified here is an error, and not passing one + makes the respective field go away from the rendering + output (default: None) ============= Running: jw-pkg.py -t ../../../.. --log-level info projects create-pkg-config --help usage: jw-pkg.py projects create-pkg-config [-h] [-F PROJECT_DESCR_FILE] [-d DESCRIPTION] [-n NAME] -- 2.55.0 From 1349665c39dc30c98d00d1f70fc19267c7e39f28 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 12 Sep 2026 19:33:54 +0200 Subject: [PATCH 2/3] cmds.projects.CmdCreateFile: Add --field-separator option List values in a rendered template are joined with a fixed separator of ",\n" (comma plus newline), and there is no way to change that. Templates sometimes want a different separator, e.g. a plain newline or a single-line comma-separated list. Add a --field-separator option that controls how list values are joined in the rendered output. %n expands to a newline, and the default remains ",\n" for both formats. render_tmpl() applies the separator to the rendered template, while render_pyright() rejects any other value: the built-in pyrightconfig.json template is a fixed JSON document, and no alternative separator is supported for it. Update the help output test expectation accordingly. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.85.1 Signed-off-by: Jan Lindemann --- .../jw/pkg/cmds/projects/CmdCreateFile.py | 47 +++++++++++++++---- .../integration/jw-pkg/help/test-expected.txt | 4 ++ .../pkg/cmds/projects/CmdCreateFile/Makefile | 8 ++++ .../pkg/cmds/projects/CmdCreateFile/test.py | 25 ++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/Makefile create mode 100644 test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/test.py diff --git a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py index 007d9734..f1d09a17 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py +++ b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py @@ -55,32 +55,56 @@ class CmdCreateFile(Cmd): # export p = 'render_' return [name.removeprefix(p) for name in dir(self) if name.startswith(p)] + def __format_separator(self, separator: str | None, default: str) -> str: + if separator is None: + return default + format_chars = { + '%n': '\n', + } + for src, dst in format_chars.items(): + separator = separator.replace(src, dst) + return separator + def __render( self, template_name: str, values: list[RenderValues], li_quote: bool = False, - li_delimiter: str = '\n', + li_separator: str = '\n', ) -> str: return tmpl_render( template_name, values, li_quote = li_quote, - li_delimiter = li_delimiter, + li_delimiter = li_separator, search_path = self.app.args.search_path.split(':'), ) - def render_tmpl(self, module: str, extra_fields: RenderValues) -> str: + def render_tmpl( + self, + module: str, + extra_fields: RenderValues, + separator: str | None, + ) -> str: template_name = self.app.args.template_name if template_name is None: raise Exception('Can\'t render template without name') return self.__render( - template_name, [extra_fields], + template_name, + [extra_fields], li_quote = self.app.args.quote, - li_delimiter = ',\n' + li_separator = self.__format_separator(separator, ',\n'), ) - def render_pyright(self, module: str, extra_fields: RenderValues) -> str: + def render_pyright( + self, + module: str, + extra_fields: RenderValues, + separator: str | None, + ) -> str: + separator = self.__format_separator(separator, ',\n') + if separator != ',\n': + raise Exception(f'Unsupported separator for pyright config: "{separator}"') extra_paths = [] for m in self.__jw_required(include_self = True): path = self.app.find_dir(m, search_subdirs = ['src/python', 'tools/python']) @@ -94,7 +118,7 @@ class CmdCreateFile(Cmd): # export return self.__render( 'pyrightconfig.json', [values, extra_fields], li_quote = True, - li_delimiter = ',\n' + li_separator = ',\n' ) def __init__(self, parent: Parent) -> None: @@ -142,6 +166,13 @@ class CmdCreateFile(Cmd): # export 'rendering output' ) ) + parser.add_argument( + '--field-separator', + help = ( + 'Field separator. %%n expands to newline. Default value depends on ' + '--format' + ) + ) parser.add_argument('module', help = 'The module to generate the file for') @override @@ -166,4 +197,4 @@ class CmdCreateFile(Cmd): # export raise Exception(f'Unsupported output format {args.format}') # -- Render - sys.stdout.write(method(args.module, fields)) + sys.stdout.write(method(args.module, fields, args.field_separator)) diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index ae035045..5bd788a1 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -337,6 +337,7 @@ usage: jw-pkg.py projects create-file [-h] --format {pyright,tmpl} [--template-name TEMPLATE_NAME] [--quote] [-f KEY=VALUE] [--field-keys FIELD_KEYS] + [--field-separator FIELD_SEPARATOR] module Generate a file from project metadata @@ -365,6 +366,9 @@ options: not specified here is an error, and not passing one makes the respective field go away from the rendering output (default: None) + --field-separator FIELD_SEPARATOR + Field separator. %n expands to newline. Default value + depends on --format (default: None) ============= Running: jw-pkg.py -t ../../../.. --log-level info projects create-pkg-config --help usage: jw-pkg.py projects create-pkg-config [-h] [-F PROJECT_DESCR_FILE] [-d DESCRIPTION] [-n NAME] diff --git a/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/Makefile b/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/Makefile new file mode 100644 index 00000000..47708b2e --- /dev/null +++ b/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/Makefile @@ -0,0 +1,8 @@ +TOPDIR = ../../../../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: + +test: run diff --git a/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/test.py b/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/test.py new file mode 100644 index 00000000..c5000bd0 --- /dev/null +++ b/test/unit/python/jw/pkg/cmds/projects/CmdCreateFile/test.py @@ -0,0 +1,25 @@ +from jw.pkg.cmds.projects.CmdCreateFile import CmdCreateFile + +# __format_separator() is a private method that does not use self, so an +# uninitialised instance is a sufficient receiver for the bound method +inst = CmdCreateFile.__new__(CmdCreateFile) +fmt = getattr(inst, '_CmdCreateFile__format_separator') + +# -- __format_separator -- + +# A None separator falls back to the format's default +assert fmt(None, ',\n') == ',\n' + +# %n expands to a newline, wherever it appears in the separator +assert fmt('%n', ',\n') == '\n' +assert fmt(',%n', '\n') == ',\n' +assert fmt('a%n b', '') == 'a\n b' + +# Several %n expand independently +assert fmt('%n%n', '') == '\n\n' + +# Anything that is not %n passes through untouched +assert fmt('%', '') == '%' +assert fmt('%x', '') == '%x' + +print('All CmdCreateFile tests passed') -- 2.55.0 From 6fe8f5dc65d5289dd115175822d744e19495fccf Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sun, 13 Sep 2026 17:05:07 +0200 Subject: [PATCH 3/3] test/integration/projects/create-file: Add CmdCreateFile renders a file from project metadata, yet nothing runs it end to end. The "tmpl" format substitutes --field values into template markers, and the "pyright" format computes extra paths from the jw run dependencies of a module. Both formats are untested, and so is the handling of missing templates, unknown --field keys, and malformed command-line arguments. Add an integration test under test/integration/jw-pkg/projects that drives the command through the real CLI and diffs the rendered output against a reference. It covers single- and multi-value field substitution, the default and custom field separators including the %n newline escape, value quoting, and --field-keys filling in keys the caller omitted. The pyright case uses a relative topdir so the computed project paths stay relocatable. The error cases check that the command rejects a missing template name, an unknown template file, a --field key outside --field-keys, an unsupported pyright separator, and invalid --format and --field arguments with the expected message and exit status. Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.85.1 Signed-off-by: Jan Lindemann --- .../jw-pkg/projects/create-file/Makefile | 23 ++++ .../projects/create-file/templates/basic.tmpl | 2 + .../projects/create-file/templates/list.tmpl | 1 + .../projects/create-file/test-expected.txt | 55 +++++++++ .../jw-pkg/projects/create-file/test.sh | 108 ++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100644 test/integration/jw-pkg/projects/create-file/Makefile create mode 100644 test/integration/jw-pkg/projects/create-file/templates/basic.tmpl create mode 100644 test/integration/jw-pkg/projects/create-file/templates/list.tmpl create mode 100644 test/integration/jw-pkg/projects/create-file/test-expected.txt create mode 100644 test/integration/jw-pkg/projects/create-file/test.sh diff --git a/test/integration/jw-pkg/projects/create-file/Makefile b/test/integration/jw-pkg/projects/create-file/Makefile new file mode 100644 index 00000000..47a63b62 --- /dev/null +++ b/test/integration/jw-pkg/projects/create-file/Makefile @@ -0,0 +1,23 @@ +TOPDIR = ../../../../.. + +OUTPUT = test-out.txt +REFERENCE = test-expected.txt + +# Relative topdir keeps the rendered project paths relocatable; dropping the +# log position keeps the error output free of volatile line numbers. +TEST_CMD_ARGS = --topdir-format relative --log-flags prio,stderr + +include $(TOPDIR)/make/proj.mk +include $(TOPDIR)/make/test-jw-pkg.mk + +all: + +$(OUTPUT): Makefile test.sh templates/basic.tmpl templates/list.tmpl + bash ./test.sh $(TEST_CMD_LINE) > $(OUTPUT).tmp 2>&1 + diff $(REFERENCE) $(OUTPUT).tmp + mv $(OUTPUT).tmp $(OUTPUT) + +test: $(OUTPUT) +clean: test.integration.in-tree.clean +test.integration.in-tree.clean: + rm -f $(OUTPUT) $(OUTPUT).tmp diff --git a/test/integration/jw-pkg/projects/create-file/templates/basic.tmpl b/test/integration/jw-pkg/projects/create-file/templates/basic.tmpl new file mode 100644 index 00000000..e55fc8af --- /dev/null +++ b/test/integration/jw-pkg/projects/create-file/templates/basic.tmpl @@ -0,0 +1,2 @@ +name={name} +path={path} diff --git a/test/integration/jw-pkg/projects/create-file/templates/list.tmpl b/test/integration/jw-pkg/projects/create-file/templates/list.tmpl new file mode 100644 index 00000000..85d16b63 --- /dev/null +++ b/test/integration/jw-pkg/projects/create-file/templates/list.tmpl @@ -0,0 +1 @@ +start={items}end diff --git a/test/integration/jw-pkg/projects/create-file/test-expected.txt b/test/integration/jw-pkg/projects/create-file/test-expected.txt new file mode 100644 index 00000000..88e91be6 --- /dev/null +++ b/test/integration/jw-pkg/projects/create-file/test-expected.txt @@ -0,0 +1,55 @@ +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name basic.tmpl --search-path templates --field name=foo jw-pkg +name=foo +path={path} +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name list.tmpl --search-path templates --field items=a --field items=b jw-pkg +start=a, +start=bend +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name list.tmpl --search-path templates --field items=a --field items=b --field-separator | jw-pkg +start=a|bend +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name list.tmpl --search-path templates --field items=a --field items=b --field-separator %n jw-pkg +start=a +start=bend +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name list.tmpl --search-path templates --field items=a --field items=b --quote --field-separator | jw-pkg +start="a"|"b"end +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name basic.tmpl --search-path templates --field-keys name,path --field name=foo jw-pkg +name=foo +path= +============= Running: jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format pyright --field base=./conf/project/pyrightconfig-base.json --field include=src/python jw-pkg +{ + "extends": "./conf/project/pyrightconfig-base.json", + "include": [ + "src/python" + ], + "exclude": [ + "**/__pycache__", + "**/.pytest_cache", + "**/.mypy_cache", + "**/.ruff_cache", + "**/.venv", + "**/build", + "**/dist" + ], + "extraPaths": [ + "src/python" + ], + "typeCheckingMode": "basic", + "pythonPlatform": "Linux" +} +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl jw-pkg + Failed: Can't render template without name +============= Exit status: 1 +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name nosuch.tmpl --search-path templates jw-pkg + Failed: Failed to find template "nosuch.tmpl" +============= Exit status: 1 +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name basic.tmpl --search-path templates --field-keys name --field path=x jw-pkg + Failed: The following keys in --field are not in --field-keys: path +============= Exit status: 1 +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format pyright --field base=x --field include=y --field-separator | jw-pkg + Failed: Unsupported separator for pyright config: "|" +============= Exit status: 1 +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format bogus jw-pkg +jw-pkg.py projects create-file: error: argument --format: invalid choice: 'bogus' (choose from 'pyright', 'tmpl') +============= Exit status: 2 +============= Running (expect failure): jw-pkg.py -t ../../../../.. --log-level info --topdir-format relative --log-flags prio,stderr projects create-file --format tmpl --template-name basic.tmpl --search-path templates --field noval jw-pkg +jw-pkg.py projects create-file: error: argument -f/--field: Expected KEY=VALUE +============= Exit status: 2 diff --git a/test/integration/jw-pkg/projects/create-file/test.sh b/test/integration/jw-pkg/projects/create-file/test.sh new file mode 100644 index 00000000..58fb7a4f --- /dev/null +++ b/test/integration/jw-pkg/projects/create-file/test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# shellcheck disable=SC2048,SC2086 +# Unquoted $* is intentional — the jw-pkg command line + +export LC_ALL="C" +set -euo pipefail + +jw_pkg_py="$*" + +# Print a header and run a command whose output is part of the expected +# result. Under `set -e` a non-zero exit aborts the test. +run() +{ + local log_cmd + # shellcheck disable=SC2001 + log_cmd=$(echo "$*" | sed 's|.*python3[0-9.]*\s\+\(\.\.\/\)*scripts/||') + printf '============= Running: %s\n' "$log_cmd" + "$@" +} + +# Run a command that must exit non-zero and report the given message. Prints +# the command, the resulting error message, and the observed exit status. +# Fails the test if the command unexpectedly succeeds or the message is +# missing. Only the last output line is shown: for the app-level errors it is +# the error itself, and for argparse rejections it is the error line, which +# keeps the expected file free of the volatile usage text. +expect_fail() +{ + local expected out rc log_cmd + expected=$1 + shift + # shellcheck disable=SC2001 + log_cmd=$(echo "$*" | sed 's|.*python3[0-9.]*\s\+\(\.\.\/\)*scripts/||') + printf '============= Running (expect failure): %s\n' "$log_cmd" + set +e + out=$("$@" 2>&1) + rc=$? + set -e + printf '%s\n' "$(tail -n 1 <<<"$out")" + printf '============= Exit status: %s\n' "$rc" + if [ "$rc" -eq 0 ]; then + echo "ERROR: expected failure, but the command succeeded" >&2 + return 1 + fi + if ! grep -qF -- "$expected" <<<"$out"; then + echo "ERROR: expected message not found: $expected" >&2 + return 1 + fi +} + +cmd_test() +{ + # -- tmpl: substitute a field; a marker without a value stays literal + run $jw_pkg_py projects create-file --format tmpl \ + --template-name basic.tmpl --search-path templates \ + --field name=foo jw-pkg + + # -- tmpl: multi-value field, default separator (comma, newline) + run $jw_pkg_py projects create-file --format tmpl \ + --template-name list.tmpl --search-path templates \ + --field items=a --field items=b jw-pkg + + # -- tmpl: multi-value field, custom separator + run $jw_pkg_py projects create-file --format tmpl \ + --template-name list.tmpl --search-path templates \ + --field items=a --field items=b --field-separator '|' jw-pkg + + # -- tmpl: multi-value field, "%n" separator + run $jw_pkg_py projects create-file --format tmpl \ + --template-name list.tmpl --search-path templates \ + --field items=a --field items=b --field-separator '%n' jw-pkg + + # -- tmpl: multi-value field, quoted values + run $jw_pkg_py projects create-file --format tmpl \ + --template-name list.tmpl --search-path templates \ + --field items=a --field items=b --quote --field-separator '|' jw-pkg + + # -- tmpl: --field-keys adds keys that were not passed as empty fields + run $jw_pkg_py projects create-file --format tmpl \ + --template-name basic.tmpl --search-path templates \ + --field-keys name,path --field name=foo jw-pkg + + # -- pyright: built-in template, extra paths from project metadata + run $jw_pkg_py projects create-file --format pyright \ + --field base=./conf/project/pyrightconfig-base.json \ + --field include=src/python jw-pkg + + # -- errors + expect_fail "Can't render template without name" \ + $jw_pkg_py projects create-file --format tmpl jw-pkg + expect_fail 'Failed to find template "nosuch.tmpl"' \ + $jw_pkg_py projects create-file --format tmpl \ + --template-name nosuch.tmpl --search-path templates jw-pkg + expect_fail "not in --field-keys: path" \ + $jw_pkg_py projects create-file --format tmpl \ + --template-name basic.tmpl --search-path templates \ + --field-keys name --field path=x jw-pkg + expect_fail 'Unsupported separator for pyright config: "|"' \ + $jw_pkg_py projects create-file --format pyright \ + --field base=x --field include=y --field-separator '|' jw-pkg + expect_fail "invalid choice: 'bogus'" \ + $jw_pkg_py projects create-file --format bogus jw-pkg + expect_fail "Expected KEY=VALUE" \ + $jw_pkg_py projects create-file --format tmpl \ + --template-name basic.tmpl --search-path templates --field noval jw-pkg +} + +cmd_test -- 2.55.0