conf/templates/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/dirs.mk

conf/templates/pyproject.toml

@@ -1,3 +1,7 @@
+[tool.mypy]
+
+  {mypypath}
+
 [tool.isort]
 
   lines_between_sections = 1

make/defs.mk

@@ -264,6 +264,7 @@ CVS_RSH                     ?= /usr/bin/ssh
 JW_PKG_DIR                   = $(JWBDIR)
 JW_PKG_CONF_BASE_DIR        ?= $(firstword $(wildcard $(JW_PKG_DIR)/conf /etc/opt/jw-pkg))
 JW_PKG_CONF_DIR             ?= $(firstword $(wildcard $(JW_PKG_CONF_BASE_DIR)/project $(JW_PKG_CONF_BASE_DIR)))
+JW_PKG_TMPL_DIR             ?= $(JW_PKG_CONF_BASE_DIR)/templates
 PROJECT_STEM                 = $(PROJECT)-$(DIST_VERSION)
 
 # only works if checked out true to CVS, i.e. below proj

make/py-topdir.mk

@@ -1,10 +1,21 @@
-TD_COPY_FILES               += pyproject.toml
+include $(JWBDIR)/make/ldlibpath.mk
+
+TD_GENERATE_FILES           += pyproject.toml
 
 PY_CHECK_EXCLUDE            ?=
-PY_SRC_ROOT                 += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
+
+MYPY_CONFIG_PATH             = $(subst :,:$$MYPY_CONFIG_FILE_DIR/,:$(PYTHONPATH))
+MYPY_PATH_DIRECTIVE          = mypy_path = "$(MYPY_CONFIG_PATH)"
+
+ifndef PY_CHECK_ROOTS
+  PY_CHECK_ROOTS            += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
+endif
 
 ifndef PY_CHECK_RUFF
   PY_CHECK_RUFF             := $(shell which ruff 2>/dev/null)
+  ifneq ($(PY_CHECK_RUFF),)
+    PY_CHECK_RUFF           += --config pyproject.toml
+  endif
 endif
 
 ifndef PY_CHECK_YAPF
@@ -25,27 +36,27 @@ check-format: py-check-format
 py-check: py-check-syntax py-check-format
 py-check-syntax:
 ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT)
+	$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
 endif
-	mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT)
+	mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
 ifneq ($(PY_CHECK_PYRIGHT),)
-	pyright
+	pyright $(PY_CHECK_ROOTS)
 endif
 
 py-check-format:
 ifneq ($(PY_CHECK_YAPF),)
-	$(PY_CHECK_YAPF) --diff --recursive .
+	$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
 endif
 
 py-format:
 	find . -type f -name '*.py' -print0 | \
 		xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
 ifneq ($(PY_CHECK_YAPF),)
-	$(PY_CHECK_YAPF) --in-place --recursive .
+	$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
 endif
 
 py-format-assignments:
-	find . \
+	find $(PY_CHECK_ROOTS) \
 	  -path './.git' -prune -o \
 	  -type f -name '*.py' \
 	  -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
@@ -53,19 +64,24 @@ py-format-assignments:
 
 py-check-annotation-imports:
 ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes .
+	$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
 endif
 
 py-format-annotation-imports:
 ifneq ($(PY_CHECK_RUFF),)
-	$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes .
+	$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
 endif
 
 clean.topdir: clean.py-check
 clean.py-check:
 	rm -rf .mypy_cache
 
+pyproject.toml:
+	$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \
+		--template-name $@ --search-path $(JW_PKG_CONF_BASE_DIR)/templates --field mypypath='$(MYPY_PATH_DIRECTIVE)' $(PROJECT) > $@.tmp
+	mv $@.tmp $@
+
 pyrightconfig.json:
 	$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \
-		$(PROJECT) --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) > $@.tmp
+		--field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp
 	mv $@.tmp $@

src/python/jw/pkg/cmds/projects/CmdCreateFile.py

@@ -1,11 +1,10 @@
 from argparse import ArgumentParser, ArgumentTypeError, Namespace
 from enum import Enum, auto
-from typing import Iterable, TypeAlias
 
 from ...lib.log import WARNING, log
 from .Cmd import Cmd, Parent
 from .lib.pkg_relations import VersionSyntax, pkg_relations
-from .lib.templates import tmpl_render
+from .lib.templates import ListDict, RenderValues, tmpl_render
 
 def key_value(s):
     try:
@@ -18,27 +17,14 @@ def key_value(s):
 class Fmt(Enum):
     Pyright = auto()
 
-TupleList: TypeAlias = Iterable[tuple[str, str]]
-ListDict: TypeAlias = dict[str, list[str]]
-
 class CmdCreateFile(Cmd):  # export
 
-    def __tuple_list_to_dict(self, src: TupleList) -> ListDict:
+    def __jw_required(self, module: str | None = None) -> list[str]:
-        ret: ListDict = {}
+        if module is None:
-        for key, val in src:
+            module = self.app.args.module
-            entry = ret.setdefault(key, [])
+        if module is None:
-            entry.append(val)
+            raise Exception('Can\'t get required packages without module name')
-        return ret
+        return pkg_relations(
-
-    def __update_dict(self, dst: ListDict, src: TupleList) -> ListDict:
-        ret = dst
-        for key, val in src:
-            entry = ret.setdefault(key, [])
-            entry.append(val)
-        return ret
-
-    def render_pyright(self, module: str, extra_fields: TupleList) -> str:
-        prereq = pkg_relations(
             self.app,
             rel_type = 'requires',
             flavours = ['run'],
@@ -52,9 +38,35 @@ class CmdCreateFile(Cmd):  # export
             hide_jw_pkg = False,
         )
 
+    def __render(
+        self,
+        template_name: str,
+        values: list[RenderValues],
+        li_quote = False,
+        li_delimiter = '\n',
+    ) -> str:
+        return tmpl_render(
+            template_name,
+            values,
+            li_quote = li_quote,
+            li_delimiter = li_delimiter,
+            search_path = self.app.args.search_path.split(':'),
+        )
+
+    def render_tmpl(self, module: str, extra_fields: RenderValues) -> str:
+        template_name = self.app.args.template_name
+        if template_name is None:
+            raise Exception('Can\'t render template without name')
+        return self.__render(
+            template_name, [extra_fields],
+            li_quote = self.app.args.quote,
+            li_delimiter = ',\n'
+        )
+
+    def render_pyright(self, module: str, extra_fields: RenderValues) -> str:
         extra_paths = []
-        for m in prereq:
+        for m in self.__jw_required():
-            path = self.app.find_dir(m)
+            path = self.app.find_dir(m, search_subdirs = ['src/python', 'tools/python'])
             if path is None:
                 log(WARNING, f'No project directory for module "{m}"')
                 continue
@@ -62,10 +74,10 @@ class CmdCreateFile(Cmd):  # export
         values: ListDict = {
             'extra_paths': extra_paths,
         }
-        self.__update_dict(values, extra_fields)
+        return self.__render(
-
+            'pyrightconfig.json', [values, extra_fields],
-        return tmpl_render(
+            li_quote = True,
-            'pyrightconfig.json', values, li_quote = True, li_delimiter = ',\n'
+            li_delimiter = ',\n'
         )
 
     def __init__(self, parent: Parent) -> None:
@@ -77,8 +89,21 @@ class CmdCreateFile(Cmd):  # export
         super().add_arguments(parser)
         parser.add_argument(
             '--format',
-            choices = [fmt.name.lower() for fmt in Fmt],
+            help = (
-            help = 'Output format'
+                'Output format, for example: '
+                ', '.join([fmt.name.lower() for fmt in Fmt])
+            )
+        )
+        parser.add_argument(
+            '--search-path',
+            default = '/etc/opt/jw-pkg/templates',
+            help = 'Template search path, colon separated',
+        )
+        parser.add_argument('--template-name', help = 'Template file name')
+        parser.add_argument(
+            '--quote',
+            action = 'store_true',
+            help = 'Enclose variable values in double quotes before substituting'
         )
         parser.add_argument(
             '-f',

src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from .Cmd import Cmd, Parent
 from .lib.templates import tmpl_render
-from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
@@ -61,24 +62,26 @@ class CmdCreatePkgConfig(Cmd):  # export
 
         contents = tmpl_render(
             'pkg-config',
-            {
+            [
-                'prefix': args.prefix,
+                {
-                'name': args.name,
+                    'prefix': args.prefix,
-                'description': merged['summary'],
+                    'name': args.name,
-                'version': args.version,
+                    'description': merged['summary'] or '',
-            },
+                    'version': args.version,
+                }
+            ]
         )
 
         if args.cflags is not None:
             contents += f'Cflags: {args.cflags}\n'
         if args.libflags is not None:
             contents += f'Libs: {args.libflags}\n'
-        if merged['requires_run'] is not None:
+        val = merged.get('requires_run')
-            contents += f'Requires: {self.__cleanup_requires(merged["requires_run"])}'
+        if val is not None:
-        if merged['requires_build'] is not None:
+            contents += f'Requires: {self.__cleanup_requires(val)}'
-            contents += (
+        val = merged.get('requires_build')
-                f'Requires.private: {self.__cleanup_requires(merged["requires_build"])}'
+        if val is not None:
-            )
+            contents += (f'Requires.private: {self.__cleanup_requires(val)}')
         # not sure what to do with requires_devel
 
         print(contents)

src/python/jw/pkg/cmds/projects/lib/templates.py

@@ -1,21 +1,98 @@
 import textwrap
+from typing import Iterable, TypeAlias, TypeGuard
 
-def format_lines(template, values, li_quote, li_delimiter):
+TupleList: TypeAlias = Iterable[tuple[str, str]]
+ListDict: TypeAlias = dict[str, list[str]]
+StrDict: TypeAlias = dict[str, str]
+RenderValues: TypeAlias = ListDict | StrDict | TupleList
+
+def is_str_dict(values: RenderValues) -> TypeGuard[StrDict]:
+    if not isinstance(values, dict):
+        return False
+    for key, val in values.items():
+        if not isinstance(key, str):
+            return False
+        if not isinstance(val, str):
+            return False
+    return True
+
+def is_list_dict(values: RenderValues) -> TypeGuard[ListDict]:
+    if not isinstance(values, dict):
+        return False
+    for key, val in values.items():
+        if not isinstance(key, str):
+            return False
+        if isinstance(val, str):
+            continue
+        if not isinstance(val, list):
+            return False
+        for entry in val:
+            if not isinstance(entry, str):
+                return False
+    return True
+
+def is_tuple_list(values: RenderValues) -> TypeGuard[TupleList]:
+    if not isinstance(values, list):
+        return False
+    for item in values:
+        if not isinstance(item, tuple):
+            return False
+        if not len(item) == 2:
+            return False
+        if not isinstance(item[0], str):
+            return False
+        if not isinstance(item[1], str):
+            return False
+    return True
+
+def render_values_to_list_dict(values: RenderValues) -> ListDict:
+
+    def __tuple_list_to_dict(src: TupleList) -> ListDict:
+        ret: ListDict = {}
+        for key, val in src:
+            entry = ret.setdefault(key, [])
+            entry.append(val)
+        return ret
+
+    if is_list_dict(values):
+        return values
+    if is_tuple_list(values):
+        return __tuple_list_to_dict(values)
+    raise Exception('Unsupported template value layout')
+
+def merge_values(*values: RenderValues) -> ListDict:
+    ret: ListDict = {}
+    for rhs in values:
+        rhs_dict = render_values_to_list_dict(rhs)
+        for key, val in rhs_dict.items():
+            entry = ret.setdefault(key, [])
+            if isinstance(val, list):
+                entry += val
+            else:
+                entry.append(val)
+    return ret
+
+def format_list_dict(
+    template: str, values: ListDict | dict[str, str], li_quote: bool, li_delimiter: str
+) -> str:
 
     def __format_value(val):
         if not li_quote:
             return str(val)
         return f'"{val}"'
 
+    fmt_dict: dict[str, str] = {}
     for key, value in values.items():
         if isinstance(value, (list, tuple)):
-            values[key] = li_delimiter.join(map(__format_value, value))
+            fmt_dict[key] = li_delimiter.join(map(__format_value, value))
+        elif isinstance(value, str):
+            fmt_dict[key] = value
 
-    ret = []
+    ret: list[str] = []
     parts = template.splitlines(keepends = True)
 
     for line in parts:
-        for key, value in values.items():
+        for key, value in fmt_dict.items():
             marker = '{' + key + '}'
             if marker in line:
                 indent = line[:line.index(marker)]
@@ -25,6 +102,11 @@ def format_lines(template, values, li_quote, li_delimiter):
 
     return ''.join(ret)
 
+def format_lines(
+    template: str, values: list[RenderValues], li_quote: bool, li_delimiter: str
+) -> str:
+    return format_list_dict(template, merge_values(*values), li_quote, li_delimiter)
+
 _templates = {
     'pkg-config':
     """\
@@ -63,11 +145,31 @@ _templates = {
 }
 
 def tmpl_render(
-    template_name: str, values, li_quote = False, li_delimiter = '\n'
+    template_name: str,
+    values: list[RenderValues],
+    li_quote = False,
+    li_delimiter = '\n',
+    search_path: list[str] = []
 ) -> str:
-    return format_lines(
+
-        textwrap.dedent(_templates[template_name]),
+    def __format(template: str) -> str:
-        values,
+        return format_lines(
-        li_quote = li_quote,
+            template,
-        li_delimiter = li_delimiter,
+            values,
-    )
+            li_quote = li_quote,
+            li_delimiter = li_delimiter,
+        )
+
+    for d in search_path:
+        path = d + '/' + template_name
+        try:
+            with open(path, 'r') as f:
+                template = f.read()
+                return __format(template)
+        except FileNotFoundError:
+            pass
+
+    raw = _templates.get(template_name, None)
+    if raw is None:
+        raise Exception(f'Failed to find template "{template_name}"')
+    return __format(textwrap.dedent(raw))