janware/jw-pkg
1
Code Pull requests Activity Actions

Improve Python config file template substitution #8

Merged
Jan Lindemann merged 10 commits from jan/feature/20260609-pyproject-toml-add-mypypath into master 2026-06-09 08:13:09 +02:00 AGit
Conversation 0 Commits 10 Files changed 7 +219 -64
7 changed files with 219 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

4
conf/templates/Makefile Normal file
View file

@ -0,0 +1,4 @@
TOPDIR = ../..
include $(TOPDIR)/make/proj.mk
include $(JWBDIR)/make/dirs.mk

4
conf/topdir/pyproject.toml → conf/templates/pyproject.toml
View file

@ -1,3 +1,7 @@
[tool.mypy]
{mypypath}
[tool.isort]
lines_between_sections = 1

1
make/defs.mk
View file

@ -264,6 +264,7 @@ CVS_RSH ?= /usr/bin/ssh
JW_PKG_DIR = $(JWBDIR)
JW_PKG_CONF_BASE_DIR ?= $(firstword $(wildcard $(JW_PKG_DIR)/conf /etc/opt/jw-pkg))
JW_PKG_CONF_DIR ?= $(firstword $(wildcard $(JW_PKG_CONF_BASE_DIR)/project $(JW_PKG_CONF_BASE_DIR)))
JW_PKG_TMPL_DIR ?= $(JW_PKG_CONF_BASE_DIR)/templates
PROJECT_STEM = $(PROJECT)-$(DIST_VERSION)
# only works if checked out true to CVS, i.e. below proj

38
make/py-topdir.mk
View file

@ -1,10 +1,21 @@
TD_COPY_FILES += pyproject.toml
include $(JWBDIR)/make/ldlibpath.mk
TD_GENERATE_FILES += pyproject.toml
PY_CHECK_EXCLUDE ?=
PY_SRC_ROOT += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
MYPY_CONFIG_PATH = $(subst :,:$$MYPY_CONFIG_FILE_DIR/,:$(PYTHONPATH))
MYPY_PATH_DIRECTIVE = mypy_path = "$(MYPY_CONFIG_PATH)"
ifndef PY_CHECK_ROOTS
PY_CHECK_ROOTS += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools)
endif
ifndef PY_CHECK_RUFF
PY_CHECK_RUFF := $(shell which ruff 2>/dev/null)
ifneq ($(PY_CHECK_RUFF),)
PY_CHECK_RUFF += --config pyproject.toml
endif
endif
ifndef PY_CHECK_YAPF
@ -25,27 +36,27 @@ check-format: py-check-format
py-check: py-check-syntax py-check-format
py-check-syntax:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT)
$(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
endif
mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT)
mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS)
ifneq ($(PY_CHECK_PYRIGHT),)
pyright
pyright $(PY_CHECK_ROOTS)
endif
py-check-format:
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --diff --recursive .
$(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS)
endif
py-format:
find . -type f -name '*.py' -print0 | \
xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}'
ifneq ($(PY_CHECK_YAPF),)
$(PY_CHECK_YAPF) --in-place --recursive .
$(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS)
endif
py-format-assignments:
find . \
find $(PY_CHECK_ROOTS) \
-path './.git' -prune -o \
-type f -name '*.py' \
-execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+'
@ -53,19 +64,24 @@ py-format-assignments:
py-check-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes .
$(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS)
endif
py-format-annotation-imports:
ifneq ($(PY_CHECK_RUFF),)
$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes .
$(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS)
endif
clean.topdir: clean.py-check
clean.py-check:
rm -rf .mypy_cache
pyproject.toml:
$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \
--template-name $@ --search-path $(JW_PKG_CONF_BASE_DIR)/templates --field mypypath='$(MYPY_PATH_DIRECTIVE)' $(PROJECT) > $@.tmp
mv $@.tmp $@
pyrightconfig.json:
$(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \
$(PROJECT) --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) > $@.tmp
--field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp
mv $@.tmp $@

83
src/python/jw/pkg/cmds/projects/CmdCreateFile.py
View file

@ -1,11 +1,10 @@
from argparse import ArgumentParser, ArgumentTypeError, Namespace
from enum import Enum, auto
from typing import Iterable, TypeAlias
from ...lib.log import WARNING, log
from .Cmd import Cmd, Parent
from .lib.pkg_relations import VersionSyntax, pkg_relations
from .lib.templates import tmpl_render
from .lib.templates import ListDict, RenderValues, tmpl_render
def key_value(s):
try:
@ -18,27 +17,14 @@ def key_value(s):
class Fmt(Enum):
Pyright = auto()
TupleList: TypeAlias = Iterable[tuple[str, str]]
ListDict: TypeAlias = dict[str, list[str]]
class CmdCreateFile(Cmd): # export
def __tuple_list_to_dict(self, src: TupleList) -> ListDict:
ret: ListDict = {}
for key, val in src:
entry = ret.setdefault(key, [])
entry.append(val)
return ret
def __update_dict(self, dst: ListDict, src: TupleList) -> ListDict:
ret = dst
for key, val in src:
entry = ret.setdefault(key, [])
entry.append(val)
return ret
def render_pyright(self, module: str, extra_fields: TupleList) -> str:
prereq = pkg_relations(
def __jw_required(self, module: str | None = None) -> list[str]:
if module is None:
module = self.app.args.module
if module is None:
raise Exception('Can\'t get required packages without module name')
return pkg_relations(
self.app,
rel_type = 'requires',
flavours = ['run'],
@ -52,9 +38,35 @@ class CmdCreateFile(Cmd): # export
hide_jw_pkg = False,
)
def __render(
self,
template_name: str,
values: list[RenderValues],
li_quote = False,
li_delimiter = '\n',
) -> str:
return tmpl_render(
template_name,
values,
li_quote = li_quote,
li_delimiter = li_delimiter,
search_path = self.app.args.search_path.split(':'),
)
def render_tmpl(self, module: str, extra_fields: RenderValues) -> str:
template_name = self.app.args.template_name
if template_name is None:
raise Exception('Can\'t render template without name')
return self.__render(
template_name, [extra_fields],
li_quote = self.app.args.quote,
li_delimiter = ',\n'
)
def render_pyright(self, module: str, extra_fields: RenderValues) -> str:
extra_paths = []
for m in prereq:
path = self.app.find_dir(m)
for m in self.__jw_required():
path = self.app.find_dir(m, search_subdirs = ['src/python', 'tools/python'])
if path is None:
log(WARNING, f'No project directory for module "{m}"')
continue
@ -62,10 +74,10 @@ class CmdCreateFile(Cmd): # export
values: ListDict = {
'extra_paths': extra_paths,
}
self.__update_dict(values, extra_fields)
return tmpl_render(
'pyrightconfig.json', values, li_quote = True, li_delimiter = ',\n'
return self.__render(
'pyrightconfig.json', [values, extra_fields],
li_quote = True,
li_delimiter = ',\n'
)
def __init__(self, parent: Parent) -> None:
@ -77,8 +89,21 @@ class CmdCreateFile(Cmd): # export
super().add_arguments(parser)
parser.add_argument(
'--format',
choices = [fmt.name.lower() for fmt in Fmt],
help = 'Output format'
help = (
'Output format, for example: '
', '.join([fmt.name.lower() for fmt in Fmt])
)
)
parser.add_argument(
'--search-path',
default = '/etc/opt/jw-pkg/templates',
help = 'Template search path, colon separated',
)
parser.add_argument('--template-name', help = 'Template file name')
parser.add_argument(
'--quote',
action = 'store_true',
help = 'Enclose variable values in double quotes before substituting'
)
parser.add_argument(
'-f',

21
src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py
View file

@ -1,8 +1,9 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from .Cmd import Cmd, Parent
from .lib.templates import tmpl_render
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from argparse import ArgumentParser, Namespace
@ -61,24 +62,26 @@ class CmdCreatePkgConfig(Cmd): # export
contents = tmpl_render(
'pkg-config',
[
{
'prefix': args.prefix,
'name': args.name,
'description': merged['summary'],
'description': merged['summary'] or '',
'version': args.version,
},
}
]
)
if args.cflags is not None:
contents += f'Cflags: {args.cflags}\n'
if args.libflags is not None:
contents += f'Libs: {args.libflags}\n'
if merged['requires_run'] is not None:
contents += f'Requires: {self.__cleanup_requires(merged["requires_run"])}'
if merged['requires_build'] is not None:
contents += (
f'Requires.private: {self.__cleanup_requires(merged["requires_build"])}'
)
val = merged.get('requires_run')
if val is not None:
contents += f'Requires: {self.__cleanup_requires(val)}'
val = merged.get('requires_build')
if val is not None:
contents += (f'Requires.private: {self.__cleanup_requires(val)}')
# not sure what to do with requires_devel
print(contents)

114
src/python/jw/pkg/cmds/projects/lib/templates.py
View file

@ -1,21 +1,98 @@
import textwrap
from typing import Iterable, TypeAlias, TypeGuard
def format_lines(template, values, li_quote, li_delimiter):
TupleList: TypeAlias = Iterable[tuple[str, str]]
ListDict: TypeAlias = dict[str, list[str]]
StrDict: TypeAlias = dict[str, str]
RenderValues: TypeAlias = ListDict | StrDict | TupleList
def is_str_dict(values: RenderValues) -> TypeGuard[StrDict]:
if not isinstance(values, dict):
return False
for key, val in values.items():
if not isinstance(key, str):
return False
if not isinstance(val, str):
return False
return True
def is_list_dict(values: RenderValues) -> TypeGuard[ListDict]:
if not isinstance(values, dict):
return False
for key, val in values.items():
if not isinstance(key, str):
return False
if isinstance(val, str):
continue
if not isinstance(val, list):
return False
for entry in val:
if not isinstance(entry, str):
return False
return True
def is_tuple_list(values: RenderValues) -> TypeGuard[TupleList]:
if not isinstance(values, list):
return False
for item in values:
if not isinstance(item, tuple):
return False
if not len(item) == 2:
return False
if not isinstance(item[0], str):
return False
if not isinstance(item[1], str):
return False
return True
def render_values_to_list_dict(values: RenderValues) -> ListDict:
def __tuple_list_to_dict(src: TupleList) -> ListDict:
ret: ListDict = {}
for key, val in src:
entry = ret.setdefault(key, [])
entry.append(val)
return ret
if is_list_dict(values):
return values
if is_tuple_list(values):
return __tuple_list_to_dict(values)
raise Exception('Unsupported template value layout')
def merge_values(*values: RenderValues) -> ListDict:
ret: ListDict = {}
for rhs in values:
rhs_dict = render_values_to_list_dict(rhs)
for key, val in rhs_dict.items():
entry = ret.setdefault(key, [])
if isinstance(val, list):
entry += val
else:
entry.append(val)
return ret
def format_list_dict(
template: str, values: ListDict | dict[str, str], li_quote: bool, li_delimiter: str
) -> str:
def __format_value(val):
if not li_quote:
return str(val)
return f'"{val}"'
fmt_dict: dict[str, str] = {}
for key, value in values.items():
if isinstance(value, (list, tuple)):
values[key] = li_delimiter.join(map(__format_value, value))
fmt_dict[key] = li_delimiter.join(map(__format_value, value))
elif isinstance(value, str):
fmt_dict[key] = value
ret = []
ret: list[str] = []
parts = template.splitlines(keepends = True)
for line in parts:
for key, value in values.items():
for key, value in fmt_dict.items():
marker = '{' + key + '}'
if marker in line:
indent = line[:line.index(marker)]
@ -25,6 +102,11 @@ def format_lines(template, values, li_quote, li_delimiter):
return ''.join(ret)
def format_lines(
template: str, values: list[RenderValues], li_quote: bool, li_delimiter: str
) -> str:
return format_list_dict(template, merge_values(*values), li_quote, li_delimiter)
_templates = {
'pkg-config':
"""\
@ -63,11 +145,31 @@ _templates = {
}
def tmpl_render(
template_name: str, values, li_quote = False, li_delimiter = '\n'
template_name: str,
values: list[RenderValues],
li_quote = False,
li_delimiter = '\n',
search_path: list[str] = []
) -> str:
def __format(template: str) -> str:
return format_lines(
textwrap.dedent(_templates[template_name]),
template,
values,
li_quote = li_quote,
li_delimiter = li_delimiter,
)
for d in search_path:
path = d + '/' + template_name
try:
with open(path, 'r') as f:
template = f.read()
return __format(template)
except FileNotFoundError:
pass
raw = _templates.get(template_name, None)
if raw is None:
raise Exception(f'Failed to find template "{template_name}"')
return __format(textwrap.dedent(raw))