diff --git a/conf/templates/Makefile b/conf/templates/Makefile new file mode 100644 index 00000000..b1ec41b6 --- /dev/null +++ b/conf/templates/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/conf/topdir/pyproject.toml b/conf/templates/pyproject.toml similarity index 98% rename from conf/topdir/pyproject.toml rename to conf/templates/pyproject.toml index 88acff3d..a9f1e109 100644 --- a/conf/topdir/pyproject.toml +++ b/conf/templates/pyproject.toml @@ -1,3 +1,7 @@ +[tool.mypy] + + {mypypath} + [tool.isort] lines_between_sections = 1 diff --git a/make/defs.mk b/make/defs.mk index 1622367e..f8beacec 100644 --- a/make/defs.mk +++ b/make/defs.mk @@ -264,6 +264,7 @@ CVS_RSH ?= /usr/bin/ssh JW_PKG_DIR = $(JWBDIR) JW_PKG_CONF_BASE_DIR ?= $(firstword $(wildcard $(JW_PKG_DIR)/conf /etc/opt/jw-pkg)) JW_PKG_CONF_DIR ?= $(firstword $(wildcard $(JW_PKG_CONF_BASE_DIR)/project $(JW_PKG_CONF_BASE_DIR))) +JW_PKG_TMPL_DIR ?= $(JW_PKG_CONF_BASE_DIR)/templates PROJECT_STEM = $(PROJECT)-$(DIST_VERSION) # only works if checked out true to CVS, i.e. below proj diff --git a/make/py-topdir.mk b/make/py-topdir.mk index 4a298642..a58e1207 100644 --- a/make/py-topdir.mk +++ b/make/py-topdir.mk @@ -1,10 +1,21 @@ -TD_COPY_FILES += pyproject.toml +include $(JWBDIR)/make/ldlibpath.mk + +TD_GENERATE_FILES += pyproject.toml PY_CHECK_EXCLUDE ?= -PY_SRC_ROOT += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) + +MYPY_CONFIG_PATH = $(subst :,:$$MYPY_CONFIG_FILE_DIR/,:$(PYTHONPATH)) +MYPY_PATH_DIRECTIVE = mypy_path = "$(MYPY_CONFIG_PATH)" + +ifndef PY_CHECK_ROOTS + PY_CHECK_ROOTS += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) +endif ifndef PY_CHECK_RUFF PY_CHECK_RUFF := $(shell which ruff 2>/dev/null) + ifneq ($(PY_CHECK_RUFF),) + PY_CHECK_RUFF += --config pyproject.toml + endif endif ifndef PY_CHECK_YAPF @@ -25,27 +36,27 @@ check-format: py-check-format py-check: py-check-syntax py-check-format py-check-syntax: ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT) + $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) endif - mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_SRC_ROOT) + mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) ifneq ($(PY_CHECK_PYRIGHT),) - pyright + pyright $(PY_CHECK_ROOTS) endif py-check-format: ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --diff --recursive . + $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) endif py-format: find . -type f -name '*.py' -print0 | \ xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}' ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --in-place --recursive . + $(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS) endif py-format-assignments: - find . \ + find $(PY_CHECK_ROOTS) \ -path './.git' -prune -o \ -type f -name '*.py' \ -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+' @@ -53,19 +64,24 @@ py-format-assignments: py-check-annotation-imports: ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes . + $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS) endif py-format-annotation-imports: ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes . + $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) endif clean.topdir: clean.py-check clean.py-check: rm -rf .mypy_cache +pyproject.toml: + $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \ + --template-name $@ --search-path $(JW_PKG_CONF_BASE_DIR)/templates --field mypypath='$(MYPY_PATH_DIRECTIVE)' $(PROJECT) > $@.tmp + mv $@.tmp $@ + pyrightconfig.json: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \ - $(PROJECT) --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) > $@.tmp + --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp mv $@.tmp $@ diff --git a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py index 16a0f305..ce714b62 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCreateFile.py +++ b/src/python/jw/pkg/cmds/projects/CmdCreateFile.py @@ -1,11 +1,10 @@ from argparse import ArgumentParser, ArgumentTypeError, Namespace from enum import Enum, auto -from typing import Iterable, TypeAlias from ...lib.log import WARNING, log from .Cmd import Cmd, Parent from .lib.pkg_relations import VersionSyntax, pkg_relations -from .lib.templates import tmpl_render +from .lib.templates import ListDict, RenderValues, tmpl_render def key_value(s): try: @@ -18,27 +17,14 @@ def key_value(s): class Fmt(Enum): Pyright = auto() -TupleList: TypeAlias = Iterable[tuple[str, str]] -ListDict: TypeAlias = dict[str, list[str]] - class CmdCreateFile(Cmd): # export - def __tuple_list_to_dict(self, src: TupleList) -> ListDict: - ret: ListDict = {} - for key, val in src: - entry = ret.setdefault(key, []) - entry.append(val) - return ret - - def __update_dict(self, dst: ListDict, src: TupleList) -> ListDict: - ret = dst - for key, val in src: - entry = ret.setdefault(key, []) - entry.append(val) - return ret - - def render_pyright(self, module: str, extra_fields: TupleList) -> str: - prereq = pkg_relations( + def __jw_required(self, module: str | None = None) -> list[str]: + if module is None: + module = self.app.args.module + if module is None: + raise Exception('Can\'t get required packages without module name') + return pkg_relations( self.app, rel_type = 'requires', flavours = ['run'], @@ -52,9 +38,35 @@ class CmdCreateFile(Cmd): # export hide_jw_pkg = False, ) + def __render( + self, + template_name: str, + values: list[RenderValues], + li_quote = False, + li_delimiter = '\n', + ) -> str: + return tmpl_render( + template_name, + values, + li_quote = li_quote, + li_delimiter = li_delimiter, + search_path = self.app.args.search_path.split(':'), + ) + + def render_tmpl(self, module: str, extra_fields: RenderValues) -> str: + template_name = self.app.args.template_name + if template_name is None: + raise Exception('Can\'t render template without name') + return self.__render( + template_name, [extra_fields], + li_quote = self.app.args.quote, + li_delimiter = ',\n' + ) + + def render_pyright(self, module: str, extra_fields: RenderValues) -> str: extra_paths = [] - for m in prereq: - path = self.app.find_dir(m) + for m in self.__jw_required(): + path = self.app.find_dir(m, search_subdirs = ['src/python', 'tools/python']) if path is None: log(WARNING, f'No project directory for module "{m}"') continue @@ -62,10 +74,10 @@ class CmdCreateFile(Cmd): # export values: ListDict = { 'extra_paths': extra_paths, } - self.__update_dict(values, extra_fields) - - return tmpl_render( - 'pyrightconfig.json', values, li_quote = True, li_delimiter = ',\n' + return self.__render( + 'pyrightconfig.json', [values, extra_fields], + li_quote = True, + li_delimiter = ',\n' ) def __init__(self, parent: Parent) -> None: @@ -77,8 +89,21 @@ class CmdCreateFile(Cmd): # export super().add_arguments(parser) parser.add_argument( '--format', - choices = [fmt.name.lower() for fmt in Fmt], - help = 'Output format' + help = ( + 'Output format, for example: ' + ', '.join([fmt.name.lower() for fmt in Fmt]) + ) + ) + parser.add_argument( + '--search-path', + default = '/etc/opt/jw-pkg/templates', + help = 'Template search path, colon separated', + ) + parser.add_argument('--template-name', help = 'Template file name') + parser.add_argument( + '--quote', + action = 'store_true', + help = 'Enclose variable values in double quotes before substituting' ) parser.add_argument( '-f', diff --git a/src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py b/src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py index 372fabd4..30f075c6 100644 --- a/src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py +++ b/src/python/jw/pkg/cmds/projects/CmdCreatePkgConfig.py @@ -1,8 +1,9 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from .Cmd import Cmd, Parent from .lib.templates import tmpl_render -from typing import TYPE_CHECKING if TYPE_CHECKING: from argparse import ArgumentParser, Namespace @@ -61,24 +62,26 @@ class CmdCreatePkgConfig(Cmd): # export contents = tmpl_render( 'pkg-config', - { - 'prefix': args.prefix, - 'name': args.name, - 'description': merged['summary'], - 'version': args.version, - }, + [ + { + 'prefix': args.prefix, + 'name': args.name, + 'description': merged['summary'] or '', + 'version': args.version, + } + ] ) if args.cflags is not None: contents += f'Cflags: {args.cflags}\n' if args.libflags is not None: contents += f'Libs: {args.libflags}\n' - if merged['requires_run'] is not None: - contents += f'Requires: {self.__cleanup_requires(merged["requires_run"])}' - if merged['requires_build'] is not None: - contents += ( - f'Requires.private: {self.__cleanup_requires(merged["requires_build"])}' - ) + val = merged.get('requires_run') + if val is not None: + contents += f'Requires: {self.__cleanup_requires(val)}' + val = merged.get('requires_build') + if val is not None: + contents += (f'Requires.private: {self.__cleanup_requires(val)}') # not sure what to do with requires_devel print(contents) diff --git a/src/python/jw/pkg/cmds/projects/lib/templates.py b/src/python/jw/pkg/cmds/projects/lib/templates.py index eec74085..37fd3d2b 100644 --- a/src/python/jw/pkg/cmds/projects/lib/templates.py +++ b/src/python/jw/pkg/cmds/projects/lib/templates.py @@ -1,21 +1,98 @@ import textwrap +from typing import Iterable, TypeAlias, TypeGuard -def format_lines(template, values, li_quote, li_delimiter): +TupleList: TypeAlias = Iterable[tuple[str, str]] +ListDict: TypeAlias = dict[str, list[str]] +StrDict: TypeAlias = dict[str, str] +RenderValues: TypeAlias = ListDict | StrDict | TupleList + +def is_str_dict(values: RenderValues) -> TypeGuard[StrDict]: + if not isinstance(values, dict): + return False + for key, val in values.items(): + if not isinstance(key, str): + return False + if not isinstance(val, str): + return False + return True + +def is_list_dict(values: RenderValues) -> TypeGuard[ListDict]: + if not isinstance(values, dict): + return False + for key, val in values.items(): + if not isinstance(key, str): + return False + if isinstance(val, str): + continue + if not isinstance(val, list): + return False + for entry in val: + if not isinstance(entry, str): + return False + return True + +def is_tuple_list(values: RenderValues) -> TypeGuard[TupleList]: + if not isinstance(values, list): + return False + for item in values: + if not isinstance(item, tuple): + return False + if not len(item) == 2: + return False + if not isinstance(item[0], str): + return False + if not isinstance(item[1], str): + return False + return True + +def render_values_to_list_dict(values: RenderValues) -> ListDict: + + def __tuple_list_to_dict(src: TupleList) -> ListDict: + ret: ListDict = {} + for key, val in src: + entry = ret.setdefault(key, []) + entry.append(val) + return ret + + if is_list_dict(values): + return values + if is_tuple_list(values): + return __tuple_list_to_dict(values) + raise Exception('Unsupported template value layout') + +def merge_values(*values: RenderValues) -> ListDict: + ret: ListDict = {} + for rhs in values: + rhs_dict = render_values_to_list_dict(rhs) + for key, val in rhs_dict.items(): + entry = ret.setdefault(key, []) + if isinstance(val, list): + entry += val + else: + entry.append(val) + return ret + +def format_list_dict( + template: str, values: ListDict | dict[str, str], li_quote: bool, li_delimiter: str +) -> str: def __format_value(val): if not li_quote: return str(val) return f'"{val}"' + fmt_dict: dict[str, str] = {} for key, value in values.items(): if isinstance(value, (list, tuple)): - values[key] = li_delimiter.join(map(__format_value, value)) + fmt_dict[key] = li_delimiter.join(map(__format_value, value)) + elif isinstance(value, str): + fmt_dict[key] = value - ret = [] + ret: list[str] = [] parts = template.splitlines(keepends = True) for line in parts: - for key, value in values.items(): + for key, value in fmt_dict.items(): marker = '{' + key + '}' if marker in line: indent = line[:line.index(marker)] @@ -25,6 +102,11 @@ def format_lines(template, values, li_quote, li_delimiter): return ''.join(ret) +def format_lines( + template: str, values: list[RenderValues], li_quote: bool, li_delimiter: str +) -> str: + return format_list_dict(template, merge_values(*values), li_quote, li_delimiter) + _templates = { 'pkg-config': """\ @@ -63,11 +145,31 @@ _templates = { } def tmpl_render( - template_name: str, values, li_quote = False, li_delimiter = '\n' + template_name: str, + values: list[RenderValues], + li_quote = False, + li_delimiter = '\n', + search_path: list[str] = [] ) -> str: - return format_lines( - textwrap.dedent(_templates[template_name]), - values, - li_quote = li_quote, - li_delimiter = li_delimiter, - ) + + def __format(template: str) -> str: + return format_lines( + template, + values, + li_quote = li_quote, + li_delimiter = li_delimiter, + ) + + for d in search_path: + path = d + '/' + template_name + try: + with open(path, 'r') as f: + template = f.read() + return __format(template) + except FileNotFoundError: + pass + + raw = _templates.get(template_name, None) + if raw is None: + raise Exception(f'Failed to find template "{template_name}"') + return __format(textwrap.dedent(raw))