From 7b303c3aee1a8539b9c8b27e489c0a097af070b2 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 09:14:43 +0200 Subject: [PATCH 1/9] ldlibpath.mk: Don't include py-path.mk anylonger For backwards compatibility, ldlibpath.mk kept py-path.mk included. The projects depending on that have been fixed by now, this is no longer needed, don't include it anylonger, thereby reducing the complexity of the include graph. Signed-off-by: Jan Lindemann --- make/ldlibpath.mk | 2 -- 1 file changed, 2 deletions(-) diff --git a/make/ldlibpath.mk b/make/ldlibpath.mk index b0373d74..f11406cf 100644 --- a/make/ldlibpath.mk +++ b/make/ldlibpath.mk @@ -16,5 +16,3 @@ ifeq ($(origin JW_PKG_EXE_PATH),undefined) JW_PKG_EXE_PATH := $(call proj_query, exepath --delimiter ' ' $(PROJECT) $(PREREQ_RUN)) endif export PATH := $(subst $(space),:,$(JW_PKG_EXE_PATH)):$(EXE_SEARCH_PATH_ENV) - -include $(JWBDIR)/make/py-path.mk -- 2.55.0 From 150866015add49c0a08fd9caf2dfd8bc8984904a Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:36:23 +0200 Subject: [PATCH 2/9] py-defs.mk: Allow inclusion without defs.mk py-defs.mk uses the variables ECHO and SED defined in defs.mk. The complexity this introduces doesn't justify the reduced redundancy, though, so this commit defines them redundantly in py-defs.mk and to allow inclusion without prior defs.mk. Signed-off-by: Jan Lindemann --- make/py-defs.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/make/py-defs.mk b/make/py-defs.mk index 3f02d78c..b6068922 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -37,6 +37,8 @@ endif # deduce PY_INSTALL_DIR_PY from working directory below .. python/ ifeq ($(PY_INSTALL_DIR_PY),) + ECHO ?= echo + SED ?= sed PY_INSTALL_PKG_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) 's%.*/python/%%; s%/.*%%') PY_INSTALL_SUB_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) "s%.*/$(PY_INSTALL_PKG_MOD)\(/\|$$\)%%") ifneq ($(PY_INSTALL_SUB_MOD),) -- 2.55.0 From ee697d203d9f99e89cd0b83d94a89348c6aa0363 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 17 Jul 2026 12:57:17 +0200 Subject: [PATCH 3/9] py-check.mk: Add file Add py-check.mk and use it from py-topdir.mk and py-rules.mk. This removes redundant check definitions, making sure that that checks run from a repo's subdirectory match the checks run from its toplevel directory. Signed-off-by: Jan Lindemann --- make/py-check.mk | 89 +++++++++++++++++++++++++++++++++++++++++++++++ make/py-defs.mk | 2 -- make/py-rules.mk | 3 +- make/py-topdir.mk | 64 ++-------------------------------- 4 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 make/py-check.mk diff --git a/make/py-check.mk b/make/py-check.mk new file mode 100644 index 00000000..05687d7c --- /dev/null +++ b/make/py-check.mk @@ -0,0 +1,89 @@ +.PHONY: \ + all \ + check \ + format \ + check-syntax \ + check-format \ + py-check \ + py-check-syntax \ + py-check-format \ + py-format \ + py-format-assignments \ + py-check-annotation-imports \ + py-format-annotation-imports \ + clean \ + clean.py-check + +ifndef PY_CHECK_ROOTS + PY_CHECK_ROOTS = . +endif + +ifndef PY_CHECK_MYPY + PY_CHECK_MYPY := mypy +endif + +ifndef PY_CHECK_RUFF + PY_CHECK_RUFF := $(shell which ruff 2>/dev/null) + ifneq ($(PY_CHECK_RUFF),) + PY_CHECK_RUFF += --config $(TOPDIR)/pyproject.toml + endif +endif + +ifndef PY_CHECK_YAPF + PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3)) +endif + +ifndef PY_CHECK_PYRIGHT + PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null) + TD_GENERATE_FILES += pyrightconfig.json +endif + +all: +check: py-check +format: py-format +check-syntax: py-check-syntax +check-format: py-check-format + +py-check: py-check-syntax py-check-format + +py-check-syntax: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) +endif + $(PY_CHECK_MYPY) $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) +ifneq ($(PY_CHECK_PYRIGHT),) + $(PY_CHECK_PYRIGHT) $(PY_CHECK_ROOTS) +endif + +py-check-format: +ifneq ($(PY_CHECK_YAPF),) + $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) +endif + +py-format: + find . -type f -name '*.py' -print0 | \ + xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}' +ifneq ($(PY_CHECK_YAPF),) + $(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS) +endif + +py-format-assignments: + find $(PY_CHECK_ROOTS) \ + -path './.git' -prune -o \ + -type f -name '*.py' \ + -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+' + git diff --exit-code + +py-check-annotation-imports: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS) +endif + +py-format-annotation-imports: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) +endif + +clean: clean.py-check +clean.py-check: + rm -rf .mypy_cache .ruff_cache .pytest_cache diff --git a/make/py-defs.mk b/make/py-defs.mk index b6068922..f6db9459 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -19,8 +19,6 @@ else PY_SITE_PACKAGES_PATH := $(shell $(PYTHON) -c "import site; print([d for d in site.getsitepackages() if d.find('/local/') == -1][0])") endif -PY_MYPY ?= mypy --ignore-missing-imports --no-namespace-packages - PY_SRC_PY ?= $(wildcard *.py) PY_ALL_PY = $(PY_SRC_PY) diff --git a/make/py-rules.mk b/make/py-rules.mk index ce6f16a3..acf463ff 100644 --- a/make/py-rules.mk +++ b/make/py-rules.mk @@ -29,8 +29,7 @@ $(PY_INSTALL_DIR_PY)/%.pyc: %.pyc endif $(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@ -check: - $(PY_MYPY) $(shell /bin/bash $(JWB_SCRIPT_DIR)/scm.sh ls-files | grep '\.py$$') +include $(JWBDIR)/make/py-check.mk $(PY_INSTALL_DIR_PY)/py.typed: py.typed $(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@ diff --git a/make/py-topdir.mk b/make/py-topdir.mk index 65f170f3..c6b0f3ac 100644 --- a/make/py-topdir.mk +++ b/make/py-topdir.mk @@ -11,71 +11,9 @@ ifndef PY_CHECK_ROOTS PY_CHECK_ROOTS = $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) endif -ifndef PY_CHECK_RUFF - PY_CHECK_RUFF := $(shell which ruff 2>/dev/null) - ifneq ($(PY_CHECK_RUFF),) - PY_CHECK_RUFF += --config pyproject.toml - endif -endif - -ifndef PY_CHECK_YAPF - PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3)) -endif - -ifndef PY_CHECK_PYRIGHT - PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null) - TD_GENERATE_FILES += pyrightconfig.json -endif - all: - -format: py-format -check-syntax: py-check-syntax -check-format: py-check-format - -py-check: py-check-syntax py-check-format -py-check-syntax: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) -endif - mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) -ifneq ($(PY_CHECK_PYRIGHT),) - pyright $(PY_CHECK_ROOTS) -endif - -py-check-format: -ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) -endif - -py-format: - find . -type f -name '*.py' -print0 | \ - xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}' -ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS) -endif - -py-format-assignments: - find $(PY_CHECK_ROOTS) \ - -path './.git' -prune -o \ - -type f -name '*.py' \ - -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+' - git diff --exit-code - -py-check-annotation-imports: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS) -endif - -py-format-annotation-imports: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) -endif - clean: clean.topdir clean.topdir: clean.py-check -clean.py-check: - rm -rf .mypy_cache .ruff_cache .pytest_cache pyproject.toml: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \ @@ -86,3 +24,5 @@ pyrightconfig.json: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \ --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp mv $@.tmp $@ + +include $(JWBDIR)/make/py-check.mk -- 2.55.0 From 9fa654374d8d0c8363a8d8454feb987aeab4b3c3 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 09:16:39 +0200 Subject: [PATCH 4/9] py-path.mk: Add target py-path Add target py-path to py-path.mk. It's mostly there for documentation purposes, giving people and machines a defined, easily accessible and omni-present way to determine how Python imports should be resolved. Signed-off-by: Jan Lindemann --- make/py-path.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/make/py-path.mk b/make/py-path.mk index 11a115cd..64c03dab 100644 --- a/make/py-path.mk +++ b/make/py-path.mk @@ -11,3 +11,7 @@ all: clean: py-tools.clean py-tools.clean: $(RM) -rf $(wildcard *.pyc) __pycache__ .mypy_cache .ruff_cache .pytest_cache + +py-path: + @echo "PYTHONPATH=$(PYTHONPATH)" + @#echo "MYPYPATH=$(MYPYPATH)" -- 2.55.0 From b53fdd2fea75aadcebf87bcc5bc70210f7519a56 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 06:57:03 +0200 Subject: [PATCH 5/9] rules.mk / defs.mk: Disable BUILD_MAKEDIR BUILD_MAKEDIR is a variable which exists for consistency's sake. On the other hand, nor jw-pkg nor any downstream package ever copies makefile snippets during build time. The rule introduced by BUILD_MAKEDIR is quite costly in terms of performance and makes caching harder to understand. This commit disables the variable. Signed-off-by: Jan Lindemann --- make/defs.mk | 3 ++- make/rules.mk | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/make/defs.mk b/make/defs.mk index 5127b63b..2e497b86 100644 --- a/make/defs.mk +++ b/make/defs.mk @@ -405,7 +405,8 @@ INSTALLED_INIT += $(addprefix $(INSTALL_INITDIR)/,$(INIT_SCRIPTS)) # -- MAKE INSTALLATION_FILE_TYPES += MAKE -BUILD_MAKEDIR = $(TOPDIR)/make +#BUILD_MAKEDIR is currently not used anywhere and costs performance. Disabled for the time being. +#BUILD_MAKEDIR = $(TOPDIR)/make MKFILES += $(filter-out pckg-defs.mk pckg-deps.mk local.mk,$(filter-out $(DONT_INSTALL),$(wildcard *.mk))) INSTALL_MAKEDIR ?= $(PREFIX)/make INSTALLED_MAKE += $(addprefix $(INSTALL_MAKEDIR)/,$(MKFILES)) diff --git a/make/rules.mk b/make/rules.mk index bbdadda3..0179fbbb 100644 --- a/make/rules.mk +++ b/make/rules.mk @@ -390,12 +390,14 @@ $$(TOPDIR)/dir_install_$(1).done: mkdir -p $$(INSTALL_$(1)DIR) touch $$@ +ifneq ($$(BUILD_$(1)DIR),) $$(BUILD_$(1)DIR)/%: % | $$(TOPDIR)/dir_build_$(1).done $(Q)if [ ! $$< -ef $$@ -a "`echo $$< | $(SED) 's/\..*//'`" != local ]; then \ echo $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\ $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\ $(RM) -f $$(TOPDIR)/dirs-*.done ;\ fi +endif $$(INSTALL_$(1)DIR): ifeq ($(PACKAGE_INSTALL_DIR),true) -- 2.55.0 From 68cda8197e414370c2a36e4381dfdbd479b038c9 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:18:58 +0200 Subject: [PATCH 6/9] cmds.projects.CmdBuild: Support --dep-flavours CmdBuild caters for the needs of the targets all, clean and pkg-*. It uses builtin dependency flavours matching those targets to resolve build order. To make it more flexible in general and support more targets, e.g. check and test, add a --dep-flavours option. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/cmds/projects/CmdBuild.py | 17 ++++++-- .../integration/jw-pkg/help/test-expected.txt | 40 ++++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/python/jw/pkg/cmds/projects/CmdBuild.py b/src/python/jw/pkg/cmds/projects/CmdBuild.py index c8e51c0e..9f2f3f43 100644 --- a/src/python/jw/pkg/cmds/projects/CmdBuild.py +++ b/src/python/jw/pkg/cmds/projects/CmdBuild.py @@ -53,6 +53,14 @@ class CmdBuild(Cmd): # export 'on the command line' ), ) + parser.add_argument( + '--dep-flavours', + default = 'auto', + help = ( + 'Dependency flavours to take into consideration for build, ' + 'comma or space separated' + ) + ) parser.add_argument( '--env-reinit', action = 'store_true', @@ -247,9 +255,12 @@ class CmdBuild(Cmd): # export # -- build order: list[str] = [] - dep_flavours = ['build'] - if re.match('pkg-.*', target) is not None: - dep_flavours = ['build', 'run', 'release', 'devel'] + if args.dep_flavours != 'auto': + dep_flavours = re.split(r'[\s,]', args.dep_flavours) + else: + dep_flavours = ['build'] + if re.match('pkg-.*', target) is not None: + dep_flavours.extend(['run', 'release', 'devel']) if target != 'order' and not args.build_order: log(NOTICE, 'Using prerequisite flavours ' + ' '.join(dep_flavours)) diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 66b44f81..0ff7611a 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -254,31 +254,35 @@ Available subcommands of projects: tmpl-dir Print directory containing templates of a given module ============= Running: jw-pkg.py --log-level info projects build --help usage: jw-pkg.py projects build [-h] [--exclude EXCLUDE] [-n] [-O] [-I] - [--env-reinit] [--env-keep ENV_KEEP] + [--dep-flavours DEP_FLAVOURS] [--env-reinit] + [--env-keep ENV_KEEP] target modules [modules ...] janware software project build tool positional arguments: - target Build target - modules Modules to be built + target Build target + modules Modules to be built options: - -h, --help show this help message and exit - --exclude EXCLUDE Space seperated ist of modules to be excluded from - build (default: ) - -n, --dry-run Don't build anything, just print what would be done. - (default: False) - -O, --build-order Don't build anything, just print the build order. - (default: False) - -I, --ignore-deps Don't build dependencies, i.e. build only modules - specified on the command line (default: False) - --env-reinit Source /etc/profile before each build step. Discard - environment unless --env-keep is specified (default: - False) - --env-keep ENV_KEEP Comma seperated list of environment variables to keep, - "all" or "none", only meaningful if --env-reinit is - specified (default: none) + -h, --help show this help message and exit + --exclude EXCLUDE Space seperated ist of modules to be excluded from + build (default: ) + -n, --dry-run Don't build anything, just print what would be done. + (default: False) + -O, --build-order Don't build anything, just print the build order. + (default: False) + -I, --ignore-deps Don't build dependencies, i.e. build only modules + specified on the command line (default: False) + --dep-flavours DEP_FLAVOURS + Dependency flavours to take into consideration for + build, comma or space separated (default: auto) + --env-reinit Source /etc/profile before each build step. Discard + environment unless --env-keep is specified (default: + False) + --env-keep ENV_KEEP Comma seperated list of environment variables to keep, + "all" or "none", only meaningful if --env-reinit is + specified (default: none) ============= Running: jw-pkg.py --log-level info projects canonicalize-remotes --help usage: jw-pkg.py projects canonicalize-remotes [-h] [-n] -- 2.55.0 From 15c9863b95f598a67aea079ccf13a84cb70f9951 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:36:01 +0200 Subject: [PATCH 7/9] projects-dir.mk: Add target test projects-dir.mk has no test target, fix that. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index 96d0a2ee..c05d7898 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -155,7 +155,8 @@ PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(add # --- mandatory targets -all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) +all: +all test: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) $(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS) clean: clean-dirs distclean: clean-all-dirs done.clean -- 2.55.0 From ad6ae4a6f262752ea746a415fd651bef8ab55cc9 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sun, 12 Jul 2026 13:40:09 +0200 Subject: [PATCH 8/9] projects-dir.mk: Support check targets Add support for the targets check and check-post to projects-dir.mk to make them usable from inside the projects directory. Note that check-pre is intentionally left out: Running make check in a repo before its prerequisite repos have built their __init__.py files will fail due to broken import resolution. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index c05d7898..115a0063 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -156,7 +156,7 @@ PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(add # --- mandatory targets all: -all test: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) +all test check check-pre check-post: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) $(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS) clean: clean-dirs distclean: clean-all-dirs done.clean -- 2.55.0 From 9ff92bd541542ffa236468488f478985718c3fb6 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 14:47:49 +0200 Subject: [PATCH 9/9] install-files.mk: Add missing target "test" install-files.mk lacks a "test" target, so this commit adds a stub. Signed-off-by: Jan Lindemann --- make/install-files.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/make/install-files.mk b/make/install-files.mk index a2d898cf..af4addb8 100644 --- a/make/install-files.mk +++ b/make/install-files.mk @@ -25,6 +25,7 @@ all: install: install.done clean: done.clean distclean: +test: done.clean: $(RM) -f *.done -- 2.55.0