From 701d8a33e76b757ae642d3aa5be1185ffdc9afb6 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 24 Jul 2026 09:59:42 +0000 Subject: [PATCH 01/16] lib.ec.ssh.Paramiko: Tolerate import-not-found Tolera$te missing paramiko imports. jw-pkg is designed to work with what it finds and use plain /usr/bin/ssh if need be. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/ec/ssh/Paramiko.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py index 018b6796..09e59358 100644 --- a/src/python/jw/pkg/lib/ec/ssh/Paramiko.py +++ b/src/python/jw/pkg/lib/ec/ssh/Paramiko.py @@ -2,9 +2,11 @@ from __future__ import annotations from typing import TYPE_CHECKING -import paramiko # type: ignore[import-untyped] # error: Library stubs not installed for "paramiko" -import paramiko.agent # type: ignore[import-untyped] -import paramiko.SCPClient # type: ignore[import-untyped] +# Tolerate missing paramiko imports. jw-pkg is designed to work with what it +# finds. +import paramiko # type: ignore[import-untyped,import-not-found] # error: Library stubs not installed for "paramiko" +import paramiko.agent # type: ignore[import-untyped,import-not-found] +import paramiko.SCPClient # type: ignore[import-untyped,import-not-found] from ...base import Result from ...log import ERR, log -- 2.55.0 From 1062be24db0f85dbc12fcfb60a7ef8b492864f92 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 24 Jul 2026 12:41:58 +0200 Subject: [PATCH 02/16] App: Remove hardcoded local/src from App and defs.mk local/src is a janware-specific path, remove it from App and defs.mk. It's still in pkg.sh as a safety measure. Will have to go, too, but is kept in for now until further audit. Signed-off-by: Jan Lindemann --- make/defs.mk | 1 - make/test-jw-pkg.mk | 1 + src/python/jw/pkg/App.py | 9 +- .../integration/jw-pkg/help/test-expected.txt | 102 +++++++++--------- 4 files changed, 58 insertions(+), 55 deletions(-) diff --git a/make/defs.mk b/make/defs.mk index 5127b63b..8b416b55 100644 --- a/make/defs.mk +++ b/make/defs.mk @@ -282,7 +282,6 @@ ifeq ($(VCS),cvs) endif REV_SUBDIRS = $(call reverse $(SUBDIRS)) -FRESH_CVSDIR = $(HOME)/local/src/cvs.fresh PCKG_DEFS_DIR = $(JWBDIR)/make/defs.d HDRDIR_SCOPE_SUFFIX ?= $(PROJECT) diff --git a/make/test-jw-pkg.mk b/make/test-jw-pkg.mk index b13538a2..32f1d651 100644 --- a/make/test-jw-pkg.mk +++ b/make/test-jw-pkg.mk @@ -1,6 +1,7 @@ EXE ?= $(TOPDIR)/scripts/jw-pkg.py LOG_LEVEL ?= info +TEST_CMD_GLOBAL_OPTS ?= -t $(TOPDIR) ifneq ($(LOG_LEVEL),) TEST_OPTS_LOG_LEVEL := --log-level $(LOG_LEVEL) endif diff --git a/src/python/jw/pkg/App.py b/src/python/jw/pkg/App.py index 3dfee93c..f161253d 100644 --- a/src/python/jw/pkg/App.py +++ b/src/python/jw/pkg/App.py @@ -5,7 +5,6 @@ from __future__ import annotations import os -import pwd import re import sys @@ -115,6 +114,8 @@ class App(Base): return self.__pretty_topdir return self.__topdir for d in [self.__projs_root, self.___opt_root]: + if d is None: + continue ret = d + '/' + name if os.path.exists(ret): return ret @@ -353,7 +354,7 @@ class App(Base): # -- Members with default values self.__topdir_fmt = 'absolute' - self.__projs_root = pwd.getpwuid(os.getuid()).pw_dir + '/local/src/jw.dev/proj' + self.__projs_root: str | None = None self.___opt_root = '/opt' self.__pretty_projs_root = None @@ -485,7 +486,9 @@ class App(Base): return self.__top_name @property - def projs_root(self): + def projs_root(self) -> str: + if self.__projs_root is None: + raise Exception('Tried to get unknown projects root directory') return self.__projs_root @property diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 66b44f81..ce70c996 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -1,4 +1,4 @@ -============= Running: jw-pkg.py --log-level info --help +============= Running: jw-pkg.py -t ../../../.. --log-level info --help usage: jw-pkg.py [--log-flags LOG_FLAGS] [--log-level LOG_LEVEL] [--log-file LOG_FILE] [--backtrace] [--write-profile WRITE_PROFILE] [-t TOPDIR] @@ -47,7 +47,7 @@ Available subcommands: POSIX utility interface projects Project metadata evaluation for building packages secrets Manage package secrets -============= Running: jw-pkg.py --log-level info packages --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages --help usage: jw-pkg.py packages [-h] ... System package manager wrapper @@ -65,7 +65,7 @@ Available subcommands of packages: reboot-required Check whether the machine needs rebooting refresh Refresh the distribution's notion of available packages select Select packages by filter -============= Running: jw-pkg.py --log-level info packages delete --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages delete --help usage: jw-pkg.py packages delete [-h] [names ...] Delete packages by name @@ -75,7 +75,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info packages dup --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages dup --help usage: jw-pkg.py packages dup [-h] [--download-only] Upgrade distribution @@ -84,7 +84,7 @@ options: -h, --help show this help message and exit --download-only Only download packages from the repos, don't install them, yet (default: False) -============= Running: jw-pkg.py --log-level info packages install --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages install --help usage: jw-pkg.py packages install [-h] [--only-update] [-F] [names ...] Install the distribution's notion of available packages @@ -97,7 +97,7 @@ options: --only-update Only update the listed packages, don't install them (default: False) -F, --fixed-strings Don't expand macros in (default: False) -============= Running: jw-pkg.py --log-level info packages ls --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages ls --help usage: jw-pkg.py packages ls [-h] [names ...] List package contents @@ -107,7 +107,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info packages meta --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages meta --help usage: jw-pkg.py packages meta [-h] [names ...] List package metadata @@ -117,21 +117,21 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info packages reboot-required --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages reboot-required --help usage: jw-pkg.py packages reboot-required [-h] Check whether the machine needs rebooting options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info packages refresh --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages refresh --help usage: jw-pkg.py packages refresh [-h] Refresh the distribution's notion of available packages options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info packages select --help +============= Running: jw-pkg.py -t ../../../.. --log-level info packages select --help usage: jw-pkg.py packages select [-h] filter Select packages by filter @@ -141,7 +141,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info platform --help +============= Running: jw-pkg.py -t ../../../.. --log-level info platform --help usage: jw-pkg.py platform [-h] ... Miscellaneous platform-related commands @@ -152,7 +152,7 @@ options: Available subcommands of platform: info Retrieve information about target platform -============= Running: jw-pkg.py --log-level info platform info --help +============= Running: jw-pkg.py -t ../../../.. --log-level info platform info --help usage: jw-pkg.py platform info [-h] [--format FORMAT] Retrieve information about target platform @@ -162,7 +162,7 @@ options: --format FORMAT Format string, expanding macros %{os}, %{id}, %{name}, %{codename}, %{gnu-triplet}, %{os-cascade}, %{os-release}, %{pkg-ext} (default: %{cascade}) -============= Running: jw-pkg.py --log-level info posix --help +============= Running: jw-pkg.py -t ../../../.. --log-level info posix --help usage: jw-pkg.py posix [-h] ... Perform various operations on a distro through its POSIX utility interface @@ -174,7 +174,7 @@ Available subcommands of posix: copy Copy files tar Handle tar archives -============= Running: jw-pkg.py --log-level info posix copy --help +============= Running: jw-pkg.py -t ../../../.. --log-level info posix copy --help usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst Copy files @@ -189,7 +189,7 @@ options: -g, --group GROUP Destination file group (default: None) -m, --mode MODE Destination file mode (default: None) -F, --fixed-strings Don't expand macros in and (default: False) -============= Running: jw-pkg.py --log-level info posix tar --help +============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar --help usage: jw-pkg.py posix tar [-h] ... Handle tar archives @@ -200,7 +200,7 @@ options: Available subcommands of tar: x Extract a tar archive -============= Running: jw-pkg.py --log-level info posix tar x --help +============= Running: jw-pkg.py -t ../../../.. --log-level info posix tar x --help usage: jw-pkg.py posix tar x [-h] -f ARCHIVE_PATH dst Extract a tar archive @@ -212,7 +212,7 @@ options: -h, --help show this help message and exit -f, --archive-path ARCHIVE_PATH Archive path -============= Running: jw-pkg.py --log-level info projects --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects --help usage: jw-pkg.py projects [-h] ... Project metadata evaluation for building packages @@ -252,7 +252,7 @@ Available subcommands of projects: summary Print summary description of given modules test Test tmpl-dir Print directory containing templates of a given module -============= Running: jw-pkg.py --log-level info projects build --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects build --help usage: jw-pkg.py projects build [-h] [--exclude EXCLUDE] [-n] [-O] [-I] [--env-reinit] [--env-keep ENV_KEEP] target modules [modules ...] @@ -279,7 +279,7 @@ options: --env-keep ENV_KEEP Comma seperated list of environment variables to keep, "all" or "none", only meaningful if --env-reinit is specified (default: none) -============= Running: jw-pkg.py --log-level info projects canonicalize-remotes --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects canonicalize-remotes --help usage: jw-pkg.py projects canonicalize-remotes [-h] [-n] Streamline janware Git remotes @@ -287,7 +287,7 @@ Streamline janware Git remotes options: -h, --help show this help message and exit -n, --dry-run Only log what would be done (default: False) -============= Running: jw-pkg.py --log-level info projects cflags --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects cflags --help usage: jw-pkg.py projects cflags [-h] [module ...] cflags @@ -297,7 +297,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects check --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects check --help usage: jw-pkg.py projects check [-h] ... Run miscellaneous code and project checks @@ -308,7 +308,7 @@ options: Available subcommands of check: deps Check for circular dependencies between given modules -============= Running: jw-pkg.py --log-level info projects check deps --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects check deps --help usage: jw-pkg.py projects check deps [-h] [-f [FLAVOUR]] [module ...] Check for circular dependencies between given modules @@ -319,14 +319,14 @@ positional arguments: options: -h, --help show this help message and exit -f, --flavour [FLAVOUR] -============= Running: jw-pkg.py --log-level info projects commands --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects commands --help usage: jw-pkg.py projects commands [-h] List available commands options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects create-file --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects create-file --help usage: jw-pkg.py projects create-file [-h] [--format FORMAT] [--search-path SEARCH_PATH] [--template-name TEMPLATE_NAME] @@ -351,7 +351,7 @@ options: -f, --field KEY=VALUE Additional fields to insert into the output file (default: []) -============= Running: jw-pkg.py --log-level info projects create-pkg-config --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects create-pkg-config --help usage: jw-pkg.py projects create-pkg-config [-h] [-F PROJECT_DESCR_FILE] [-d DESCRIPTION] [-n NAME] [-s SUMMARY] [-p PREFIX] @@ -375,7 +375,7 @@ options: -r, --requires-run REQUIRES_RUN -R, --requires-build REQUIRES_BUILD -V, --variables [VARIABLES ...] -============= Running: jw-pkg.py --log-level info projects exepath --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects exepath --help usage: jw-pkg.py projects exepath [-h] [-d [DELIMITER]] [module ...] exepath @@ -387,7 +387,7 @@ options: -h, --help show this help message and exit -d, --delimiter [DELIMITER] Output words delimiter (default: :) -============= Running: jw-pkg.py --log-level info projects get-auth-info --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects get-auth-info --help usage: jw-pkg.py projects get-auth-info [-h] [--only-values] [--username] [--password] [--remote-owner-base] [--remote-base] @@ -402,7 +402,7 @@ options: --remote-owner-base Show remote base URL for owner jw-pkg was cloned from (default: False) --remote-base Show remote base URL (default: False) -============= Running: jw-pkg.py --log-level info projects getval --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects getval --help usage: jw-pkg.py projects getval [-h] [--project PROJECT] section key Get value from project config @@ -415,7 +415,7 @@ options: -h, --help show this help message and exit --project PROJECT Project name, default is name of project's topdir (default: None) -============= Running: jw-pkg.py --log-level info projects htdocs-dir --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects htdocs-dir --help usage: jw-pkg.py projects htdocs-dir [-h] [module ...] Print source directory containing document root of a given module @@ -425,7 +425,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects ldflags --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects ldflags --help usage: jw-pkg.py projects ldflags [-h] [--exclude EXCLUDE] [-s] [module ...] ldflags @@ -438,7 +438,7 @@ options: --exclude EXCLUDE Exclude Modules (default: []) -s, --add-self Include libflags of specified modules, too, not only their dependencies (default: False) -============= Running: jw-pkg.py --log-level info projects ldlibpath --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects ldlibpath --help usage: jw-pkg.py projects ldlibpath [-h] [-d [DELIMITER]] [module ...] ldlibpath @@ -450,7 +450,7 @@ options: -h, --help show this help message and exit -d, --delimiter [DELIMITER] Output words delimiter (default: :) -============= Running: jw-pkg.py --log-level info projects libname --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects libname --help usage: jw-pkg.py projects libname [-h] [module ...] libname @@ -460,7 +460,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects list-repos --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects list-repos --help usage: jw-pkg.py projects list-repos [-h] [--username USERNAME] [--askpass ASKPASS] [--from-owner FROM_OWNER] @@ -480,7 +480,7 @@ options: (default: None) --from-owner FROM_OWNER List from-owner's projects (default: janware) -============= Running: jw-pkg.py --log-level info projects modules --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects modules --help usage: jw-pkg.py projects modules [-h] [-F [FILTER]] Query existing janware packages @@ -490,7 +490,7 @@ options: -F, --filter [FILTER] Key-value pairs, seperated by commas, to be searched for in project.conf (default: None) -============= Running: jw-pkg.py --log-level info projects path --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects path --help usage: jw-pkg.py projects path [-h] [module ...] path @@ -500,7 +500,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects pkg-conflicts --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects pkg-conflicts --help usage: jw-pkg.py projects pkg-conflicts [-h] [-S [SUBSECTIONS]] [-d [DELIMITER]] [-p] [--dont-strip-revision] @@ -552,7 +552,7 @@ options: (default: False) --quote Put double quotes around each listed dependency (default: False) -============= Running: jw-pkg.py --log-level info projects pkg-provides --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects pkg-provides --help usage: jw-pkg.py projects pkg-provides [-h] [-S [SUBSECTIONS]] [-d [DELIMITER]] [-p] [--dont-strip-revision] @@ -603,7 +603,7 @@ options: (default: False) --quote Put double quotes around each listed dependency (default: False) -============= Running: jw-pkg.py --log-level info projects pkg-requires --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects pkg-requires --help usage: jw-pkg.py projects pkg-requires [-h] [-S [SUBSECTIONS]] [-d [DELIMITER]] [-p] [--dont-strip-revision] @@ -654,7 +654,7 @@ options: (default: False) --quote Put double quotes around each listed dependency (default: False) -============= Running: jw-pkg.py --log-level info projects proj-dir --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects proj-dir --help usage: jw-pkg.py projects proj-dir [-h] [module ...] Print directory of a given package @@ -664,7 +664,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects pythonpath --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects pythonpath --help usage: jw-pkg.py projects pythonpath [-h] [--subdir SUBDIR] [--delimiter DELIMITER] [--prefix PATH_COMPONENT_PREFIX] @@ -685,7 +685,7 @@ options: --prefix PATH_COMPONENT_PREFIX Prefix to prepend before every path component (default: None) -============= Running: jw-pkg.py --log-level info projects required-os-pkg --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects required-os-pkg --help usage: jw-pkg.py projects required-os-pkg [-h] [--skip-excluded] [--quote] flavours [modules ...] @@ -701,7 +701,7 @@ options: (default: False) --quote Put double quotes around each listed dependency (default: False) -============= Running: jw-pkg.py --log-level info projects summary --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects summary --help usage: jw-pkg.py projects summary [-h] [module ...] Print summary description of given modules @@ -711,7 +711,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects test --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects test --help usage: jw-pkg.py projects test [-h] blah Test @@ -721,7 +721,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info projects tmpl-dir --help +============= Running: jw-pkg.py -t ../../../.. --log-level info projects tmpl-dir --help usage: jw-pkg.py projects tmpl-dir [-h] [module ...] Print directory containing templates of a given module @@ -731,7 +731,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info secrets --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets --help usage: jw-pkg.py secrets [-h] ... Manage package secrets @@ -750,7 +750,7 @@ Available subcommands of secrets: list-templates List package template files rm-compilation-output Remove package compilation output files -============= Running: jw-pkg.py --log-level info secrets compile-templates --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets compile-templates --help usage: jw-pkg.py secrets compile-templates [-h] [--owner OWNER] [--group GROUP] [--mode MODE] [packages ...] @@ -765,7 +765,7 @@ options: --owner, -o OWNER Default output file owner (default: None) --group, -g GROUP Default output file group (default: None) --mode, -m MODE Default output file mode (default: None) -============= Running: jw-pkg.py --log-level info secrets install --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets install --help usage: jw-pkg.py secrets install [-h] [--only-missing] src [packages ...] Install secrets from various sources as static secrets onto the target @@ -778,7 +778,7 @@ options: -h, --help show this help message and exit --only-missing Install only secrets not already on the target (default: False) -============= Running: jw-pkg.py --log-level info secrets list-compilation-output --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets list-compilation-output --help usage: jw-pkg.py secrets list-compilation-output [-h] [--all] [packages ...] List package compilation output files @@ -790,7 +790,7 @@ options: -h, --help show this help message and exit --all Show all output targets, including non-existent files (default: False) -============= Running: jw-pkg.py --log-level info secrets list-secrets --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets list-secrets --help usage: jw-pkg.py secrets list-secrets [-h] [--all] [packages ...] List package secret files @@ -802,7 +802,7 @@ options: -h, --help show this help message and exit --all Show all secret paths, including non-existent files (default: False) -============= Running: jw-pkg.py --log-level info secrets list-templates --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets list-templates --help usage: jw-pkg.py secrets list-templates [-h] [packages ...] List package template files @@ -812,7 +812,7 @@ positional arguments: options: -h, --help show this help message and exit -============= Running: jw-pkg.py --log-level info secrets rm-compilation-output --help +============= Running: jw-pkg.py -t ../../../.. --log-level info secrets rm-compilation-output --help usage: jw-pkg.py secrets rm-compilation-output [-h] [packages ...] Remove package compilation output files -- 2.55.0 From c8ffae57fc252339f67199055bd59978f66a1ee0 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 23 Jul 2026 18:10:30 +0200 Subject: [PATCH 03/16] test/integration/jw-pkg/projects: Add directory Add a directory for integration tests of "jw-pkg projects". Signed-off-by: Jan Lindemann --- test/integration/jw-pkg/projects/Makefile | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test/integration/jw-pkg/projects/Makefile diff --git a/test/integration/jw-pkg/projects/Makefile b/test/integration/jw-pkg/projects/Makefile new file mode 100644 index 00000000..5e331b8e --- /dev/null +++ b/test/integration/jw-pkg/projects/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk -- 2.55.0 From 100d8f9abbb0d607abdeaef16645c8aea53ef328 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 23 Jul 2026 18:10:53 +0200 Subject: [PATCH 04/16] test/integration/jw-pkg/projects/cache: Add directory Add tests for jw-pkg.py calls run during makefile caching. Broken caching can compromise the build without causing it to fail entirely, i.e. in non-obvious ways. Signed-off-by: Jan Lindemann --- .../jw-pkg/projects/cache/Makefile | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/integration/jw-pkg/projects/cache/Makefile diff --git a/test/integration/jw-pkg/projects/cache/Makefile b/test/integration/jw-pkg/projects/cache/Makefile new file mode 100644 index 00000000..3ac59759 --- /dev/null +++ b/test/integration/jw-pkg/projects/cache/Makefile @@ -0,0 +1,29 @@ +# This file runs commands typically used during initial Makefile caching. +# +# It's especially useful to run them in an explicit test, because they are used +# to fill Makefile variables, and if they fail, they leave an empty variable +# behind instead of failing the build entirely. + +export JW_DEFAULT_SHOW_BACKTRACE = true + +TOPDIR = ../../../../.. + +include $(TOPDIR)/make/proj.mk +include $(TOPDIR)/make/test-jw-pkg.mk + +all: + +test: test.integration.in-tree +test.integration.in-tree: + $(TEST_CMD_LINE) --topdir-format absolute platform info --format "%{gnu-triplet} %{cascade}" + $(TEST_CMD_LINE) --topdir-format absolute platform info --format %{id}-%{codename} + $(TEST_CMD_LINE) --topdir-format absolute projects pkg-requires --no-subpackages --subsections=jw --syntax names-only --delimiter " " "build devel" jw-pkg + $(TEST_CMD_LINE) --topdir-format absolute projects pkg-requires --no-subpackages --subsections=jw --syntax names-only --delimiter " " run jw-pkg + $(TEST_CMD_LINE) --topdir-format absolute projects proj-dir jw-pkg + $(TEST_CMD_LINE) --topdir-format absolute projects pythonpath --delimiter " " jw-pkg + $(TEST_CMD_LINE) --topdir-format relative projects pythonpath --prefix '$$MYPY_CONFIG_FILE_DIR/' jw-pkg + $(TEST_CMD_LINE) --topdir-format unaltered projects create-file --format tmpl --template-name pyproject.toml --search-path $(TOPDIR)/conf/templates --field "mypypath=mypy_path = "$MYPY_CONFIG_FILE_DIR/src/python"" jw-pkg > test-pyproject.toml + $(TEST_CMD_LINE) --topdir-format unaltered projects create-file --format pyright --field base=./conf/project/pyrightconfig-base.json --field include=src/python jw-pkg > test-pyrightconfig.json +clean: test.integration.in-tree.clean +test.integration.in-tree.clean: + rm -f test-*.* -- 2.55.0 From d59dba37e02a08c9e0bc52a325d3f974b2d1e659 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 09:14:43 +0200 Subject: [PATCH 05/16] ldlibpath.mk: Don't include py-path.mk anylonger For backwards compatibility, ldlibpath.mk kept py-path.mk included. The projects depending on that have been fixed by now and this is no longer needed. So, move it to the py-defs.mk where it belongs and has narrower scope. Signed-off-by: Jan Lindemann --- make/ldlibpath.mk | 2 -- make/py-defs.mk | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/make/ldlibpath.mk b/make/ldlibpath.mk index b0373d74..f11406cf 100644 --- a/make/ldlibpath.mk +++ b/make/ldlibpath.mk @@ -16,5 +16,3 @@ ifeq ($(origin JW_PKG_EXE_PATH),undefined) JW_PKG_EXE_PATH := $(call proj_query, exepath --delimiter ' ' $(PROJECT) $(PREREQ_RUN)) endif export PATH := $(subst $(space),:,$(JW_PKG_EXE_PATH)):$(EXE_SEARCH_PATH_ENV) - -include $(JWBDIR)/make/py-path.mk diff --git a/make/py-defs.mk b/make/py-defs.mk index 3f02d78c..915491a2 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -93,4 +93,4 @@ endif PY_DEFS_MK_INCLUDED := true -include $(JWBDIR)/make/ldlibpath.mk +include $(JWBDIR)/make/py-path.mk -- 2.55.0 From 6743808000b1fbc26e2b53848e7fa637ae833288 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 24 Jul 2026 10:58:51 +0200 Subject: [PATCH 06/16] py-run.mk: Remove dead code Remove disabled included statements which are not going to be enabled anymore. Signed-off-by: Jan Lindemann --- make/py-run.mk | 2 -- 1 file changed, 2 deletions(-) diff --git a/make/py-run.mk b/make/py-run.mk index f029ef8e..85ebeba1 100644 --- a/make/py-run.mk +++ b/make/py-run.mk @@ -1,7 +1,5 @@ include $(JWBDIR)/make/defs.mk include $(JWBDIR)/make/py-defs.mk -#include $(JWBDIR)/make/scripts-targets.mk -#include $(JWBDIR)/make/rules.mk EXE ?= $(firstword $(wildcard main.py runme.py test.py *.py)) EXE_ARGS ?= -- 2.55.0 From c63b3cc06ffc6b7cd79ebc796eb59af041a33e80 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:36:23 +0200 Subject: [PATCH 07/16] py-defs.mk: Allow inclusion without defs.mk py-defs.mk uses the variables ECHO and SED defined in defs.mk. The complexity this introduces doesn't justify the reduced redundancy, though, so this commit defines them redundantly in py-defs.mk and to allow inclusion without prior defs.mk. Signed-off-by: Jan Lindemann --- make/py-defs.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/make/py-defs.mk b/make/py-defs.mk index 915491a2..de6da233 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -37,6 +37,8 @@ endif # deduce PY_INSTALL_DIR_PY from working directory below .. python/ ifeq ($(PY_INSTALL_DIR_PY),) + ECHO ?= echo + SED ?= sed PY_INSTALL_PKG_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) 's%.*/python/%%; s%/.*%%') PY_INSTALL_SUB_MOD ?= $(shell $(ECHO) $(CWD) | $(SED) "s%.*/$(PY_INSTALL_PKG_MOD)\(/\|$$\)%%") ifneq ($(PY_INSTALL_SUB_MOD),) -- 2.55.0 From eea3e5f14b796fc91cb0a291b5b475163da81c7c Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 17 Jul 2026 12:57:17 +0200 Subject: [PATCH 08/16] py-check.mk: Add file Add py-check.mk and use it from py-topdir.mk and py-rules.mk. This removes redundant check definitions, making sure that that checks run from a repo's subdirectory match the checks run from its toplevel directory. Signed-off-by: Jan Lindemann --- make/py-check.mk | 89 +++++++++++++++++++++++++++++++++++++++++++++++ make/py-defs.mk | 2 -- make/py-rules.mk | 3 +- make/py-topdir.mk | 64 ++-------------------------------- 4 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 make/py-check.mk diff --git a/make/py-check.mk b/make/py-check.mk new file mode 100644 index 00000000..05687d7c --- /dev/null +++ b/make/py-check.mk @@ -0,0 +1,89 @@ +.PHONY: \ + all \ + check \ + format \ + check-syntax \ + check-format \ + py-check \ + py-check-syntax \ + py-check-format \ + py-format \ + py-format-assignments \ + py-check-annotation-imports \ + py-format-annotation-imports \ + clean \ + clean.py-check + +ifndef PY_CHECK_ROOTS + PY_CHECK_ROOTS = . +endif + +ifndef PY_CHECK_MYPY + PY_CHECK_MYPY := mypy +endif + +ifndef PY_CHECK_RUFF + PY_CHECK_RUFF := $(shell which ruff 2>/dev/null) + ifneq ($(PY_CHECK_RUFF),) + PY_CHECK_RUFF += --config $(TOPDIR)/pyproject.toml + endif +endif + +ifndef PY_CHECK_YAPF + PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3)) +endif + +ifndef PY_CHECK_PYRIGHT + PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null) + TD_GENERATE_FILES += pyrightconfig.json +endif + +all: +check: py-check +format: py-format +check-syntax: py-check-syntax +check-format: py-check-format + +py-check: py-check-syntax py-check-format + +py-check-syntax: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) +endif + $(PY_CHECK_MYPY) $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) +ifneq ($(PY_CHECK_PYRIGHT),) + $(PY_CHECK_PYRIGHT) $(PY_CHECK_ROOTS) +endif + +py-check-format: +ifneq ($(PY_CHECK_YAPF),) + $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) +endif + +py-format: + find . -type f -name '*.py' -print0 | \ + xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}' +ifneq ($(PY_CHECK_YAPF),) + $(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS) +endif + +py-format-assignments: + find $(PY_CHECK_ROOTS) \ + -path './.git' -prune -o \ + -type f -name '*.py' \ + -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+' + git diff --exit-code + +py-check-annotation-imports: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS) +endif + +py-format-annotation-imports: +ifneq ($(PY_CHECK_RUFF),) + $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) +endif + +clean: clean.py-check +clean.py-check: + rm -rf .mypy_cache .ruff_cache .pytest_cache diff --git a/make/py-defs.mk b/make/py-defs.mk index de6da233..a947ef26 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -19,8 +19,6 @@ else PY_SITE_PACKAGES_PATH := $(shell $(PYTHON) -c "import site; print([d for d in site.getsitepackages() if d.find('/local/') == -1][0])") endif -PY_MYPY ?= mypy --ignore-missing-imports --no-namespace-packages - PY_SRC_PY ?= $(wildcard *.py) PY_ALL_PY = $(PY_SRC_PY) diff --git a/make/py-rules.mk b/make/py-rules.mk index ce6f16a3..acf463ff 100644 --- a/make/py-rules.mk +++ b/make/py-rules.mk @@ -29,8 +29,7 @@ $(PY_INSTALL_DIR_PY)/%.pyc: %.pyc endif $(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@ -check: - $(PY_MYPY) $(shell /bin/bash $(JWB_SCRIPT_DIR)/scm.sh ls-files | grep '\.py$$') +include $(JWBDIR)/make/py-check.mk $(PY_INSTALL_DIR_PY)/py.typed: py.typed $(INSTALL) -p -m $(PYMODMODE) -o $(PYMODOWNER) -g $(PYMODGROUP) $< $@ diff --git a/make/py-topdir.mk b/make/py-topdir.mk index 65f170f3..c6b0f3ac 100644 --- a/make/py-topdir.mk +++ b/make/py-topdir.mk @@ -11,71 +11,9 @@ ifndef PY_CHECK_ROOTS PY_CHECK_ROOTS = $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) endif -ifndef PY_CHECK_RUFF - PY_CHECK_RUFF := $(shell which ruff 2>/dev/null) - ifneq ($(PY_CHECK_RUFF),) - PY_CHECK_RUFF += --config pyproject.toml - endif -endif - -ifndef PY_CHECK_YAPF - PY_CHECK_YAPF := $(firstword $(wildcard /usr/bin/yapf /usr/bin/yapf3)) -endif - -ifndef PY_CHECK_PYRIGHT - PY_CHECK_PYRIGHT := $(shell which pyright 2>/dev/null) - TD_GENERATE_FILES += pyrightconfig.json -endif - all: - -format: py-format -check-syntax: py-check-syntax -check-format: py-check-format - -py-check: py-check-syntax py-check-format -py-check-syntax: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) -endif - mypy $(addprefix --exclude ,$(PY_CHECK_EXCLUDE)) $(PY_CHECK_ROOTS) -ifneq ($(PY_CHECK_PYRIGHT),) - pyright $(PY_CHECK_ROOTS) -endif - -py-check-format: -ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) -endif - -py-format: - find . -type f -name '*.py' -print0 | \ - xargs -0 sed -i -E '1{/^# -\*- coding: utf-8 -\*-$$/{:a;N;/\n[[:space:]]*$$/ba;s/^# -\*- coding: utf-8 -\*-\n([[:space:]]*\n)*/ /;s/^ //}}' -ifneq ($(PY_CHECK_YAPF),) - $(PY_CHECK_YAPF) --in-place --recursive $(PY_CHECK_ROOTS) -endif - -py-format-assignments: - find $(PY_CHECK_ROOTS) \ - -path './.git' -prune -o \ - -type f -name '*.py' \ - -execdir /usr/bin/sed -i 's/^\(\s\+[a-zA-Z0-9_]\+\)=\([^,[:space:]]\+\)\([,(]\)*\s*$$/\1 = \2\3/g' {} '+' - git diff --exit-code - -py-check-annotation-imports: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --diff --unsafe-fixes $(PY_CHECK_ROOTS) -endif - -py-format-annotation-imports: -ifneq ($(PY_CHECK_RUFF),) - $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) -endif - clean: clean.topdir clean.topdir: clean.py-check -clean.py-check: - rm -rf .mypy_cache .ruff_cache .pytest_cache pyproject.toml: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \ @@ -86,3 +24,5 @@ pyrightconfig.json: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format pyright \ --field base=$(JW_PKG_CONF_BASE_DIR)/project/pyrightconfig-base.json $(addprefix --field include=,$(wildcard src/python tools/python)) $(PROJECT) > $@.tmp mv $@.tmp $@ + +include $(JWBDIR)/make/py-check.mk -- 2.55.0 From dcbe959843846964ec8ffd5ca82e6820c1ea4e84 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 24 Jul 2026 10:33:42 +0200 Subject: [PATCH 09/16] lib: Fix misplaced "# export" markers A couple of "# export" markers after function prototypes have been pushed along with the closing parenthesis onto the wrong line by the code formatter, fix that. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/lib/App.py | 4 ++-- src/python/jw/pkg/lib/ec/SSHClient.py | 4 ++-- src/python/jw/pkg/lib/pm/dpkg.py | 8 ++++---- src/python/jw/pkg/lib/pm/rpm.py | 8 ++++---- src/python/jw/pkg/lib/util.py | 12 ++++++------ 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/python/jw/pkg/lib/App.py b/src/python/jw/pkg/lib/App.py index c9fab8e0..54c6abec 100644 --- a/src/python/jw/pkg/lib/App.py +++ b/src/python/jw/pkg/lib/App.py @@ -292,8 +292,8 @@ class App: # export self.__async_runner = None return ret -def run_sub_commands( +def run_sub_commands( # export description = '', name_filter = '^Cmd.*', modules = None, argv = None -): # export +): app = App(description, name_filter, modules) return app.run(argv = argv) diff --git a/src/python/jw/pkg/lib/ec/SSHClient.py b/src/python/jw/pkg/lib/ec/SSHClient.py index 39a5b4bb..e920a25b 100644 --- a/src/python/jw/pkg/lib/ec/SSHClient.py +++ b/src/python/jw/pkg/lib/ec/SSHClient.py @@ -109,9 +109,9 @@ class SSHClient(ExecContext): def password(self) -> str | None: return self.uri.password -def ssh_client( +def ssh_client( # export *args, type: str | list[str] | None = None, **kwargs -) -> SSHClient: # export +) -> SSHClient: from importlib import import_module errors: list[str] = [] diff --git a/src/python/jw/pkg/lib/pm/dpkg.py b/src/python/jw/pkg/lib/pm/dpkg.py index 6ed15fb9..41596211 100644 --- a/src/python/jw/pkg/lib/pm/dpkg.py +++ b/src/python/jw/pkg/lib/pm/dpkg.py @@ -33,20 +33,20 @@ async def _run( if sudo else await run_cmd(cmd, ec = ec, cmd_input = InputMode.NonInteractive) ).stdout_str -async def run_dpkg( +async def run_dpkg( # export args: list[str], sudo: bool = False, ec: ExecContext | None = None -) -> str: # export +) -> str: cmd = ['/usr/bin/dpkg'] cmd.extend(args) return await _run(cmd, sudo, ec) -async def run_dpkg_query( +async def run_dpkg_query( # export args: list[str], sudo: bool = False, ec: ExecContext | None = None -) -> str: # export +) -> str: cmd = ['/usr/bin/dpkg-query'] cmd.extend(args) return await _run(cmd, sudo, ec) diff --git a/src/python/jw/pkg/lib/pm/rpm.py b/src/python/jw/pkg/lib/pm/rpm.py index b52c8271..8635a9d5 100644 --- a/src/python/jw/pkg/lib/pm/rpm.py +++ b/src/python/jw/pkg/lib/pm/rpm.py @@ -25,13 +25,13 @@ def meta_map(): ) return _meta_map -async def run_rpm( +async def run_rpm( # export args: list[str], sudo: bool = False, ec: ExecContext | None = None, mode: InputMode = InputMode.OptInteractive, **kwargs, -) -> str: # export +) -> str: cmd = ['/usr/bin/rpm'] cmd.extend(args) result = ( @@ -40,10 +40,10 @@ async def run_rpm( ) return result.stdout_str -async def query_packages( +async def query_packages( # export names: Iterable[str] = [], ec: ExecContext | None = None, -) -> Iterable[Package]: # export +) -> Iterable[Package]: fmt_str = ( '|'.join([(f'%{{{tag}}}' if tag else '') for tag in meta_map().values()]) + r'\n' diff --git a/src/python/jw/pkg/lib/util.py b/src/python/jw/pkg/lib/util.py index 3a809305..3390b58b 100644 --- a/src/python/jw/pkg/lib/util.py +++ b/src/python/jw/pkg/lib/util.py @@ -209,12 +209,12 @@ async def copy( return e assert False, 'Unreachable code' -async def get_username( +async def get_username( # export args: Namespace | None = None, url: str | None = None, askpass_env: list[str] = [], ec: ExecContext | None = None, -) -> str | None: # export +) -> str | None: url_user = None if url is None else Uri(url).username if args is not None: if args.username is not None: @@ -228,12 +228,12 @@ async def get_username( return url_user return await run_askpass(askpass_env, AskpassKey.Username, ec = ec) -async def get_password( +async def get_password( # export args: Namespace | None = None, url: str | None = None, askpass_env: list[str] = [], ec: ExecContext | None = None, -) -> str | None: # export +) -> str | None: if args is None and url is None and not askpass_env: raise Exception( 'Neither URL nor command-line arguments nor askpass environment variable ' @@ -251,11 +251,11 @@ async def get_password( return ret return await run_askpass(askpass_env, AskpassKey.Password, ec = ec) -async def get_profile_env( +async def get_profile_env( # export throw: bool = True, keep: Iterable[str] | bool = False, ec: ExecContext | None = None, -) -> dict[str, str]: # export +) -> dict[str, str]: """ Get a fresh environment from /etc/profile -- 2.55.0 From 4bf6445412846787e081ddf9a1267be268f6ea39 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Fri, 24 Jul 2026 08:36:09 +0200 Subject: [PATCH 10/16] py-check.mk: Add target py-check-bad-patterns Add target py-check-bad-patterns, which currently looks for leftover breakpoints and orphaned "export" comments on the closing line of a function prototype. Signed-off-by: Jan Lindemann --- make/py-check.mk | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/make/py-check.mk b/make/py-check.mk index 05687d7c..1c4b4faf 100644 --- a/make/py-check.mk +++ b/make/py-check.mk @@ -44,7 +44,7 @@ format: py-format check-syntax: py-check-syntax check-format: py-check-format -py-check: py-check-syntax py-check-format +py-check: py-check-syntax py-check-format py-check-bad-patterns py-check-syntax: ifneq ($(PY_CHECK_RUFF),) @@ -55,6 +55,10 @@ ifneq ($(PY_CHECK_PYRIGHT),) $(PY_CHECK_PYRIGHT) $(PY_CHECK_ROOTS) endif +py-check-bad-patterns: + if find $(PY_CHECK_ROOTS) -type f -name '*.py' -print0 | xargs -0 grep breakpoint; then exit 1; fi + if find $(PY_CHECK_ROOTS) -type f -name '*.py' -print0 | xargs -0 grep "^\s*).*#\s*export"; then exit 1; fi + py-check-format: ifneq ($(PY_CHECK_YAPF),) $(PY_CHECK_YAPF) --diff --recursive $(PY_CHECK_ROOTS) -- 2.55.0 From 79edaff1dd1c28178c620ab194518d9a05b68ba9 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 09:16:39 +0200 Subject: [PATCH 11/16] py-path.mk: Add target py-path Add target py-path to py-path.mk. At this point, it's mostly introduced for documentation purposes, giving people and machines a defined, easily accessible and omni-present way to determine how Python imports should be resolved. Signed-off-by: Jan Lindemann --- make/py-path.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/make/py-path.mk b/make/py-path.mk index 11a115cd..64c03dab 100644 --- a/make/py-path.mk +++ b/make/py-path.mk @@ -11,3 +11,7 @@ all: clean: py-tools.clean py-tools.clean: $(RM) -rf $(wildcard *.pyc) __pycache__ .mypy_cache .ruff_cache .pytest_cache + +py-path: + @echo "PYTHONPATH=$(PYTHONPATH)" + @#echo "MYPYPATH=$(MYPYPATH)" -- 2.55.0 From b7431e5bc6dd5215be81e293f677e8184b9738d9 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 06:57:03 +0200 Subject: [PATCH 12/16] rules.mk / defs.mk: Disable BUILD_MAKEDIR BUILD_MAKEDIR is a variable which exists for consistency's sake. On the other hand, nor jw-pkg nor any downstream package ever copies makefile snippets during build time. The rule introduced by BUILD_MAKEDIR is quite costly in terms of performance and makes caching harder to understand. This commit disables the variable. Signed-off-by: Jan Lindemann --- make/defs.mk | 3 ++- make/rules.mk | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/make/defs.mk b/make/defs.mk index 8b416b55..2d135613 100644 --- a/make/defs.mk +++ b/make/defs.mk @@ -404,7 +404,8 @@ INSTALLED_INIT += $(addprefix $(INSTALL_INITDIR)/,$(INIT_SCRIPTS)) # -- MAKE INSTALLATION_FILE_TYPES += MAKE -BUILD_MAKEDIR = $(TOPDIR)/make +#BUILD_MAKEDIR is currently not used anywhere and costs performance. Disabled for the time being. +#BUILD_MAKEDIR = $(TOPDIR)/make MKFILES += $(filter-out pckg-defs.mk pckg-deps.mk local.mk,$(filter-out $(DONT_INSTALL),$(wildcard *.mk))) INSTALL_MAKEDIR ?= $(PREFIX)/make INSTALLED_MAKE += $(addprefix $(INSTALL_MAKEDIR)/,$(MKFILES)) diff --git a/make/rules.mk b/make/rules.mk index bbdadda3..0179fbbb 100644 --- a/make/rules.mk +++ b/make/rules.mk @@ -390,12 +390,14 @@ $$(TOPDIR)/dir_install_$(1).done: mkdir -p $$(INSTALL_$(1)DIR) touch $$@ +ifneq ($$(BUILD_$(1)DIR),) $$(BUILD_$(1)DIR)/%: % | $$(TOPDIR)/dir_build_$(1).done $(Q)if [ ! $$< -ef $$@ -a "`echo $$< | $(SED) 's/\..*//'`" != local ]; then \ echo $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\ $(BIN_INSTALL) -D -p -m $($(1)MODE) $$< $$@ ;\ $(RM) -f $$(TOPDIR)/dirs-*.done ;\ fi +endif $$(INSTALL_$(1)DIR): ifeq ($(PACKAGE_INSTALL_DIR),true) -- 2.55.0 From 41ca202142bfeee472bc6e7d3ed58b6f767d206b Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:18:58 +0200 Subject: [PATCH 13/16] cmds.projects.CmdBuild: Support --dep-flavours CmdBuild supports the targets all, clean and pkg-*. It uses builtin dependency flavours matching those targets to resolve build order. To make it more flexible in general, and allow it to support more targets, e.g. "check" and "test", this commit adds a --dep-flavours option. Signed-off-by: Jan Lindemann --- src/python/jw/pkg/cmds/projects/CmdBuild.py | 17 ++++++-- .../integration/jw-pkg/help/test-expected.txt | 40 ++++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/python/jw/pkg/cmds/projects/CmdBuild.py b/src/python/jw/pkg/cmds/projects/CmdBuild.py index c8e51c0e..9f2f3f43 100644 --- a/src/python/jw/pkg/cmds/projects/CmdBuild.py +++ b/src/python/jw/pkg/cmds/projects/CmdBuild.py @@ -53,6 +53,14 @@ class CmdBuild(Cmd): # export 'on the command line' ), ) + parser.add_argument( + '--dep-flavours', + default = 'auto', + help = ( + 'Dependency flavours to take into consideration for build, ' + 'comma or space separated' + ) + ) parser.add_argument( '--env-reinit', action = 'store_true', @@ -247,9 +255,12 @@ class CmdBuild(Cmd): # export # -- build order: list[str] = [] - dep_flavours = ['build'] - if re.match('pkg-.*', target) is not None: - dep_flavours = ['build', 'run', 'release', 'devel'] + if args.dep_flavours != 'auto': + dep_flavours = re.split(r'[\s,]', args.dep_flavours) + else: + dep_flavours = ['build'] + if re.match('pkg-.*', target) is not None: + dep_flavours.extend(['run', 'release', 'devel']) if target != 'order' and not args.build_order: log(NOTICE, 'Using prerequisite flavours ' + ' '.join(dep_flavours)) diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index ce70c996..95651dcb 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -254,31 +254,35 @@ Available subcommands of projects: tmpl-dir Print directory containing templates of a given module ============= Running: jw-pkg.py -t ../../../.. --log-level info projects build --help usage: jw-pkg.py projects build [-h] [--exclude EXCLUDE] [-n] [-O] [-I] - [--env-reinit] [--env-keep ENV_KEEP] + [--dep-flavours DEP_FLAVOURS] [--env-reinit] + [--env-keep ENV_KEEP] target modules [modules ...] janware software project build tool positional arguments: - target Build target - modules Modules to be built + target Build target + modules Modules to be built options: - -h, --help show this help message and exit - --exclude EXCLUDE Space seperated ist of modules to be excluded from - build (default: ) - -n, --dry-run Don't build anything, just print what would be done. - (default: False) - -O, --build-order Don't build anything, just print the build order. - (default: False) - -I, --ignore-deps Don't build dependencies, i.e. build only modules - specified on the command line (default: False) - --env-reinit Source /etc/profile before each build step. Discard - environment unless --env-keep is specified (default: - False) - --env-keep ENV_KEEP Comma seperated list of environment variables to keep, - "all" or "none", only meaningful if --env-reinit is - specified (default: none) + -h, --help show this help message and exit + --exclude EXCLUDE Space seperated ist of modules to be excluded from + build (default: ) + -n, --dry-run Don't build anything, just print what would be done. + (default: False) + -O, --build-order Don't build anything, just print the build order. + (default: False) + -I, --ignore-deps Don't build dependencies, i.e. build only modules + specified on the command line (default: False) + --dep-flavours DEP_FLAVOURS + Dependency flavours to take into consideration for + build, comma or space separated (default: auto) + --env-reinit Source /etc/profile before each build step. Discard + environment unless --env-keep is specified (default: + False) + --env-keep ENV_KEEP Comma seperated list of environment variables to keep, + "all" or "none", only meaningful if --env-reinit is + specified (default: none) ============= Running: jw-pkg.py -t ../../../.. --log-level info projects canonicalize-remotes --help usage: jw-pkg.py projects canonicalize-remotes [-h] [-n] -- 2.55.0 From e1ccbc111523aab85d3b38d357eddc4be6b3f21d Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sat, 11 Jul 2026 12:36:01 +0200 Subject: [PATCH 14/16] projects-dir.mk: Add target test projects-dir.mk lacks a "test" target, fix that. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index 96d0a2ee..c05d7898 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -155,7 +155,8 @@ PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(add # --- mandatory targets -all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) +all: +all test: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) $(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS) clean: clean-dirs distclean: clean-all-dirs done.clean -- 2.55.0 From d306e8cbdce8fd817017e0f6d5ca911e6262bb5c Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sun, 12 Jul 2026 13:40:09 +0200 Subject: [PATCH 15/16] projects-dir.mk: Support check targets Add support for the targets "check" and "check-post" to projects-dir.mk to make them usable from inside the projects directory. Note that "check-pre" is intentionally left out: Running "make check" in a project before its prerequisite projects have built their __init__.py files will fail due to broken import resolution. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index c05d7898..115a0063 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -156,7 +156,7 @@ PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(add # --- mandatory targets all: -all test: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) +all test check check-pre check-post: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) $(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS) clean: clean-dirs distclean: clean-all-dirs done.clean -- 2.55.0 From a4306ad0c5316a7a30e294b1526034bfdfab3249 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 14:47:49 +0200 Subject: [PATCH 16/16] install-files.mk: Add missing target "test" install-files.mk lacks a "test" target, so this commit adds a stub. Signed-off-by: Jan Lindemann --- make/install-files.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/make/install-files.mk b/make/install-files.mk index a2d898cf..af4addb8 100644 --- a/make/install-files.mk +++ b/make/install-files.mk @@ -25,6 +25,7 @@ all: install: install.done clean: done.clean distclean: +test: done.clean: $(RM) -f *.done -- 2.55.0