From 5f98de8f5a7925364992205d6063d650c19dc22a Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 21 Sep 2026 05:34:47 +0200 Subject: [PATCH 1/5] cmds.projects.CmdBuild: Add --pkg-relations The relations the build closure of projects build follows are hard-coded in read_deps(): [pkg.requires.jw] only. A workspace that also wants to build the projects its seeds recommend in [pkg.recommends.jw] has no way to say so. Add a --pkg-relations option that lists the relations between jw packages to take into consideration for the build, comma or space separated, defaulting to requires. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.86.1 --- src/python/jw/pkg/cmds/projects/CmdBuild.py | 26 ++++++++++++++----- .../integration/jw-pkg/help/test-expected.txt | 7 ++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/python/jw/pkg/cmds/projects/CmdBuild.py b/src/python/jw/pkg/cmds/projects/CmdBuild.py index 3a29b899..e92a5b22 100644 --- a/src/python/jw/pkg/cmds/projects/CmdBuild.py +++ b/src/python/jw/pkg/cmds/projects/CmdBuild.py @@ -62,6 +62,14 @@ class CmdBuild(Cmd): # export 'comma or space separated' ) ) + parser.add_argument( + '--pkg-relations', + default = 'requires', + help = ( + 'Relations between packages to take into consideration for build, ' + 'comma or space separated' + ) + ) parser.add_argument( '--env-reinit', action = 'store_true', @@ -103,7 +111,11 @@ class CmdBuild(Cmd): # export ) @lru_cache(maxsize = None) - def read_deps(cur: str, dep_flavour: str) -> list[str]: + def read_deps( + cur: str, + dep_flavour: str, + relations: tuple[str, ...], + ) -> Iterable[str]: # dep cache doesn't make a difference at all if dep_flavour in dep_cache: if cur in dep_cache[dep_flavour]: @@ -113,13 +125,13 @@ class CmdBuild(Cmd): # export ret = self.app.get_project_refs( [cur], - ['pkg.requires.jw'], + [f'pkg.{rel}.jw' for rel in relations], dep_flavour, scope = Scope.Subtree, add_self = False, names_only = True, ) - log(DEBUG, f'Prerequisites: {" ".join(ret)}') + log(DEBUG, f'Prerequisites ({"/".join(relations)}): {" ".join(ret)}') if cur in ret: ret.remove(cur) @@ -134,6 +146,7 @@ class CmdBuild(Cmd): # export def add_dep_tree( cur: str, dep_flavours: Iterable[str], + relations: tuple[str, ...], tree: DepNode, all_deps: set[str], ) -> int: @@ -145,9 +158,9 @@ class CmdBuild(Cmd): # export all_deps.add(cur) for t in dep_flavours: log(DEBUG, f'Checking deps of type "{t}"') - deps.update(read_deps(cur, t)) + deps.update(read_deps(cur, t, relations)) for d in deps: - add_dep_tree(d, dep_flavours, tree, all_deps) + add_dep_tree(d, dep_flavours, relations, tree, all_deps) tree[cur] = deps return len(deps) @@ -158,9 +171,10 @@ class CmdBuild(Cmd): # export ) -> int: all_deps: set[str] = set() dep_tree: DepNode = {} + relations = tuple(re.split(r'[,\s]+', args.pkg_relations)) for m in modules: log(DEBUG, f'--- Adding dependency tree of module "{m}"') - add_dep_tree(m, dep_flavours, dep_tree, all_deps) + add_dep_tree(m, dep_flavours, relations, dep_tree, all_deps) while len(all_deps): # Find any leaf, deterministically: sorted, so that the # order is reproducible across runs diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index 8560a11c..c7b895c9 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -256,7 +256,8 @@ Available subcommands of projects: tmpl-dir Print directory containing templates of a given module ============= Running: jw-pkg.py -t ../../../.. --log-level info projects build --help usage: jw-pkg.py projects build [-h] [--exclude EXCLUDE] [-n] [-O] [-I] - [--dep-flavours DEP_FLAVOURS] [--env-reinit] + [--dep-flavours DEP_FLAVOURS] + [--pkg-relations PKG_RELATIONS] [--env-reinit] [--env-keep ENV_KEEP] target modules [modules ...] @@ -279,6 +280,10 @@ options: --dep-flavours DEP_FLAVOURS Dependency flavours to take into consideration for build, comma or space separated (default: auto) + --pkg-relations PKG_RELATIONS + Relations between packages to take into consideration + for build, comma or space separated (default: + requires) --env-reinit Source /etc/profile before each build step. Discard environment unless --env-keep is specified (default: False) -- 2.55.0 From 32dd507cf54418c3a83ec5c4691b68168f6f0c9f Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 22 Sep 2026 06:12:29 +0200 Subject: [PATCH 2/5] cmds.projects.CmdRequiredOsPkg: Add --pkg-relations The OS package list of required-os-pkg, like the build closure, is derived from the [pkg.requires.jw] section only, hard-coded in _run(). Add a --pkg-relations option to required-os-pkg, mirroring the one of CmdBuild. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.86.1 --- .../jw/pkg/cmds/projects/CmdRequiredOsPkg.py | 13 ++++++++++++- test/integration/jw-pkg/help/test-expected.txt | 18 +++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py index c19ebb47..9aec61dc 100644 --- a/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py +++ b/src/python/jw/pkg/cmds/projects/CmdRequiredOsPkg.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re + from typing import TYPE_CHECKING, override from ...App import Scope @@ -36,6 +38,14 @@ class CmdRequiredOsPkg(Cmd): # export default = False, help = 'Put double quotes around each listed dependency', ) + parser.add_argument( + '--pkg-relations', + default = 'requires', + help = ( + 'Relations between packages to take into consideration, ' + 'comma or space separated' + ) + ) @override async def _run(self, args: Namespace) -> None: @@ -48,9 +58,10 @@ class CmdRequiredOsPkg(Cmd): # export if 'release' in flavours: flavours |= set(['run', 'devel', 'build']) log(DEBUG, 'flavours = ' + args.flavours) + relations = tuple(re.split(r'[,\s]+', args.pkg_relations)) deps = self.app.get_project_refs( modules, - ['pkg.requires.jw'], + [f'pkg.{rel}.jw' for rel in relations], list(flavours), scope = Scope.Subtree, add_self = True, diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index c7b895c9..2ab2fac0 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -763,20 +763,24 @@ options: (default: None) ============= Running: jw-pkg.py -t ../../../.. --log-level info projects required-os-pkg --help usage: jw-pkg.py projects required-os-pkg [-h] [--skip-excluded] [--quote] + [--pkg-relations PKG_RELATIONS] flavours [modules ...] List distribution packages required for a package positional arguments: - flavours Dependency flavours - modules Modules (default: None) + flavours Dependency flavours + modules Modules (default: None) options: - -h, --help show this help message and exit - --skip-excluded Output empty prerequisite list for excluded modules - (default: False) - --quote Put double quotes around each listed dependency (default: - False) + -h, --help show this help message and exit + --skip-excluded Output empty prerequisite list for excluded modules + (default: False) + --quote Put double quotes around each listed dependency + (default: False) + --pkg-relations PKG_RELATIONS + Relations between packages to take into consideration, + comma or space separated (default: requires) ============= Running: jw-pkg.py -t ../../../.. --log-level info projects summary --help usage: jw-pkg.py projects summary [-h] [module ...] -- 2.55.0 From 11511043287868c7a9eafd9639aa8d692f29cfac Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 21 Sep 2026 05:42:08 +0200 Subject: [PATCH 3/5] projects-dir.mk: Derive DEP_PROJECTS from the build order DEP_PROJECTS is computed with pkg-requires --recursive, which follows [pkg.requires.jw] only. That set does not match what the build actually builds, and it omits seed projects that no other project of the closure depends on. Derive DEP_PROJECTS from projects build --build-order with the same dependency flavours as make all, the very command the build-order make target and the workspace mount closure (mount-projects-dir.sh) use. DEP_PROJECTS is now exactly the build closure, so the file listings, the text-files cache and the git helpers of a workspace cover the same project set as make all. Introduce the PKG_RELATIONS_BUILD variable, defaulting to requires, and pass it through JW_PKG_PY_BUILD to every build invocation of the workspace, so the relations of the build closure are a workspace property. Signed-off-by: Jan Lindemann Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.86.1 --- make/projects-dir.mk | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index d37fc50a..614f171e 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -124,7 +124,8 @@ ifeq ($(TIME),) endif endif JW_PKG_PY_PROJECTS = $(JW_PKG_PY) projects -JW_PKG_PY_BUILD = $(JW_PKG_PY_PROJECTS) build $(JW_PKG_PY_EXTRA_BUILD_OPTS) +PKG_RELATIONS_BUILD ?= requires +JW_PKG_PY_BUILD = $(JW_PKG_PY_PROJECTS) build --pkg-relations "$(PKG_RELATIONS_BUILD)" $(JW_PKG_PY_EXTRA_BUILD_OPTS) PKG_MANAGER ?= $(TIME) $(JW_PKG_PY) --interactive=$(INTERACTIVE) pkg ifneq ($(origin PROJECTS_DIR_REMOTE_BASE),undefined) @@ -152,7 +153,7 @@ JANWARE_PACKAGE_FILTER = url =~ janware # ------------ projects to be built TARGET_PROJECTS = $(filter-out $(EXCLUDE_FROM_BUILD),$(PROJECTS)) -DEP_PROJECTS = $(shell $(JW_PKG_PY_PROJECTS) pkg-requires --recursive --syntax names-only --no-subpackages --delimiter ' ' --subsections jw run,build,devel,release $(TARGET_PROJECTS)) +DEP_PROJECTS = $(shell JW_DEFAULT_LOG_LEVEL=WARNING $(JW_PKG_PY_BUILD) --build-order --dep-flavours="$(BUILD_DEP_FLAVOURS)" all $(TARGET_PROJECTS)) GIT_PROJECTS = $(patsubst %/,%,$(dir $(wildcard $(addsuffix /.git,$(DEP_PROJECTS))))) PROJECTS_WITH_PROJECT_CONF = $(patsubst %/make/project.conf,%,$(wildcard $(addsuffix /make/project.conf,$(DEP_PROJECTS)))) -- 2.55.0 From fa8b9d51562f3bebab158709675a1ea6fadc066c Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Tue, 22 Sep 2026 09:16:42 +0200 Subject: [PATCH 4/5] projects-dir.mk: Add JW_PKG_PY_REQUIRED_OS_PKG Add and use JW_PKG_PY_REQUIRED_OS_PKG. The variable and its use effectively ties more targets to act on the same package set as the make all, influenced by the contents of $(PKG_RELATIONS_BUILD): echo-build-deps echo-install-deps echo-release-deps pkg-install-build-deps pkg-install-release-deps Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index 614f171e..84af96b7 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -126,6 +126,7 @@ endif JW_PKG_PY_PROJECTS = $(JW_PKG_PY) projects PKG_RELATIONS_BUILD ?= requires JW_PKG_PY_BUILD = $(JW_PKG_PY_PROJECTS) build --pkg-relations "$(PKG_RELATIONS_BUILD)" $(JW_PKG_PY_EXTRA_BUILD_OPTS) +JW_PKG_PY_REQUIRED_OS_PKG = $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded --pkg-relations "$(PKG_RELATIONS_BUILD)" PKG_MANAGER ?= $(TIME) $(JW_PKG_PY) --interactive=$(INTERACTIVE) pkg ifneq ($(origin PROJECTS_DIR_REMOTE_BASE),undefined) @@ -199,13 +200,13 @@ build-order-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) build-order: build-order-all echo-build-deps: - $(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build" $(TARGET_PROJECTS) + $(Q)$(JW_PKG_PY_REQUIRED_OS_PKG) "build" $(TARGET_PROJECTS) echo-install-deps: - $(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run" $(TARGET_PROJECTS) + $(Q)$(JW_PKG_PY_REQUIRED_OS_PKG) "build,run" $(TARGET_PROJECTS) echo-release-deps: - $(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run,release" $(TARGET_PROJECTS) + $(Q)$(JW_PKG_PY_REQUIRED_OS_PKG) "build,run,release" $(TARGET_PROJECTS) echo-os: $(Q)$(JW_PKG_PY) platform info @@ -262,10 +263,10 @@ pkg-manager-dup: $(PKG_MANAGER) dup pkg-install-build-deps: - $(PKG_MANAGER) install $(shell $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build" $(TARGET_PROJECTS)) + $(PKG_MANAGER) install $(shell $(JW_PKG_PY_REQUIRED_OS_PKG) "build" $(TARGET_PROJECTS)) pkg-install-release-deps: - $(PKG_MANAGER) install $(shell $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run,release" $(TARGET_PROJECTS)) + $(PKG_MANAGER) install $(shell $(JW_PKG_PY_REQUIRED_OS_PKG) "build,run,release" $(TARGET_PROJECTS)) pkg-release-reinstall: $(PREREQ_RELEASE) -- 2.55.0 From c7c9779e40850fd93f6568c36f10305266f8efbe Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 21 Sep 2026 17:06:58 +0200 Subject: [PATCH 5/5] projects-dir.mk: PKG_RELATIONS_BUILD += recommends Add recommended projects into the build closure. Would be nice if they got built along. Let's see if nightly survives it. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/projects-dir.mk b/make/projects-dir.mk index 84af96b7..d6e97d75 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -124,7 +124,7 @@ ifeq ($(TIME),) endif endif JW_PKG_PY_PROJECTS = $(JW_PKG_PY) projects -PKG_RELATIONS_BUILD ?= requires +PKG_RELATIONS_BUILD ?= requires,recommends JW_PKG_PY_BUILD = $(JW_PKG_PY_PROJECTS) build --pkg-relations "$(PKG_RELATIONS_BUILD)" $(JW_PKG_PY_EXTRA_BUILD_OPTS) JW_PKG_PY_REQUIRED_OS_PKG = $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded --pkg-relations "$(PKG_RELATIONS_BUILD)" PKG_MANAGER ?= $(TIME) $(JW_PKG_PY) --interactive=$(INTERACTIVE) pkg -- 2.55.0