From 6325f9ea19baf75f4299d05ec663ab8966e11c03 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Wed, 8 Jul 2026 19:34:41 +0200 Subject: [PATCH 1/4] platform.mk: Fix make -C $(TOPDIR)/make clean all If run from $(TOPDIR), "make clean all" runs fine, because it recurses twice into $(TOPDIR)/make, once for every target. If invoked directly from $(TOPDIR)/make, it can break in two different ways: If the cache files don't exist, "make clean all" in $(TOPDIR)/make tries to create them too early as implicit make target. This leaves variables empty which should have a value. If the cache files do exist, "make clean all" in $(TOPDIR)/make includes them, cleans them, and re-creates them from the same variables just read from cache. Undesirable for cache purging. Signed-off-by: Jan Lindemann --- make/platform.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/make/platform.mk b/make/platform.mk index 117b4dc0..2caad545 100644 --- a/make/platform.mk +++ b/make/platform.mk @@ -12,8 +12,10 @@ endef CACHE_PROJECTS_MK := $(JWBDIR)/cache-projects.mk +ifeq ($(filter clean distclean,$(MAKECMDGOALS)),) $(eval $(call try_include,$(TOPDIR)/make/.cache-project.mk)) $(eval $(call try_include,$(CACHE_PROJECTS_MK))) +endif $(eval $(call try_include,$(JWBDIR)/make/pre-local.mk)) $(eval $(call try_include,$(TOPDIR)/make/pre-local.mk)) $(eval $(call try_include,pre-local.mk)) -- 2.55.0 From 193c2ee0bd9c1f4b6275cca5f28c93e41a77c033 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 6 Jul 2026 20:44:38 +0200 Subject: [PATCH 2/4] py-topdir.mk: PY_CHECK_ROOTS: Don't += assign py-topdir.mk has this: ifndef PY_CHECK_ROOTS PY_CHECK_ROOTS += ... endif That is too involved: Either PY_CHECK_ROOTS is defined, then nothing is appended, or it's undefined, then a simple "=" would be just fine. Use that instead. Signed-off-by: Jan Lindemann --- make/py-topdir.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/py-topdir.mk b/make/py-topdir.mk index acbcb8aa..9b2ab591 100644 --- a/make/py-topdir.mk +++ b/make/py-topdir.mk @@ -8,7 +8,7 @@ MYPY_CONFIG_PATH = $(shell $(JW_PKG_PY) --topdir-format relative pro MYPY_PATH_DIRECTIVE = mypy_path = "$(MYPY_CONFIG_PATH)" ifndef PY_CHECK_ROOTS - PY_CHECK_ROOTS += $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) + PY_CHECK_ROOTS = $(wildcard $(TOPDIR)/src $(TOPDIR)/tools) endif ifndef PY_CHECK_RUFF -- 2.55.0 From d90c0ebf18a139bf88a976f1d954cf358a345e8c Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 07:44:35 +0200 Subject: [PATCH 3/4] cache.mk: Include from topdir.mk, not make.mk During a topdir "make all", caching variables is currently not the first thing that happens. Instead, variables are cached as soon as a project recurses into the make subdirectory. That was necessary, because some makefiles were regenerated in the make subdirectory by autoconf, potentially contributing variables that needed to be cached. As of now, autoconf is long gone and this is no longer true. And for some variables, the two step process becomes involved, notably for PYTHONPATH, which coding agents would like to look at from the topdir very early on. This commit moves the .project-cache.mk creation to topdir.mk, and .projects-cache.mk creation to jw-pkg/Makefile to address that. Signed-off-by: Jan Lindemann --- .gitignore | 2 ++ Makefile | 14 ++++++++++++-- make/Makefile | 10 ---------- make/cache.mk | 18 ++++++++++++++---- make/make.mk | 9 --------- make/pkg-dist.mk | 2 +- make/platform.mk | 19 ++++++++++++++----- make/topdir.mk | 3 ++- scripts/jw-pkg-create-project.sh | 3 +-- 9 files changed, 46 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 6336c5b8..3b34a6fa 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ pkg-build.dist pkgconfig/ rebase.sh __pycache__ +.cache-project.mk +.cache-projects.mk diff --git a/Makefile b/Makefile index 4511fe1d..105c83d0 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,24 @@ TOPDIR = . PKG_UPLOAD_URLPREFIX ?= rsync_ssh://$(JANWARE_USER)@pkg.janware.com:/srv/dav/pub/packages/jw-foss/$(PKG_UPLOAD_OS_NAME)/$(PKG_UPLOAD_OS_VERSION) +all: +all: cache-projects + include $(TOPDIR)/make/proj.mk include $(JWBDIR)/make/topdir.mk include $(JWBDIR)/make/srcdist.mk include $(JWBDIR)/make/git-cvs-bridge.mk include $(JWBDIR)/make/py-topdir.mk -all: clean: clean.pkgconfig - clean.pkgconfig: rm -rf pkgconfig + +# -- Cache for all projects +cache-projects: $(CACHE_PROJECTS_MK) +clean.cache-projects: + rm -f $(CACHE_PROJECTS_MK) +topdir.clean: clean.cache-projects +$(CACHE_PROJECTS_MK): $(CACHE_PROJECT_MK) + sed -E '/^[[:blank:]]*ifndef[[:blank:]]+($(subst $(space),|,$(CACHED_VARS_ONLY_PROJECT)))[[:blank:]]*$$/,/^[[:blank:]]*endif[[:blank:]]*$$/d' $< > $@.tmp + mv $@.tmp $@ diff --git a/make/Makefile b/make/Makefile index 913d9978..f7497a62 100644 --- a/make/Makefile +++ b/make/Makefile @@ -13,19 +13,9 @@ endif include $(JWBDIR)/make/dev-utils.mk -all: cache-projects -clean: clean-cache-projects - ifeq ($(TARGET),mingw) install: $(INSTALL_MAKEDIR)/winres.rc.tmpl endif grep-vars: $(Q)sed '/^ *[A-Z]/!d; s/^[ ]*//g; s/[=+?:].*//g; s/[ ]*//g' *.mk | sort -u - -cache-projects: $(CACHE_PROJECTS_MK) -clean-cache-projects: - rm -f $(CACHE_PROJECTS_MK) -$(CACHE_PROJECTS_MK): $(CACHE_PROJECT_MK) - sed -E '/^[[:blank:]]*ifndef[[:blank:]]+($(subst $(space),|,$(CACHED_VARS_ONLY_PROJECT)))[[:blank:]]*$$/,/^[[:blank:]]*endif[[:blank:]]*$$/d' $< > $@.tmp - mv $@.tmp $@ diff --git a/make/cache.mk b/make/cache.mk index 4c945942..ad46fb5d 100644 --- a/make/cache.mk +++ b/make/cache.mk @@ -1,9 +1,15 @@ # ----- define these variables # CACHED_FILES = $(TOPDIR)/VERSION # CACHED_VARS = VERSION -.PHONY: cache clean-cache clean-cache-projects +.PHONY: cache clean.cache + +CACHED_FILES ?= $(VERSION_FILE) $(wildcard $(TOPDIR)/make/project.conf) +CACHED_VARS ?= PROJECT PREREQ_BUILD PREREQ_RUN PREREQ VERSION HEX_VERSION \ + FULL_NAME WHICH PYTHON ECHO TEST BROWSER SED RM PWD ID CUT TR \ + AWK GETENT XARGS FIND PRINTF PLATFORM_INFO OS_NAME_VERSION \ + CAT BIN_INSTALL SUDO \ + JW_PKG_LD_LIBRARY_PATH JW_PKG_EXE_PATH JW_PKG_PYTHON_PATH -CACHE_PROJECT_MK ?= .cache-project.mk CACHED_VARS_ONLY_PROJECT ?= \ PROJECT \ PREREQ_BUILD \ @@ -15,9 +21,13 @@ CACHED_VARS_ONLY_PROJECT ?= \ JW_PKG_LD_LIBRARY_PATH \ JW_PKG_EXE_PATH \ JW_PKG_PYTHON_PATH + +# We need this to cache JW_PKG_PYTHON_PATH +include $(JWBDIR)/make/py-path.mk + all: cache -clean: clean-cache -clean-cache: +clean: clean.cache +clean.cache: rm -f $(CACHE_PROJECT_MK) cache: $(CACHE_PROJECT_MK) diff --git a/make/make.mk b/make/make.mk index f350d46b..d71ca4d8 100644 --- a/make/make.mk +++ b/make/make.mk @@ -8,15 +8,6 @@ include $(JWBDIR)/make/ldlibpath.mk include $(JWBDIR)/make/defs-dirs.mk include $(JWBDIR)/make/dev-utils.mk -CACHED_FILES ?= $(VERSION_FILE) $(wildcard $(TOPDIR)/make/project.conf) -CACHED_VARS ?= PROJECT PREREQ_BUILD PREREQ_RUN PREREQ VERSION HEX_VERSION \ - FULL_NAME WHICH PYTHON ECHO TEST BROWSER SED RM PWD ID CUT TR \ - AWK GETENT XARGS FIND PRINTF PLATFORM_INFO OS_NAME_VERSION \ - CAT BIN_INSTALL SUDO \ - JW_PKG_LD_LIBRARY_PATH JW_PKG_EXE_PATH JW_PKG_PYTHON_PATH - -include $(JWBDIR)/make/cache.mk - ifneq ($(CREATE_DEVEL),false) MKFILES += $(wildcard project.conf) DEVEL_TARGETS += install_MAKE $(INSTALLED_PKG_MK) diff --git a/make/pkg-dist.mk b/make/pkg-dist.mk index e8b511ff..ab3ca837 100644 --- a/make/pkg-dist.mk +++ b/make/pkg-dist.mk @@ -128,7 +128,7 @@ pkg-update: pkgupdate.dist pkg-upload-local: pkg-upload-local.dist clean.rpm: - $(RM) -rf $(DIST_DIR) *.dist make/.cache-project.mk + $(RM) -rf $(DIST_DIR) *.dist pkg-build.dist: $(VERSION_FILE) $(PKG_SH) build $(PKG_SH_BUILD_OPTS) diff --git a/make/platform.mk b/make/platform.mk index 2caad545..22d6846e 100644 --- a/make/platform.mk +++ b/make/platform.mk @@ -3,6 +3,16 @@ ifndef PLATFORM_MK_INCLUDED PLATFORM_MK_INCLUDED = true +CACHE_PROJECT_MK ?= $(TOPDIR)/.cache-project.mk +CACHE_PROJECTS_MK := $(JWBDIR)/.cache-projects.mk +JW_PKG_NO_CACHE ?= false + +ifneq ($(filter clean distclean,$(MAKECMDGOALS)),) + ifeq ($(TOPDIR),.) + JW_PKG_NO_CACHE = true + endif +endif + # "-include" tries search path, we don't want that define try_include ifneq ($$(wildcard $(1)),) @@ -10,12 +20,11 @@ define try_include endif endef -CACHE_PROJECTS_MK := $(JWBDIR)/cache-projects.mk - -ifeq ($(filter clean distclean,$(MAKECMDGOALS)),) -$(eval $(call try_include,$(TOPDIR)/make/.cache-project.mk)) -$(eval $(call try_include,$(CACHE_PROJECTS_MK))) +ifneq ($(JW_PKG_NO_CACHE),true) + $(eval $(call try_include,$(CACHE_PROJECT_MK))) + $(eval $(call try_include,$(CACHE_PROJECTS_MK))) endif + $(eval $(call try_include,$(JWBDIR)/make/pre-local.mk)) $(eval $(call try_include,$(TOPDIR)/make/pre-local.mk)) $(eval $(call try_include,pre-local.mk)) diff --git a/make/topdir.mk b/make/topdir.mk index 238b5837..0573e7f0 100644 --- a/make/topdir.mk +++ b/make/topdir.mk @@ -104,6 +104,7 @@ include $(JWBDIR)/make/rules.mk include $(JWBDIR)/make/doc-rules.mk include $(JWBDIR)/make/list-files.mk include $(JWBDIR)/make/projects+project.mk +include $(JWBDIR)/make/cache.mk STREAMLINE_PROJECT ?= bash $(JWB_SCRIPT_DIR)/streamline-project.sh @@ -115,7 +116,7 @@ TD_COPY_FILES ?= $(filter-out Makefile,$(notdir $(wildcard $(TD_CO # -- Internal variables, define JW_PKG_TOPDIR_COPY_PATH if you want to # add more files appended to the target files TD_COPY_SRC_PATH = $(subst :,$(space),$(JW_PKG_TOPDIR_COPY_PATH)) -TD_GENERATE_FILES += $(TD_COPY_FILES) +TD_GENERATE_FILES += $(TD_COPY_FILES) $(CACHE_PROJECT_MK) test: all format: topdir diff --git a/scripts/jw-pkg-create-project.sh b/scripts/jw-pkg-create-project.sh index 881e35b4..fc149892 100644 --- a/scripts/jw-pkg-create-project.sh +++ b/scripts/jw-pkg-create-project.sh @@ -103,8 +103,7 @@ cat_gitignore() *.done *.dist *.dep.mk - .cache.mk - make/.cache-project.mk + .cache-project.mk *.o *.so.* *.so -- 2.55.0 From 36af4a590fa868640fb928612693f9c05dfdb57a Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 9 Jul 2026 07:47:25 +0200 Subject: [PATCH 4/4] py-xxx.mk: Clean more python tool caches .mypy_cache, .ruff_cache and .pytest_cache are not consistently cleaned in all subdirectories. Fix that. Signed-off-by: Jan Lindemann --- make/py-defs.mk | 3 --- make/py-path.mk | 6 ++++++ make/py-run.mk | 6 +++--- make/py-test.mk | 6 +++--- make/py-topdir.mk | 3 ++- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/make/py-defs.mk b/make/py-defs.mk index b96d5cf4..3f02d78c 100644 --- a/make/py-defs.mk +++ b/make/py-defs.mk @@ -73,7 +73,6 @@ endif ifneq ($(PY_DEFS_MK_INCLUDED),true) all: -clean: pyc.clean echo-py: @echo 'PYTHON = $(PYTHON)' @@ -86,8 +85,6 @@ echo-py: @echo 'PY_INSTALL_MOD = $(PY_INSTALL_MOD)' @echo 'PY_INSTALL_SUB_MOD = $(PY_INSTALL_SUB_MOD)' -pyc.clean: - $(RM) -rf $(wildcard *.pyc) __pycache__ libpath: py-libpath py-libpath: @echo 'export PYTHONPATH = $(PYTHONPATH)' diff --git a/make/py-path.mk b/make/py-path.mk index d6fee0cb..11a115cd 100644 --- a/make/py-path.mk +++ b/make/py-path.mk @@ -5,3 +5,9 @@ ifeq ($(origin JW_PKG_PYTHON_PATH),undefined) endif export PYTHONPATH := $(subst $(space),:,$(JW_PKG_PYTHON_PATH)) export MYPYPATH := $(PYTHONPATH) + +all: + +clean: py-tools.clean +py-tools.clean: + $(RM) -rf $(wildcard *.pyc) __pycache__ .mypy_cache .ruff_cache .pytest_cache diff --git a/make/py-run.mk b/make/py-run.mk index 43c210ed..f029ef8e 100644 --- a/make/py-run.mk +++ b/make/py-run.mk @@ -20,6 +20,6 @@ run.sh: echo -e "#!/bin/bash\n\nexport PYTHONPATH=$(PYTHONPATH)\nset -x\nexec $(PYTHON) $(EXE)" '"$$@"' > $@.tmp chmod 755 $@.tmp mv $@.tmp $@ -clean-run-sh: - rm -f run.sh -clean: clean-run-sh +clean.py-run: + rm -rf run.sh +clean: clean.py-run diff --git a/make/py-test.mk b/make/py-test.mk index 69634d8f..9ec652de 100644 --- a/make/py-test.mk +++ b/make/py-test.mk @@ -18,6 +18,6 @@ $(PY_CONFTEST): $(PY_CONFTEST_CMD) > $@.tmp mv $@.tmp $@ -clean: clean.generated -clean.generated: - rm -f $(PY_GENERATED) +clean: clean.py-test +clean.py-test: + rm -rf $(PY_GENERATED) .pytest-cache diff --git a/make/py-topdir.mk b/make/py-topdir.mk index 9b2ab591..65f170f3 100644 --- a/make/py-topdir.mk +++ b/make/py-topdir.mk @@ -72,9 +72,10 @@ ifneq ($(PY_CHECK_RUFF),) $(PY_CHECK_RUFF) check --select TC,FA --fix --unsafe-fixes $(PY_CHECK_ROOTS) endif +clean: clean.topdir clean.topdir: clean.py-check clean.py-check: - rm -rf .mypy_cache + rm -rf .mypy_cache .ruff_cache .pytest_cache pyproject.toml: $(PYTHON) $(JWB_SCRIPT_DIR)/jw-pkg.py -p $(PROJECTS_DIR) -t $(TOPDIR) --topdir-format unaltered projects create-file --format tmpl \ -- 2.55.0