Further improve build time: Try to avoid calling projects.py

This commit tries to remove the necessity to call projects.py from $(TOPDIR) to
speed up recursive builds over all projects yet again. This is a major
undertaking. There are two variables which are filled py projects.py in
$(TOPDIR): PREREQ and PREREQ_DIRS. Sadly, the latter is a path relative to
$(TOPDIR)/make, so this is kind of pointless. Unless the cache is maintained in
$(TOPDIR), a thing I tried to avoid. So this commit is only able to cache
$(PREREQ), not $(PREREQ_DIRS), which still is a hassle. Introduced defs-dirs.mk
for that, to make it accessible to make.mk, and modified all the other parts of
the machinery, too.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/cache.mk

@@ -1,13 +1,15 @@
 # ----- define these variables
 # CACHED_FILES     = $(TOPDIR)/VERSION
 # CACHED_VARS      = VERSION
+.PHONY: cache
 
 CACHE_FILE_MK ?= .cache.mk
 
 # if not checked against MAKECMDGOALS, a standard rule kicks in from rules.mk,
 # and $(CACHE_FILE_MK) is remade with the clean target
-ifeq ($(findstring clean,$(MAKECMDGOALS)),)
+all: cache
-all: $(CACHE_FILE_MK)
+ifneq ($(foreach g,all cache,$(findstring $(g),$(MAKECMDGOALS))),)
+cache: $(CACHE_FILE_MK)
 $(CACHE_FILE_MK): $(CACHED_FILES) $(MODDIR)/make/cache.mk
 ifeq ($(wildcard $(CACHE_FILE_MK)),)
 	@echo $(foreach v,$(CACHED_VARS),$v := $(value $(v))EOL) | $(SED) 's/EOL */\n/g;' | $(GREP) . | tee $@.tmp

make/defs-dirs.mk

@@ -0,0 +1,29 @@
+FIND_SUBDIRS           := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\
+	$(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell $(FIND) . -maxdepth 2 -mindepth 2 -a \( -iname Makefile -o -iname GNUmakefile \))))))
+SUBDIRS_TO_ITERATE      = $(filter-out $(IGNORE_SUBDIRS),$(SUBDIRS))
+
+ALLOWED_SUBDIR_TARGETS ?= all install clean distclean test link-in
+
+SUBDIRS                ?= $(ORDERED_SUBDIRS) $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS))
+#SUBDIRS                ?= $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS))
+#SUBDIRS                ?= $(FIND_SUBDIRS)
+
+ifneq ($(DONT_CHECK_PREREQ_DONE),true)
+  ifndef PREREQ
+    PREREQ           := $(call proj_query,pkg-requires --no-version --no-subpackages --subsections=jw --delimiter=' ' build $(PROJECT))
+  endif
+  ifndef PREREQ_DIRS
+    PREREQ_DIRS      := $(call proj_query,proj-dir $(PREREQ))
+  endif
+  PREREQ_DIRS_DONE   := $(addsuffix /dirs-all.done,$(filter-out $(TOPDIR) /opt/%,$(PREREQ_DIRS)))
+endif
+
+ifneq ($(SUBDIRS_TO_ITERATE),)
+
+  ifeq ($(MAKECMDGOALS),)
+    SUBDIR_TARGETS      = all
+  else
+    SUBDIR_TARGETS      = $(filter $(ALLOWED_SUBDIR_TARGETS),$(MAKECMDGOALS))
+  endif
+  
+endif

make/defs.mk

@@ -7,8 +7,19 @@ JW_BUILD_DEF_MK_INCLUDED = true
 
 MAKEFLAGS             += -r
 
--include $(MODDIR)/make/.cache.mk
+define try_include
--include $(TOPDIR)/make/.cache.mk
+  ifneq ($(wildcard $(1)),)
+    include $(1)
+  endif
+endef
+
+ifneq ($(wildcard $(MODDIR)/make/.cache.mk),)
+  include $(MODDIR)/make/.cache.mk
+endif
+ifneq ($(wildcard $(TOPDIR)/make/.cache.mk),)
+  include $(TOPDIR)/make/.cache.mk
+endif
+
 include $(MODDIR)/make/platform.mk
 include $(MODDIR)/make/projects.mk
 
@@ -87,11 +98,6 @@ ifndef PRINTF
   PRINTF              := $(shell $(WHICH) printf)
 endif
 
-# makefile helpers
-FIND_SUBDIRS          := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\
-	$(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell $(FIND) . -maxdepth 2 -mindepth 2 -a \( -iname Makefile -o -iname GNUmakefile \))))))
-SUBDIRS_TO_ITERATE     = $(filter-out $(IGNORE_SUBDIRS),$(SUBDIRS))
-
 PKG_SH_EXE            := /bin/bash $(MOD_SCRIPT_DIR)/pkg.sh
 LOG_INSTALL_SH        := $(PKG_SH_EXE) log-install
 UPDATE_REPO_SH        := /opt/packager-server/bin/packager-server schedule-rebuild

make/dirs.mk

@@ -3,10 +3,6 @@
 # contact@jannet.de
 # $Id$
 
-ALLOWED_SUBDIR_TARGETS ?= all install clean distclean test link-in
-
-reverse = $(if $(wordlist 2,2,$(1)),$(call reverse,$(wordlist 2,$(words $(1)),$(1))) $(firstword $(1)),$(1))
-
 .NOTPARALLEL:
 .PHONY: $(SUBDIRS_TO_ITERATE) $(ALLOWED_SUBDIR_TARGETS)
 
@@ -15,10 +11,7 @@ ifneq ($(FORCE_REBUILD_SUBDIRS),)
 endif
 
 include $(MODDIR)/make/defs.mk
-
+include $(MODDIR)/make/defs-dirs.mk
-SUBDIRS ?= $(ORDERED_SUBDIRS) $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS))
-#SUBDIRS ?= $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS))
-#SUBDIRS ?= $(FIND_SUBDIRS)
 
 ifeq ($(SUBDIRS_TO_ITERATE),)
 
@@ -26,18 +19,6 @@ $(ALLOWED_SUBDIR_TARGETS):
 
 else
 
-ifeq ($(MAKECMDGOALS),)
-SUBDIR_TARGETS = all
-else
-SUBDIR_TARGETS = $(filter $(ALLOWED_SUBDIR_TARGETS),$(MAKECMDGOALS))
-endif
-
-ifneq ($(DONT_CHECK_PREREQ_DONE),true)
-PREREQ := $(call proj_query,pkg-requires --no-version --no-subpackages --subsections=jw --delimiter=' ' build $(PROJECT))
-PREREQ_DIRS := $(call proj_query,proj-dir $(PREREQ))
-PREREQ_DIRS_DONE := $(addsuffix /dirs-all.done,$(filter-out $(TOPDIR) /opt/%,$(PREREQ_DIRS)))
-endif
-
 all: dirs-all.done
 install: dirs-install.done
 clean: dirs.clean

make/make.mk

@@ -4,9 +4,10 @@
 # $Id$
 
 include $(MODDIR)/make/defs.mk
+include $(MODDIR)/make/defs-dirs.mk
 
 CACHED_FILES      ?= $(VERSION_FILE)
-CACHED_VARS       ?= VERSION HEX_VERSION PROJECT
+CACHED_VARS       ?= PROJECT PREREQ VERSION HEX_VERSION
 
 include $(MODDIR)/make/cache.mk