make/[Makefile|*.mk): Improve variable caching

This commit aims at improving speed by using better caching.

  - Makefile, cache.mk: Split .cache.mk up

    To allow caching of runtime path variables which are
    project-specific, split .cache.mk up in .cache-project.mk and
    .cache-projects.mk

  - ldlibpath.mk: Cache ldlibpath, exepath and pythonpath

    Place the output of $(call proj_query ldlibpath), $(call
    proj_query, exepath) and $(call proj_query pythonpath) in
    JW_PKG_LD_LIBRARY_PATH, JW_PKG_EXE_PATH, and JW_PKG_PYTHON_PATH
    respectively, and cache the variables in make/.project-cache.mk.

  - cache.mk: Use = instead of :=

    Recursively expanded variables are nearly as fast as := variables
    if the assigned value is a fixed string. And sometimes it's not,
    rightly so, because variables get assigned below, as with
    JW_PKG_XXX for instance.

  - cache.mk: Use $(TOPDIR) as variable values

    Replace absolute references to project's topdir by $(TOPDIR) with
    sed. As soon as the project queries produce absolute paths, they
    will be transformed into relative paths which allow the code base
    to be moved to a different location and still remain functional
    without a rebuild.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/Makefile

@@ -12,7 +12,8 @@ endif
 
 include $(JWBDIR)/make/dev-utils.mk
 
-all:
+all: cache-projects
+clean: clean-cache-projects
 
 ifeq ($(TARGET),mingw)
 install: $(INSTALL_MAKEDIR)/winres.rc.tmpl

make/cache.mk

@@ -1,24 +1,39 @@
 # ----- define these variables
 # CACHED_FILES     = $(TOPDIR)/VERSION
 # CACHED_VARS      = VERSION
-.PHONY: cache
+.PHONY: cache clean-cache clean-cache-projects
 
-CACHE_FILE_MK ?= .cache.mk
+CACHE_PROJECT_MK            ?= .cache-project.mk
-
+CACHE_PROJECTS_MK           ?= .cache-projects.mk
-# if not checked against MAKECMDGOALS, a standard rule kicks in from rules.mk,
+CACHE_FILES_MK               = $(CACHE_PROJECT_MK) $(CACHE_PROJECTS_MK)
-# and $(CACHE_FILE_MK) is remade with the clean target
+CACHED_VARS_ONLY_PROJECT    ?= \
+	PROJECT \
+	PREREQ \
+	VERSION \
+	HEX_VERSION \
+	FULL_NAME \
+	JW_PKG_LD_LIBRARY_PATH \
+	JW_PKG_EXE_PATH \
+	JW_PKG_PYTHON_PATH
 all: cache
-ifneq ($(foreach g,all cache,$(findstring $(g),$(MAKECMDGOALS))),)
-cache: $(CACHE_FILE_MK)
-$(CACHE_FILE_MK): $(CACHED_FILES) $(JWBDIR)/make/cache.mk
-ifeq ($(wildcard $(CACHE_FILE_MK)),)
-	@echo $(foreach v,$(CACHED_VARS),$v := $(value $(v))EOL) | $(SED) 's/EOL */\n/g;' | $(GREP) . | tee $@.tmp
-	mv $@.tmp $@
-else
-	rm $@
-	make $@
-endif
-endif
 clean: clean-cache
 clean-cache:
-	rm -f $(CACHE_FILE_MK)
+	rm -f $(CACHE_PROJECT_MK)
+
+cache: $(CACHE_PROJECT_MK)
+$(CACHE_PROJECT_MK): $(CACHED_FILES) $(JWBDIR)/make/cache.mk
+	@echo $(foreach v,$(CACHED_VARS),$v = $(value $(v))EOL) | \
+		$(SED) 's/EOL */\n/g;' | \
+		$(SED) 's|$(realpath $(TOPDIR))|$$(TOPDIR)|g' | \
+		$(SED) 's|$(realpath $(PROJECTS_DIR))|$$(PROJECTS_DIR)|g' | \
+		$(GREP) . | \
+		tee $@.tmp
+	mv $@.tmp $@
+
+# -- Only used in jw-pkg
+cache-projects: $(CACHE_PROJECTS_MK)
+clean-cache-projects:
+	rm -f $(CACHE_PROJECTS_MK)
+$(CACHE_PROJECTS_MK): $(CACHE_PROJECT_MK)
+	sed '/\($(subst $(space),\|,$(CACHED_VARS_ONLY_PROJECT))\)\s*[:?]\?=/ d' $< > $@.tmp
+	mv $@.tmp $@

make/ldlibpath.mk

@@ -1,17 +1,25 @@
-# jan's utility modules
+# == System environment variables
-# (c) 2001-2005 jannet it services
-# contact@jannet.de
-# $Id$
-
-LD_LIB_PATH_ENV             := $(LD_LIBRARY_PATH)
-EXE_SEARCH_PATH_ENV         := $(PATH)
-LD_LIB_PATH_LDFLAGS          = $(shell echo $(FINAL_LDFLAGS) | $(SED) 's/^-[^L] *[^ ]*/ /g; s/[ 	]-[^L] *[^ ]*/ /g; s/-L[ ]*\([^ ]*\)[ ]*/\1:/g')
 
+# -- LD_LIBRARY_PATH
+LD_LIBRARY_PATH_ENV          = $(shell echo $(FINAL_LDFLAGS) | $(SED) 's/^-[^L] *[^ ]*/ /g; s/[ 	]-[^L] *[^ ]*/ /g; s/-L[ ]*\([^ ]*\)[ ]*/\1:/g')
+ifndef JW_PKG_LD_LIBRARY_PATH
+  JW_PKG_LD_LIBRARY_PATH    := $(call proj_query, ldlibpath $(PROJECT) $(PREREQ_BUILD))
+endif
+export LD_LIBRARY_PATH      := $(JW_PKG_LD_LIBRARY_PATH)
 ifeq ($(TARGET),mingw)
-DLL_PATH                     = $(shell echo $(LD_LIBRARY_PATH) | $(SED) 's/:/;/g');$(CROSS_TOOL_DIR)/bin
+  DLL_PATH                   = $(shell echo $(LD_LIBRARY_PATH) | $(SED) 's/:/;/g');$(CROSS_TOOL_DIR)/bin
 endif
 
-#export LD_LIBRARY_PATH      = $(shell echo $(strip $(LD_LIB_PATH_LDFLAGS):$(LD_LIB_PATH):$(LD_LIB_PATH_ENV)) | $(SED) 's/ /:/g; s/::/:/g')
+# -- PATH
-LD_LIBRARY_PATH             := $(call proj_query, ldlibpath $(PROJECT) $(PREREQ_BUILD))
+EXE_SEARCH_PATH_ENV         := $(PATH)
-export LD_LIBRARY_PATH
+ifndef JW_PKG_EXE_PATH
-export PATH                 := $(call proj_query, exepath $(PROJECT) $(PREREQ_BUILD)):$(EXE_SEARCH_PATH_ENV)
+  JW_PKG_EXE_PATH           := $(call proj_query, exepath $(PROJECT) $(PREREQ_BUILD)):$(EXE_SEARCH_PATH_ENV)
+endif
+export PATH                 := $(JW_PKG_EXE_PATH)
+
+# -- PYTHONPATH
+PYTHONPATH_ENV              := $(PYTHONPATH)
+ifndef JW_PKG_PYTHON_PATH
+  JW_PKG_PYTHON_PATH        := $(call proj_query, pythonpath $(PROJECT))
+endif
+export PYTHONPATH           := $(JW_PKG_PYTHON_PATH)

make/make.mk

@@ -4,6 +4,7 @@
 # $Id$
 
 include $(JWBDIR)/make/defs.mk
+include $(JWBDIR)/make/ldlibpath.mk
 include $(JWBDIR)/make/defs-dirs.mk
 include $(JWBDIR)/make/dev-utils.mk
 
@@ -11,7 +12,8 @@ CACHED_FILES                ?= $(VERSION_FILE) $(wildcard $(TOPDIR)/make/project
 CACHED_VARS                 ?= PROJECT PREREQ VERSION HEX_VERSION FULL_NAME \
 				WHICH PYTHON ECHO TEST BROWSER SED RM PWD ID CUT TR \
 				AWK GETENT XARGS FIND PRINTF HOST_TUPLE OS_NAME_VERSION \
-				OS_NAME CAT BIN_INSTALL SUDO
+				OS_NAME CAT BIN_INSTALL SUDO \
+				JW_PKG_LD_LIBRARY_PATH JW_PKG_EXE_PATH JW_PKG_PYTHON_PATH
 
 include $(JWBDIR)/make/cache.mk
 

make/platform.mk

@@ -10,8 +10,8 @@ define try_include
   endif
 endef
 
-$(eval $(call try_include,$(JWBDIR)/make/.cache.mk))
+$(eval $(call try_include,$(JWBDIR)/make/.cache-projects.mk))
-$(eval $(call try_include,$(TOPDIR)/make/.cache.mk))
+$(eval $(call try_include,$(TOPDIR)/make/.cache-project.mk))
 $(eval $(call try_include,$(JWBDIR)/make/pre-local.mk))
 $(eval $(call try_include,$(TOPDIR)/make/pre-local.mk))
 $(eval $(call try_include,pre-local.mk))

make/py-defs.mk

@@ -1,5 +1,3 @@
-ENV_PYTHONPATH              := $(PYTHONPATH)
-
 include $(JWBDIR)/make/py-version.mk
 
 ifeq ($(DEVELOPMENT),true)
@@ -77,8 +75,9 @@ ifeq ($(PY_INSTALL_REG),true)
 endif
 
 ifeq ($(origin PYTHONPATH),undefined)
-  PYTHONPATH                := $(call proj_query, pythonpath $(PROJECT) $(PY_PREREQ_RUN))
+  PYTHONPATH                := $(JW_PKG_PYTHON_PATH)
 endif
+
 #PYTHONPATH                  = $(subst $(space),,$(ENV_PYTHONPATH)$(foreach d,$(PY_MODULE_DIRS),:$(d)))
 MYPYPATH                     = $(PYTHONPATH)
 export PYTHONPATH MYPYPATH

make/rpmdist.mk

@@ -140,7 +140,7 @@ pkg-update: pkgupdate.dist
 pkg-upload-local: pkg-upload-local.dist
 
 clean.rpm:
-	$(RM) -rf $(DIST_DIR) *.dist make/.cache.mk
+	$(RM) -rf $(DIST_DIR) *.dist make/.cache-project.mk make/.cache-projects.mk
 
 #ifeq ($(PROJECT_DESCR_FILE),)
 #

make/topdir.mk

@@ -224,5 +224,4 @@ echo-hash:
 	@/bin/bash $(PKG_SH_EXE) hash
 
 recache-vars:
-	rm -f $(TOPDIR)/make/.cache.mk
+	rm -f $(TOPDIR)/make clean-cache cache
-	make -C $(TOPDIR)/make .cache.mk