From 3dd55cf067e2fa5ae13ea0f059bb86c0303a5800 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 28 Feb 2019 14:48:57 +0000 Subject: [PATCH] Further improve build time: Try to avoid calling projects.py This commit tries to remove the necessity to call projects.py from $(TOPDIR) to speed up recursive builds over all projects yet again. This is a major undertaking. There are two variables which are filled py projects.py in $(TOPDIR): PREREQ and PREREQ_DIRS. Sadly, the latter is a path relative to $(TOPDIR)/make, so this is kind of pointless. Unless the cache is maintained in $(TOPDIR), a thing I tried to avoid. So this commit is only able to cache $(PREREQ), not $(PREREQ_DIRS), which still is a hassle. Introduced defs-dirs.mk for that, to make it accessible to make.mk, and modified all the other parts of the machinery, too. Signed-off-by: Jan Lindemann --- make/cache.mk | 6 ++++-- make/defs-dirs.mk | 29 +++++++++++++++++++++++++++++ make/defs.mk | 20 +++++++++++++------- make/dirs.mk | 21 +-------------------- make/make.mk | 3 ++- 5 files changed, 49 insertions(+), 30 deletions(-) create mode 100644 make/defs-dirs.mk diff --git a/make/cache.mk b/make/cache.mk index 8070bb77..4f4a78e2 100644 --- a/make/cache.mk +++ b/make/cache.mk @@ -1,13 +1,15 @@ # ----- define these variables # CACHED_FILES = $(TOPDIR)/VERSION # CACHED_VARS = VERSION +.PHONY: cache CACHE_FILE_MK ?= .cache.mk # if not checked against MAKECMDGOALS, a standard rule kicks in from rules.mk, # and $(CACHE_FILE_MK) is remade with the clean target -ifeq ($(findstring clean,$(MAKECMDGOALS)),) -all: $(CACHE_FILE_MK) +all: cache +ifneq ($(foreach g,all cache,$(findstring $(g),$(MAKECMDGOALS))),) +cache: $(CACHE_FILE_MK) $(CACHE_FILE_MK): $(CACHED_FILES) $(MODDIR)/make/cache.mk ifeq ($(wildcard $(CACHE_FILE_MK)),) @echo $(foreach v,$(CACHED_VARS),$v := $(value $(v))EOL) | $(SED) 's/EOL */\n/g;' | $(GREP) . | tee $@.tmp diff --git a/make/defs-dirs.mk b/make/defs-dirs.mk new file mode 100644 index 00000000..f389a6ab --- /dev/null +++ b/make/defs-dirs.mk @@ -0,0 +1,29 @@ +FIND_SUBDIRS := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\ + $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell $(FIND) . -maxdepth 2 -mindepth 2 -a \( -iname Makefile -o -iname GNUmakefile \)))))) +SUBDIRS_TO_ITERATE = $(filter-out $(IGNORE_SUBDIRS),$(SUBDIRS)) + +ALLOWED_SUBDIR_TARGETS ?= all install clean distclean test link-in + +SUBDIRS ?= $(ORDERED_SUBDIRS) $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS)) +#SUBDIRS ?= $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS)) +#SUBDIRS ?= $(FIND_SUBDIRS) + +ifneq ($(DONT_CHECK_PREREQ_DONE),true) + ifndef PREREQ + PREREQ := $(call proj_query,pkg-requires --no-version --no-subpackages --subsections=jw --delimiter=' ' build $(PROJECT)) + endif + ifndef PREREQ_DIRS + PREREQ_DIRS := $(call proj_query,proj-dir $(PREREQ)) + endif + PREREQ_DIRS_DONE := $(addsuffix /dirs-all.done,$(filter-out $(TOPDIR) /opt/%,$(PREREQ_DIRS))) +endif + +ifneq ($(SUBDIRS_TO_ITERATE),) + + ifeq ($(MAKECMDGOALS),) + SUBDIR_TARGETS = all + else + SUBDIR_TARGETS = $(filter $(ALLOWED_SUBDIR_TARGETS),$(MAKECMDGOALS)) + endif + +endif diff --git a/make/defs.mk b/make/defs.mk index 4525b3ef..2b4c32d7 100644 --- a/make/defs.mk +++ b/make/defs.mk @@ -7,8 +7,19 @@ JW_BUILD_DEF_MK_INCLUDED = true MAKEFLAGS += -r --include $(MODDIR)/make/.cache.mk --include $(TOPDIR)/make/.cache.mk +define try_include + ifneq ($(wildcard $(1)),) + include $(1) + endif +endef + +ifneq ($(wildcard $(MODDIR)/make/.cache.mk),) + include $(MODDIR)/make/.cache.mk +endif +ifneq ($(wildcard $(TOPDIR)/make/.cache.mk),) + include $(TOPDIR)/make/.cache.mk +endif + include $(MODDIR)/make/platform.mk include $(MODDIR)/make/projects.mk @@ -87,11 +98,6 @@ ifndef PRINTF PRINTF := $(shell $(WHICH) printf) endif -# makefile helpers -FIND_SUBDIRS := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\ - $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell $(FIND) . -maxdepth 2 -mindepth 2 -a \( -iname Makefile -o -iname GNUmakefile \)))))) -SUBDIRS_TO_ITERATE = $(filter-out $(IGNORE_SUBDIRS),$(SUBDIRS)) - PKG_SH_EXE := /bin/bash $(MOD_SCRIPT_DIR)/pkg.sh LOG_INSTALL_SH := $(PKG_SH_EXE) log-install UPDATE_REPO_SH := /opt/packager-server/bin/packager-server schedule-rebuild diff --git a/make/dirs.mk b/make/dirs.mk index 23842f5b..38c922af 100644 --- a/make/dirs.mk +++ b/make/dirs.mk @@ -3,10 +3,6 @@ # contact@jannet.de # $Id$ -ALLOWED_SUBDIR_TARGETS ?= all install clean distclean test link-in - -reverse = $(if $(wordlist 2,2,$(1)),$(call reverse,$(wordlist 2,$(words $(1)),$(1))) $(firstword $(1)),$(1)) - .NOTPARALLEL: .PHONY: $(SUBDIRS_TO_ITERATE) $(ALLOWED_SUBDIR_TARGETS) @@ -15,10 +11,7 @@ ifneq ($(FORCE_REBUILD_SUBDIRS),) endif include $(MODDIR)/make/defs.mk - -SUBDIRS ?= $(ORDERED_SUBDIRS) $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS)) -#SUBDIRS ?= $(filter-out $(ORDERED_SUBDIRS),$(FIND_SUBDIRS)) -#SUBDIRS ?= $(FIND_SUBDIRS) +include $(MODDIR)/make/defs-dirs.mk ifeq ($(SUBDIRS_TO_ITERATE),) @@ -26,18 +19,6 @@ $(ALLOWED_SUBDIR_TARGETS): else -ifeq ($(MAKECMDGOALS),) -SUBDIR_TARGETS = all -else -SUBDIR_TARGETS = $(filter $(ALLOWED_SUBDIR_TARGETS),$(MAKECMDGOALS)) -endif - -ifneq ($(DONT_CHECK_PREREQ_DONE),true) -PREREQ := $(call proj_query,pkg-requires --no-version --no-subpackages --subsections=jw --delimiter=' ' build $(PROJECT)) -PREREQ_DIRS := $(call proj_query,proj-dir $(PREREQ)) -PREREQ_DIRS_DONE := $(addsuffix /dirs-all.done,$(filter-out $(TOPDIR) /opt/%,$(PREREQ_DIRS))) -endif - all: dirs-all.done install: dirs-install.done clean: dirs.clean diff --git a/make/make.mk b/make/make.mk index f5097359..3d3f8c60 100644 --- a/make/make.mk +++ b/make/make.mk @@ -4,9 +4,10 @@ # $Id$ include $(MODDIR)/make/defs.mk +include $(MODDIR)/make/defs-dirs.mk CACHED_FILES ?= $(VERSION_FILE) -CACHED_VARS ?= VERSION HEX_VERSION PROJECT +CACHED_VARS ?= PROJECT PREREQ VERSION HEX_VERSION include $(MODDIR)/make/cache.mk