make: Further improve build time

This commit sees several improvements to the build performance:

- Introduce cache.mk, which creates makefiles caching often used
  variables, per tree and per project.
- Define more variables with := enclosed in condistions, instead of
  defining them with ?=, because the RHS of ?= is expanded deferredly.
- Add more definitions for executables.
- Move some more specialized definitions out into specialized makefiles,
  notably htdocs.mk and tmpl.mk

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2019-02-28 13:22:12 +00:00
commit 9e245c986e
7 changed files with 176 additions and 109 deletions

View file

@ -5,22 +5,33 @@
TOPDIR = .. TOPDIR = ..
CACHED_FILES := defs.mk projects.mk platform.mk Makefile
#CACHED_VARS = $(shell sed '/^[ \t]*\([A-Z_]\+\) *\(.=\).*$$.shell/!d; s/ *.=.*//' $(CACHED_FILES))
CACHED_VARS := WHICH BROWSER SED RM PWD ID CUT TR AWK GETENT XARGS FIND PRINTF OS_TUPLE OS_NAME_VERSION OS_NAME CAT BIN_INSTALL SUDO
#CACHED_VARS += TARGET_ARCH RPM_ARCH
CACHED_VARS += TARGET_ARCH
#CACHED_VARS += DEVELOPMENT
include $(TOPDIR)/make/proj.mk include $(TOPDIR)/make/proj.mk
include $(MODDIR)/make/make.mk include $(MODDIR)/make/make.mk
ifneq ($(MAKECMDGOALS),install-links) ifneq ($(MAKECMDGOALS),install-links)
ifneq ($(MAKECMDGOALS),do-install-links) ifneq ($(MAKECMDGOALS),do-install-links)
include $(MODDIR)/make/dirs.mk include $(MODDIR)/make/dirs.mk
endif endif
endif endif
include $(MODDIR)/make/dev-utils.mk include $(MODDIR)/make/dev-utils.mk
#include $(MODDIR)/make/cache.mk
all:
ifeq ($(TARGET),mingw) ifeq ($(TARGET),mingw)
install: $(INSTALL_MAKEDIR)/winres.rc.tmpl install: $(INSTALL_MAKEDIR)/winres.rc.tmpl
endif endif
grep-vars:
@sed '/^ *[A-Z]/!d; s/^[ ]*//g; s/[=+?:].*//g; s/[ ]*//g' *.mk | sort -u
$(INSTALL_MAKEDIR)/%.tmpl: %.tmpl $(TOPDIR)/dir_install_MAKE.done $(INSTALL_MAKEDIR)/%.tmpl: %.tmpl $(TOPDIR)/dir_install_MAKE.done
$(INSTALL) -o $(MAKEOWNER) -g $(MAKEGROUP) -m $(MAKEMODE) $< $@A $(INSTALL) -o $(MAKEOWNER) -g $(MAKEGROUP) -m $(MAKEMODE) $< $@A
grep-vars:
@sed '/^ *[A-Z]/!d; s/^[ ]*//g; s/[=+?:].*//g; s/[ ]*//g' *.mk | sort -u

22
make/cache.mk Normal file
View file

@ -0,0 +1,22 @@
# ----- define these variables
# CACHED_FILES = $(TOPDIR)/VERSION
# CACHED_VARS = VERSION
CACHE_FILE_MK ?= .cache.mk
# if not checked against MAKECMDGOALS, a standard rule kicks in from rules.mk,
# and $(CACHE_FILE_MK) is remade with the clean target
ifeq ($(findstring clean,$(MAKECMDGOALS)),)
all: $(CACHE_FILE_MK)
$(CACHE_FILE_MK): $(CACHED_FILES) $(MODDIR)/make/cache.mk
ifeq ($(wildcard $(CACHE_FILE_MK)),)
@echo $(foreach v,$(CACHED_VARS),$v := $(value $(v))EOL) | $(SED) 's/EOL */\n/g;' | $(GREP) . | tee $@.tmp
mv $@.tmp $@
else
rm $@
make $@
endif
endif
clean: clean-cache
clean-cache:
$(RM) -f $(CACHE_FILE_MK)

View file

@ -7,51 +7,85 @@ JW_BUILD_DEF_MK_INCLUDED = true
MAKEFLAGS += -r MAKEFLAGS += -r
-include $(MODDIR)/make/.cache.mk
-include $(TOPDIR)/make/.cache.mk
include $(MODDIR)/make/platform.mk include $(MODDIR)/make/platform.mk
include $(MODDIR)/make/projects.mk include $(MODDIR)/make/projects.mk
ifeq ($(MAKE_BENCHMARK),true) ifeq ($(MAKE_BENCHMARK),true)
SHELL := /bin/bash $(MOD_SCRIPT_DIR)/timed-make-shell.sh export SHELL := /bin/bash $(MOD_SCRIPT_DIR)/timed-make-shell.sh
else
export SHELL := /bin/bash
endif endif
# ----- pre-local.mk # ----- pre-local.mk
ifneq ($(wildcard $(MODDIR)/make/pre-local.mk),) -include $(MODDIR)/make/pre-local.mk
include $(MODDIR)/make/pre-local.mk -include $(TOPDIR)/make/pre-local.mk
endif -include pre-local.mk
ifneq ($(wildcard $(TOPDIR)/make/pre-local.mk),)
include $(TOPDIR)/make/pre-local.mk
endif
ifneq ($(wildcard pre-local.mk),)
include pre-local.mk
endif
# ----- executables # ----- executables
WHICH := $(firstword $(wildcard /usr/bin/which) $(shell which which)) ifndef WHICH
SED := $(shell $(WHICH) sed) WHICH := $(firstword $(wildcard /usr/bin/which) $(shell which which))
CAT := /bin/cat endif
#ifeq ($(SHELL),) ifndef GREP
export SHELL := /bin/bash GREP := $(shell $(WHICH) grep)
endif
ifndef SED
SED := $(shell $(WHICH) sed)
endif
CAT := $(shell $(WHICH) cat)
ifndef BROWSER
BROWSER := $(shell $(WHICH) xdg-open)
endif
ifeq ($(origin RM),default)
RM := $(shell $(WHICH) rm)
endif
ifndef AR
AR = $(shell $(WHICH) ar)
endif
#ifndef PWD
PWD := $(shell $(WHICH) pwd)
#endif #endif
BROWSER ?= xdg-open ifndef CWD
RM := $(shell $(WHICH) rm) CWD := $(shell $(PWD))
AR = /usr/bin/ar endif
PWD := $(shell $(WHICH) pwd) ifndef BIN_INSTALL
CWD := $(shell $(PWD)) BIN_INSTALL := $(shell $(WHICH) install)
BIN_INSTALL := /usr/bin/install endif
SUDO ?= /usr/bin/sudo ifndef SUDO
MY_SHELL := /bin/bash SUDO := $(shell $(WHICH) sudo)
GET_OS_SH := $(SHELL) $(MOD_SCRIPT_DIR)/get-os.sh endif
ID := $(shell $(WHICH) id) ifndef MY_SHELL
CUT := $(shell $(WHICH) cut) MY_SHELL := /bin/bash
TR := $(shell $(WHICH) tr) endif
AWK := $(shell $(WHICH) awk) ifndef GET_OS_SH
GETENT := $(shell $(WHICH) getent) GET_OS_SH := $(SHELL) $(MOD_SCRIPT_DIR)/get-os.sh
XARGS := $(shell $(WHICH) xargs) endif
FIND := $(shell $(WHICH) find) ifndef ID
PRINTF := $(shell $(WHICH) printf) ID := $(shell $(WHICH) id)
endif
ifndef CUT
CUT := $(shell $(WHICH) cut)
endif
ifndef TR
TR := $(shell $(WHICH) tr)
endif
ifndef AWK
AWK := $(shell $(WHICH) awk)
endif
ifndef GETENT
GETENT := $(shell $(WHICH) getent)
endif
ifndef XARGS
XARGS := $(shell $(WHICH) xargs)
endif
ifndef FIND
FIND := $(shell $(WHICH) find)
endif
ifndef PRINTF
PRINTF := $(shell $(WHICH) printf)
endif
# makefile helpers # makefile helpers
FIND_SUBDIRS := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\ FIND_SUBDIRS := $(filter-out . .. nomake CVS $(IGNORE_SUBDIRS),\
@ -147,27 +181,27 @@ ifeq ($(OS_TUPLE),)
endif endif
ifneq ($(TARGET_TUPLE),) ifneq ($(TARGET_TUPLE),)
TARGET_TUPLE_WORDS ?= $(subst -, ,$(TARGET_TUPLE)) TARGET_TUPLE_WORDS := $(subst -, ,$(TARGET_TUPLE))
TARGET_ARCH ?= $(word 1,$(TARGET_TUPLE_WORDS)) TARGET_ARCH := $(word 1,$(TARGET_TUPLE_WORDS))
TARGET_VENDOR ?= $(word 2,$(TARGET_TUPLE_WORDS)) TARGET_VENDOR := $(word 2,$(TARGET_TUPLE_WORDS))
TARGET_OS ?= $(word 3,$(TARGET_TUPLE_WORDS)) TARGET_OS := $(word 3,$(TARGET_TUPLE_WORDS))
TARGET_ABI ?= $(word 4,$(TARGET_TUPLE_WORDS)) TARGET_ABI := $(word 4,$(TARGET_TUPLE_WORDS))
TARGET_TRIPLET ?= $(TARGET_ARCH)-$(TARGET_OS)-$(TARGET_ABI) TARGET_TRIPLET := $(TARGET_ARCH)-$(TARGET_OS)-$(TARGET_ABI)
else ifneq ($(TARGET_TRIPLET),) else ifneq ($(TARGET_TRIPLET),)
TARGET_TRIPLET_WORDS ?= $(subst -, ,$(TARGET_TRIPLET)) TARGET_TRIPLET_WORDS := $(subst -, ,$(TARGET_TRIPLET))
TARGET_ARCH ?= $(word 1,$(TARGET_TRIPLET_WORDS)) TARGET_ARCH := $(word 1,$(TARGET_TRIPLET_WORDS))
TARGET_VENDOR ?= unknown TARGET_VENDOR := unknown
TARGET_OS ?= $(word 2,$(TARGET_TRIPLET_WORDS)) TARGET_OS := $(word 2,$(TARGET_TRIPLET_WORDS))
TARGET_ABI ?= $(word 3,$(TARGET_TRIPLET_WORDS)) TARGET_ABI := $(word 3,$(TARGET_TRIPLET_WORDS))
TARGET_TUPLE ?= $(TARGET_ARCH)-$(TARGET_VENDOR)-$(TARGET_OS)-$(TARGET_ABI) TARGET_TUPLE := $(TARGET_ARCH)-$(TARGET_VENDOR)-$(TARGET_OS)-$(TARGET_ABI)
else else
TARGET_TUPLE ?= $(OS_TUPLE) TARGET_TUPLE := $(OS_TUPLE)
TARGET_TUPLE_WORDS ?= $(subst -, ,$(TARGET_TUPLE)) TARGET_TUPLE_WORDS := $(subst -, ,$(TARGET_TUPLE))
TARGET_ARCH ?= $(word 1,$(TARGET_TUPLE_WORDS)) TARGET_ARCH := $(word 1,$(TARGET_TUPLE_WORDS))
TARGET_VENDOR ?= $(word 2,$(TARGET_TUPLE_WORDS)) TARGET_VENDOR := $(word 2,$(TARGET_TUPLE_WORDS))
TARGET_OS ?= $(word 3,$(TARGET_TUPLE_WORDS)) TARGET_OS := $(word 3,$(TARGET_TUPLE_WORDS))
TARGET_ABI ?= $(word 4,$(TARGET_TUPLE_WORDS)) TARGET_ABI := $(word 4,$(TARGET_TUPLE_WORDS))
TARGET_TRIPLET ?= $(TARGET_ARCH)-$(TARGET_OS)-$(TARGET_ABI) TARGET_TRIPLET := $(TARGET_ARCH)-$(TARGET_OS)-$(TARGET_ABI)
endif endif
#ifeq ($(TARGET_TUPLE),) #ifeq ($(TARGET_TUPLE),)
@ -246,12 +280,12 @@ ifeq ($(PKG_FORMAT),debian)
RPM_SUFFIX ?= deb RPM_SUFFIX ?= deb
else else
RPM ?= /bin/rpm RPM ?= /bin/rpm
RPM_ARCH ?= $(ARCH) RPM_ARCH ?= $(TARGET_ARCH)
RPM_SUFFIX ?= rpm RPM_SUFFIX ?= rpm
endif endif
ifeq ($(PKG_FORMAT),debian) ifeq ($(PKG_FORMAT),debian)
export INCLUDE_REV_IN_VERSIONS = true export INCLUDE_REV_IN_VERSIONS = true
endif endif
PROJECT_DESCR_FILE ?= $(wildcard $(TOPDIR)/make/project.conf) PROJECT_DESCR_FILE ?= $(wildcard $(TOPDIR)/make/project.conf)
@ -262,9 +296,9 @@ PROJECT_SUMMARY ?= $(PROJECT_DESCR)
CUSTOMER ?= jannet CUSTOMER ?= jannet
ifneq ($(PROJECT_DESCR_FILE),) ifneq ($(PROJECT_DESCR_FILE),)
ifeq ($(shell grep "^ *subpackages *=.*devel" $(PROJECT_DESCR_FILE)),) ifeq ($(shell $(GREP) "^ *subpackages *=.*devel" $(PROJECT_DESCR_FILE)),)
CREATE_DEVEL = false CREATE_DEVEL = false
endif endif
endif endif
ifeq ($(VERSION),) ifeq ($(VERSION),)
@ -410,34 +444,6 @@ INSTALL_CGIDIR = $(CGI_DIR)
BUILD_CGI += $(addprefix $(BUILD_CGIDIR)/,$(EXE_CGI)) BUILD_CGI += $(addprefix $(BUILD_CGIDIR)/,$(EXE_CGI))
INSTALLED_CGI += $(addprefix $(INSTALL_CGIDIR)/,$(EXE_CGI)) INSTALLED_CGI += $(addprefix $(INSTALL_CGIDIR)/,$(EXE_CGI))
# -- HTML
INSTALLATION_FILE_TYPES += HTML
BUILD_HTMLDIR = $(TOPDIR)/htdocs
LOCAL_HTML += $(filter-out $(DONT_INSTALL),$(wildcard *.html *.css *.php *.js) $(FONTS) $(IMAGES))
ifeq ($(HTML_SUBDIR),)
HTML_SUBDIR := $(shell $(PWD) | $(SED) '/.*\/htdocs\(\/\|$$\)/!d; s%.*/htdocs\(/\|$$\)%%')
endif
ifeq ($(HTML_SUBDIR),)
INSTALL_HTMLDIR ?= $(PROJECT_HTMLDIR)
else
INSTALL_HTMLDIR ?= $(PROJECT_HTMLDIR)/$(HTML_SUBDIR)
endif
INSTALLED_HTML += $(addprefix $(INSTALL_HTMLDIR)/,$(LOCAL_HTML))
# -- TMPL
INSTALLATION_FILE_TYPES += TMPL
BUILD_TMPLDIR = $(TOPDIR)/tmpl
LOCAL_TMPL += $(filter-out $(DONT_INSTALL),$(wildcard *.tmpl))
ifndef TMPL_SUBDIR
TMPL_SUBDIR := $(shell $(PWD) | $(SED) '/.*\/tmpl\(\/\|$$\)/!d; s%.*/tmpl\(/\|$$\)%%')
endif
ifeq ($(TMPL_SUBDIR),)
INSTALL_TMPLDIR = $(PROJECT_TMPLDIR)
else
INSTALL_TMPLDIR = $(PROJECT_TMPLDIR)/$(TMPL_SUBDIR)
endif
INSTALLED_TMPL += $(addprefix $(INSTALL_TMPLDIR)/,$(LOCAL_TMPL))
# -- SYSTEMD # -- SYSTEMD
INSTALLATION_FILE_TYPES += SYSTEMD INSTALLATION_FILE_TYPES += SYSTEMD
BUILD_SYSTEMDDIR = $(TOPDIR)/bin/systemd BUILD_SYSTEMDDIR = $(TOPDIR)/bin/systemd
@ -562,17 +568,9 @@ INSTALL_DIR ?= $(INSTALL)
# ----- local.mk # ----- local.mk
ifneq ($(wildcard $(MODDIR)/make/local.mk),) -include $(MODDIR)/make/local.mk
include $(MODDIR)/make/local.mk -include $(TOPDIR)/make/local.mk
endif -include local.mk
ifneq ($(wildcard $(TOPDIR)/make/local.mk),)
include $(TOPDIR)/make/local.mk
endif
ifneq ($(wildcard local.mk),)
include local.mk
endif
BTOOLS_DIR ?= $(call proj_dir, dspider-btools) BTOOLS_DIR ?= $(call proj_dir, dspider-btools)
#DSPCD_DIR ?= $(call proj_dir, dspcd) #DSPCD_DIR ?= $(call proj_dir, dspcd)
@ -598,4 +596,4 @@ doc-module:
doc-project: doc-project:
$(BROWSER) $(JANWARE_URL_DOC_PROJECT) $(BROWSER) $(JANWARE_URL_DOC_PROJECT)
endif endif # ifndef JW_BUILD_DEF_MK_INCLUDED

View file

@ -3,6 +3,22 @@ LESS_GENERATED_CSS ?= $(patsubst %.less,%.css,$(LESS_SRC))
LESS_CPY ?= lesscpy LESS_CPY ?= lesscpy
include $(MODDIR)/make/defs.mk include $(MODDIR)/make/defs.mk
# -- HTML
INSTALLATION_FILE_TYPES += HTML
BUILD_HTMLDIR = $(TOPDIR)/htdocs
LOCAL_HTML += $(filter-out $(DONT_INSTALL),$(wildcard *.html *.css *.php *.js) $(FONTS) $(IMAGES))
ifeq ($(HTML_SUBDIR),)
HTML_SUBDIR := $(shell $(PWD) | $(SED) '/.*\/htdocs\(\/\|$$\)/!d; s%.*/htdocs\(/\|$$\)%%')
endif
ifeq ($(HTML_SUBDIR),)
INSTALL_HTMLDIR ?= $(PROJECT_HTMLDIR)
else
INSTALL_HTMLDIR ?= $(PROJECT_HTMLDIR)/$(HTML_SUBDIR)
endif
INSTALLED_HTML += $(addprefix $(INSTALL_HTMLDIR)/,$(LOCAL_HTML))
include $(MODDIR)/make/js.mk include $(MODDIR)/make/js.mk
include $(MODDIR)/make/rules.mk include $(MODDIR)/make/rules.mk
include $(MODDIR)/make/dev-utils.mk include $(MODDIR)/make/dev-utils.mk

View file

@ -5,6 +5,11 @@
include $(MODDIR)/make/defs.mk include $(MODDIR)/make/defs.mk
CACHED_FILES ?= $(VERSION_FILE)
CACHED_VARS ?= VERSION HEX_VERSION PROJECT
include $(MODDIR)/make/cache.mk
ifneq ($(CREATE_DEVEL),false) ifneq ($(CREATE_DEVEL),false)
MKFILES += $(wildcard project.conf) MKFILES += $(wildcard project.conf)
DEVEL_TARGETS += install_MAKE $(INSTALLED_PKG_MK) install_PKG_CONFIG DEVEL_TARGETS += install_MAKE $(INSTALLED_PKG_MK) install_PKG_CONFIG

View file

@ -1,7 +1,7 @@
# == Variables for inter-project-communication # == Variables for inter-project-communication
# -- What do I know about myself? # -- What do I know about myself?
VERSION_FILE ?= $(firstword $(wildcard VERSION) $(TOPDIR)/VERSION) VERSION_FILE := $(firstword $(wildcard VERSION) $(TOPDIR)/VERSION)
ifndef DEVELOPMENT ifndef DEVELOPMENT
ifeq ($(wildcard $(VERSION_FILE)),) ifeq ($(wildcard $(VERSION_FILE)),)
DEVELOPMENT := true DEVELOPMENT := true
@ -17,13 +17,13 @@ ifeq ($(DEVELOPMENT),true)
else else
PRJS_DIR ?= /opt PRJS_DIR ?= /opt
endif endif
MOD_SCRIPT_DIR ?= $(firstword $(wildcard $(MODDIR)/scripts $(MODDIR)/bin)) MOD_SCRIPT_DIR := $(firstword $(wildcard $(MODDIR)/scripts $(MODDIR)/bin))
# -- Query the build system about other projects: # -- Query the build system about other projects:
PYTHON ?= /usr/bin/python2 PYTHON ?= /usr/bin/python2
ifneq ($(TOPDIR),) ifneq ($(TOPDIR),)
proj_query_cmd = $(PYTHON) $(MOD_SCRIPT_DIR)/projects.py -p $(PRJS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_ARGS) proj_query_cmd = $(PYTHON) $(MOD_SCRIPT_DIR)/projects.py -p $(PRJS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_ARGS)
proj_query = $(shell $(proj_query_cmd) $(1)) proj_query = $(shell $(proj_query_cmd) $(1))
proj_dir = $(call proj_query,proj-dir $(1)) proj_dir = $(call proj_query,proj-dir $(1))
htdocs_dir = $(call proj_query,htdocs-dir $(1)) htdocs_dir = $(call proj_query,htdocs-dir $(1))
endif endif

View file

@ -1,4 +1,19 @@
include $(MODDIR)/make/defs.mk include $(MODDIR)/make/defs.mk
# -- TMPL
INSTALLATION_FILE_TYPES += TMPL
BUILD_TMPLDIR = $(TOPDIR)/tmpl
LOCAL_TMPL += $(filter-out $(DONT_INSTALL),$(wildcard *.tmpl))
ifndef TMPL_SUBDIR
TMPL_SUBDIR := $(shell $(PWD) | $(SED) '/.*\/tmpl\(\/\|$$\)/!d; s%.*/tmpl\(/\|$$\)%%')
endif
ifeq ($(TMPL_SUBDIR),)
INSTALL_TMPLDIR = $(PROJECT_TMPLDIR)
else
INSTALL_TMPLDIR = $(PROJECT_TMPLDIR)/$(TMPL_SUBDIR)
endif
INSTALLED_TMPL += $(addprefix $(INSTALL_TMPLDIR)/,$(LOCAL_TMPL))
include $(MODDIR)/make/rules.mk include $(MODDIR)/make/rules.mk
all: all: