jw-pkg/make/projects-dir.mk

# SPDX-License-Identifier: LGPL-2.0-only
#
# Makefile for managing multiple software repositories in one tree
#
# (C) Copyright 2001-2025, Jan Lindemann <jan@janware.com>
#
# This is the top-level Makefile for a software build tree organized by
# jw-pkg. It is provided under the terms of the GNU Lesser Public License,
# Version 2.
#
# Current documentation on how this Makefile is meant to be used can be found
# under https://janware.com/wiki/pub/en/sw/build/. Running "make help" might
# take you there semi-automatically.
#

# ------------ Makefile and environment variable definitions

.NOTPARALLEL:

PROJECTS_MAKEFILE_NAME      ?= $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))

# -- Find JWBDIR
DEV_PROJECTS_DIR            ?= .
JWBDIR_NAME                 ?= jw-pkg
JWBDIR_SEARCH_PATH          ?= $(DEV_PROJECTS_DIR) $(BUILD_TOOLS_PREFIX)/opt/$(FLAVOUR_PATH_PREFIX)
JWBDIR                      ?= $(firstword $(wildcard $(addsuffix /$(JWBDIR_NAME),$(JWBDIR_SEARCH_PATH))))

JW_PKG_BINDIR                = $(JWBDIR)/bin
JWB_SCRIPT_DIR               = $(firstword $(wildcard $(JWBDIR)/scripts $(JW_PKG_BINDIR)) jwb-script-dir-not-found)
JW_PKG_REMOTE_BINDIR         = /opt/jw-pkg/bin

SHELL                        = /bin/bash -o pipefail +H
PROJECTS_TXT                ?= projects.txt
JW_PKG_VERBOSE              ?= false
PREREQ_RELEASE              ?= pull
ifneq ($(JANWARE_USER),)
  export JANWARE_USER
endif

ifneq ($(GIT_ASKPASS),)
  GIT_ASKPASS               := $(realpath $(GIT_ASKPASS))
endif

ifneq ($(SSH_ASKPASS),)
  SSH_ASKPASS               := $(realpath $(SSH_ASKPASS))
endif

# ------------ evaluate Makefile and environment variables

ifneq ($(wildcard $(PROJECTS_TXT)),)
  PROJECTS                  ?= $(shell cat $(PROJECTS_TXT) | sed '/^ *\#/ d')
else
  PROJECTS                  ?= $(shell ls -d */GNUmakefile */Makefile 2>/dev/null | sed 's%/[^/]*%%' | sort -u)
endif

BUILD_DEP_FLAVOURS          ?= run,build,test,release
BUILD_DEP_FLAVOURS_LINT     ?= run,build,test,release
BUILD_DEP_FLAVOURS_TEST     ?= run,build,test,release
BUILD_DEP_FLAVOURS_ORDER    ?= run,build,test,release

ifeq ($(JW_PKG_VERBOSE),true)
  SSH_WRAPPER_TRACE         ?= -x
endif

export JW_PKG_VERBOSE

# ------------ external programs I

CWD                         := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
JW_PKG_PY                    = python3 $(JWB_SCRIPT_DIR)/jw-pkg.py --prefix $(shell pwd) $(JW_PKG_PY_EXTRA_OPTS)
SSH_WRAPPER_SH              := $(CWD)/ssh-wrapper.sh

EXCLUDES_FILE               ?= exclude.txt
EXCLUDES_FILES              := $(wildcard $(patsubst %,exclude-%.txt,$(shell $(JW_PKG_PY) platform info --format '%{cascade}')) $(EXCLUDES_FILE))

ifneq ($(EXCLUDES_FILES),)
  EXCLUDE_FROM_BUILD        += $(shell sed 's/\#.*//g' $(EXCLUDES_FILES))
endif
OFFLINE                     ?= false
OFFLINE_PROJECTS            ?= $(EXCLUDE_FROM_BUILD)

TEXT_FILES_CACHE            ?= text-files.txt

ifeq ($(OFFLINE),true)
  UNAVAILABLE_TARGETS       ?= pull.done update.done get.done
else
  UNAVAILABLE_TARGETS       ?=
endif

ifneq ($(origin JW_PKG_SSH),undefined)
  export GIT_SSH            := $(JW_PKG_SSH)
else
  export GIT_SSH            := $(SSH_WRAPPER_SH)
endif

JW_PKG_SSH_EXTRA_OPTS       += \
		-o StrictHostKeyChecking=no \
		-o BatchMode=yes \
		-o ServerAliveInterval=15 \
		-o ServerAliveCountMax=3 \
		-o ControlMaster=auto \
		-o ControlPath=/tmp/%r@jw-pkg:%h:%p \
		-o ControlPersist=3m
ifneq ($(JANWARE_USER),)
  JW_PKG_SSH_EXTRA_OPTS   += -l $(JANWARE_USER)
endif
export JW_PKG_SSH_EXTRA_OPTS

ifneq ($(EXCLUDE_FROM_BUILD),)
  JW_PKG_PY_EXTRA_BUILD_OPTS += --exclude "$(EXCLUDE_FROM_BUILD)"
endif

# non-interactive mode
INTERACTIVE                 ?= auto

# ------------ external programs II

BROWSER                     ?= xdg-open
EDITOR                      ?= xdg-open
ifeq ($(TIME),)
  TIME                      := $(shell which time)
  ifneq ($(TIME),)
    TIME                    += -p
  endif
endif
JW_PKG_PY_PROJECTS           = $(JW_PKG_PY) projects
JW_PKG_PY_BUILD              = $(JW_PKG_PY_PROJECTS) build $(JW_PKG_PY_EXTRA_BUILD_OPTS)
PKG_MANAGER                 ?= $(TIME) $(JW_PKG_PY) --interactive=$(INTERACTIVE) pkg

ifneq ($(origin PROJECTS_DIR_REMOTE_BASE),undefined)
  PGIT_SH                   += --remote-base $(PROJECTS_DIR_REMOTE_BASE)
endif
PGIT_SH_GET                 := $(PGIT_SH) get
ifneq ($(JANWARE_USER),)
  PGIT_SH_OPTS_NETWORK      += --login $(JANWARE_USER)
  PGIT_SH_GET               += $(PGIT_SH_OPTS_NETWORK) --create-remote-user-repos
endif
PGIT_SH_GET_DEFAULT          = $(PGIT_SH_GET)
ifneq ($(CLONE_FROM_USER),)
  PGIT_SH_GET_DEFAULT       += --refspec $(CLONE_FROM_USER):current-branch:current-branch
endif
ifneq ($(OFFLINE_PROJECTS),)
  export PGIT_IGNORE         = $(OFFLINE_PROJECTS)
endif

PURGE_SH                     = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/purge-stale-projects.sh $(JW_PKG_BINDIR)/purge-stale-projects.sh) purge-not-found)
CREATE_PROJECT_SH           ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-pkg-create-project.sh $(JW_PKG_BINDIR)/jw-pkg-create-project.sh) jw-pkg-create-project-not-found)
LIST_VCS_FILES_SH            = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/scm.sh $(JW_PKG_BINDIR)/scm.sh) scm-sh-not-found) ls-files
GIT_SRV_ADMIN_SH             = $(GIT_SSH) $(JANWARE_USER)@devgit.janware.com $(JW_PKG_REMOTE_BINDIR)/git-srv-admin.sh
JANWARE_PACKAGE_FILTER       = url =~ janware

# ------------ projects to be built

TARGET_PROJECTS              = $(filter-out $(EXCLUDE_FROM_BUILD),$(PROJECTS))
DEP_PROJECTS                 = $(shell $(JW_PKG_PY_PROJECTS) pkg-requires --recursive --syntax names-only --no-subpackages --delimiter ' ' --subsections jw run,build,devel,release $(TARGET_PROJECTS))
GIT_PROJECTS                 = $(patsubst %/,%,$(dir $(wildcard $(addsuffix /.git,$(DEP_PROJECTS)))))
PROJECTS_WITH_PROJECT_CONF   = $(patsubst %/make/project.conf,%,$(wildcard $(addsuffix /make/project.conf,$(DEP_PROJECTS))))

# ------------ targets

# --- mandatory targets

all:
all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(JW_PKG_PY_BUILD) --dep-flavours="$(BUILD_DEP_FLAVOURS)" $@ $(TARGET_PROJECTS)
check check-pre check-post: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(JW_PKG_PY_BUILD) --dep-flavours="$(BUILD_DEP_FLAVOURS_LINT)" $@ $(TARGET_PROJECTS)
test: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(JW_PKG_PY_BUILD) --dep-flavours="$(BUILD_DEP_FLAVOURS_TEST)" $@ $(TARGET_PROJECTS)


clean: clean-dirs
distclean: clean-all-dirs done.clean
install:
	@echo
	@echo "  Target install is not supported by this Makefile."
	@echo "  Target pkg-rebuild-reinstall might be what you are looking for."
	@echo
	$(Q)exit 1

# --- build targets

rebuild: clean purge pull subdirs-all

subdirs-%:
	FORCE_REBUILD_SUBDIRS=true make $*

# --- informative-only targets

help doc-project doc-module:
	$(BROWSER) $(firstword $(shell sed '/https:/ !d; s%.*https%https%; s/ .*//' $(firstword $(MAKEFILE_LIST))))

status: $(SSH_WRAPPER_SH)

build-order-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	@JW_DEFAULT_LOG_LEVEL=WARNING $(JW_PKG_PY_BUILD) --build-order --dep-flavours="$(BUILD_DEP_FLAVOURS_ORDER)" $* $(TARGET_PROJECTS) | sed 's/  */\n/g'

build-order: build-order-all

echo-build-deps:
	$(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build" $(TARGET_PROJECTS)

echo-install-deps:
	$(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run" $(TARGET_PROJECTS)

echo-release-deps:
	$(Q)$(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run,release" $(TARGET_PROJECTS)
echo-os:
	$(Q)$(JW_PKG_PY) platform info

echo-projects:
	@echo $(PROJECTS)

echo-target-projects:
	@echo $(TARGET_PROJECTS)

echo-excludes:
	@echo $(EXCLUDE_FROM_BUILD)

edit-%: | $(TEXT_FILES_CACHE)
	$(EDITOR) $(shell grep "/$*$$" $(TEXT_FILES_CACHE))

distclean: clean.text-files-cache
clean.text-files-cache:
	rm -f $(TEXT_FILES_CACHE)
list-files:
	$(Q)for p in $(DEP_PROJECTS); do \
		$(LIST_VCS_FILES_SH) -znf $$p | sed -z "s/^/$$p\//" | \
			xargs -0 realpath -q ;\
	done
$(TEXT_FILES_CACHE):
	$(Q)make -s text-files-update
text-files-update:
	make -s --no-print-directory list-files | tr '\n' '\0' | xargs -0 grep -d skip -Il . > $(TEXT_FILES_CACHE).tmp 2>/dev/null || true
	mv $(TEXT_FILES_CACHE).tmp $(TEXT_FILES_CACHE)
text-files-update-all:
	$(Q)PROJECTS_TXT= make text-files-update
text-files-list list-text-files: | $(TEXT_FILES_CACHE)
	$(Q)cat $(TEXT_FILES_CACHE)
text-files-list-0 list-text-files-0: | $(TEXT_FILES_CACHE)
	$(Q)cat $(TEXT_FILES_CACHE) | tr '\n' '\0'

cloc:
	for p in $(GIT_PROJECTS); do \
		git -C $$p submodule status | sed "s|^ *\([^ ]\+\) \+\([^ ]\+\)  *.*|$$p/\2|" ;\
	done > cloc-ignore.txt
	for p in $(foreach s,dist include bin lib,$(addsuffix /$s,$(DEP_PROJECTS))); do \
		echo $$p >> cloc-ignore.txt ;\
	done
	cloc --exclude-list-file=cloc-ignore.txt $(DEP_PROJECTS)

# --- package-related targets

pkg-delete-ours:
	$(PKG_MANAGER) select "$(JANWARE_PACKAGE_FILTER)" | xargs -r $(PKG_MANAGER) delete

pkg-manager-refresh:
	$(PKG_MANAGER) refresh

pkg-manager-dup:
	$(PKG_MANAGER) dup

pkg-install-build-deps:
	$(PKG_MANAGER) install $(shell $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build" $(TARGET_PROJECTS))

pkg-install-release-deps:
	$(PKG_MANAGER) install $(shell $(JW_PKG_PY_PROJECTS) required-os-pkg --quote --skip-excluded "build,run,release" $(TARGET_PROJECTS))

pkg-release-reinstall: $(PREREQ_RELEASE)

pkg-release-all:
	/bin/bash ./packager-client/scripts/packager-client-2.sh

pkg-init-%:
	$(CREATE_PROJECT_SH) $*

pkg-%install: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(JW_PKG_PY_BUILD) --env-reinit --env-keep=HOME,SSH_AUTH_SOCK $@ $(TARGET_PROJECTS)

pkg-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(JW_PKG_PY_BUILD) $@ $(TARGET_PROJECTS)

# --- generic cleanup targets

clean-all-dirs:
	$(JW_PKG_PY_BUILD) clean $(sort $(subst /,,$(dir $(wildcard */*.done))) $(PROJECTS))

clean-dirs:
	$(JW_PKG_PY_BUILD) clean $(PROJECTS)

purge: $(SSH_WRAPPER_SH)
ifneq ($(PURGE_SH),/bin/bash purge-not-found)
	$(PURGE_SH) --vcs git
endif

$(PROJECTS_TXT):
	echo $(PROJECTS) | sed 's/ /\n/g; s%/%%g' > $@

done.clean:
	rm -f *.done

# --- collab targets

list-maintainers: $(SSH_WRAPPER_SH)
	$(GIT_SRV_ADMIN_SH) $@

update pull: purge git-get
	touch pull.done

sync: pull push

sync-all: pull-all push-all

sync-%:
	ssh $* make -C $(shell pwd) sync

pull-all: purge git-get git-pull-all
	touch get.done
	touch pull.done

diff-all diff: $(SSH_WRAPPER_SH)
	$(PGIT_SH) --porcelain diff
diff-projects:
	PGIT_SH_PROJECTS="$(patsubst %/.git,%,$(wildcard $(addsuffix /.git,$(shell make -s build-order))))" $(PGIT_SH) --porcelain diff

get-official: git-get-official
get-maintainer: git-get-maintainer
get-pub: git-get-pub
get-%: git-get-%
	@:

# --- git targets

git-push push: $(SSH_WRAPPER_SH)
	$(PGIT_SH) push $(PGIT_SH_OPTS_NETWORK) --force-with-lease

git-push-all: $(SSH_WRAPPER_SH)
	$(PGIT_SH) push $(PGIT_SH_OPTS_NETWORK) --force-with-lease --all --recurse-submodules=on-demand

git-diff: $(SSH_WRAPPER_SH)
	$(PGIT_SH) --porcelain diff

git-short-diff: $(SSH_WRAPPER_SH)
	$(PGIT_SH) --porcelain diff --shortstat

git-status:
	$(PGIT_SH) status -uno

git-log-%:
	$(PGIT_SH) log -$* 2>&1

git-get: $(SSH_WRAPPER_SH)
	$(PGIT_SH_GET_DEFAULT)
	touch get.done

git-get-mini: $(SSH_WRAPPER_SH)
	PGIT_SH_PROJECTS="$(patsubst %/.git,%,$(wildcard $(addsuffix /.git,$(shell make -s build-order))))" $(PGIT_SH_GET_DEFAULT)

git-pull-all: $(SSH_WRAPPER_SH)
	$(PGIT_SH) pull $(PGIT_SH_OPTS_NETWORK) --all

git-get-maintainer: $(SSH_WRAPPER_SH)
	PGIT_SH_PROJECTS="$(PROJECTS_WITH_PROJECT_CONF)" $(PGIT_SH) exec make $@

git-get-pub:
	PGIT_SH_PROJECTS="$(PROJECTS_WITH_PROJECT_CONF)" $(PGIT_SH) exec make $@

git-get-official: $(SSH_WRAPPER_SH)
	PGIT_SH_PROJECTS="$(PROJECTS_WITH_PROJECT_CONF)" $(PGIT_SH) exec make $@

git-get-%: $(SSH_WRAPPER_SH)
	$(PGIT_SH_GET) --refspec "$*:master:current-branch"

git-show-non-master-branches:
	$(Q)$(PGIT_SH) --porcelain branch 2>&1 | \
		sed '/^#\|^*/!d; s/.*git -C //; s/ *branch *//; s/ *\* *//' | \
		while read p; do \
			read b ;\
			if [ "$$b" != "master" ]; then \
				echo " * $$p:	$$b" ;\
			fi ;\
		done

git-show-pushable:
	$(Q)for p in $(DEP_PROJECTS); do \
		if ! git -C $$p rev-parse --abbrev-ref --symbolic-full-name @{upstream} >/dev/null 2>&1; then \
			echo ======================= $$p ;\
			git -C $$p log --oneline master.. ;\
		elif git -C $$p log --oneline @{upstream}..HEAD | grep . >/dev/null 2>&1; then \
			echo ======================= $$p ;\
			git -C $$p log --oneline @{u}..HEAD ;\
		fi ;\
	done

git-show-ahead-of-master:
	$(Q)for p in $(DEP_PROJECTS); do \
		if git -C $$p log --oneline origin/master.. | grep . >/dev/null; then \
			echo ======================= $$p ;\
			git -C $$p log --oneline origin/master.. ;\
		fi ;\
	done

# git-echo-link-<filename> returns a string functioning as hyperlink to
# matching files in git when embedded into a janware wiki or ticket.
git-echo-links-%: | $(TEXT_FILES_CACHE)
	sed "/$*$$/!d; s%$(CWD)%%; s|^|\n  \[\[jgit>/proj/$(JANWARE_USER)/|; s/$$/|$*\]\]\n/" $(TEXT_FILES_CACHE)

git-update-project-descriptions: $(SSH_WRAPPER_SH)
	$(GIT_SRV_ADMIN_SH) -j update-descriptions all

git-commit:
	$(PGIT_SH) commit

# --- rules

$(SSH_WRAPPER_SH): $(JWB_SCRIPT_DIR)/ssh-wrapper.sh
	cp $< $@.tmp
	chmod 700 $@.tmp
	mv $@.tmp $@
ssh-wrapper: $(SSH_WRAPPER_SH)
clean.ssh-wrapper:
	rm -f $(SSH_WRAPPER_SH)
distclean: clean.ssh-wrapper

pull.done: $(filter-out $(UNAVAILABLE_TARGETS),get.done)
	touch $@

get.done: $(filter-out $(UNAVAILABLE_TARGETS),$(SSH_WRAPPER_SH))
	$(PGIT_SH_GET_DEFAULT)
	touch $@