diff --git a/make/projects-dir-include.mk b/make/projects-dir-include.mk new file mode 100644 index 00000000..4c3f3043 --- /dev/null +++ b/make/projects-dir-include.mk @@ -0,0 +1,3 @@ +all: + +include $(JWBDIR)/make/projects-dir.mk diff --git a/make/projects-dir-minimal.mk b/make/projects-dir-minimal.mk new file mode 100644 index 00000000..df517fe9 --- /dev/null +++ b/make/projects-dir-minimal.mk @@ -0,0 +1,41 @@ +# +# SPDX-License-Identifier: LGPL-2.0-only +# +# Makefile for managing multiple software repositories in one tree +# +# (C) Copyright 2001-2025, Jan Lindemann +# +# This is the top-level Makefile for a janware software build tree. It is +# provided under the terms of the GNU Lesser Public License, Version 2. +# +# Some of its targets download software from janware GmbH servers. For those, +# you will need a janware.com user account. Ask admin@janware.com if you want +# one, then define the JANWARE_USER = environment variable. +# +# Current documentation on how this Makefile is meant to be used can be found +# under https://janware.com/wiki/pub/en/sw/build/. Running "make help" might +# take you there semi-automatically. +# + +PROJECTS_MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) +JWBDIR_GIT_REMOTE ?= ssh://$(JANWARE_USER)@git.janware.com/srv/git/jan/proj/jw-build +JWBDIR ?= $(notdir $(JWBDIR_GIT_REMOTE)) +PROJECTS_DIR_INCLUDE_MK = $(JWBDIR)/make/projects-dir-include.mk + +ifndef JANWARE_USER + JANWARE_USER = $(shell id -un) + $(warning Assuming JANWARE_USER=$(JANWARE_USER) from id -un) + $(warning Explicitly set environment variable JANWARE_USER to turn off this warning!) +endif + +-include local.mk + +all: + +include $(PROJECTS_DIR_INCLUDE_MK) + +$(PROJECTS_DIR_INCLUDE_MK): + git clone $(JWBDIR_GIT_REMOTE) $(JWBDIR) + [ -L $(PROJECTS_MAKEFILE_NAME) ] || \ + ln -sf `find $(JWBDIR) -type f -print0 | xargs -0 grep -l some-random-string-to-id-this-makefile` \ + $(PROJECTS_MAKEFILE_NAME) diff --git a/make/projects-dir.mk b/make/projects-dir.mk new file mode 100644 index 00000000..ee889764 --- /dev/null +++ b/make/projects-dir.mk @@ -0,0 +1,442 @@ +# +# SPDX-License-Identifier: GPL-2.0+ +# +# Manage multiple software repositories in one tree +# +# (C) Copyright 2001-2019, Jan Lindemann +# +# This is the top-level Makefile for a janware software build tree. It is +# provided under the terms of the GNU Lesser Public License, Version 2. +# +# Some of its targets download software from janware GmbH servers. For those, +# you will need a janware.com user account. Ask admin@janware.com if you want +# one, then define the JANWARE_USER = environment variable. +# +# Current documentation on how this Makefile is meant to be used can be found +# under https://janware.com/wiki/pub/en/sw/build/. Running "make help" might +# take you there semi-automatically. +# + +# ------------ Makefile and environment variable definitions + +.NOTPARALLEL: + +-include local.mk +PROJECTS_MAKEFILE_NAME ?= $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) + +# -- Find JWBDIR +DEV_PROJECTS_DIR ?= . +JWBDIR_NAME ?= jw-build +JWBDIR_SEARCH_PATH ?= $(DEV_PROJECTS_DIR) $(BUILD_TOOLS_PREFIX)/opt/$(FLAVOUR_PATH_PREFIX) +JWBDIR ?= $(firstword $(wildcard $(addsuffix /$(JWBDIR_NAME),$(JWBDIR_SEARCH_PATH)))) + +JW_BUILD_BINDIR = $(JWBDIR)/bin +JWB_SCRIPT_DIR = $(firstword $(wildcard ./$(JWBDIR_NAME)/scripts $(JW_BUILD_BINDIR)) jwb-script-dir-not-found) +JW_BUILD_REMOTE_BINDIR = /opt/jw-build/bin + +SHELL = /bin/bash -o pipefail +H +PROJECTS_TXT ?= projects.txt +JW_BUILD_VERBOSE ?= false +BASE_PKGS = git cvs make sudo time time xdg-utils python3 +PREREQ_RELEASE ?= pull + +# ------------ evaluate Makefile and environment variables + +ifneq ($(wildcard $(PROJECTS_TXT)),) + PROJECTS ?= $(shell cat $(PROJECTS_TXT) | sed '/^ *\#/ d') +else + PROJECTS ?= $(shell ls -d */GNUmakefile */Makefile 2>/dev/null | sed 's%/[^/]*%%' | sort -u) +endif + +ifeq ($(JW_BUILD_VERBOSE),true) + SSH_WRAPPER_TRACE ?= -x +endif + +export JW_BUILD_VERBOSE + +# ------------ external programs I + +CWD := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +GET_OS_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/get-os.sh $(JW_BUILD_BINDIR)/get-os.sh) get-os-sh-not-found) +SSH_WRAPPER_SH := $(CWD)/ssh-wrapper.sh + +EXCLUDES_FILE ?= exclude.txt +EXCLUDES_FILES = $(wildcard exclude-$(shell $(GET_OS_SH) name 2>/dev/null).txt exclude-$(shell $(GET_OS_SH) 2>/dev/null).txt $(EXCLUDES_FILE)) +ifneq ($(EXCLUDES_FILES),) + EXCLUDE_FROM_BUILD += $(shell sed 's/\#.*//g' $(EXCLUDES_FILES)) +endif +OFFLINE_PROJECTS ?= $(EXCLUDE_FROM_BUILD) + +TEXT_FILES_CACHE ?= text-files.txt + +ifndef JANWARE_USER + ifneq ($(wildcard CVS/Root),) + JANWARE_USER = $(shell sed '/^:ext/ !d; s/:ext:\([^@]\+\)@.*/\1/' CVS/Root) + $(warning Assuming JANWARE_USER=$(JANWARE_USER) from CVS/Root) + else + JANWARE_USER = $(shell id -un) + $(warning Assuming JANWARE_USER=$(JANWARE_USER) from id -un) + endif + $(warning Explicitly set environment variable JANWARE_USER to turn off this warning!) +endif + +ifeq ($(JANWARE_USER),) + OFFLINE ?= true +else + export JANWARE_USER +endif + +ifeq ($(OFFLINE),true) + UNAVAILABLE_TARGETS ?= pull.done update.done clone.done +else + UNAVAILABLE_TARGETS ?= + export CVSROOT = :ext:$(JANWARE_USER)@cvs.janware.com:/srv/cvs +endif + +ifneq ($(JW_BUILD_SSH),) + export CVS_RSH := $(JW_BUILD_SSH) +else + export CVS_RSH := $(SSH_WRAPPER_SH) +endif + +export GIT_SSH := $(CVS_RSH) +ifeq ($(filter pkg-%,$(MAKECMDGOALS)),) + export JW_BUILD_SSH_EXTRA_OPTS += -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPath=/tmp/%r@jw-build:%h:%p -o ControlPersist=3m -l $(JANWARE_USER) +endif + +ifneq ($(CLONE_FROM_USER),) + export PGIT_CLONE_FROM_USER = $(CLONE_FROM_USER) +else + export PGIT_CLONE_FROM_USER = $(JANWARE_USER) +endif + +ifneq ($(OFFLINE_PROJECTS),) + export PGIT_IGNORE = $(OFFLINE_PROJECTS) +endif + +ifneq ($(EXCLUDE_FROM_BUILD),) + PROJECTS_PY_EXTRA_BUILD_OPTS += --exclude "$(EXCLUDE_FROM_BUILD)" +endif + +# non-interactive mode +INTERACTIVE ?= true +ifneq ($(INTERACTIVE),true) + DASH_Y := -y +endif + +# ------------ external programs II + +BROWSER ?= xdg-open +EDITOR ?= xdg-open +ifeq ($(TIME),) + TIME = $(shell which time) -p +endif +PROJECTS_PY = python3 $(JWB_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS) +PROJECTS_PY_BUILD = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS) +PGIT_SH = /bin/bash $(JWB_SCRIPT_DIR)/pgit.sh +PURGE_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/purge-stale-projects.sh $(JW_BUILD_BINDIR)/purge-stale-projects.sh) purge-not-found) +PKG_MANAGER_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/pkg-manager.sh $(JW_BUILD_BINDIR)/pkg-manager.sh) pkg-manager-not-found) +CREATE_PROJECT_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-build-create-project.sh $(JW_BUILD_BINDIR)/jw-build-create-project.sh) jw-build-create-project-not-found) +LIST_VCS_FILES_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/scm.sh $(JW_BUILD_BINDIR)/scm.sh) scm-sh-not-found) ls-files +JW_PKG = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-pkg $(JW_BUILD_BINDIR)/jw-pkg) jw-pkg-not-found) +GIT_SRV_ADMIN_SH = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com $(JW_BUILD_REMOTE_BINDIR)/git-srv-admin.sh +CVS_ADMIN_SH = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@cvs.janware.com $(JW_BUILD_REMOTE_BINDIR)/cvs-admin.sh + +# ------------ projects to be built + +TARGET_PROJECTS = $(filter-out $(EXCLUDE_FROM_BUILD),$(PROJECTS)) +BUILD_PROJECTS = $(shell $(PROJECTS_PY_BUILD) --build-order all $(TARGET_PROJECTS)) +GIT_PROJECTS = $(patsubst %/,%,$(dir $(wildcard $(addsuffix /.git,$(BUILD_PROJECTS))))) +CVS_PROJECTS = $(patsubst %/,%,$(dir $(wildcard $(addsuffix /CVS,$(BUILD_PROJECTS))))) + +# ------------ targets + +# --- mandatory targets + +all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) + $(TIME) $(PROJECTS_PY_BUILD) $@ $(TARGET_PROJECTS) +clean: clean-dirs +distclean: clean-all-dirs done.clean +install: + @echo + @echo " Target install is not supported by this Makefile." + @echo " Target pkg-rebuild-reinstall might be what you are looking for." + @echo + @exit 1 + +# --- build targets + +rebuild: clean purge pull subdirs-all + +subdirs-%: + FORCE_REBUILD_SUBDIRS=true make $* + +# --- informative-only targets + +help doc-project doc-module: + $(BROWSER) $(firstword $(shell sed '/https:/ !d; s%.*https%https%; s/ .*//' $(firstword $(MAKEFILE_LIST)))) + +status: $(SSH_WRAPPER_SH) + for p in $(CVS_PROJECTS); do test -f $$d/CVS || echo $$p; done + cvs status $(addsuffix VERSION,$(CVS_PROJECTS)) + $(PGIT_SH) -uno status + +build-order-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) + $(PROJECTS_PY_BUILD) --build-order $* $(TARGET_PROJECTS) | sed 's/ */\n/g' + +build-order: build-order-all + +echo-build-deps: + @$(PROJECTS_PY) required-os-pkg --skip-excluded --flavours "build" $(TARGET_PROJECTS) + +echo-install-deps: + @$(PROJECTS_PY) required-os-pkg --skip-excluded --flavours "build run" $(TARGET_PROJECTS) + +echo-release-deps: + @$(PROJECTS_PY) required-os-pkg --skip-excluded --flavours "build run release" $(TARGET_PROJECTS) +echo-os: + @$(GET_OS_SH) + +echo-projects: + @echo $(PROJECTS) + +echo-target-projects: + @echo $(TARGET_PROJECTS) + +echo-excludes: + @echo $(EXCLUDE_FROM_BUILD) + +edit-%: | $(TEXT_FILES_CACHE) + $(EDITOR) $(shell grep "/$*$$" $(TEXT_FILES_CACHE)) + +distclean: clean.text-files-cache +clean.text-files-cache: + rm -f $(TEXT_FILES_CACHE) +list-files: + @realpath PROJECTS_MAKEFILE_NAME + @for p in $(BUILD_PROJECTS); do \ + $(LIST_VCS_FILES_SH) -znf $$p | sed -z "s/^/$$p\//" | \ + xargs -0 realpath ;\ + done +$(TEXT_FILES_CACHE): + @make -s text-files-update +text-files-update: + make -s --no-print-directory list-files | tr '\n' '\0' | xargs -0 file -N | sed "/:.*text/I !d; s/:.*//" > $(TEXT_FILES_CACHE).tmp + mv $(TEXT_FILES_CACHE).tmp $(TEXT_FILES_CACHE) +text-files-update-all: + @PROJECTS_TXT= make text-files-update +text-files-list list-text-files: | $(TEXT_FILES_CACHE) + @cat $(TEXT_FILES_CACHE) +text-files-list-0 list-text-files-0: | $(TEXT_FILES_CACHE) + @cat $(TEXT_FILES_CACHE) | tr '\n' '\0' + +cloc: + for p in $(GIT_PROJECTS); do \ + git -C $$p submodule status | sed "s|^ *\([^ ]\+\) \+\([^ ]\+\) *.*|$$p/\2|" ;\ + done > cloc-ignore.txt + for p in $(foreach s,dist include bin lib,$(addsuffix /$s,$(BUILD_PROJECTS))); do \ + echo $$p >> cloc-ignore.txt ;\ + done + cloc --exclude-list-file=cloc-ignore.txt $(BUILD_PROJECTS) + +# --- package-related targets + +pkg-manager-refresh: + $(PKG_MANAGER_SH) refresh $(DASH_Y) + +pkg-install-build-deps: + $(PKG_MANAGER_SH) install $(DASH_Y) "$(BASE_PKGS) $(shell $(PROJECTS_PY) required-os-pkg --skip-excluded --flavours build $(TARGET_PROJECTS))" + +pkg-install-release-deps: + $(PKG_MANAGER_SH) install $(DASH_Y) "$(BASE_PKGS) $(shell $(PROJECTS_PY) required-os-pkg --skip-excluded --flavours 'build run release' $(TARGET_PROJECTS))" + +pkg-exclude-built-today: + touch $(EXCLUDES_FILE) + $(JW_PKG) built-today > built-today.tmp + cat $(EXCLUDES_FILE) built-today.tmp | sed 's/ */\n/g' | sort -u > $(EXCLUDES_FILE).tmp + mv $(EXCLUDES_FILE).tmp $(EXCLUDES_FILE) +distclean: clean.pkg-exclude-built-today +clean.pkg-exclude-built-today: + rm -f $(EXCLUDES_FILE).tmp built-today.tmp + +pkg-exclude-installed: + $(JW_PKG) list-projects | while read p; do sed -i "s/^# *$$p$$/$$p/" $(EXCLUDES_FILE) ; done + +pkg-list-groups: + @find . -name project.conf | xargs sed '/^ *group *=/!d; s/group *= *//; s/"//g' | sort -u + +pkg-release-reinstall: $(PREREQ_RELEASE) + +pkg-release-all: + /bin/bash ./packager-client/scripts/packager-client-2.sh + +pkg-fetch-from-%: + ssh $* /opt/jw-base/bin/jw-pkg list -s > $(PROJECTS_TXT).tmp + mv $(PROJECTS_TXT).tmp $(PROJECTS_TXT) + +pkg-init-%: + $(CREATE_PROJECT_SH) $* + +pkg-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done) + $(PROJECTS_PY_BUILD) $@ $(TARGET_PROJECTS) + +# --- generic cleanup targets + +clean-dirs: + echo $(sort $(dir $(wildcard */*.done))) | xargs -r $(PROJECTS_PY_BUILD) clean + +clean-all-dirs: + $(PROJECTS_PY_BUILD) clean $(PROJECTS) + make clean-dirs + +purge: $(SSH_WRAPPER_SH) +ifneq ($(PURGE_SH),/bin/bash purge-not-found) + $(PURGE_SH) +endif + +$(PROJECTS_TXT): + echo $(PROJECTS) | sed 's/ /\n/g; s%/%%g' > $@ + +done.clean: + rm -f *.done + +# --- collab targets + +list-maintainers: $(SSH_WRAPPER_SH) + $(GIT_SRV_ADMIN_SH) $@ + +update pull: purge cvs-update git-clone + touch pull.done + +sync: pull push + +sync-all: pull-all push-all + +sync-%: + ssh $* make -C $(shell pwd) sync + +pull-all: purge cvs-update git-clone git-pull-all + touch clone.done + touch pull.done + +diff-all diff: $(SSH_WRAPPER_SH) + cvs diff -u; $(PGIT_SH) diff + +cvs-diff: $(SSH_WRAPPER_SH) + cvs diff -u + +cvs-update: $(SSH_WRAPPER_SH) + rm -f $@.done + make $@.done + +git-push push: $(SSH_WRAPPER_SH) + $(PGIT_SH) push + +git-push-all: $(SSH_WRAPPER_SH) + $(PGIT_SH) push --all --recurse-submodules=on-demand + +git-diff: $(SSH_WRAPPER_SH) + $(PGIT_SH) diff + +git-short-diff: $(SSH_WRAPPER_SH) + $(PGIT_SH) diff --shortstat + +git-status: + $(PGIT_SH) status -uno + + +git-pull: $(SSH_WRAPPER_SH) + $(PGIT_SH) clone + +git-pull-mini: $(SSH_WRAPPER_SH) + PGIT_CLONE_PROJECTS="$(patsubst %/.git,%,$(wildcard $(addsuffix /.git,$(shell make -s build-order))))" $(PGIT_SH) clone + +git-pull-all: $(SSH_WRAPPER_SH) + $(PGIT_SH) pull --all + +git-clone: $(SSH_WRAPPER_SH) + $(PGIT_SH) clone + touch clone.done + +git-clone-%: $(SSH_WRAPPER_SH) + PGIT_CLONE_FROM_USER=$* $(PGIT_SH) clone + +git-show-non-master-branches: + @$(PGIT_SH) branch 2>&1 | \ + sed '/^#\|^*/!d; s/.*git -C //; s/ *branch *//; s/ *\* *//' | \ + while read p; do \ + read b ;\ + if [ "$$b" != "master" ]; then \ + echo " * $$p: $$b" ;\ + fi ;\ + done + +# git-echo-link- returns a string functioning as hyperlink to +# matching files in git when embedded into a janware wiki or ticket. +git-echo-links-%: | $(TEXT_FILES_CACHE) + sed "/$*$$/!d; s%$(CWD)%%; s|^|\n \[\[jgit>/proj/$(JANWARE_USER)/|; s/$$/|$*\]\]\n/" $(TEXT_FILES_CACHE) + +git-update-project-descriptions: $(SSH_WRAPPER_SH) + $(GIT_SRV_ADMIN_SH) -j update-descriptions all + +git-pull-%: $(SSH_WRAPPER_SH) cvs-update.done + PGIT_CLONE_FROM_USER=$* $(PGIT_SH) clone + +git-commit: + $(PGIT_SH) commit + +git-conv-%: $(SSH_WRAPPER_SH) + [ -e $*/.git ] || { \ + mv $* old/ ;\ + if PROJECTS="$*" make clone; then \ + sed -i "/^D\/$*\// d" CVS/Entries ;\ + else \ + mv old/$* . ;\ + fi \ + } + +git-check-conv: $(SSH_WRAPPER_SH) + for p in `$(GIT_SRV_ADMIN_SH) -u jan -j list-personal-projects`; do \ + make git-conv-$$p ;\ + done + +# --- rules + +$(SSH_WRAPPER_SH): $(PROJECTS_MAKEFILE_NAME) + /bin/echo -e '#!/bin/bash $(SSH_WRAPPER_TRACE)\n\nexec /usr/bin/ssh $$JW_BUILD_SSH_EXTRA_OPTS "$$@"' > $@.tmp + chmod 700 $@.tmp + mv $@.tmp $@ +ssh-wrapper: $(SSH_WRAPPER_SH) +clean.ssh-wrapper: + rm -f $(SSH_WRAPPER_SH) +distclean: clean.ssh-wrapper + +define check_cvs_user + @if ! grep -q ":ext:$(JANWARE_USER)@" CVS/Root; then \ + echo "Wrong username in CVS/Root, change that to: $(CVSROOT)" ;\ + exit 1; \ + fi +endef + +cvsdir.done: $(SSH_WRAPPER_SH) + if [ ! -d CVS ]; then \ + mkdir CVS && \ + echo $(CVSROOT) > CVS/Root && \ + echo proj > CVS/Repository && \ + touch CVS/Entries ;\ + fi + touch $@ + +cvs-update.done: cvsdir.done $(SSH_WRAPPER_SH) + $(check_cvs_user) + cvs update -dP Makefile $(shell $(CVS_ADMIN_SH) list-projects) + touch $@ + +update.done: purge + +pull.done: $(filter-out $(UNAVAILABLE_TARGETS),cvs-update.done clone.done) + touch $@ + +clone.done: $(filter-out $(UNAVAILABLE_TARGETS),$(SSH_WRAPPER_SH)) + $(PGIT_SH) clone + touch $@