jw-pkg/make/projects-dir.mk

#
# $Id$
#
# This is the top-level Makefile for a janware software build tree. It is
# provided under the terms of the GNU Lesser Public License, Version 2.
#
# Some of its targets download software from janware GmbH servers. For those,
# you will need a janware.com user account. Ask admin@janware.com if you want
# one, then define the JANWARE_USER=<janware user name> environment variable.
#
# Current documentation on how this Makefile is meant to be used can be found
# under https://janware.com/wiki/pub/en/sw/build/. Running "make doc-module"
# might take you there semi-automatically.
#

# ------------ Makefile and environment variables

-include local.mk

SHELL = /bin/bash -o pipefail

ifneq ($(PROJECTS),)
  export PGIT_CLONE_PROJECTS = $(PROJECTS)
endif

PROJECTS_TXT         ?= projects.txt

ifneq ($(wildcard $(PROJECTS_TXT)),)
  PROJECTS ?= $(shell cat $(PROJECTS_TXT) | sed '/^ *\#/ d')
  # TODO: this could be nicer
  CVS_PROJECTS = $(PROJECTS)
else
  PROJECTS ?= $(shell ls -d */Makefile 2>/dev/null | sed 's%/[^/]*%%')
  CVS_PROJECTS = $(filter $(dir $(wildcard */CVS)),$(PROJECTS))
endif

BASE_PACKAGES = cvs git make sudo

EXCLUDE_FROM_BUILD += \
	dspider-btools \
	dspider-shared \
	feedfs-dspcd \
	casview \
	dspfs \
	jux3 \
	emscren-v8

# ------------ external programs I

CWD                  := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
MOD_SCRIPT_DIR        = ./jw-build/scripts
GET_OS_SH             = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/get-os.sh /opt/jw-build/bin/get-os.sh) get-os-sh-not-found)
SSH_WRAPPER_SH       := $(CWD)/ssh-wrapper.sh

EXCLUDES_FILE        ?= exclude.txt
EXCLUDES_FILES        = exclude-$(shell $(GET_OS_SH) name).txt exclude-$(shell $(GET_OS_SH)).txt $(EXCLUDES_FILE)
TEXT_FILES_CACHE     ?= text-files.txt

ifneq ($(wildcard $(EXCLUDES_FILES)),)
  EXCLUDE_FROM_BUILD += $(shell sed 's/\#.*//g' $(wildcard $(EXCLUDES_FILES)))
endif

BUILD_PROJECTS = $(filter-out $(EXCLUDE_FROM_BUILD),$(PROJECTS))

ifndef JANWARE_USER
  ifneq ($(wildcard CVS/Root),)
    JANWARE_USER = $(shell sed '/^:ext/ !d; s/:ext:\([^@]\+\)@.*/\1/' CVS/Root)
    $(warning Assuming JANWARE_USER=$(JANWARE_USER) from CVS/Root)
  else
    JANWARE_USER = $(shell id -un)
    $(warning Assuming JANWARE_USER=$(JANWARE_USER) from id -un)
  endif
  $(warning Explicitly set environment variable JANWARE_USER to turn off this warning!)
  export JANWARE_USER
endif

ifeq ($(JANWARE_USER),)
  UNAVAILABLE_TARGETS ?= pull.done update.done clone.done
else
  UNAVAILABLE_TARGETS ?=
  export CVSROOT = :ext:$(JANWARE_USER)@cvs.janware.com:/home/jannet/arc/cvs
endif

ifneq ($(JW_BUILD_SSH),)
  export CVS_RSH := $(JW_BUILD_SSH)
else
  export CVS_RSH := $(SSH_WRAPPER_SH)
endif

export GIT_SSH := $(CVS_RSH)
ifeq ($(filter pkg-%,$(MAKECMDGOALS)),)
  export JW_BUILD_SSH_EXTRA_OPTS += -o ControlMaster=auto -o ControlPath=/tmp/%r@jw-build:%p -o ControlPersist=3m -l $(JANWARE_USER)
endif

ifneq ($(CLONE_FROM_USER),)
  export PGIT_CLONE_FROM_USER = $(CLONE_FROM_USER)
else
  export PGIT_CLONE_FROM_USER = $(JANWARE_USER)
endif

ifneq ($(EXCLUDE_FROM_BUILD),)
  PROJECTS_PY_EXTRA_BUILD_OPTS += --exclude "$(EXCLUDE_FROM_BUILD)"
endif

# ------------ external programs II

PGIT                  = /bin/bash $(MOD_SCRIPT_DIR)/pgit.sh
PROJECTS_PY           = python2 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS)
PROJECTS_PY_BUILD     = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS)
PURGE_SH              = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/purge-stale-projects.sh /opt/jw-build/bin/purge-stale-projects.sh) purge-not-found)
PKG_MANAGER_SH       ?= /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/pkg-manager.sh /opt/jw-build/bin/pkg-manager.sh) pkg-manager-not-found)
CREATE_PROJECT_SH    ?= /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/jw-build-create-project.sh /opt/jw-build/bin/jw-build-create-project.sh) jw-build-create-project-not-found)
LIST_VCS_FILES        = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/list-cvs-files.sh /opt/jw-build/bin/list-cvs-files.sh) list-cvs-files-not-found)
JW_PKG                = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/jw-pkg /opt/jw-build/bin/jw-pkg) jw-pkg-not-found)
BROWSER              ?= xdg-open
EDITOR               ?= xdg-open
GIT_SRV_ADMIN_SH      = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com /opt/jw-build/bin/git-srv-admin.sh 
CVS_ADMIN_SH          = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com /opt/jw-build/bin/cvs-admin.sh 

# ------------ targets
  
# --- mandatory targets

all: $(filter-out $(UNAVAILABLE_TARGETS),pull.done links.done)
	$(PROJECTS_PY_BUILD) $@ $(BUILD_PROJECTS)
clean: clean-dirs done.clean
distclean: clean-all-dirs done.clean
install:
	@echo
	@echo "  Target install is not supported by this Makefile."
	@echo "  Target pkg-rebuild-reinstall might be what you are looking for."
	@echo
	@exit 1

# --- build targets

rebuild: clean purge pull subdirs-all

subdirs-%:
	FORCE_REBUILD_SUBDIRS=true make $*

# --- informative-only targets

doc-project doc-module:
	$(BROWSER) $(firstword $(shell sed '/https:/ !d; s%.*https%https%; s/ .*//' $(firstword $(MAKEFILE_LIST))))

status: $(SSH_WRAPPER_SH)
	for p in $(CVS_PROJECTS); do test -f $$d/CVS || echo $$p; done 
	cvs status $(addsuffix VERSION,$(CVS_PROJECTS))
	$(PGIT) -uno status

build-order-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(PROJECTS_PY_BUILD) --build-order $* $(BUILD_PROJECTS) | sed 's/  */\n/g'

build-order: build-order-all

echo-prereq-build:
	@$(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build" $(BUILD_PROJECTS)

echo-prereq-install:
	@$(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build run" $(BUILD_PROJECTS)

echo-prereq-release:
	@$(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build run release" $(BUILD_PROJECTS)
echo-os:
	@$(GET_OS_SH)

echo-excludes:
	@echo $(EXCLUDE_FROM_BUILD)

edit-%: | $(TEXT_FILES_CACHE)
	$(EDITOR) $(shell grep "/$*$$" $(TEXT_FILES_CACHE))

list-files:
	@for p in $(PROJECTS); do \
		$(LIST_VCS_FILES) -znf $$p | sed -z "s/^/$$p\//" | \
			xargs -0 realpath ;\
	done

clean: clean.text-files-cache
clean.text-files-cache:
	rm -f $(TEXT_FILES_CACHE)
$(TEXT_FILES_CACHE):
	@make text-files-update
text-files-update:
	@make --no-print-directory list-files | xargs file -N | grep ":.*text" | cut -d: -f1 > $(TEXT_FILES_CACHE).tmp
	mv $(TEXT_FILES_CACHE).tmp $(TEXT_FILES_CACHE)
text-files-update-all:
	@PROJECTS_TXT= make text-files-update
text-files-list list-text-files: | $(TEXT_FILES_CACHE)
	@cat $(TEXT_FILES_CACHE)

loc-all:
	@make --no-print-directory list-text-files | \
		grep -v "iow-standalone\|jux3\|ntv2/contrib\|\.dia$$\|\.dat\$$\|bootsrap\.css\|mpegplayer/contrib\|gitweb.cgi" | \
		xargs wc -l

find-code-files:
	find . -name '*.c' -o -name '*.cpp' -o -name '*.h' \
		-o -name Makefile -o -name '*.mk' \
		-o -name '*.sh' -o -name '*.pl' -o -name '*.py' \
		| grep -v "/dist/\|\.dep\.mk\|local.mk\|coordgen\|^\./[^/]\+/include/" \
		| grep -v "contrib/wt\|/bin/\|contrib/bootstrap\|contrib/chntpw/src\|contrib/jquery/src\|contrib/ntv2linux" \
		| grep -v "copilot-suse-tw-cf/initrd-tree" \
		| grep -v "iow-standalone\|jux3" \
		| grep -v "generated\|tmp\|old\|orig\|purged"

# --- package-related targets

pkg-manager-refresh:
	$(PKG_MANAGER_SH) refresh -y

pkg-install-prereq-build:
	$(PKG_MANAGER_SH) install -y $(BASE_PACKAGES) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build" $(BUILD_PROJECTS))

pkg-install-prereq-release:
	$(PKG_MANAGER_SH) install -y $(BASE_PACKAGES) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build run release" $(BUILD_PROJECTS))

pkg-exclude-built-today:
	touch $(EXCLUDES_FILE)
	$(JW_PKG) built-today > built-today.tmp
	cat $(EXCLUDES_FILE) built-today.tmp | sed 's/  */\n/g' | sort -u > $(EXCLUDES_FILE).tmp
	mv $(EXCLUDES_FILE).tmp $(EXCLUDES_FILE)
clean: clean.pkg-exclude-built-today
clean.pkg-exclude-built-today:
	rm -f $(EXCLUDES_FILE).tmp built-today.tmp

pkg-exclude-installed:
	$(JW_PKG) list-projects | while read p; do sed -i "s/^# *$$p$$/$$p/" $(EXCLUDES_FILE) ; done

pkg-list-groups:
	@find . -name project.conf | xargs sed '/^ *group *=/!d; s/group *= *//; s/"//g' | sort -u

pkg-release-reinstall: pull

pkg-release-all:
	/bin/bash ./packager-client/scripts/packager-client-2.sh

pkg-fetch-from-%:
	ssh $* /opt/jw-base/bin/jw-pkg list -s > $(PROJECTS_TXT).tmp
	mv $(PROJECTS_TXT).tmp $(PROJECTS_TXT)

pkg-init-%:
	$(CREATE_PROJECT_SH) $*

pkg-%: $(filter-out $(UNAVAILABLE_TARGETS),pull.done)
	$(PROJECTS_PY_BUILD) $@ $(BUILD_PROJECTS)

# --- generic cleanup targets

clean-dirs:
	if [ -n "$(wildcard */*.done)" ]; then ls */*.done 2>/dev/null | sed 's%/.*%%' | xargs -r $(PROJECTS_PY_BUILD) --exclude "" clean; fi

clean-all-dirs:
	$(PROJECTS_PY_BUILD) clean $(PROJECTS)

purge: $(SSH_WRAPPER_SH)
ifneq ($(wildcard CVS),)
	$(PURGE_SH)
endif

$(PROJECTS_TXT):
	echo $(PROJECTS) | sed 's/ /\n/g; s%/%%g' > $@

done.clean:
	rm -f *.done

# --- collab targets

list-maintainers: $(SSH_WRAPPER_SH)
	$(GIT_SRV_ADMIN_SH) $@

update pull: purge cvs-update git-clone
	touch pull.done

sync: pull push

sync-all: pull-all push-all

sync-%:
	ssh $* make -C $(shell pwd) sync

pull-all: purge cvs-update git-clone git-pull-all
	touch clone.done
	touch pull.done

diff-all diff: $(SSH_WRAPPER_SH)
	cvs diff -u; $(PGIT) diff

cvs-diff: $(SSH_WRAPPER_SH)
	cvs diff -u

cvs-update: $(SSH_WRAPPER_SH)
	rm -f $@.done
	make $@.done

git-push push: $(SSH_WRAPPER_SH)
	$(PGIT) push

git-push-all: $(SSH_WRAPPER_SH)
	$(PGIT) push --all --recurse-submodules=on-demand

git-diff: $(SSH_WRAPPER_SH)
	$(PGIT) diff

git-short-diff: $(SSH_WRAPPER_SH)
	$(PGIT) diff --shortstat

git-status:
	$(PGIT) status -uno


git-pull: $(SSH_WRAPPER_SH)
	$(PGIT) clone

git-pull-all: $(SSH_WRAPPER_SH)
	$(PGIT) pull --all

git-clone: $(SSH_WRAPPER_SH)
	$(PGIT) clone
	touch clone.done

git-clone-%: $(SSH_WRAPPER_SH)
	PGIT_CLONE_FROM_USER=$* $(PGIT) clone

git-show-non-master-branches:
	@$(PGIT) branch 2>&1 | \
		sed '/^#\|^*/!d; s/.*git -C //; s/ *branch *//; s/ *\* *//' | \
		while read p; do \
			read b ;\
			if [ "$$b" != "master" ]; then \
				echo " * $$p:	$$b" ;\
			fi ;\
		done

# git-echo-link-<filename> returns a string functioning as hyperlink to
# matching files in git when embedded into a janware wiki or ticket.
git-echo-links-%: | $(TEXT_FILES_CACHE)
	sed "/$*$$/!d; s%$(CWD)%%; s|^|\n  \[\[jgit>/proj/$(JANWARE_USER)/|; s/$$/|$*\]\]\n/" $(TEXT_FILES_CACHE)

git-update-project-descriptions: $(SSH_WRAPPER_SH)
	$(GIT_SRV_ADMIN_SH) -j update-descriptions all

git-pull-%: $(SSH_WRAPPER_SH)
	PGIT_CLONE_FROM_USER=$* $(PGIT) clone

git-commit:
	$(PGIT) commit

git-conv-%: $(SSH_WRAPPER_SH)
	[ -e $*/.git ] || { \
		mv $* old/ ;\
		if PROJECTS="$*" make clone; then \
			sed -i "/^D\/$*\// d" CVS/Entries ;\
		else \
			mv old/$* . ;\
		fi \
	}

git-check-conv: $(SSH_WRAPPER_SH)
	for p in `$(GIT_SRV_ADMIN_SH) -u jan -j list-personal-projects`; do \
		make git-conv-$$p ;\
	done

# --- rules

links.done:
	if [ -d btools ]; then ln -sf btools dspider-btools; fi
	if [ -d dspc/src/shared ]; then ln -sf dspc/src/shared dspider-shared; fi
	touch $@

$(SSH_WRAPPER_SH): Makefile
	/bin/echo -e '#!/bin/bash -x\n\nexec /usr/bin/ssh $$JW_BUILD_SSH_EXTRA_OPTS "$$@"' > $@.tmp
	chmod 700 $@.tmp
	mv $@.tmp $@
ssh-wrapper: $(SSH_WRAPPER_SH)
clean.ssh-wrapper:
	rm -f $(SSH_WRAPPER_SH)
clean: clean.ssh-wrapper

define check_cvs_user
	@if ! grep -q ":ext:$(JANWARE_USER)@" CVS/Root; then \
		echo "Wrong username in CVS/Root, change that to: $(CVSROOT)" ;\
		exit 1; \
	fi
endef

cvsdir.done: $(SSH_WRAPPER_SH)
	if [ ! -d CVS ]; then \
		mkdir CVS && \
		echo $(CVSROOT) > CVS/Root && \
		echo proj > CVS/Repository && \
		echo "/Makefile/$Revision$ ///" | sed 's/ *evision: *\([0-9.]*\) */\1/; s/ *$$ *//' > CVS/Entries ;\
	fi
	touch $@

cvs-update.done: cvsdir.done $(SSH_WRAPPER_SH)
	$(check_cvs_user)
	cvs update -dP Makefile $(shell $(CVS_ADMIN_SH) list-projects)
	touch $@

update.done: purge

pull.done: $(filter-out $(UNAVAILABLE_TARGETS),cvs-update.done clone.done links.done)
	touch $@

clone.done: $(filter-out $(UNAVAILABLE_TARGETS),$(SSH_WRAPPER_SH))
	$(PGIT) clone
	touch $@