From 0c9fd980090a53288a39a2a7fe892f7ca07fd3b7 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Mon, 17 Nov 2025 13:05:43 +0100 Subject: [PATCH] projects-dir.mk, scripts/Makefile, jw-pkg: Move jw-pkg from jw-build to jw-base jw-pkg is related to, but strictly speaking not indispensible for building and packaging software. So, in the attempt have a minimal jw-build, move jw-pkg to jw-base, and fix all packages that use it. Signed-off-by: Jan Lindemann --- make/projects-dir.mk | 22 +- scripts/Makefile | 4 +- scripts/jw-pkg | 503 ------------------------------------------- 3 files changed, 2 insertions(+), 527 deletions(-) delete mode 100644 scripts/jw-pkg diff --git a/make/projects-dir.mk b/make/projects-dir.mk index d7c064f5..754d1d99 100644 --- a/make/projects-dir.mk +++ b/make/projects-dir.mk @@ -37,7 +37,7 @@ JW_BUILD_REMOTE_BINDIR = /opt/jw-build/bin SHELL = /bin/bash -o pipefail +H PROJECTS_TXT ?= projects.txt JW_BUILD_VERBOSE ?= false -BASE_PKGS = git make sudo time time xdg-utils python3 +BASE_PKGS = git make sudo time xdg-utils python3 PREREQ_RELEASE ?= pull # ------------ evaluate Makefile and environment variables @@ -131,7 +131,6 @@ PURGE_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR PKG_MANAGER_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/pkg-manager.sh $(JW_BUILD_BINDIR)/pkg-manager.sh) pkg-manager-not-found) CREATE_PROJECT_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-build-create-project.sh $(JW_BUILD_BINDIR)/jw-build-create-project.sh) jw-build-create-project-not-found) LIST_VCS_FILES_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/scm.sh $(JW_BUILD_BINDIR)/scm.sh) scm-sh-not-found) ls-files -JW_PKG = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-pkg $(JW_BUILD_BINDIR)/jw-pkg) jw-pkg-not-found) GIT_SRV_ADMIN_SH = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com $(JW_BUILD_REMOTE_BINDIR)/git-srv-admin.sh # ------------ projects to be built @@ -238,30 +237,11 @@ pkg-install-build-deps: pkg-install-release-deps: $(PKG_MANAGER_SH) install $(DASH_Y) "$(BASE_PKGS) $(shell $(PROJECTS_PY) required-os-pkg --skip-excluded --flavours 'build run release' $(TARGET_PROJECTS))" -pkg-exclude-built-today: - touch $(EXCLUDES_FILE) - $(JW_PKG) built-today > built-today.tmp - cat $(EXCLUDES_FILE) built-today.tmp | sed 's/ */\n/g' | sort -u > $(EXCLUDES_FILE).tmp - mv $(EXCLUDES_FILE).tmp $(EXCLUDES_FILE) -distclean: clean.pkg-exclude-built-today -clean.pkg-exclude-built-today: - rm -f $(EXCLUDES_FILE).tmp built-today.tmp - -pkg-exclude-installed: - $(JW_PKG) list-projects | while read p; do sed -i "s/^# *$$p$$/$$p/" $(EXCLUDES_FILE) ; done - -pkg-list-groups: - @find . -name project.conf | xargs sed '/^ *group *=/!d; s/group *= *//; s/"//g' | sort -u - pkg-release-reinstall: $(PREREQ_RELEASE) pkg-release-all: /bin/bash ./packager-client/scripts/packager-client-2.sh -pkg-fetch-from-%: - ssh $* /opt/jw-base/bin/jw-pkg list -s > $(PROJECTS_TXT).tmp - mv $(PROJECTS_TXT).tmp $(PROJECTS_TXT) - pkg-init-%: $(CREATE_PROJECT_SH) $* diff --git a/scripts/Makefile b/scripts/Makefile index e886a5d2..44e2f969 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -1,7 +1,5 @@ TOPDIR = .. include $(TOPDIR)/make/proj.mk - -EXE_SH += $(wildcard jannet jcs jw-pkg) - +EXE_SH += $(wildcard jannet jcs) include $(JWBDIR)/make/scripts.mk diff --git a/scripts/jw-pkg b/scripts/jw-pkg deleted file mode 100644 index 6afb0759..00000000 --- a/scripts/jw-pkg +++ /dev/null @@ -1,503 +0,0 @@ -#!/bin/bash - -log() -{ - if [ "$1" = -n ]; then - shift - echo -n "$myname: $*" - return - fi - - if [ "$1" = -c ]; then - shift - echo "$*" - return - fi - - if [ "$1" = -C ]; then - shift - echo -n "$*" - return - fi - - echo "$myname: $*" -} - -channel_present() -{ - if smart channel --show $1 2>&1 | grep -q baseurl >/dev/null; then - return 0 - fi - return 1 -} - -usage() -{ -cat << EOT - - $myname [global options] command [options] [package ...] - - command is one of - - help: show this help screen - init: initialize - uninit: undo initialization - install: install packages in [arguments] - update: update all jannet software on the system - restore: restore a previous jannet software version state - list: query information about installed software packages - list-projects: query information about installed software projects - checklog: update the installation log file if necessary - rpmnew: remove superfluous rpmnew files and show conflicts - for others - build-date: show installed janware packages along with build date - built-today: show installed janware packages built today - - list-changed-files: list files changed since installation - - list-templates: list templates - list-template-tables: list templates tables - list-template-output: list template output files - rm-template-output: remove template output files - compile-templates: compile templates - - global options are - - -v be verbose - - package - - optional whitespace-separated list of package names, generally limiting the - scope of the command to the specified packages, defaults to all installed - janware packages - -EOT - [ "$1" ] && exit $1 -} - -create_ldconfig_state() -{ - echo "=== automatically created by $myname update script" - echo "--- files" - ls /etc/ld.so.conf.d - echo "--- directories" - find /etc/ld.so.conf.d -type f | xargs cat | sort -u -} - -check_ldconfig() -{ - local state=/var/log/jw-build-ldconf.state - [ -f $state ] && create_ldconfig_state | diff $state - >/dev/null 2>&1 && return - echo -n "running ldconfig ... " - /sbin/ldconfig - create_ldconfig_state > $state - echo "done." -} - -list_packages() -{ - local query_tags="" - - eval set -- `getopt -- VPNUD "$@"` - while [ "$1" != -- ]; do - case $1 in - -V) - query_tags="$query_tags %{Vendor}" - ;; - -P) - query_tags="$query_tags %{Packager}" - ;; - -U) - query_tags="$query_tags %{URL}" - ;; - -D) - query_tags="$query_tags %{Distribution}" - ;; - *) - echo -e "Unexpected argument >$1<\n" >&2 - usage 1 - ;; - esac - shift - done - shift - - [ "$query_tags" ] || query_tags="%{URL}" - - local names="$1" - local rpm_args - if [ -n "$names" ]; then - rpm_args="$rpm_args $names" - else - rpm_args="$rpm_args -a" - fi - rpm -q --queryformat "%{Name}: $query_tags\n" $rpm_args | \ - grep -i '^[^ ]\+:.*\(jannet\|janware\)' | \ - sed 's/\(^[^ ]\+\) *:.*/\1/; s/[ ]*$//' |\ - sort -u -} - -list_projects() -{ - list_packages "$@" | xargs -r rpm -q --queryformat '%{SOURCERPM}\n' | \ - sed 's/[ ]*//; s/-[0-9]\+.*//' |\ - sort -u -} - -list_files() -{ - # Need to sort -u, because apparently, during installation, rpm -ql - # returns every file in the package _twice_. Funny. - list_packages "$@" | sort -u | xargs -r rpm -ql -} - -cmd_rpmnew() -{ - local file - - list_files "$@" | - while read file; do - if [ -e "$file.rpmnew" ]; then - if diff -q $file $file.rpmnew; then - if [ -L "$file" ]; then - echo ========== skipping link $file - diff $file.rpmnew $file - continue - fi - echo mv $file.rpmnew $file - sudo mv $file.rpmnew $file - else - echo =========== $file - diff $file.rpmnew $file - fi - fi - done -} - -cmd_diff() -{ - local file - - list_packages "$@" | - xargs -r rpm -qV | - sed '/^....L\|^..5/ !d; s%[^/]*/%/%' | - while read file; do - if diff -q $file $file.rpmnew; then - if [ -L "$file" ]; then - echo ========== skipping link $file - diff $file.rpmnew $file - continue - fi - echo mv $file.rpmnew $file - sudo mv $file.rpmnew $file - else - echo =========== $file - diff $file.rpmnew $file - fi - done -} - -cmd_build_date() -{ - list_packages "$@" | - xargs -r rpm -q --queryformat '%{BUILDTIME} %{NAME}\n' | - sort -n | - sed 's/-run\|-devel//' | - sort -u | - while read t p; do - echo `date --date=@$t --rfc-3339=seconds` $p - done -} - -cmd_list_templates() -{ - local ext_from="$template_exts" # TODO: support more than one - local ext_from_re=`echo $ext_from | sed 's/\./\\./g'` - local f - - list_files "$@" | while read f; do - [ -f "$f" -o -L "$f" ] || continue - echo $f | grep -q "$ext_from_re$" || continue - echo $f - done -} - -cmd_list_template_tables() -{ - local ext_from="$table_exts" - local e f t - list_files "$@" | while read f; do - [ -f "$f" -o -L "$f" ] || continue - for e in $ext_from; do - t="${f%.*}$e" - [ -f "$t" ] || continue - echo $t - done - done -} - -cmd_list_template_output() -{ - local re=`echo $template_exts | sed 's/\./\\\\./g; s/^ *//; s/ *$//; s/ */$\\\\|/g; s/\(.*\)/\\\\(\1$\\\\)/'` - local f - list_files "$@" | sed "/$re/ !d; s/$re//" | while read f; do - [ -f "$f" -o -L "$f" ] || continue - echo $f - done -} - -cmd_rm_template_output() -{ - local restore=0 - while [ "${1:0:1}" = '-' ]; do - case "$1" in - -r) - restore=1 - ;; - *) - usage 1 - ;; - esac - shift - done - - local f - cmd_list_template_output $@ | while read f; do - log "removing $f" - rm $f - bak=$f$template_bak_ext - if [ "$restore" = 1 -a -f "$bak" ]; then - log "restoring $f from $bak" - mv $bak $f - fi - done -} - -cmd_compile_templates() -{ - __md() { - stat --format '%U:%G %a' "$1" 2>/dev/null - } - - local ext_from="$template_exts" # TODO: support more than one - local ext_from_re=`echo $template_exts | sed 's/\./\\./g'` - local ext_to="" - local ext_tables="$table_exts" - local default_group=root - local default_owner=root - local default_mode=600 - local conf_patt="^[ ]*# *conf:" - local missing_table=0 - local backup=0 - - umask 0077 - - while [ "${1:0:1}" = '-' ]; do - case "$1" in - -o) - default_owner="$2" - shift - ;; - -m) - default_mode="$2" - shift - ;; - -g) - default_group="$2" - shift - ;; - -b) - backup=1 - ;; - -c) - conf_patt="$2" - shift - ;; - *) - usage 1 - ;; - esac - shift - done - - local f - while read f; do - local base=`echo $f | sed "s/$ext_from_re$//"` - [ "$base" ] || continue - local to=$base$ext_to - local table="" - local ext - for ext in $ext_tables; do - table=$base$ext - [ -f "$table" ] && break - done - [ -f $table ] || { - log "WARNING: No key-value table found for template $f, not compiling." >&2 - ((missing_table++)) - continue - } - # TODO: use mktemp -d and keep temporary files in read-only dir - local tmp=$to.tmp - log -n "Applying macros in $table to $f" - local mode=$default_mode - local owner=$default_owner - local group=$default_group - local diff="" - eval `sed "/$conf_patt/ !d; s/$conf_patt//" $f` - eval `sed "/$conf_patt/ !d; s/$conf_patt//" $table` - sed 's|^[ ]*#.*||; s|/|\\/|g; s|\([^ =]\+\)[ =]\+\(.*\)|s/\1/\2/g|' $table | sed -f - $f > $tmp - chmod $mode $tmp - chown $owner $tmp - chgrp $group $tmp - diff -q $tmp $to >/dev/null 2>&1 || diff="@content" - local md_to=$(__md "$to") - local md_tmp=$(__md "$tmp") - [ "$md_to" = "$md_tmp" ] || diff="$diff@metadata ($md_to -> $md_tmp)" - if [ -z "$diff" ]; then - log -c ", no changes." - rm $tmp - continue - fi - local bak=$to$template_bak_ext - if [ "$backup" = 1 -a -f $to ] && ! diff -q $to $bak >/dev/null 2>&1; then - log -C ", saving backup to $to to $bak" - cp -p $to $bak - fi - mv $tmp $to - diff=${diff##@} - log -c ", updating ${diff//@/ + }, done." - done <<< $(cmd_list_templates "$@") - - if [ "$missing_table" != 0 ]; then - log "WARNING: $missing_table missing tables found. You might want to add them and run sudo $cmdline again." - fi -} - -cmd_get_value_from_tables() -{ - local key show_file - eval set -- `getopt -- k: "$@"` - while [ "$1" != -- ]; do - case $1 in - -k) - key=$2 - shift - ;; - -f) - show_file=1 - shift - ;; - *) - echo -e "Unexpected argument >$1<\n" >&2 - usage 1 - ;; - esac - shift - done - shift - - local f - while read f; do - grep -q "^\s*$key\s\+" $f || continue - [ "$show_file" = 1 ] && echo -n "$f: " - sed "s|^[ ]*#.*||; /^\s*$key\(\s\|=\)\+/ !d; s/^\s*$key\(\s\|=\)\+//" $f - done <<< $(cmd_list_template_tables "$@") -} - -cmd_built_today() -{ - local today=`date +'%Y-%m-%d'` - cmd_build_date | grep $today | cut -d' ' -f3 -} - -cmd_list_changed_files() -{ - list_projects "$@" | xargs rpm -qV | grep '^..5' | sed 's|[^/]*/|/|' -} - -# ----- here we go - -cmdline="$0 $@" -myname=`basename $0` -longname=$0 -opts="v" -table_exts=".jw-secret .jw-vars" -template_exts=".jw-tmpl" -template_bak_ext=".jw-pkg.bak" - -while [ "${1:0:1}" = '-' ]; do -case "$1" in --v) - opt_verbose=true - shift - ;; -*) - usage 1 - ;; -esac -shift -done - -cmd="${1//-/_}" -shift - -case $cmd in - -# -- TODO: re-implement this using pkg-manager.sh, or integrate with pkg-manager.sh -#init) -# if ! channel_present ftp.jannet.de; then -# echo -n "adding installation source ... " -# sudo smart channel -y --add ftp.jannet.de \ -# type=yast2 \ -# baseurl=ftp://dspadm:dspasswd@ftp.jannet.de/pub/packages/linux/suse/10.1 \ -# >/dev/null 2>&1 -# if channel_present; then echo done; else echo failed; fi -# fi -# ;; -# -#uninit) -# if channel_present ftp.jannet.de; then -# echo -n "removing installation source ... " -# sudo smart channel -y --remove ftp.jannet.de >/dev/null 2>&1 -# echo done -# fi -# ;; -# -#update) -# sudo smart update ftp.jannet.de -# sudo smart upgrade -y -# check_ldconfig -# ;; -# -#install) -# sudo smart update ftp.jannet.de -# if [ -f $1 ]; then -# sudo smart install -y `cat $1` -# else -# sudo smart install -y $* -# fi -# check_ldconfig -# ;; -#restore) -# echo "not yet implemented, sorry" >&2 -# ;; -list) - if [ "$opt_verbose" = true ]; then - list_packages "$@" | xargs -r rpm -qi - else - list_packages "$@" | xargs -r rpm -q - fi - ;; -list_projects) - list_projects "$@" - ;; -compile_templates|list_templates|list_template_tables|list_template_output|rm_template_output|diff|build_date|built_today|rpmnew|get_value_from_tables|list_changed_files) - cmd_$cmd "$@" - ;; -help) - usage 0 - ;; -*) - usage 1 - ;; -esac