projects-dir.mk, scripts/Makefile, jw-pkg: Move jw-pkg from jw-build to jw-base

jw-pkg is related to, but strictly speaking not indispensible for building and
packaging software. So, in the attempt have a minimal jw-build, move jw-pkg to
jw-base, and fix all packages that use it.

Signed-off-by: Jan Lindemann <jan@janware.com>
This commit is contained in:
Jan Lindemann 2025-11-17 13:05:43 +01:00
commit 0c9fd98009
3 changed files with 2 additions and 527 deletions

View file

@ -37,7 +37,7 @@ JW_BUILD_REMOTE_BINDIR = /opt/jw-build/bin
SHELL = /bin/bash -o pipefail +H SHELL = /bin/bash -o pipefail +H
PROJECTS_TXT ?= projects.txt PROJECTS_TXT ?= projects.txt
JW_BUILD_VERBOSE ?= false JW_BUILD_VERBOSE ?= false
BASE_PKGS = git make sudo time time xdg-utils python3 BASE_PKGS = git make sudo time xdg-utils python3
PREREQ_RELEASE ?= pull PREREQ_RELEASE ?= pull
# ------------ evaluate Makefile and environment variables # ------------ evaluate Makefile and environment variables
@ -131,7 +131,6 @@ PURGE_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR
PKG_MANAGER_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/pkg-manager.sh $(JW_BUILD_BINDIR)/pkg-manager.sh) pkg-manager-not-found) PKG_MANAGER_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/pkg-manager.sh $(JW_BUILD_BINDIR)/pkg-manager.sh) pkg-manager-not-found)
CREATE_PROJECT_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-build-create-project.sh $(JW_BUILD_BINDIR)/jw-build-create-project.sh) jw-build-create-project-not-found) CREATE_PROJECT_SH ?= /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-build-create-project.sh $(JW_BUILD_BINDIR)/jw-build-create-project.sh) jw-build-create-project-not-found)
LIST_VCS_FILES_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/scm.sh $(JW_BUILD_BINDIR)/scm.sh) scm-sh-not-found) ls-files LIST_VCS_FILES_SH = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/scm.sh $(JW_BUILD_BINDIR)/scm.sh) scm-sh-not-found) ls-files
JW_PKG = /bin/bash $(firstword $(wildcard $(JWB_SCRIPT_DIR)/jw-pkg $(JW_BUILD_BINDIR)/jw-pkg) jw-pkg-not-found)
GIT_SRV_ADMIN_SH = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com $(JW_BUILD_REMOTE_BINDIR)/git-srv-admin.sh GIT_SRV_ADMIN_SH = JW_BUILD_SSH_EXTRA_OPTS="$(JW_BUILD_SSH_EXTRA_OPTS)" $(GIT_SSH) $(JANWARE_USER)@git.janware.com $(JW_BUILD_REMOTE_BINDIR)/git-srv-admin.sh
# ------------ projects to be built # ------------ projects to be built
@ -238,30 +237,11 @@ pkg-install-build-deps:
pkg-install-release-deps: pkg-install-release-deps:
$(PKG_MANAGER_SH) install $(DASH_Y) "$(BASE_PKGS) $(shell $(PROJECTS_PY) required-os-pkg --skip-excluded --flavours 'build run release' $(TARGET_PROJECTS))" $(PKG_MANAGER_SH) install $(DASH_Y) "$(BASE_PKGS) $(shell $(PROJECTS_PY) required-os-pkg --skip-excluded --flavours 'build run release' $(TARGET_PROJECTS))"
pkg-exclude-built-today:
touch $(EXCLUDES_FILE)
$(JW_PKG) built-today > built-today.tmp
cat $(EXCLUDES_FILE) built-today.tmp | sed 's/ */\n/g' | sort -u > $(EXCLUDES_FILE).tmp
mv $(EXCLUDES_FILE).tmp $(EXCLUDES_FILE)
distclean: clean.pkg-exclude-built-today
clean.pkg-exclude-built-today:
rm -f $(EXCLUDES_FILE).tmp built-today.tmp
pkg-exclude-installed:
$(JW_PKG) list-projects | while read p; do sed -i "s/^# *$$p$$/$$p/" $(EXCLUDES_FILE) ; done
pkg-list-groups:
@find . -name project.conf | xargs sed '/^ *group *=/!d; s/group *= *//; s/"//g' | sort -u
pkg-release-reinstall: $(PREREQ_RELEASE) pkg-release-reinstall: $(PREREQ_RELEASE)
pkg-release-all: pkg-release-all:
/bin/bash ./packager-client/scripts/packager-client-2.sh /bin/bash ./packager-client/scripts/packager-client-2.sh
pkg-fetch-from-%:
ssh $* /opt/jw-base/bin/jw-pkg list -s > $(PROJECTS_TXT).tmp
mv $(PROJECTS_TXT).tmp $(PROJECTS_TXT)
pkg-init-%: pkg-init-%:
$(CREATE_PROJECT_SH) $* $(CREATE_PROJECT_SH) $*

View file

@ -1,7 +1,5 @@
TOPDIR = .. TOPDIR = ..
include $(TOPDIR)/make/proj.mk include $(TOPDIR)/make/proj.mk
EXE_SH += $(wildcard jannet jcs)
EXE_SH += $(wildcard jannet jcs jw-pkg)
include $(JWBDIR)/make/scripts.mk include $(JWBDIR)/make/scripts.mk

View file

@ -1,503 +0,0 @@
#!/bin/bash
log()
{
if [ "$1" = -n ]; then
shift
echo -n "$myname: $*"
return
fi
if [ "$1" = -c ]; then
shift
echo "$*"
return
fi
if [ "$1" = -C ]; then
shift
echo -n "$*"
return
fi
echo "$myname: $*"
}
channel_present()
{
if smart channel --show $1 2>&1 | grep -q baseurl >/dev/null; then
return 0
fi
return 1
}
usage()
{
cat << EOT
$myname [global options] command [options] [package ...]
command is one of
help: show this help screen
init: initialize
uninit: undo initialization
install: install packages in [arguments]
update: update all jannet software on the system
restore: restore a previous jannet software version state
list: query information about installed software packages
list-projects: query information about installed software projects
checklog: update the installation log file if necessary
rpmnew: remove superfluous rpmnew files and show conflicts
for others
build-date: show installed janware packages along with build date
built-today: show installed janware packages built today
list-changed-files: list files changed since installation
list-templates: list templates
list-template-tables: list templates tables
list-template-output: list template output files
rm-template-output: remove template output files
compile-templates: compile templates
global options are
-v be verbose
package
optional whitespace-separated list of package names, generally limiting the
scope of the command to the specified packages, defaults to all installed
janware packages
EOT
[ "$1" ] && exit $1
}
create_ldconfig_state()
{
echo "=== automatically created by $myname update script"
echo "--- files"
ls /etc/ld.so.conf.d
echo "--- directories"
find /etc/ld.so.conf.d -type f | xargs cat | sort -u
}
check_ldconfig()
{
local state=/var/log/jw-build-ldconf.state
[ -f $state ] && create_ldconfig_state | diff $state - >/dev/null 2>&1 && return
echo -n "running ldconfig ... "
/sbin/ldconfig
create_ldconfig_state > $state
echo "done."
}
list_packages()
{
local query_tags=""
eval set -- `getopt -- VPNUD "$@"`
while [ "$1" != -- ]; do
case $1 in
-V)
query_tags="$query_tags %{Vendor}"
;;
-P)
query_tags="$query_tags %{Packager}"
;;
-U)
query_tags="$query_tags %{URL}"
;;
-D)
query_tags="$query_tags %{Distribution}"
;;
*)
echo -e "Unexpected argument >$1<\n" >&2
usage 1
;;
esac
shift
done
shift
[ "$query_tags" ] || query_tags="%{URL}"
local names="$1"
local rpm_args
if [ -n "$names" ]; then
rpm_args="$rpm_args $names"
else
rpm_args="$rpm_args -a"
fi
rpm -q --queryformat "%{Name}: $query_tags\n" $rpm_args | \
grep -i '^[^ ]\+:.*\(jannet\|janware\)' | \
sed 's/\(^[^ ]\+\) *:.*/\1/; s/[ ]*$//' |\
sort -u
}
list_projects()
{
list_packages "$@" | xargs -r rpm -q --queryformat '%{SOURCERPM}\n' | \
sed 's/[ ]*//; s/-[0-9]\+.*//' |\
sort -u
}
list_files()
{
# Need to sort -u, because apparently, during installation, rpm -ql
# returns every file in the package _twice_. Funny.
list_packages "$@" | sort -u | xargs -r rpm -ql
}
cmd_rpmnew()
{
local file
list_files "$@" |
while read file; do
if [ -e "$file.rpmnew" ]; then
if diff -q $file $file.rpmnew; then
if [ -L "$file" ]; then
echo ========== skipping link $file
diff $file.rpmnew $file
continue
fi
echo mv $file.rpmnew $file
sudo mv $file.rpmnew $file
else
echo =========== $file
diff $file.rpmnew $file
fi
fi
done
}
cmd_diff()
{
local file
list_packages "$@" |
xargs -r rpm -qV |
sed '/^....L\|^..5/ !d; s%[^/]*/%/%' |
while read file; do
if diff -q $file $file.rpmnew; then
if [ -L "$file" ]; then
echo ========== skipping link $file
diff $file.rpmnew $file
continue
fi
echo mv $file.rpmnew $file
sudo mv $file.rpmnew $file
else
echo =========== $file
diff $file.rpmnew $file
fi
done
}
cmd_build_date()
{
list_packages "$@" |
xargs -r rpm -q --queryformat '%{BUILDTIME} %{NAME}\n' |
sort -n |
sed 's/-run\|-devel//' |
sort -u |
while read t p; do
echo `date --date=@$t --rfc-3339=seconds` $p
done
}
cmd_list_templates()
{
local ext_from="$template_exts" # TODO: support more than one
local ext_from_re=`echo $ext_from | sed 's/\./\\./g'`
local f
list_files "$@" | while read f; do
[ -f "$f" -o -L "$f" ] || continue
echo $f | grep -q "$ext_from_re$" || continue
echo $f
done
}
cmd_list_template_tables()
{
local ext_from="$table_exts"
local e f t
list_files "$@" | while read f; do
[ -f "$f" -o -L "$f" ] || continue
for e in $ext_from; do
t="${f%.*}$e"
[ -f "$t" ] || continue
echo $t
done
done
}
cmd_list_template_output()
{
local re=`echo $template_exts | sed 's/\./\\\\./g; s/^ *//; s/ *$//; s/ */$\\\\|/g; s/\(.*\)/\\\\(\1$\\\\)/'`
local f
list_files "$@" | sed "/$re/ !d; s/$re//" | while read f; do
[ -f "$f" -o -L "$f" ] || continue
echo $f
done
}
cmd_rm_template_output()
{
local restore=0
while [ "${1:0:1}" = '-' ]; do
case "$1" in
-r)
restore=1
;;
*)
usage 1
;;
esac
shift
done
local f
cmd_list_template_output $@ | while read f; do
log "removing $f"
rm $f
bak=$f$template_bak_ext
if [ "$restore" = 1 -a -f "$bak" ]; then
log "restoring $f from $bak"
mv $bak $f
fi
done
}
cmd_compile_templates()
{
__md() {
stat --format '%U:%G %a' "$1" 2>/dev/null
}
local ext_from="$template_exts" # TODO: support more than one
local ext_from_re=`echo $template_exts | sed 's/\./\\./g'`
local ext_to=""
local ext_tables="$table_exts"
local default_group=root
local default_owner=root
local default_mode=600
local conf_patt="^[ ]*# *conf:"
local missing_table=0
local backup=0
umask 0077
while [ "${1:0:1}" = '-' ]; do
case "$1" in
-o)
default_owner="$2"
shift
;;
-m)
default_mode="$2"
shift
;;
-g)
default_group="$2"
shift
;;
-b)
backup=1
;;
-c)
conf_patt="$2"
shift
;;
*)
usage 1
;;
esac
shift
done
local f
while read f; do
local base=`echo $f | sed "s/$ext_from_re$//"`
[ "$base" ] || continue
local to=$base$ext_to
local table=""
local ext
for ext in $ext_tables; do
table=$base$ext
[ -f "$table" ] && break
done
[ -f $table ] || {
log "WARNING: No key-value table found for template $f, not compiling." >&2
((missing_table++))
continue
}
# TODO: use mktemp -d and keep temporary files in read-only dir
local tmp=$to.tmp
log -n "Applying macros in $table to $f"
local mode=$default_mode
local owner=$default_owner
local group=$default_group
local diff=""
eval `sed "/$conf_patt/ !d; s/$conf_patt//" $f`
eval `sed "/$conf_patt/ !d; s/$conf_patt//" $table`
sed 's|^[ ]*#.*||; s|/|\\/|g; s|\([^ =]\+\)[ =]\+\(.*\)|s/\1/\2/g|' $table | sed -f - $f > $tmp
chmod $mode $tmp
chown $owner $tmp
chgrp $group $tmp
diff -q $tmp $to >/dev/null 2>&1 || diff="@content"
local md_to=$(__md "$to")
local md_tmp=$(__md "$tmp")
[ "$md_to" = "$md_tmp" ] || diff="$diff@metadata ($md_to -> $md_tmp)"
if [ -z "$diff" ]; then
log -c ", no changes."
rm $tmp
continue
fi
local bak=$to$template_bak_ext
if [ "$backup" = 1 -a -f $to ] && ! diff -q $to $bak >/dev/null 2>&1; then
log -C ", saving backup to $to to $bak"
cp -p $to $bak
fi
mv $tmp $to
diff=${diff##@}
log -c ", updating ${diff//@/ + }, done."
done <<< $(cmd_list_templates "$@")
if [ "$missing_table" != 0 ]; then
log "WARNING: $missing_table missing tables found. You might want to add them and run sudo $cmdline again."
fi
}
cmd_get_value_from_tables()
{
local key show_file
eval set -- `getopt -- k: "$@"`
while [ "$1" != -- ]; do
case $1 in
-k)
key=$2
shift
;;
-f)
show_file=1
shift
;;
*)
echo -e "Unexpected argument >$1<\n" >&2
usage 1
;;
esac
shift
done
shift
local f
while read f; do
grep -q "^\s*$key\s\+" $f || continue
[ "$show_file" = 1 ] && echo -n "$f: "
sed "s|^[ ]*#.*||; /^\s*$key\(\s\|=\)\+/ !d; s/^\s*$key\(\s\|=\)\+//" $f
done <<< $(cmd_list_template_tables "$@")
}
cmd_built_today()
{
local today=`date +'%Y-%m-%d'`
cmd_build_date | grep $today | cut -d' ' -f3
}
cmd_list_changed_files()
{
list_projects "$@" | xargs rpm -qV | grep '^..5' | sed 's|[^/]*/|/|'
}
# ----- here we go
cmdline="$0 $@"
myname=`basename $0`
longname=$0
opts="v"
table_exts=".jw-secret .jw-vars"
template_exts=".jw-tmpl"
template_bak_ext=".jw-pkg.bak"
while [ "${1:0:1}" = '-' ]; do
case "$1" in
-v)
opt_verbose=true
shift
;;
*)
usage 1
;;
esac
shift
done
cmd="${1//-/_}"
shift
case $cmd in
# -- TODO: re-implement this using pkg-manager.sh, or integrate with pkg-manager.sh
#init)
# if ! channel_present ftp.jannet.de; then
# echo -n "adding installation source ... "
# sudo smart channel -y --add ftp.jannet.de \
# type=yast2 \
# baseurl=ftp://dspadm:dspasswd@ftp.jannet.de/pub/packages/linux/suse/10.1 \
# >/dev/null 2>&1
# if channel_present; then echo done; else echo failed; fi
# fi
# ;;
#
#uninit)
# if channel_present ftp.jannet.de; then
# echo -n "removing installation source ... "
# sudo smart channel -y --remove ftp.jannet.de >/dev/null 2>&1
# echo done
# fi
# ;;
#
#update)
# sudo smart update ftp.jannet.de
# sudo smart upgrade -y
# check_ldconfig
# ;;
#
#install)
# sudo smart update ftp.jannet.de
# if [ -f $1 ]; then
# sudo smart install -y `cat $1`
# else
# sudo smart install -y $*
# fi
# check_ldconfig
# ;;
#restore)
# echo "not yet implemented, sorry" >&2
# ;;
list)
if [ "$opt_verbose" = true ]; then
list_packages "$@" | xargs -r rpm -qi
else
list_packages "$@" | xargs -r rpm -q
fi
;;
list_projects)
list_projects "$@"
;;
compile_templates|list_templates|list_template_tables|list_template_output|rm_template_output|diff|build_date|built_today|rpmnew|get_value_from_tables|list_changed_files)
cmd_$cmd "$@"
;;
help)
usage 0
;;
*)
usage 1
;;
esac