calculate_hash(): Derive hash from VCS file ids

The source hash is an md5sum over all versioned files in the working tree.
md5sum follows symlinks, so a project with links to files outside the tree
may produce different hashes depending on files outside the repo.

Compute the hash over the VCS file ids instead. Add a -s option to scm.sh
ls-files that lists each versioned file together with its content id. The
listing is sorted by path rather than by id, so an entry keeps its position
when its content changes.

For Git repos, git ls-tree -r HEAD reports the blob of a symlink's target
string and the pinned commit of a submodule, so the resulting hash depends
only on the committed tree, never on the working tree or on the machine.

calculate_hash() and the HASH variable in make/pkg-dist.mk feed the id
listing, minus the release metadata files, directly to md5sum.

calculate_hash() fails loudly when the pipeline fails, with its callers
refusing to proceed on an empty result.

The scheme change invalidates the stored HASH values, so every project
gets one more release the first time it runs against the new code, and the
hashes are stable again afterwards.

Signed-off-by: Jan Lindemann <jan@janware.com>
Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v0.85.1

scripts/pkg.sh

@@ -107,12 +107,16 @@ scm_files()
 
 calculate_hash()
 {
-	(
+	local h
+	h=`(
+		set -o pipefail
 		cd $TOPDIR
-		scm_files -z | \
+		scm_files -s -z | \
 			grep -vz "CHANGES\|VERSION\|HASH\|MD5SUMS\|RELEASES" | \
-			xargs -0 md5sum | md5sum | sed 's/ .*//'
-	)
+			md5sum | sed 's/ .*//'
+	)` || fatal "failed to calculate source hash"
+	[ -n "$h" ] || fatal "failed to calculate source hash"
+	echo $h
 }
 
 check_scm()
@@ -180,6 +184,7 @@ check_next_version()
 		return
 	fi
 	local h=`calculate_hash`
+	[ -n "$h" ] || fatal "failed to calculate source hash"
 	if [ ! -f $TOPDIR/HASH ]; then
 		echo "+ $PKG_PROJECT has no HASH file, needs release" >&2
 		echo $v
@@ -602,6 +607,7 @@ upload_pkg()
 	local p
 	local v=`read_map $TOPDIR/VERSION | sed 's/-dev//'`
 	local h=`calculate_hash`
+	[ -n "$h" ] || fatal "failed to calculate source hash"
 	case $PKG_FORMAT in
 	rpm)
 		server=pkg.janware.com

scripts/scm.sh

@@ -150,7 +150,33 @@ cmd_ls_files()
 		list_dirents_git $1
 	}
 
-	set -- `getopt fnzta "$@"`
+	list_dirents_git_ids()
+	{
+		local entry meta mode sha
+		while IFS= read -r -d '' entry; do
+			meta=${entry%%$'\t'*}
+			mode=${meta%% *}
+			sha=${meta##* }
+			[ "$opt_no_submodules" = 1 ] && [ "$mode" = 160000 ] && continue
+			[ "$opt_only_regular_files" = 1 ] && [ "$mode" != 100644 ] && [ "$mode" != 100755 ] && continue
+			output "$sha ${entry#*$'\t'}"
+		done < <(git ls-tree -r -z HEAD "$1")
+	}
+
+	list_dirents_ids()
+	{
+		if [ -d $1/CVS ]; then
+			echo "failed to list file ids in $(realpath $1): $scm has no file ids" >&2
+			exit 1
+		fi
+		git -C $1 status >/dev/null 2>&1 || {
+			echo "failed to list file ids in $(realpath $1): no VCS" >&2
+			exit 1
+		}
+		list_dirents_git_ids $1
+	}
+
+	set -- `getopt fnzsta "$@"`
 
 	while [ "$1" != -- ]; do
 	case $1 in
@@ -160,6 +186,9 @@ cmd_ls_files()
 	-n)
 		opt_no_submodules=1
 		;;
+	-s)
+		show_ids=1
+		;;
 	-z)
 		zero_terminate=1
 		git_ls_files_opts="$git_ls_files_opts -z"
@@ -181,6 +210,8 @@ cmd_ls_files()
 
 	if [ "$text_files" ]; then
 		list_dirents . | sort $opt_sort | xargs file -N | grep ":.*text" | cut -d: -f1
+	elif [ "$show_ids" = 1 ]; then
+		list_dirents_ids . | sort $opt_sort -k2
 	else
 		list_dirents . | sort $opt_sort
 	fi