From 25e4c197d33f840d617d67b0814e8b9f0a02628c Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Sun, 31 May 2015 11:52:08 +0000 Subject: [PATCH] build-package.sh: Rename to pkg.sh Rename build-package.sh to pkg.sh, prepare it for handling more generic tasks. Signed-off-by: Jan Lindemann --- make/rpmdist.mk | 8 +- scripts/pkg.sh | 633 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 637 insertions(+), 4 deletions(-) create mode 100644 scripts/pkg.sh diff --git a/make/rpmdist.mk b/make/rpmdist.mk index 9c53dbce..a213e562 100644 --- a/make/rpmdist.mk +++ b/make/rpmdist.mk @@ -84,7 +84,7 @@ ifeq ($(CREATE_DEVEL),true) BINARY_RPM += $(DIST_PCKG_DIR)/$(PCKG_RPM_DEVEL_I386) endif DIST_PCKG_SRPM = $(BINARY_RPM) $(DIST_PCKG_DIR)/$(PCKG_RPM_SRC) -# to be replaced by build-package.sh +# to be replaced by pkg.sh RPM_REQUIRES_DEVEL += $(RPM_PROJECT)-run = __NEXT_VERSION__ unexport RPM_VERSION VERSION DIST_SRC_DIR @@ -92,7 +92,7 @@ unexport RPM_VERSION VERSION DIST_SRC_DIR include $(MODDIR)/make/tag-rules.mk include $(MODDIR)/make/upload-rules.mk -BUILD_PACKAGE_SH = /bin/bash $(MOD_SCRIPT_DIR)/build-package.sh \ +PKG_SH = /bin/bash $(MOD_SCRIPT_DIR)/pkg.sh \ -N $(RPM_PROJECT) \ -R "$(RPM_REQUIRES_RUN)" -D "$(RPM_REQUIRES_DEVEL)" -P $(PROJECT) \ -m $(MOD_SCRIPT_DIR) -a $(RPM_ARCH) -p $(DIST_PCKG_DIR) -F $(PKG_FORMAT) @@ -139,7 +139,7 @@ clean.rpm: #endif pkg-build.dist: - $(BUILD_PACKAGE_SH) build + $(PKG_SH) build touch $@ pkg-update-local-repo: @@ -320,7 +320,7 @@ pkg-release: pkg-release-reinstall # pkg-release-update: pkg-release pkg-update pkg-release-reinstall: update - $(BUILD_PACKAGE_SH) release + $(PKG_SH) release ifneq ($(TARGET_HOST),) pkg-remote-install: diff --git a/scripts/pkg.sh b/scripts/pkg.sh new file mode 100644 index 00000000..47c07d46 --- /dev/null +++ b/scripts/pkg.sh @@ -0,0 +1,633 @@ +#!/bin/bash + +log() +{ + echo $@ +} + +goodbye() +{ + : + rm -f $tmp_mkdef_sh scm_status.tmp +} + +quit() +{ + goodbye + exit $1 +} + +usage() +{ + cat << EOT | sed 's/^ |//' + |usage: $myname cmd +EOT + [ "$1" ] && { + quit $1 + } +} + +fatal() +{ + log "$@" + log "Giving up." + quit 1 +} + +os() +{ + /bin/bash $MOD_SCRIPT_DIR/get_os.sh +} + +scm_commit() +{ + case $SCM in + cvs) + cvs commit "$@" + ;; + git) + git commit -uno "$@" + ;; + *) + fatal "tried to commit with unknown source code management tool $SCM" + ;; + esac +} + +platform() +{ + echo `os`/$RPM_ARCH +} + +abspath() +{ + (cd $1; /bin/pwd) +} + +read_map() +{ + sh $MOD_SCRIPT_DIR/platform-map-file.sh -m $MOD_SCRIPT_DIR -p `platform` $@ read | sed 's/-dev//' +} + +write_map() +{ + sh $MOD_SCRIPT_DIR/platform-map-file.sh -m $MOD_SCRIPT_DIR -p `platform` $1 write $2 +} + +find_path() +{ + local p + for p in $@; do + [ -e "$p" ] && { + echo $p + return 0 + } + done + fatal "none of the paths \"$@\" exists" +} + +scm_files() +{ + ( + cd $TOPDIR + /bin/bash $MOD_SCRIPT_DIR/list-cvs-files.sh -f + ) +} + +calculate_hash() +{ + ( + cd $TOPDIR + scm_files | \ + grep -v "CHANGES\|VERSION\|HASH\|MD5SUMS\|_RPM_RUN" | \ + xargs md5sum | md5sum | sed 's/ .*//' + ) +} + +check_scm() +{ + if [ -d .git ]; then + log + checking git + git pull --no-edit || fatal "git pull failed, giving up" + git status --porcelain | grep -v '^??' | grep -q . && { + git status + fatal "git has locally modified files, giving up" + } + else + log + checking cvs + cvs update -dP + cvs status > scm_status.tmp 2>&1 + if [ $? != 0 ]; then + cat scm_status.tmp + fatal "\n======== cvs status failed, giving up." + fi + grep -qi "locally modified" scm_status.tmp && \ + fatal "cvs copy has locally modified files, giving up" + grep -qi "needs" scm_status.tmp && \ + fatal "+ cvs copy is out-of-date, giving up" + fi +} + +check_cwd() +{ + pwd | grep -q $HOME/local/src +} + +create_empty_dir() +{ + [ $# != 2 ] && \ + fatal "Unable to create empty directory \"$@\"." + + local dir="$1" + shift + local desc="$@" + + mkdir -p "$dir" + + [ -w "$dir" ] || + fatal "No write-access to $desc \"$dir\"" + + [ `ls "$dir" | wc -l` -eq 0 ] || + fatal "$desc \"$dir\" isn't empty" +} + +increase_build_no() +{ + local build_no=`echo $1 | sed 's/.*-\([0-9]\+\)\(-dev\)*/\1/'` + build_no=`expr $build_no + 1` + echo $1 | sed "s/-\([0-9]\+\)\(-dev\)*/-$build_no\2/" +} + +check_next_version() +{ + local v=`read_map $TOPDIR/VERSION | sed 's/-dev//'` + local n=`read_map -n $TOPDIR/LAST_RPM_VERSION | sed 's/-dev//'` + if [ "$n" != "$v" ]; then + echo "+ $RPM_PROJECT version is already at $v, no need to increase for release" >&2 + echo $v + return + fi + local h=`calculate_hash` + if [ ! -f $TOPDIR/HASH ]; then + echo "+ $RPM_PROJECT has no HASH file, needs release" >&2 + echo $v + return + fi + if ! read_map $TOPDIR/HASH | grep -q $h; then + echo "+ $RPM_PROJECT has no matching HASH value, needs release" >&2 + increase_build_no $v + return + fi + echo "+ $RPM_PROJECT has unchanged source" >&2 + if [ "$reinstall_check_failed" = 1 ]; then + echo "version $v is uninstallable, needs release" >&2 + increase_build_no $v + return + fi + if [ "$reinstall_check_succeeded" = 1 ]; then + echo $v + return + fi + if ! check_pkg_reinstall >&2; then + echo "version $v is uninstallable, needs release" >&2 + increase_build_no $v + return + fi + + echo "version $v doesn't match any release criteria, not releasing" >&2 + echo $v +} + +check_pkg_install() +{ + [ "$1" = -c ] && { + shift + [ ! -f "$1" ] && return 0 + } + echo "+ installing $1" + sudo $RPM -U $1 || { + echo "failed to install $1" + return 1 + } +} + +check_pkg_reinstall() +{ + local v=`read_map $TOPDIR/VERSION` + local s + for s in devel run; do + if $RPM -q $RPM_PROJECT-$s > /dev/null 2>&1 ; then + echo + removing $RPM_PROJECT-$s + sudo $RPM -e $RPM_PROJECT-$s --nodeps --allmatches || { + echo "failed to uninstall $RPM_PROJECT-$s, ignored" + } + fi + done + + local suffix + case "$PKG_FORMAT" in + rpm) + suffix=-$v.$RPM_ARCH.rpm + ;; + *) + suffix=_$v""_$RPM_ARCH.deb + ;; + esac + + check_pkg_install $TOPDIR/$DIST_PCKG_DIR/$RPM_PROJECT-run$suffix || return 1 + check_pkg_install -c $TOPDIR/$DIST_PCKG_DIR/$RPM_PROJECT-devel$suffix || return 1; +} + +check_release_is_current() +{ + local reinstall_pkg= + local next=`check_next_version` + local v=`read_map $TOPDIR/VERSION` + + set -- `getopt 'r' $*` + while [ "$1" != -- ] ; do + case "$1" in + -r) + reinstall_pkg=1 + ;; + esac + shift + done + shift + + if ! read_map -n $TOPDIR/LAST_RPM_VERSION | grep -Fq $next; then + echo "next version $next is not in LAST_RPM_VERSION, needs release" >&2 + return 1 + fi + + if [ "$reinstall_pkg" ]; then + if ! check_pkg_reinstall; then + echo "version $v is uninstallable, needs release" >&2 + reinstall_check_failed=1 + return 1 + fi + reinstall_check_succeeded=1 + fi + + return 0 +} + +check_update_version() +{ + ( + cd $TOPDIR + local v=`read_map VERSION` + local next=`check_next_version` + if [ "$next" != "$v" ]; then + echo "$next-dev" > VERSION + scm_commit -m "o $next" VERSION + return 0 + fi + return 1 + ) +} + +build_pkg() +{ + set -e + + local version=`read_map $TOPDIR/VERSION | sed 's/-dev//'` + local src_base=$RPM_PROJECT-$version + local src_tree=$DIST_SRC_DIR/$src_base + local tar_archive=$src_base.tar.bz2 + local tar_archive_orig=$src_base.orig.tar.bz2 + local RPM_REQUIRES_RUN=`echo $RPM_REQUIRES_RUN | sed "s/__NEXT_VERSION__/$version/g"` + local RPM_REQUIRES_DEVEL=`echo $RPM_REQUIRES_DEVEL | sed "s/__NEXT_VERSION__/$version/g"` + + # --- create source directory tree + create_empty_dir $src_tree "Source files compilation directory" + + # --- copy source files over + scm_files | + grep -v VERSION | + cpio -o -H newc | + ( cd $src_tree; cpio --make-directories -i) + echo $version > $src_tree/VERSION + + local pkgfmt + for pkgfmt in $PKG_FORMAT; do + + local deffmt + case $pkgfmt in + rpm) + deffmt=spec;; + debian|deb) + deffmt=debian;; + *) + ;; + esac + + # --- generate spec file + local mkdef_sh=$TOPDIR/make/mk$deffmt.sh + local mkdef_wrapper_sh=$MOD_SCRIPT_DIR/mkspec_wrapper.sh + + [ ! -r $mkdef_wrapper_sh ] && \ + fatal "Failed to read package definition deffmt wrapper script \"$mkdef_wrapper_sh\"." + + scm_files | grep -q make/mk$deffmt.sh || { + local create_mkdef_sh=$MOD_SCRIPT_DIR/create-mk$deffmt.sh + [ ! -r "$PROJECT_DESCR_FILE" ] && \ + fatal "Failed to read project description file \"$PROJECT_DESCR_FILE\"." + [ ! -r "$create_mkdef_sh" ] && \ + fatal "Failed to read package build definition creation script \"$create_mkdef_sh\"." + /bin/bash $create_mkdef_sh $PROJECT_DESCR_FILE > $mkdef_sh.tmp + mv $mkdef_sh.tmp $mkdef_sh + tmp_mkdef_sh="$tmp_mkdef_sh $mkdef_sh" + } + + mkdef_sh=`readlink -f $mkdef_sh` + mkdef_wrapper_sh=`readlink -f $mkdef_wrapper_sh` + + ( + cd $src_tree + /bin/bash $mkdef_wrapper_sh $mkdef_sh \ + -V $version \ + -S $tar_archive \ + -N $RPM_PROJECT \ + -R "$RPM_REQUIRES_RUN" \ + -D "$RPM_REQUIRES_DEVEL" \ + -P $PROJECT \ + > $RPM_PROJECT.$deffmt + ) + done + + # --- tar up source directory tree + create_empty_dir $DIST_PCKG_DIR "Package directory" + + local excluded=`find $TOPDIR -name '*.in' | sed "s/\.in$//; s%$DIST_SRC_DIR%%; s%^[./]*%%; s/^/--exclude /g"` + tar --anchored $addprefix $excluded \ + -czvf $DIST_PCKG_DIR/$tar_archive -C $DIST_SRC_DIR `basename $src_tree` + + case "$PKG_FORMAT" in + rpm) + # --- build RPMs and copy them to source directory + rpmbuild -ta $DIST_PCKG_DIR/$tar_archive + cp $SRPMS_DIR/$src_base.src.rpm $DIST_PCKG_DIR/ + cp $RPMS_DIR/$RPM_PROJECT-*-$version.$RPM_ARCH.rpm $DIST_PCKG_DIR/ + ;; + *) + #exit + #cp $DIST_PCKG_DIR/$tar_archive $src_tree/../$tar_archive_orig + ( + local mod_script_dir=`readlink -f $MOD_SCRIPT_DIR` + cd $src_tree + #export ENV_PREFIX=`readlink -f $TOPDIR`/$DIST_INST_DIR + #export INSTALL_LOG=blah + #strace -f -s 128 -o /tmp/strace.out debuild --no-tgz-check --prepend-path=$mod_script_dir -us -uc + debuild --no-tgz-check --prepend-path=$mod_script_dir -us -uc + ) + cp $DIST_SRC_DIR/*.deb $DIST_PCKG_DIR/ + ;; + esac +} + +upload_file() +{ + local f=$1 + local t=$2 + local upload_spec=644:755:`id -un`.207 + if [ ! -f $DIST_PCKG_DIR/$f ]; then + echo "+ $DIST_PCKG_DIR/$f doesn't exit, skipping" >&2 + return 1 + fi + local os_name=`os | sed 's/-.*//'` + local os_version=`os | sed 's/[^-]\+-//'` + + local target_base=rsync_ssh://root@ftp.jannet.de:/srv/ftp/pub/packages/linux/$os_name/$os_version + local target + if [ "$t" = tgz ]; then + target=$target_base/inst-source/$t/$f:$upload_spec + else + target=$target_base/inst-source/rpm/$t/$f:$upload_spec + fi + echo "+ uploading $target" + sh $MOD_SCRIPT_DIR/upload.sh $DIST_PCKG_DIR/$f $target +} + +upload_pkg() +{ + local server + set -e + local p + local v=`read_map $TOPDIR/VERSION | sed 's/-dev//'` + local h=`calculate_hash` + case $PKG_FORMAT in + rpm) + server=ftp.jannet.de + upload_file $RPM_PROJECT-$v.tar.bz2 tgz + upload_file $RPM_PROJECT-$v.src.rpm src + upload_file $RPM_PROJECT-run-$v.$RPM_ARCH.rpm $RPM_ARCH + upload_file $RPM_PROJECT-devel-$v.$RPM_ARCH.rpm $RPM_ARCH || true + ;; + debian|deb) + server=apt.janware.com + local conf=`mktemp "/tmp/$myname"_XXXXXX` + cat <<-EOT > $conf + [DEFAULT] + default_host = janware-debian + [janware-debian] + fqdn = $server + # login = + incoming = /srv/ftp/pub/packages/linux/debian/mini-dinstall/incoming/ + method = rsync + hash = sha + allow_unsigned_uploads = yes + distributions = debian-8 + allowed_distributions = unstable + delayed = + run_lintian = no + run_dinstall = no + check_version = yes + passive_ftp = yes + #progress_indicator + scp_compress = yes + ssh_config_options = ForwardAgent=yes,Compression=yes,ServerAliveInterval=30,ConnectTimeout=120 + #post_upload_command = + #pre_upload_command = + #default_host_main = + EOT + dput -c $conf janware-debian dist/src/"$RPM_PROJECT"_"$v"_"$RPM_ARCH".changes + rm -f $conf + ;; + *) + fatal "Tried to upload with unknown package format \"$PKG_FORMAT\"" + ;; + esac + + write_map $TOPDIR/LAST_RPM_VERSION $v + $SCM add $TOPDIR/LAST_RPM_VERSION || true + write_map $TOPDIR/HASH $h + $SCM add $TOPDIR/HASH || true + echo "+ scheduling rebuild" + $SSH -l root $server /opt/packager-server/bin/packager-server schedule-rebuild + scm_commit -m "o $v@`platform`" $TOPDIR/LAST_RPM_VERSION $TOPDIR/HASH + if [ "$SCM" = git ]; then + git push || true + fi +} + +cmd_build() +{ + build_pkg +} + +cmd_upload() +{ + upload_pkg +} + +# ---- here we go +umask 0022 +trap goodbye SIGINT SIGKILL + +# -- default values +TOPDIR=. +MOD_SCRIPT_DIR=/opt/ytools/bin +CHECK_CVS_SYNC_BEFORE_RPM_RELEASE=false +#LAST_RPM_VERSION=`read_map $TOPDIR/LAST_RPM_VERSION` +myname=`basename $0` +tmp_mkdef_sh="" +PROJECT_DESCR_FILE=$TOPDIR/doc/share/project.txt +DIST_SRC_DIR=dist/src +DIST_INST_DIR=dist/src +DIST_PCKG_DIR=dist/pckg +PROJECT=`abspath $TOPDIR | xargs basename` +RPM_PROJECT="$PROJECT" +RPM_REQUIRES_RUN="" +RPM_REQUIRES_DEVEL="" +RPM_ARCH=`arch` +PKG_FORMAT=rpm +SSH=ssh +SCM=cvs +[ "$RSYNC_RSH" ] && SSH=$RSYNC_RSH +[ "$CVS_RSH" ] && SSH=$CVS_RSH +[ -d .git ] && SCM=git + +eval set -- `getopt -- 'ht:p:m:N:R:D:P:a:F:' "$@"` +while [ "$1" != -- ] ; do + case "$1" in + -h) + usage 0;; + -t) + eval DIST_SRC_DIR=\"$2\" + shift + ;; + -p) + eval DIST_PCKG_DIR=\"$2\" + shift + ;; + -m) + eval MOD_SCRIPT_DIR=\"$2\" + shift + ;; + -N) + eval RPM_PROJECT=\"$2\" + shift + ;; + -R) + eval RPM_REQUIRES_RUN=\"$2\" + shift + ;; + -D) + eval RPM_REQUIRES_DEVEL=\"$2\" + shift + ;; + -P) + eval PROJECT=\"$2\" + shift + ;; + -a) + eval RPM_ARCH=\"$2\" + shift + ;; + -F) + eval PKG_FORMAT=\"$2\" + shift + ;; + esac + shift +done +shift + +SRPMS_DIR=`find_path $HOME/rpmbuild/SRPMS /usr/src/packages/SRPMS` +RPMS_DIR=`find_path $HOME/rpmbuild/RPMS/$RPM_ARCH /usr/src/packages/RPMS/$RPM_ARCH` + +cmd=$1 +shift + +[ $# -ne 0 -o ! "$cmd" ] && usage 1 + +case $PKG_FORMAT in +rpm) + RPM=/bin/rpm + ;; +deb|debian) + RPM="/bin/bash $MOD_SCRIPT_DIR/dpm.sh" + ;; +*) + fatal "Unknown package format \"$PKG_FORMAT\"" + ;; +esac + +case $cmd in +check-release) + cmd_check_release + ;; +need-release) + if check_release_is_current; then + echo no >&2 + exit 1 + else + echo yes >&1 + exit 0 + fi + ;; +update-version) + check_update_version || exit 0 + ;; +build) + check_cwd + rm -rf ./$DIST_SRC_DIR ./$DIST_PCKG_DIR + cmd_build + ;; +release-reinstall) + check_update_version + cmd_build + cmd_upload + ;; +release) + cd $TOPDIR + check_scm + check_release_is_current -r || { + set -e + check_cwd + rm -rf ./$DIST_SRC_DIR ./$DIST_PCKG_DIR + check_update_version || true + cmd_build + check_pkg_reinstall + cmd_upload + } + ;; +upload) + cmd_upload + ;; +reinstall) + check_pkg_reinstall + ;; +release) + ;; +hash) + calculate_hash + ;; +*) + usage 1 + ;; +esac + +goodbye +