jw-pkg/scripts/integrate-distro.sh

#!/bin/sh

# -- defaults
# install_dev=/dev/sdb
arch=`uname -m`
o_name=suse-11.4
cmd=install
initrd_modules="aufs via-rhine e1000 e1000e myri10ge forcedeth usbcore ohci-hcd ehci-hcd uhci-hcd hid usbhid ahci"
# ata_piix (as opposed to ahci) doesn't seem to support sata-hotplugging
#kernel_version=2.6.37.4
kernel_version=2.6.37.6-jng135-default
rpm_kernel_version=2.6.37.6-jng135-4
umask 0022

unused_pkgs="
	sysvinit-systemd
	kernel
	kernel-default-$rpm_kernel_version
	kernel-syms-$rpm_kernel_version
	kernel-source-$rpm_kernel_version
	aufs-kmp
"

# -- pciutils
# used by jux, should be done by grepping /proc-files
special_pkgs="
	aufs
	aufs-kmp-default
	kernel-default-$rpm_kernel_version
	rpm
"


# -- sysvinit
# systemd-sysvinit boots, but doesn't work cleanly as of now

payload_pkgs="

	kernel
	aufs
	mkinitrd
	sysvinit
	grub
	syslinux
	splashy

	kbd
	sensors

	db-utils

	insserv
	pam_ldap
	pam_apparmor
	gnome-keyring-pam

	portmap
	proftpd
	proftpd-ldap
	xntp
	bind
	dhcp-server
	rsyslog
	openldap2
	courier-authlib-ldap
	xinetd
	apache2
	apache2-mod_php5
	nfs-kernel-server
	tftp

	java-1_6_0-sun.i586
	java-1_6_0-sun-plugin.i586

	unscd
	nss_ldap

	fuse
	mdadm
	smartmontools
	device-mapper
	insserv
	suspend

	dhcpcd
	nfs-client
	openldap2-client
	samba-client
	sudo
	sysconfig
	glibc-locale
	openvpn

	xorg-x11
	xorg-x11-driver-video
	xorg-x11-fonts-100dpi
	xorg-x11-fonts-75dpi
	xorg-x11-fonts-cyrillic
	xorg-x11-fonts-scalable
	xorg-x11-fonts-syriac
	xorg-x11-libs
	xorg-x11-libxcb
	xorg-x11-man
	xorg-x11-sdk
	xorg-x11-server
	xorg-x11-server-glx
	xorg-x11-Xnest
	xorg-x11-Xvfb
	xorg-x11-Xvnc
	unclutter

	xterm

	man
	findutils-locate
	vim
	grep

	iputils strace
	lsof

	wget lukemftp
	openssh
	parted

	tar zip unzip
	rsync cpio

	gcc-c++ libtool autoconf automake make
	cvs
	patch

	pciutils

	ytools-run
	jux-run
	jux-client-run
	feedfsd-run
	feedfs-local-run
	feedfs-net-run
	netscan-run
	snmp-manager-run
	ftp-server-run
	bprof-run

	fetchmsttfonts
"

# -- functions
usage()
{
	echo "

	usage: $myname -h
	       $myname [options] [install|mount|unmount]

	options:
		-d target-device
		-n target-name
		-p file with root password
		-a architecture
"
	[ "$1" ] && exit $1
}

get_opts()
{
	set -- `getopt 'hd:n:a:p:' $*`
	while [ "$1" != -- ] ; do
		case "$1" in
			-h)
				usage 0
				;;
			-d)
				install_dev=$2
				shift
				;;
			-n)
				o_name=$2
				shift
				;;
			-a)
				arch=$2
				shift
				;;
			-p)
				root_password_file=$2
				shift
				;;
			*)
				usage 1
				;;
		esac
		shift
	done
	shift

	[ "$1" ] && cmd="$1"
}

log()
{
	echo $@ >&2
}

run()
{
	local exit_on_error=0

	while [[ "$1" =~ ^- ]]; do
		case $1 in
		-e)
			exit_on_error=1
		esac
		shift
	done
	log =========== running $@
	$@
	local r=$?
	if [ "$exit_on_error" = 1 -a $r != 0 ]; then
		die "exiting after error during command \"$@\""
	fi
	return $r
}

die()
{
	log "$@"
	exit 1
}

_cat()
{
	cat | sed 's/^[ 	]*|//'
}

check_var_set()
{
	local var val
	for var in $@; do
		eval val=\$$var
		[ "$val" ] || {
			log "$var is not set"
			return 1
		}
	done
	return 0
}

_run_chroot()
{
	local cmd
	set +x
	while read cmd; do
		cmd=`echo $cmd | sed 's/[	 ]*#.*//'`
		[ "$cmd" ] || continue
		echo == running LANG=POSIX chroot $root $cmd
		LANG=POSIX chroot $root $cmd || {
			die "failed to run >$cmd<"
		}
	done
	set -x
}

run_chroot()
{
	echo == running LANG=POSIX chroot $root "$@"
	LANG=POSIX chroot $root "$@" || {
		die "failed to run >$@<, exiting"
	}
}

run_parted()
{
	while read cmd; do
		run parted --script $install_dev $cmd
	done
}

create_partition_table()
{
	assert_not_mounted

	_cat << EOT | run_parted
	|mklabel gpt
	|mkpart primary 1M 2M
	|name 1 grub
	|set 1 bios_grub
	|mkpart primary 2M 10G
	|name 2 boot
	|mkpart primary 10G 40G
	|name 3 swap
	|mkpart primary 40G 100%
	|name 4 root
EOT

	grub_boot_partition_num=0
	grub_slash_boot_partition_num=1
}

create_file_systems()
{
	run mkfs -t ext4 $install_dev_boot
	run mkswap $install_dev_swap
	run mkfs -t ext4 $install_dev_root
}

canonicalize()
{
	readlink -f $1 2>/dev/null
}

check_mount()
{
	local canonicalized=`canonicalize $root/$1`
	[ ! "$canonicalized" ] && return 1
	grep -q " $canonicalized " /proc/mounts || return 1
	return 0
}

assert_not_mounted()
{
	if grep $install_dev /proc/mounts; then
		echo "$install_dev is mounted, exiting"
		exit 1
	fi
}

setup_bind_mounts()
{
	local fs
	for fs in /dev /sys /proc; do
		check_mount $fs || {
			run -e mkdir -p -m 755 $root$fs
			run -e mount -o bind $fs $root$fs
		}
	done
}

mount_devices()
{
	check_mount / || {
		run -e install -d -m 755 $root
		run -e mount $install_dev_root $root
	}

	check_mount /boot || {
		run -e mkdir -p -m 755 $root/boot
		run -e mount $install_dev_boot $root/boot
	}
}

unmount_devices()
{
	local devices="/proc /sys /dev /boot /"
	[ "$1" ] && devices="$@"
	local d
	for d in $devices; do
		check_mount $d && run umount $root$d
	done
}

zypper_repos_base()
{
	_cat << EOT | grep -v "	#"
	|distro-oss     ftp://ftp/pub/mirror/suse/distribution/11.4/repo/oss
EOT
}

zypper_repos_update()
{
	_cat << EOT | grep -v "^ *#"
	|distro-non-oss ftp://ftp/pub/mirror/suse/distribution/11.4/repo/non-oss
	|update         ftp://ftp/pub/mirror/suse/update/11.4
EOT
}

zypper_repos_payload()
{
	_cat << EOT | grep -v "^ *#"
	|#ftp.jannet.de  ftp://dspadm:dspasswd@ftp.jannet.de/pub/packages/linux/suse/11.4/inst-source
        |ftp.jannet.de  ftp://dspadm:dspasswd@ftp/pub/mirror/jannet/ftp/pub/packages/linux/suse/11.4/inst-source
	|ftp.priv.lcl   ftp://dspadm:dspasswd@ftp/pub/local/packages/suse/11.4
EOT
}

zypper_repos_special()
{
	_cat << EOT | grep -v "^ *#"
	|jengelh        ftp://ftp/pub/mirror/gwdg/pub/linux/misc/suser-jengelh/openSUSE_11.4
EOT
}

zypper_repos()  
{
	local repo
	for repo in $@; do
		zypper_repos_$repo
	done
}

setup_zypper_repos()
{
	zypper_repos $@ | while read name uri; do
		chroot $root zypper $zypper_global_opts ar $uri $name
	done
}

host_zypper()
{
	local repos="$1"
	shift
	local distros=`zypper_repos $repos | awk '{print "--plus-repo " $2 }'`
	zypper $zypper_global_opts --root $root $distros $@
}

init_root_password_file()
{
	[ "$root_password_file" ] ||
		root_password_file=$HOME/.$myname/root-pass

	[ ! -f $root_password_file ] && {
		local dir=`dirname $root_password_file`
		mkdir -m 0755 $dir
		dd  if=/dev/random of=/dev/stdout bs=1 count=4 2>/dev/null | 
			hexdump | sed '/^0000000/ !d; s/0000000 //; s/ //g' > $root_password_file
		chmod 600 $root_password_file
	}
}

make_etc_fstab()
{
	_cat<<EOT
	|proc                 /proc                proc       defaults              0 0
	|sysfs                /sys                 sysfs      noauto                0 0
	|debugfs              /sys/kernel/debug    debugfs    noauto                0 0
	|usbfs                /proc/bus/usb        usbfs      noauto                0 0
	|devpts               /dev/pts             devpts     mode=0620,gid=5       0 0
	|/usr/share/syslinux  /srv/nfs/boot/pxe/bin auto      bind                  0 0
EOT
}

make_etc_sysctl_conf()
{
	_cat<<EOT
	|# Disable response to broadcasts.
	|# You don't want yourself becoming a Smurf amplifier.
	|net.ipv4.icmp_echo_ignore_broadcasts = 1
	|# enable route verification on all interfaces
	|net.ipv4.conf.all.rp_filter = 1
	|# disable IPv6 completely
	|# net.ipv6.conf.all.disable_ipv6 = 1
	|# enable IPv6 forwarding
	|#net.ipv6.conf.all.forwarding = 1
	|# increase the number of possible inotify(7) watches
	|fs.inotify.max_user_watches = 65536
	|# avoid deleting secondary IPs on deleting the primary IP
	|net.ipv4.conf.default.promote_secondaries = 1
	|net.ipv4.conf.all.promote_secondaries = 1
	|
	|# by jan
	|kernel.core_pattern = /var/cores/core.%h.%e.%p
	|net.ipv4.ip_forward = 1
	|net.ipv4.tcp_tw_recycle = 1
	|net.ipv4.tcp_tw_reuse = 1
	|dev.rtc.max-user-freq = 4096
EOT
}

make_etc_fstab_local_boot()
{
	_cat<<EOT
	|$run_dev_swap        swap                 swap       defaults              0 0
	|$run_dev_root        /                    ext4       acl,user_xattr        1 1
	|$run_dev_boot        /boot                ext4       acl,user_xattr        1 2
EOT
}

make_etc_hosts()
{
	_cat<<EOT
	|127.0.0.1       localhost
	|
	|# special IPv6 addresses
	|::1             localhost ipv6-localhost ipv6-loopback
	|
	|fe00::0         ipv6-localnet
	|
	|ff00::0         ipv6-mcastprefix
	|ff02::1         ipv6-allnodes
	|ff02::2         ipv6-allrouters
	|ff02::3         ipv6-allhosts
	|127.0.0.1       ldap.lcl
	|127.0.0.1       ftp.priv.lcl
EOT
}

make_etc_resolv_conf()
{
	cat /etc/resolv.conf
}

make_etc_ldap_conf()
{
	cat /etc/ldap.conf
}

make_etc_openldap_ldap_conf()
{
	_cat << EOT
	TLS_CACERT /etc/certs/root-ca-1.pem
	TLS_REQCERT allow
EOT
}

make_netboot_etc_hosts()
{
	cat /etc/hosts
	_cat<<EOT
EOT
}

make_netboot_etc_ldap_conf()
{
	cat /etc/ldap.conf | sed '
		s/^host .*/host __DHCPSIADDR__/
		s%^uri .*%uri ldap://__DHCPSIADDR__%
	'
}

make_netboot_etc_openldap_ldap_conf()
{
	cat /etc/openldap/ldap.conf | sed '
		s/^HOST .*/HOST __DHCPSIADDR__/
		s%^URI .*%URI ldap://__DHCPSIADDR__%
	'
}

make_netboot_etc_fstab()
{
	make_etc_fstab
	_cat<<EOT
	|
	|# netboot client
	|/var/cores           bootsrv:/var/cores   nfs        rw,rsize=8192,wsize=8192,async   0 0
EOT
}

make_sysconfig_bootloader()
{
	_cat << EOT
	|# automatically created by $myname on $date"
	|
	|LOADER_TYPE="grub"
	|DEFAULT_APPEND="resume=$run_dev_swap splash=silent quiet showopts"
	|DEFAULT_VGA=""
	|FAILSAFE_APPEND="showopts apm=off noresume edd=off powersaved=off nohz=off highres=off processor.max_cstate=1 nomodeset x11failsafe"
	|FAILSAFE_VGA=""
	|XEN_KERNEL_APPEND="resume=$run_dev_swap splash=silent quiet showopts"
	|XEN_APPEND=""
	|XEN_VGA=""
	|CYCLE_DETECTION="no"
	|CYCLE_NEXT_ENTRY="1"
	|LOADER_LOCATION=""
EOT
}

make_boot_menu()
{
	check_var_set kernel_version || \
		die "failed to detect kernel version for creating boot menu"

	local disk=hd$run_grub_dev_num

	_cat << EOT
	|default 0
	|timeout 8
	|##YaST - generic_mbr
	|gfxmenu ($disk,$grub_slash_boot_partition_num)/message
	|
	|title Default
	|    root ($disk,$grub_slash_boot_partition_num)
	|    kernel /vmlinuz root=$run_dev_root resume=$run_dev_swap splash=silent quiet showopts
	|    initrd /initrd-local
	|
	|title Failsafe
	|    root ($disk,$grub_slash_boot_partition_num)
	|    kernel /vmlinuz root=$run_dev_root showopts apm=off noresume edd=off powersaved=off nohz=off highres=off processor.max_cstate=1 nomodeset x11failsafe
	|    initrd /initrd-local
	|
	|title Default $kernel_version
	|    root ($disk,$grub_slash_boot_partition_num)
	|    kernel /vmlinuz-$kernel_version root=$run_dev_root resume=$run_dev_swap splash=silent quiet showopts
	|    initrd /initrd-local-$kernel_version
	|
	|title Failsafe $kernel_version
	|    root ($disk,$grub_slash_boot_partition_num)
	|    kernel /vmlinuz-$kernel_version root=$run_dev_root showopts apm=off noresume edd=off powersaved=off nohz=off highres=off processor.max_cstate=1 nomodeset x11failsafe
	|    initrd /initrd-local-$kernel_version
EOT

}

make_etc_issue()
{
	_cat << EOT
	|Welcome to the Jux Playout System - Kernel \r (\l).
EOT
}

make_etc_issue_net()
{
	_cat << EOT
	|Welcome to the Jux Playout System - Kernel %r (%t).
EOT
}

setup_boot_loader_local()
{
	check_var_set grub_boot_partition_num grub_slash_boot_partition_num || \
		die "not setting up boot loader, exiting"

	local drivename=hd$install_grub_dev_num
	local boot="($drivename,$grub_boot_partition_num)"
	local slash_boot="($drivename,$grub_slash_boot_partition_num)"
	local map=/tmp/$myname-grub-device.map

	echo "($drivename) $install_dev" > $map
	#dd if=$root/boot/grub/stage1 of=$install_dev count=512 count=1
	_cat << EOT | sed 's/#.*//' | run grub --batch --device-map=$map
	|root $slash_boot
	|setup ($drivename)
	|#setup --stage2=/boot/grub/stage2 --force-lba $boot $slash_boot
	|quit
EOT

	make_sysconfig_bootloader > $root/etc/sysconfig/bootloader
	make_boot_menu > $root/boot/grub/menu.lst

	run_chroot /sbin/mkinitrd \
		-B \
		-k /boot/vmlinuz-$kernel_version \
		-i /boot/initrd-local-$kernel_version \
		-M /boot/System.map-$kernel_version \
		-m "$initrd_modules" \
		-d $install_dev_root

	(cd $root/boot; ln -sf initrd-local-$kernel_version initrd-local)
}

setup_boot_loader_net()
{
	run_chroot /sbin/mkinitrd \
		-B \
		-k /boot/vmlinuz-$kernel_version \
		-i /boot/initrd-netboot-$kernel_version \
		-M /boot/System.map-$kernel_version \
		-m "$initrd_modules" \
		-d jan://blub/dings

		# -s "853x480" is illegal in newer mkinitrd (as of SuSE 11.4)

	(cd $root/boot; ln -sf initrd-netboot-$kernel_version initrd-netboot)
}

undo_rpmnew()
{
	set +x
	local n o
	# reset config files to rpm versions
	find $root -name '*.rpmnew' | while read n; do
		o=`echo $n | sed 's/\.rpmnew//'`
		log renaming $n to $o
		mv $n $o
	done
	set -x
}

setup_root_directory()
{
	if [ -w "$install_dev" ]; then

		if [ ! "$non_interactive" ]; then

			local answer
			echo -n "Continuing will delete ALL data on $install_dev. Are you sure [y|n]? "
			read answer
			if [ "$answer" != y ]; then
				exit 1
			fi
		fi

		create_partition_table
		create_file_systems
		mount_devices
	else
		
		if [ ! "$non_interactive" ]; then

			if [ -e $root ]; then

				local answer
				echo -n "Continuing will delete ALL data below $root. Are you sure [y|n]? "
				read answer
				if [ "$answer" != y ]; then
					exit 1
				fi
			fi
		fi
		
		if [ ! "$root" ]; then
			echo "cowardly not removing empty root directory"
			exit 1
		fi

		if [[ "$root" =~ home ]]; then
			echo "cowardly not removing directory matching \"home\""
			exit 1
		fi

		set +e
		unmount_devices /proc /sys /dev
		grep -q $root /proc/mounts && {
			echo "====== there are filesystems mounted below $root: >"
			grep $root /proc/mounts
			echo "====== unmount them manually"
			exit 1
		}

		run rm -rf $root
		run install -d -m 755 $root
	fi
}

insserv_all()
{
	local services="
		boot.juxearly
		boot.bprof
		sshd
		jux
	"

	run_chroot /sbin/insserv -de $services
}

set_sysconf_value()
{
	local file="$1"
	local key="$2"
	local value="`echo $3 | sed 's%/%\\\\/%g'`"
	sed "s/^ *$key *=.*/$key=\"$value\"/" $root/$file > $root/$file.tmp
	mv $root/$file.tmp $root/$file
}

cmd_install()
{
	if true; then
	local c

	set -e

 	[ -r "$root_password_file" ] || {
 		die "root password file doesn't exist, do echo -n my-secret > $root_password_file"
 	}

	setup_root_directory
	setup_bind_mounts

	mkdir -p -m 755 $root/etc
	mkdir -p -m 755 $root/etc/openldap
	for c in \
		/etc/passwd \
		/etc/group \
		/etc/nsswitch.conf \
		/etc/certs \
		/etc/openldap/slapd.conf \
		/etc/sysconfig/keyboard \
		/etc/sysconfig/clock \
		/etc/openldap/schema/authldap.schema \
		/etc/opt/bprof/bprof.secret \
	; do
		install -d -m 755 `dirname $root$c`
		cp -rp $c $root$c || die "failed to copy $c to $root $c"
	done

	install -d -m 755 $root/etc/dhcpd.conf.d
	make_netboot_etc_ldap_conf > $root/etc/ldap.conf.netboot
	make_netboot_etc_openldap_ldap_conf > $root/etc/openldap/ldap.conf.netboot
	make_netboot_etc_hosts > $root/etc/hosts.netboot
	make_netboot_etc_fstab > $root/etc/fstab.netboot

	make_etc_openldap_ldap_conf > $root/etc/openldap/ldap.conf
	chmod 644 $root/etc/openldap/ldap.conf

	make_etc_ldap_conf > $root/etc/hosts
	make_etc_hosts > $root/etc/hosts
	make_etc_resolv_conf > $root/etc/resolv.conf
	make_etc_fstab > $root/etc/fstab
	make_etc_sysctl_conf > $root/etc/sysctl.conf
	make_etc_issue > $root/etc/issue
	make_etc_issue_net > $root/etc/issue.net

	host_zypper "base" install --auto-agree-with-licenses coreutils
	host_zypper "base update" install --auto-agree-with-licenses zypper

	#undo_rpmnew
	setup_zypper_repos base update payload

	chroot $root zypper $zypper_global_opts refresh
	chroot $root zypper $zypper_global_opts install --auto-agree-with-licenses $payload_pkgs

	fi # test
	setup_zypper_repos special
	chroot $root zypper $zypper_global_opts install --auto-agree-with-licenses $special_pkgs

	# chroot $root zypper $zypper_global_opts -t srcpackage --download-only $payload_pkgs
	#kernel_version=`readlink -f $root/boot/vmlinuz 2>/dev/null | xargs basename | sed 's/vmlinuz-//'`

	# seed the ldap database
	slapcat | chroot $root /usr/sbin/slapadd -qw

	[ "$install_dev" ] && setup_boot_loader_local
	setup_boot_loader_net
	(cd $root/boot; ln -sf vmlinuz-$kernel_version vmlinuz)

	insserv_all

	set_sysconf_value etc/sysconfig/boot RUN_PARALLEL no
	set_sysconf_value etc/sysconfig/dhcpd DHCPD_INTERFACE eth0 # TODO: bprof this
	set_sysconf_value etc/sysconfig/dhcpd DHCPD_CONF_INCLUDE_FILES \
		"/etc/hosts /etc/ldap.conf /etc/openldap/ldap.conf /etc/certs /etc/nsswitch.conf /etc/dhcpd.conf.d"

	cat $root_password_file | chroot $root /usr/bin/passwd --stdin
	chroot $root /sbin/ldconfig
	chroot $root /sbin/SuSEconfig
	chroot $root /opt/jux/bin/jux-init-namespace.sh
	install -m 755 -d $root/srv/nfs/boot/pxe/bin
	install -m 755 -d $root/srv/nfs/var
	install -m 777 -d $root/srv/nfs/var/cores
	install -m 777 -d $root/var/cores

	for c in \
		/etc/pam.d/* \
	; do
		cp -rp $c $root$c
	done

	unmount_devices
	[ -w "$install_dev" ] || {
		install -d -m 755 $root/../rw
		/etc/init.d/nfsserver status >/dev/null 2>&1 && {
			/etc/init.d/nfsserver restart
		}
	}
}

# -- here we go

myname=`basename $0`
zypper_global_opts="--non-interactive --gpg-auto-import-keys --no-gpg-checks"
date=`date`

get_opts "$@"

init_root_password_file

if [ `whoami` != root ]; then
	sudo -i $0 $@ -p $root_password_file
	exit $?
fi

name=$o_name-$arch
root=/srv/nfs/boot/distros/$name/ro

# -- set up variables denoting devices

if [ "$install_dev" ]; then

	install_dev_boot=$install_dev""2
	install_dev_swap=$install_dev""3
	install_dev_root=$install_dev""4

	case $install_dev in
	/dev/sda) install_grub_dev_num=0;;
	/dev/sdb) install_grub_dev_num=1;;
	/dev/sdc) install_grub_dev_num=2;;
	/dev/sdd) install_grub_dev_num=3;;
	/dev/sde) install_grub_dev_num=4;;
	*)
		log "device \"$install_dev\" doesn't map into a known grub device number, edit $myname to add support"
		exit 1;;
	esac

	run_dev=/dev/sda
	run_dev_boot=$run_dev""2
	run_dev_swap=$run_dev""3
	run_dev_root=$run_dev""4

	run_grub_dev_num=0

	## test >
	#grub_boot_partition_num=0
	#grub_slash_boot_partition_num=1
	#mount_devices
	## test <
fi

case $cmd in
install)
	cmd_install
	exit $?
	;;

mount)
	[ "$install_dev" ] && mount_devices
	setup_bind_mounts
	exit $?
	;;
unmount)
	unmount_devices
	exit $?
	;;
*)
	log "unknown command \"$cmd\""
	usage 1
	;;
esac
exit 0