build-flashrom

#!/bin/bash

set -o pipefail

. buildbot.conf
. common.sh

show_help() {
	echo "Usage:
${0} [-n] [-p/-np] [-s sourcepath] [-j numthreads] [-nc] [-c compiler]... [flashrom build options]

Options
    -h or --help
        Display this message.
    -n or --dry-run
        Do not create any directories or files.
    -p or --private
        do not make results public
    -np or --public
        *do* make results public
    -s or --source
        path to flashrom sources to build
    -c or --compiler
        adds a compiler to the list of compilers to use (replaces the default list, can be given multiple times)
    -j or --threads
        maximum number of concurrent build jobs
    -nc or --no-cleanup
        do not cleanup working directories

    Possible compilers are:"
	printf '\t\t%s\n' "${!available_compilers[@]}" | sort
	exit 1
}

#default values
flashrom_source=.
declare -A compilers=()
flashrom_opts=()
priv=-1
priv_def=0 # default privacy setting
cleanup=1
dryrun=0
j=7 # default threads

# constants
workdir_base="/tmp"
build_details_file="build_details.txt"
results_file="results.html"

while [ $# -gt 0 ];
do
	case ${1} in
	-h|--help)
		show_help;
		shift;;
	-n|--dry-run)
		dryrun=1;
		shift;;
	-s|--source)
		check_arg $1 "$2"
		if [ ! -d "$2" ]; then
			msg_err "$2 is not a directory"
		fi
		flashrom_source="$2"
		shift 2;;
	-c|--compiler)
		check_arg $1 "$2"
		key_var=
		if is_key_in "$2" "$(declare -p available_compilers)" ; then
			compilers["$2"]="${available_compilers["$2"]}"
		else
			msg_err "$2 is not in the list of available compiler (configurations)"
		fi
		shift 2;;
	-p|--private)
		if [ "$priv" -eq 0 ]; then
			msg_err "-p and -np are mutial exclusive"
		fi
		priv=1
		shift;;
	-np|--public)
		if [ "$priv" -eq 1 ]; then
			msg_err "-p and -np are mutial exclusive"
		fi
		priv=0
		shift;;
	-nc|--no-cleanup)
		cleanup=0
		shift;;
	-j|--threads)
		check_arg $1 "$2"
		if [ "$2" -lt 1 ]; then
			msg_err "Less than 1 concurrent process is a bad idea. $2 requested"
		fi
		j="$2"
		shift 2;;
	-*)
		show_help;
		msg_err "invalid option: $1"
		;;
	*)	# everything else are flashrom build parameters
		flashrom_opts+=("$1")
		shift;;
	esac;
done

if [ "$priv" -eq -1 ]; then
	priv=$priv_def
fi

if [ ! -d "${flashrom_source}" -o ! -e "${flashrom_source}/flashrom.c" ]; then
	msg_err "${flashrom_source} is not a valid flashrom source directory"
fi

if [ "${#compilers[@]}" -eq 0 ]; then
	# By default enable all available compilers.
	for k in "${!available_compilers[@]}" ; do
		compilers["$k"]="${available_compilers["$k"]}"
	done
fi

# get the script path
# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in
script_path="${BASH_SOURCE[0]}"
while [ -h "$script_path" ]; do # resolve $script_path until the file is no longer a symlink
  buildbot_dir="$( cd -P "$( dirname "$script_path" )" && pwd )"
  script_path="$(readlink "$script_path")"
  # if $script_path was a relative symlink, we need to resolve it relative to the path where the symlink file was located
  [[ $script_path != /* ]] && script_path="$buildbot_dir/$script_path"
done
buildbot_dir="$( cd -P "$( dirname "$script_path" )" && pwd )"


echo "buildbot_dir=${buildbot_dir}"
echo "flashrom_source=${flashrom_source}"
echo "compilers=${!compilers[@]}"
echo "flashrom_opts=${flashrom_opts[@]}"
echo

PATH="${buildbot_dir}:${PATH}"
unset LANG

if [ ! -r ${www_root}/latest-build.txt ]; then
	lastid=-1
else
	lastid=$(sed 's/.*-\(.*\)-.*/\1/' ${www_root}/latest-build.txt)
fi
newid=$(printf "%06d" $(expr $lastid + 1))

if [ "$dryrun" -eq 1 ]; then
	echo "Simulating Build #${newid} at $(date '+%Y-%m-%d %k:%M:%S')"
	results_path=/dev/null
else
	echo "Starting Build #${newid} at $(date '+%Y-%m-%d %k:%M:%S')"
	mkdir -p "${www_root}"
	resultdir=$(mktemp -d "${www_root}/flashrom-${newid}-XXX")
	if [ ! -d "${resultdir}" ]; then
		msg_err "Could not create temporary directory"
	fi
	results_path="${resultdir}/${results_file}"
	resultbase=$(basename "${resultdir}")
	echo -n "${resultbase}" >"${www_root}/latest-build.txt"
	ln -s -f -T "${resultbase}" "${www_root}/latest-build"
	if [ $priv -ne 1 ]; then
		chmod go+rx "$resultdir"
		echo -n "${resultbase}" >"${www_root}/latest-public-build.txt"
		ln -s -f -T "${resultbase}" "${www_root}/latest-public-build"
	fi
	echo "<pre>Build was requested in \"${flashrom_source}\" at $(date '+%Y-%m-%d %k:%M:%S')" >>"${results_path}"
	echo "with arguments \"${flashrom_opts[@]}\" for ${#compilers[@]} compiler configurations:" >>"${results_path}"
	for ck in $(echo "${!compilers[@]}" | tr " " "\n" | sort | tr "\n" " "); do
		echo " - ${ck}" >>"${results_path}"
	done
	echo >>"${results_path}"
fi

# Parameter parsing and verification done.
# The real action starts below: first some helper functions to deal with managing VMs, then command abstractions to access the OS inside VMs, and finally main()

sanitize_vbox_vm () {
	vb_config=$(vboxmanage showvminfo ${1} --machinereadable | sed -ne 's/^CfgFile="\(.*\)"$/\1/p')
	chmod 660 "vb_config"
}

stop_vbox_vm () {
	local ck=$1
	local vmname="${vbox_names[$ck]}"
	local lock_file="/var/lock/build-flashrom-$vmname"
	{
		if ! flock -n 43 ; then
			echo "Could not acquire vm_usage lock immediately, waiting..."
			while ! flock -w 20 43 ; do
				echo "Could not acquire vm_usage lock yet, still waiting..."
			done
		fi
		local usage=$(cat "$lock_file")
		usage=$((${usage}-1))
		echo "${usage}" >"$lock_file"
		if [ ${usage} -gt 0 ]; then
			echo "$ck is not the last user of $vmname but ${usage} remain, leaving VM alone."
			return 0
		fi
	} 43>>"$lock_file"
	exec 43>&-

	local vm_was_running="${vms_were_running[$ck]}"
	local vmip=${vbox_ips[$ck]}
	local vmhaltcmd=${halt_cmds[$vmname]}
	# default to halt -p
	if [ -z "$vmhaltcmd" ]; then
		vmhaltcmd="halt -p"
	fi

	local deadline=$(date -d 2mins +%s)
	VBoxManage list runningvms|grep -q "^\"${vmname}\"" || {
		echo "${vmname} VM is already stopped. Nothing done."
		return 0
	}
	if [ -n "$vm_was_running" ] && [ "$vm_was_running" -eq 1 ]; then
		echo "${vmname} VM was not started by this build run. Leaving it alone."
		return 0
	fi

	if VBoxManage controlvm ${vmname} savestate >/dev/null 2>&1 ; then
		echo "${vmname} VM state saved successfully."
		return 0
	else
		echo "Saving ${vmname} VM state failed. Trying to shut it down instead..."
	fi
	ssh root@${vmip} ${vmhaltcmd} >/dev/null 2>&1

	while ping -c 1 ${vmip} >/dev/null 2>&1; do
		if [ $(date +%s) -ge ${deadline} ]; then
			echo "Still reachable after timeout, enforcing shutdown"
			VBoxManage controlvm ${vmname} acpipowerbutton >/dev/null
			sleep 15
			VBoxManage controlvm ${vmname} poweroff >/dev/null
			sanitize_vbox_vm ${vmname}
			return 1
		fi
		echo "Waiting for ${vmname} VM to get disconnected (for $((${deadline}-$(date +%s))) more secs)..."
		sleep 5
	done

	while VBoxManage list runningvms|grep -q "^\"${vmname}\""; do
		if [ $(date +%s) -ge ${deadline} ]; then
			echo "VM still running after timeout, aborting"
			sanitize_vbox_vm ${vmname}
			return 1
		fi
		echo "Waiting for ${vmname} VM to stop (for $((${deadline}-$(date +%s))) more secs)..."
		sleep 5
	done
	sanitize_vbox_vm ${vmname}
	echo "${vmname} VM stopped."
}

build_prepare () {
	local ck="$1"
	shift 1
	
	case "${build_types[$ck]}" in
		vbox)
			start_vbox_vm "$ck" || return 1
			local workdir="${vm_workdir}/${resultbase}/${ck}"
			cat "${workdir_base}/${resultbase}/source.tar" | ssh ${vm_user}@${vbox_ips[$ck]} "mkdir -p ${workdir} && cd ${workdir} && tar xf -"
			;;
		*)
			local workdir="${workdir_base}/${resultbase}/${ck}"
			mkdir -p "${workdir}" && tar xf "${workdir_base}/${resultbase}/source.tar" -C "${workdir}"
			;;
	esac
}

build_execute () {
	local ck="$1"
	shift 1
	local workdir="${workdir_base}/${resultbase}/${ck}"
	case "${build_types[$ck]}" in
		vbox)
			ssh ${vm_user}@${vbox_ips[$ck]} "cd ${vm_workdir}/${resultbase}/${ck} && $@"
			;;
		*)
			cd "${workdir}" && "$@"
			;;
	esac
}

build_fetch () {
	local ck="$1"
	local dest="$2"
	shift 2
	local workdir="${workdir_base}/${resultbase}/${ck}"
	for file in "$@"; do
		case "${build_types[$ck]}" in
			vbox)
				scp -p "${vm_user}@${vbox_ips[$ck]}:\"${vm_workdir}/${resultbase}/${ck}/${file}\"" "$dest/"
				;;
			*)
				cp -a -t "$dest" "${workdir}/${file}"
				;;
		esac
	done
	return 0 # FIXME: ?
}

build_destroy () {
	local ck="$1"
	shift 1
	case "${build_types[$ck]}" in
		vbox)
			test "$cleanup" -eq 1 && ssh ${vm_user}@${vbox_ips[$ck]} "rm -rf ${vm_workdir}/${resultbase}/${ck}"
			stop_vbox_vm "$ck"
			;;
		*)
			test "$cleanup" -eq 1 && rm -rf "${workdir_base}/${resultbase}/${ck}"
			return 0
			;;
	esac
}

declare -A make_cmds

add_make_cmd () {
	local ck=$1
	if [ -z "${build_types[$ck]}" ]; then
		fill_build_types
	fi
	case "${build_types[$ck]}" in
		vbox)
			if ssh ${vm_user}@${vbox_ips[$ck]} "command -v gmake" >/dev/null 2>&1 ; then
				make_cmds[$ck]=gmake
			else
				make_cmds[$ck]=make
			fi
			;;
		*)
			make_cmds[$ck]=make
			;;
	esac
}

build_it () {
	local ck=$1
	local cc="${compilers[$ck]}"
	local cur_err=0
	echo "Starting $ck ($cc) build of ${flashrom_source}"
	if [ "$dryrun" -ne 1 ]; then
		local outdir="$resultdir/$ck"
		mkdir "$outdir" || cur_err=1
		[ "$cur_err" -ne 1 ] && { build_prepare "$ck"  >> "$outdir/buildlog.txt" 2>&1 ;[ $? -eq 0 ] || cur_err=1; }
		[ "$cur_err" -eq 1 ] && return $cur_err
		add_make_cmd "$ck"

		local exec_args=("$ck" "${make_cmds[$ck]}" CC="$cc")
		printf "Executing for %s: %q %q" "$ck" "${make_cmds[$ck]}" CC="$cc" | tee -a "$outdir/buildlog.txt"
		if is_key_in "$ck" "$(declare -p lib_dirs)" ; then
			local libs_path
			case "${build_types[$ck]}" in
				vbox)
					libs_path="LIBS_BASE=${lib_dirs[${ck}]}"
					;;
				*)
					libs_path="LIBS_BASE=${libs_base}/${lib_dirs[${ck}]}"
					;;
			esac
			exec_args+=("$libs_path")
			printf " $libs_path" | tee -a "$outdir/buildlog.txt"
		fi
		if [ "${#flashrom_opts[@]}" -eq 0 ]; then
			if is_key_in "$ck" "$(declare -p default_args)" ; then
				exec_args+=(${default_args[${ck}]})
				printf " ${default_args[${ck}]}" | tee -a "$outdir/buildlog.txt"
			fi
		else
			for opt in "${flashrom_opts[@]}" ; do
				printf " " | tee -a "$outdir/buildlog.txt"
				printf "%q" "$opt" | tee -a "$outdir/buildlog.txt"
				exec_args+=("$opt")
			done
		fi
		printf "\n" | tee -a "$outdir/buildlog.txt"
		[ "$cur_err" -ne 1 ] && { build_execute "${exec_args[@]}" >> "$outdir/buildlog.txt" 2>&1 ;[ ${PIPESTATUS[0]} -eq 0 ] || cur_err=1; }
		build_fetch "$ck" "$outdir/" "${build_details_file}" flashrom.8 flashrom flashrom.exe util/ich_descriptors_tool/ich_descriptors_tool util/ich_descriptors_tool/ich_descriptors_tool.exe >> "$outdir/buildlog.txt" 2>/dev/null
		[ "$cur_err" -ne 1 ] && [ "$cleanup" -eq 1 ] && { build_execute "$ck" "${make_cmds[$ck]}" distclean >> "$outdir/buildlog.txt" 2>&1 ;[ ${PIPESTATUS[0]} -eq 0 ] || cur_err=1; }
		build_destroy "$ck" || cur_err=1 # unconditionally clean up
	fi
	return $cur_err
}

# stores the build type for every selected compiler
#	- vbox if the compiler is hosted in a virtual box vm
#	- host if we can compile on the host machine (possibly with a cross compiler)
declare -A build_types

fill_build_types () {
	declare -A loca vm_usage
	for ck in "${!compilers[@]}"; do
		local build_type
		local vmname="${vbox_names[$ck]}"
		if [ -n "$vmname" ] ; then
			build_type="vbox"
			vm_usage["$vmname"]=$((${vm_usage[$vmname]}+1))
			echo "${vm_usage[$vmname]}" >"/var/lock/build-flashrom-$vmname"
		else
			build_type="host"
		fi
		build_types["$ck"]=$build_type
	done
}

main () {
	local errors=0
	local error_builds=()
	local -A cc_pids=()

	startdate=$(($(date +'%s * 1000 + %-N / 1000000')))

	local exdir
	########################################
	# prepare source and working directory #
	########################################
	if [ "$dryrun" -ne 1 ]; then
		pushd "${flashrom_source}" >/dev/null
		exdir="$("make" --no-print-directory ${flashrom_opts[@]} export 2>/dev/null | awk 'END {print $NF}' | xargs basename)"
		test $? -eq 0 -a -d "$exdir" || { echo "Could not export flashrom source." >&2 ; exit 1 ; }
		mkdir -p "${workdir_base}/${resultbase}" >/dev/null && tar cf "${workdir_base}/${resultbase}/source.tar" --format=ustar -C "$exdir" . >/dev/null 2>&1 || exit 1
		popd >/dev/null
	fi

	####################################
	# prepare some local map variables #
	####################################
	fill_build_types
	fill_vbox_arrs

	##########################################
	# then schedule compile runs in parallel #
	##########################################
	for ck in "${!compilers[@]}"; do
		while [ $(jobs 2>&1 | grep -c Running) -ge "$j" ]; do
			sleep 1
		done
		build_it "$ck" &
		cc_pids+=(["$ck"]="$!")
	done

	######################################
	# then wait for all jobs to complete #
	######################################
	for ck in "${!compilers[@]}"; do
		pid=${cc_pids[$ck]}
		local cur_err=0
		if [ -z "$pid" ]; then
			echo "No Job ID known for the $ck process"
			cur_err=1
		else
			wait $pid
			cur_err=$?
		fi
		if [ "$cur_err" -ne 0 ]; then
			errors=$(($errors + 1))
			echo "The build with $ck (${compilers[$ck]}) above failed with return value $cur_err."
			error_builds+=("$ck")
		fi
	done

	enddate=$(($(date +'%s * 1000 + %-N / 1000000')))

	#################################
	# cleanup and result processing #
	#################################
	datediff=$(echo "($enddate - $startdate) / 1000" | bc -l)
	printf 'Done in %.3f seconds (%d errors) at %s\n' "$datediff" "$errors" "$(date '+%Y-%m-%d %k:%M:%S')"
	printf '%d error(s) in %.3f seconds</pre>\n' "$errors" "$datediff" >>"${results_path}"
	if [ $errors -ne 0 ]; then
		echo "The following builds failed:</br>" | tee -a "${results_path}"
		local fails=$(echo "${error_builds[@]}" | tr " " "\n" | sort | tr "\n" " ")
		for ck in $fails; do
			echo "<a href=$ck/buildlog.txt>$ck (${compilers[$ck]})</a>&nbsp;<a href=$ck/${build_details_file}>Build Details</a></br>" | tee -a "${results_path}"
		done
	fi
	test "$dryrun" -eq 1 || echo "Results in ${resultdir}"
	# Build another archive this time including a top-level directory
	test "$dryrun" -eq 1 || tar czf "${resultdir}/source.tgz" --format=ustar -C "${flashrom_source}" "$exdir" >/dev/null 2>&1
	test "$dryrun" -eq 1 -o $cleanup -eq 0 || rm -r "${workdir_base}/${resultbase}/" 2>/dev/null # not created for VM-only builds
	exit $errors
}

# Synchronize whole main function via flock
{
	if ! flock -n 42 ; then
		echo "Could not acquire buildbot lock immediately, waiting..."
		while ! flock -w 20 42 ; do
			echo "Could not acquire buildbot lock yet, still waiting..."
		done
	fi
	main
} 42>>/var/lock/build-flashrom
exec 42>&-