#!/bin/sh -e

PROGNAME=`basename $0`
CUSTOMTMP="yes"
OUTDIR="./"
NUM_CORES="1"

# TODO: add --keep-indep
usage() {
	echo "usage: $PROGNAME [OPTIONS] [architecture Packages Sources]" >&2
	echo >&2
	echo " -h, --help             Show this help message and exit" >&2
	echo " -k, --keep             Keep the temporary files" >&2
	echo " -c, --self-contained   Do not work on the full repository but on a smaller" >&2
	echo "                        self-contained repository" >&2
	echo " -O, --optuniv          Calculate a self-contained repository with a minimal" >&2
	echo "                        number of source packages" >&2
	echo " -G, --optgraph         Calculate a dependency graph where each installation" >&2
	echo "                        set contains the minimal number of unavailable" >&2
	echo "                        binary packages" >&2
	echo " -T, --timers           time all program executions" >&2
	echo " -m, --allowsrcmismatch Allow binary packages to be associated with a source " >&2
	echo "                        package of a different version than themselves" >&2
	echo " -e, --ignoresrclessbin Ignore binary packages without a source package" >&2
	echo " -p, --no-drop          Do not drop known droppable build dependencies" >&2
	echo " -s, --strong           Calculate strong dependencies" >&2
	echo " -b, --sapsb            Calculate strong articulation points and bridges" >&2
	echo " -f, --no-fixpoint      Do not run fixpoint algorithm" >&2
	echo " -n, --clean            Clean the repository beforehand" >&2
	echo " -l, --latest           Select only the highest package versions" >&2
	echo " -w, --online           Produce stat.html for online consumption" >&2
	echo " -a, --available=FILE   Deb822 list of available binary packages" >&2
	echo " -o, --output=DIR       Output directory. Default is the current directory" >&2
	echo " -t, --tmp=DIR          Temporary directory. Default is created by mktemp(1)" >&2
	echo " -j, --jobs=NUM         Number of threads or processes to run in parallel" >&2
	echo "                        where applicable" >&2
	echo " -v, --verbose          Be more verbose" >&2
	echo " -d, --debug            Maximum verbosity" >&2
	echo " -D, --develop          Execute tools from the source checkout instead of \$PATH" >&2
}

die() {
	echo " + error $@"
	if [ -n "$KEEP" ] || [ "$CUSTOMTMP" = "yes" ]; then
		echo " + temporary files stored in $tmp" >&2
	else
		rm -rf "$tmp"
	fi
	exit 1
}

run() {
	if [ -n "$TIMERS" ]; then
		time --format "%e seconds" $@
	else
		$@
	fi
}

# clean up when sighup, sigint or sigterm is received
trap "die \"got signal\"" 1 2 15

getopt -T > /dev/null && exitcode=0 || exitcode=$?
if [ $exitcode -eq 4 ]; then
	# GNU enhanced getopt is available
	ARGS=`getopt --long help,keep,self-contained,optuniv,optgraph,timers,allowsrcmismatch,ignoresrclessbin,no-drop,strong,sapsb,no-fixpoint,clean,latest,online,available:,output:,tmp:,jobs:,verbose,debug,develop --options hkcOGTmepsbfnlwa:o:t:j:vdD -- "$@"`
else
	# Original getopt is available
	ARGS=`getopt hkcOGTmepsbfnlwa:o:t:j:vdD "$@"`
fi

if [ $? -ne 0 ]; then
	exit 1
fi

eval set -- $ARGS

while [ $# -gt 0 ]; do
	case "$1" in
		-h | --help)     usage; exit 0;;
		-k | --keep)     KEEP=true;;
		-c | --self-contained) SELFCONTAINED=true;;
		-O | --optuniv)  OPTUNIV=true;;
		-G | --optgraph) OPTGRAPH=--optgraph;;
		-T | --timers)   TIMERS=true;;
		-m | --allowsrcmismatch) ALLOWSRCMISMATCH=--allowsrcmismatch;;
		-e | --ignoresrclessbin) IGNORESRCLESSBIN=--ignoresrclessbin;; 
		-p | --no-drop)  NODROP=true;;
		-s | --strong)   STRONG=true;;
		-b | --sapsb)    SAPSB=--sapsb;;
		-f | --no-fixpoint) NOFIXPOINT=true;;
		-n | --clean)    CLEAN=true;;
		-l | --latest)   LATEST=true;;
		-w | --online)   ONLINE="--online";;
		-a | --available) AVAILABLE="$2"; shift;;
		-o | --output)   OUTDIR="$2"; shift;;
		-t | --tmp)      tmp="$2"; shift;;
		-j | --jobs)     NUM_CORES="$2"; shift;;
		-v | --verbose)  VERBOSE=--verbose;;
		-d | --debug)    set -x;;
		-D | --develop)  DEVELOP=true;;
		--)              shift; break;;
	esac
	shift
done

bin_coinstall=dose-deb-coinstall
bin_buildcheck=dose-builddebcheck
if [ -n "$DEVELOP" ]; then
	droppable=./droppable
	[ -f ./annotate-strong.native ] && bin_annotate_strong=./annotate-strong.native || bin_annotate_strong=./annotate-strong.d.byte
	[ -f ./bin2src.native ] && bin_bin2src=./bin2src.native || bin_bin2src=./bin2src.d.byte
	[ -f ./build-closure.native ] && bin_build_closure=./build-closure.native || bin_build_closure=./build-closure.d.byte
	[ -f ./build-fixpoint.native ] && bin_build_fixpoint=./build-fixpoint.native || bin_build_fixpoint=./build-fixpoint.d.byte
	[ -f ./buildgraph2srcgraph.native ] && bin_buildgraph_to_srcgraph=./buildgraph2srcgraph.native || bin_buildgraph_to_srcgraph=./buildgraph2srcgraph.d.byte
	[ -f ./clean-repository.native ] && bin_clean_repository=./clean-repository.native || bin_clean_repository=./clean-repository.d.byte
	[ -f ./create-graph.native ] && bin_create_graph=./create-graph.native || bin_create_graph=./create-graph.d.byte
	[ -f ./fastest.native ] && bin_fastest=./fastest.native || bin_fastest=./fastest.d.byte
	[ -f ./find-fvs.native ] && bin_find_fvs=./find-fvs.native || bin_find_fvs=./find-fvs.d.byte
	[ -f ./optuniv.native ] && bin_optuniv=./optuniv.native || bin_optuniv=./optuniv.d.byte
	[ -f ./partial-order.native ] && bin_partial_order=./partial-order.native || bin_partial_order=./partial-order.d.byte
	[ -f ./print-stats.native ] && bin_print_stats=./print-stats.native || bin_print_stats=./print-stats.d.byte
	[ -f ./src2bin.native ] && bin_src2bin=./src2bin.native || bin_src2bin=./src2bin.d.byte
	bin_draw_graph=./tools/draw-graph.py
	bin_extract_neighborhood=./tools/extract-neighborhood.py
	bin_extract_scc=./tools/extract-scc.py
	bin_fasofstats=./tools/fasofstats.py
	bin_graph_info=./tools/graph-info.py
	bin_packages_difference=./tools/packages-difference.py
	bin_packages_intersection=./tools/packages-intersection.py
	bin_packages_union=./tools/packages-union.py
	bin_profile_build_fvs=./tools/profile-build-fvs.py
	bin_latest_version=./tools/latest-version.py
	bin_stat_html=./tools/stat-html.py
	bin_download_pkgsrc=./tools/download-pkgsrc.sh
else
	droppable=/usr/share/botch/droppable
	bin_annotate_strong=botch-annotate-strong
	bin_bin2src=botch-bin2src
	bin_build_closure=botch-build-closure
	bin_build_fixpoint=botch-build-fixpoint
	bin_buildgraph_to_srcgraph=botch-buildgraph2srcgraph
	bin_clean_repository=botch-clean-repository
	bin_create_graph=botch-create-graph
	bin_fastest=botch-fastest
	bin_find_fvs=botch-find-fvs
	bin_optuniv=botch-optuniv
	bin_partial_order=botch-partial-order
	bin_print_stats=botch-print-stats
	bin_src2bin=botch-src2bin
	bin_draw_graph=botch-draw-graph
	bin_extract_neighborhood=botch-extract-neighborhood
	bin_extract_scc=botch-extract-scc
	bin_fasofstats=botch-fasofstats
	bin_graph_info=botch-graph-info
	bin_packages_difference=botch-packages-difference
	bin_packages_intersection=botch-packages-intersection
	bin_packages_union=botch-packages-union
	bin_profile_build_fvs=botch-profile-build-fvs
	bin_selfcycles=botch-selfcycles
	bin_latest_version=botch-latest-version
	bin_stat_html=botch-stat-html
	bin_download_pkgsrc=botch-download-pkgsrc
fi

mkdir -p "$OUTDIR"

case $# in
	0)
		archpkgsrc=`$bin_download_pkgsrc`
		set $archpkgsrc
		NATIVEARCH="$1"
		packages="$2"
		sources="$3"
		;;
	3)
		NATIVEARCH="$1"
		packages="$2"
		sources="$3"
		;;
	*)
		usage
		exit 1
		;;
esac

if [ -n "$tmp" ]; then
	mkdir -p "$tmp"
else
	tmp=`mktemp --directory`
	CUSTOMTMP="no"
fi

if [ -n "$LATEST" ]; then
	# we can't just blindly select packages of latest version because this
	# might break versioned dependencies, therefore we do not do this with
	# the --clean option per default
	echo " + selecting latest packages" >&2

	run $bin_latest_version $VERBOSE "$packages" - > "$tmp/packages-$NATIVEARCH-latest"
	packages="$tmp/packages-$NATIVEARCH-latest"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages" "$sources" \
		> "$tmp/sources-latest" || die "bin2src failed"
	sources="$tmp/sources-latest"

fi


if [ -n "$CLEAN" ]; then
	echo " + cleaning repository" >&2

	run $bin_clean_repository $VERBOSE --deb-native-arch=$NATIVEARCH \
		$ALLOWSRCMISMATCH "$packages" "$sources" \
		> "$tmp/packages-$NATIVEARCH" || die "clean-repository failed"
	packages="$tmp/packages-$NATIVEARCH"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages" "$sources" \
		> "$tmp/sources-clean" || die "bin2src failed"
	sources="$tmp/sources-clean"
fi

if [ -n "$OPTUNIV" ]; then
	echo " + running optuniv..." >&2

	run $bin_optuniv $VERBOSE --deb-native-arch=$NATIVEARCH \
		"$packages" "$sources" > "$tmp/opt-packages" \
		|| die "optuniv failed"
	packages="$tmp/opt-packages"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages" "$sources" > "$tmp/opt-sources" \
		|| die "bin2src failed"
	sources="$tmp/opt-sources"

	echo " + running buildcheck..." >&2

	# if buildcheck succeeds and since the source list was generated from the
	# binary list, at this point we know that the pair of opt-packages and
	# opt-sources is indeed a valid self-contained repository
	run $bin_buildcheck $VERBOSE --explain \
		--failures --deb-native-arch=$NATIVEARCH "$packages" \
		"$sources" || die "buildcheck failed"
fi

echo " + running grep-dctrl..." >&2

# these source packages are known to be cross compilable
#zcat "$sources" | \
#    grep-dctrl -P -e '^(eglibc|pcre3|attr|acl|zlib|hostname|libsepol|gpm|ifupdown|insserv|module-init-tools|python-defaults|libselinux|ncurses|bash|libpciaccess|libpng|slang|base-files|mawk|makedev|make|xz-utils|dpkg|netbase|base-files|time|manpages|libxau|libxdmcp|libpthread-stubs|diffutils|tar|cpio|diffutils|gmp|gdbm|sed|tzdata|base-passwd|coreutils|findutils|libnfnetlink|libffi|iptables|libelf|debconf|debianutils|ucf|xtrans|x11proto-kb|x11proto-input|x11proto-render|x11proto-xf86bigfont|x11proto-core|expat|net-tools|tcl8.5|tcp-wrappers|ubuntu-keyring|scowl|postgresql-common|openslp-dfsg|procps|psmisc|phpmyadmin|libusb|mcrypt|pixman|libice|bzip2|flex|db4.8|dash|iptables|keyutils|less|sysvinit|nano|db-defaults|grep|dbconfig-comon|linux-atm|util-linux|e2fsprogs|ustr|kmod|dbus|libsm|cunit|glib2.0|udev|libnih|json-c|openssl|initramfs-tools|libgcrypt11|libidn|wget|sysfsutils|iputils|ppl|binutils|busybox|libtasn1-3|libfribidi|libjson0|upstart|tcl8.5|sqlite3|db|cracklib2|pam|libcap2|libsemanage|shadow|iproute|ppl|cloog-ppl|libbsd|libedit|dbus|libgpg-error|popt|p11-kit|gnutls26|libxau|check|libxcb1|libx11|libfreetype|libfontconfig|libxrender|libxext|libxt|lzo2|cairo|libdatrie|libthai|libxft|pango1.0|libdrm|plymouth|perl|libjpeg-turbo|initramfs-tools|debconf|lsb|tzdata|makedev|pam|mountall|readline|cron|glib2.0|nspr|klibc|nano|less|gcc-4.7)$' > $OUTDIR/crossed-known
truncate --size=0 "$tmp/crossed-known"

zcat -f "$packages" | \
    run grep-dctrl -X \( \
    -FPackage build-essential --or \
    -FPackage apt --or \
    -FPackage systemd-sysv --or \
    -FPackage debhelper --or \
    -FEssential yes \
\) > "$tmp/minimal" || die "zcat failed"

echo " + running coinstall..." >&2

run $bin_coinstall $VERBOSE \
    --bg="$packages" --fg="$tmp/minimal" --deb-native-arch=$NATIVEARCH \
    > "$tmp/minimal-$NATIVEARCH" || die "coinstall failed"

# FIXME: coinstall doesnt return a non-zero exit status yet
# checking length of output file
[ ! -s "$tmp/minimal-$NATIVEARCH" ] && die "coinstall failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	"$tmp/minimal-$NATIVEARCH" "$sources" \
	> "$tmp/minimal-src" || die "bin2src failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/crossed-known" "$tmp/minimal-src" \
	"$tmp/crossed-src" || die "union failed"

echo " + running grep-dctrl..." >&2

zcat -f "$packages" | run grep-dctrl -FArchitecture all \
	> "$tmp/available-all" || die "grep-dctrl failed"

echo " + running src2bin..." >&2

run $bin_src2bin $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
    $IGNORESRCLESSBIN --bg "$sources" -A "$tmp/available-all" \
    "$packages" "$tmp/crossed-src" \
    > "$tmp/available-crossed" || die "src2bin failed"

if [ -n "$SELFCONTAINED" ]; then
	echo " + running build_closure..." >&2

	run $bin_build_closure $VERBOSE --deb-native-arch=$NATIVEARCH \
	    -m "$tmp/minimal-$NATIVEARCH" $ALLOWSRCMISMATCH --bg "$sources" \
	    "$packages" "$tmp/crossed-src" \
	    > "$tmp/minimal-closure" || die "build-closure failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	       "$tmp/minimal-closure" "$sources" \
	       > "$tmp/minimal-closure-src" || die "bin2src failed"

	packages="$tmp/minimal-closure"
	sources="$tmp/minimal-closure-src"

	echo " + running buildcheck..." >&2

	# if buildcheck succeeds and since the source list was generated from the
	# binary list, at this point we know that the pair of minimal-closure and
	# minimal-closure-src is indeed a valid self-contained repository
	run $bin_buildcheck $VERBOSE --explain-minimal \
		--failures --deb-native-arch=$NATIVEARCH "$packages" \
		"$sources" || die "buildcheck failed"

	echo " + intersecting..." >&2

	# the set of available packages must be a subset of minimal-closure
	run $bin_packages_intersection $VERBOSE "$tmp/available-crossed" "$packages" \
		"$tmp/available-crossed" || die "intersecting failed"

	echo " + running grep-dctrl..." >&2

	run grep-dctrl -FArchitecture all "$packages" \
		> "$tmp/available-all" || die "grep-dctrl failed"
fi

echo " + union..." >&2

if [ -n "$AVAILABLE" -a -f "$AVAILABLE" -a -s "$AVAILABLE" ]; then
	run $bin_packages_union $VERBOSE "$tmp/available-crossed" \
		"$tmp/available-all" "$AVAILABLE" \
		"$tmp/available-crossed-all" || die "union failed"
else
	run $bin_packages_union $VERBOSE "$tmp/available-crossed" \
		"$tmp/available-all" \
		"$tmp/available-crossed-all" || die "union failed"
fi

echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$packages" "$tmp/available-crossed-all" \
	"$tmp/not-available" || die "difference failed"

echo " + bin2src..." >&2
run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	"$tmp/not-available" "$sources" \
	> "$tmp/tocompile" || die "bin2src failed"

if [ -n "$NOFIXPOINT" ]; then
	cp "$tmp/available-crossed-all" "$tmp/available"
else
	echo " + running build-fixpoint..." >&2

	run $bin_build_fixpoint $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	    -A "$tmp/available-crossed-all" --output-order="${OUTDIR}/order1.lst" "$packages" \
	    "$tmp/tocompile" \
	    > "$tmp/compilable-src" || die "fixpoint failed"

	echo " + running src2bin..." >&2

	run $bin_src2bin $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	    $IGNORESRCLESSBIN --bg "$sources" -A "$tmp/available-crossed-all" \
	    "$packages" "$tmp/compilable-src" \
	    > "$tmp/compilable" || die "src2bin failed"

	echo " + union..." >&2

	run $bin_packages_union $VERBOSE "$tmp/available-crossed-all" \
		"$tmp/compilable" "$tmp/available" || die "union failed"

	echo " + difference..." >&2
	run $bin_packages_difference $VERBOSE "$tmp/tocompile" \
		"$tmp/compilable-src" "$tmp/tocompile" || die "difference failed"
fi

# from this point on, everything that reads GraphML must read in the same
# Packages and Sources files to create the exact same cudf universe
UNIVERSE="--deb-native-arch=$NATIVEARCH --bg $sources $packages $tmp/tocompile"

echo " + running distgraph..." >&2

# we can use deb-ignore-essential here because the set of available packages
# was made sure to contain a coinstallation set of essential earlier in this
# script
run $bin_create_graph $VERBOSE $OPTGRAPH --num_cores=$NUM_CORES \
    --deb-ignore-essential --progress --timers $STRONGARG \
    -A "$tmp/available" $ALLOWSRCMISMATCH $UNIVERSE \
    > "${OUTDIR}/buildgraph.xml" || die "create graph failed"

echo " + running buildgraph2srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE "${OUTDIR}/buildgraph.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph.xml" || die "buildgraph2srcgraph failed"

if [ -n "$STRONG" ]; then
	echo " + running annotate-strong..." >&2
	run $bin_annotate_strong $VERBOSE "${OUTDIR}/buildgraph.xml" \
		"${OUTDIR}/srcgraph.xml" $UNIVERSE || die "annotate-strong failed"
fi

echo " + printing stats and finding feedback arc set..." >&2

if [ -n "$NODROP" ]; then
	droplist=
else
	droplist="--remove-weak=${droppable}/weak-build-dependencies.list --remove-reduced=${droppable}/pehjota.list,${droppable}/thorsten-glaser.list,${droppable}/daniel-schepler.list,${droppable}/gentoo.list,${droppable}/wookey.list,${droppable}/selfcycles.list"
fi

run $bin_print_stats $VERBOSE --max-length=6 --max-length-fas=8 $ALLOWSRCMISMATCH \
	$SAPSB $droplist -A "$tmp/available" "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/srcgraph.xml" $UNIVERSE > "${OUTDIR}/stats.json" || die "print-stats failed"

echo " + generating html..." >&2

run $bin_stat_html $VERBOSE $ONLINE "${OUTDIR}/stats.json" > "${OUTDIR}/stats.html" || die "stat-html failed"

echo " + extracting the calculated feedback arc set..." >&2

run $bin_fasofstats $VERBOSE "${OUTDIR}/stats.json" > "${OUTDIR}/remove.list" || die "fasofstats failed"

echo " + extract strongly connected components..." >&2

run $bin_extract_scc $VERBOSE --outdir=${OUTDIR} "${OUTDIR}/buildgraph.xml" cyclic \
	| sort > "${OUTDIR}/fas" || die "extract-scc failed"

echo " + calculate feedback vertex sets..." >&2

if [ -n "$NODROP" ]; then
       alldrop="--remove-reduced=${OUTDIR}/remove.list"
else
       alldrop="$droplist,${OUTDIR}/remove.list"
fi

> "${OUTDIR}/feedback_vertex_set.list"
while read g; do
	echo " + processing $g..." >&2
	run $bin_find_fvs $VERBOSE $alldrop "${OUTDIR}/$g" $UNIVERSE \
		|| die "find-fvs failed"
done < "${OUTDIR}/fas" | sort > "${OUTDIR}/feedback_vertex_set.list"

echo " + profile build selected source packages..." >&2

run $bin_profile_build_fvs $VERBOSE $alldrop "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/feedback_vertex_set.list" \
	> "${OUTDIR}/buildgraph_acyclic.xml" || die "profile-build-fvs failed"

echo " + convert to srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE "${OUTDIR}/buildgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph_acyclic.xml" || die "buildgraph2srcgraph failed"

echo " + calculate partial order..." >&2

run $bin_partial_order $VERBOSE "${OUTDIR}/srcgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/order2.lst" || die "partial-order failed"

echo " + build order is saved in ${OUTDIR}/order1.lst and ${OUTDIR}/order2.lst" >&2
echo " + an overview of the dependency situation is now available in ${OUTDIR}/stats.html" >&2
echo " + the data from which stats.html was generated is saved in ${OUTDIR}/stats.json" >&2
echo " + the build order was generated using the feedback arc set in ${OUTDIR}/remove.list" >&2
echo " + without the content of remove.list, the remaining strongly connected components are:" >&2

echo -e "graph.xml\ttype\tvertices\tedges\tscc"
while read g; do
	printf "$g\t"
	run $bin_graph_info $VERBOSE "${OUTDIR}/$g" || die "graph-info failed"
done < "${OUTDIR}/fas"

if [ -n "$KEEP" ] || [ "$CUSTOMTMP" = "yes" ]; then
	echo " + temporary files stored in $tmp" >&2
else
	rm -rf "$tmp"
fi
