compile_pkgs

#!/bin/bash
# OpenWRT compilation script
# (C) 2018-2020 CZ.NIC, z.s.p.o.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
set -eu

# Step list that is run when no commands are specified
DEFAULT_STEPS=( "prepare" "compile" "sign" "store_hash" "stats" "pkgsrepo" "gen_junit" )

# Paths to turris-build sources and common utilities inclusion
src_dir="$(readlink -f "${0%/*}")"
source "$src_dir/helpers/generate_common.sh"

# Configuration
source "$src_dir/helpers/conf.sh"
OUTPUT_DIR="./pkgsrepo" # Output directory for pkgsrepo command

##################################################################################

# Technical variables
declare -A available_commands # Set of all commands defined in this script
mirror_updated="" # Variable signaling that mirrors were update in this run already
debug="" # set when debug on executed scripts should be enabled

_sh() {
	sh ${debug:+-x} "$@"
}

_perl() {
	perl ${debug:+-d:Trace} "$@"
}

_make() {
	make ${debug:+-d} "$@"
}

# Make for OpenWRT
_openwrt_make() {
	local mk
	if [ -z "$FORCE" ]; then
		mk=("make" -j "$BUILD_JOBS")
	else
		mk=("$src_dir/helpers/force-make" "-j" "$BUILD_JOBS" "-c" "$FORCE" "-f" "V=s" "--")
	fi

	"${mk[@]}" ${debug:+-d} \
		"IGNORE_ERRORS=m" "BUILD_LOG=1" ${debug:+V=s} "${BUILD_ARGS[@]}" \
		"$@"
}

# Definition of avaliable commands ###############################################

available_commands+=( ["gen_version"]="Generates Turris version package from NEWS" )
gen_version() {
	report "Generating turris-version package"
	"${src_dir}/helpers/turris-version.sh" --output "${build_dir}/package/system/turris-version" package
	pushd "${build_dir}"
	git add "${build_dir}/package/system/turris-version"
	_git commit -m 'turris-version: Create a package with release notes'
	popd
}

available_commands+=( ["conflicts"]="Show packages that we are overriding in Turris OS packages" )
conflicts() {
	report "Showing conflicting packages"
	echo "Following packages are being overridden by Turris packages:"
	echo
	mkdir -p tmp
	find feeds/turrispackages -name Makefile | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/turris-packages.list
	find feeds/ -name Makefile | grep -v '^feeds/turrispackages' | sed 's|.*/\([^/]*\)/Makefile|\1|' | sort > tmp/other-packages.list
	comm -12 tmp/turris-packages.list tmp/other-packages.list | sed 's|^| * |'
	echo
	rm -f tmp/turris-packages.list tmp/other-packages.list
}

available_commands+=( ["stats"]="	Prints statistics regarding number of built packages" )
stats() {
	mkdir -p logs
	report "Reporting statistics"
	local binpackages=0 failed=0 packages=0
	if [ -d "bin" ]; then
		binpackages="$(find bin -name '*.ipk' | wc -l)"
	fi
	if [ -f "logs/package/error.txt" ]; then
		failed="$(wc -l < logs/package/error.txt)"
		packages="$(find logs/package -name 'compile.txt' | wc -l)"
	fi
	tee logs/stats <<-EOF
		Statistics of the build:

		 * $binpackages binary packages built"
		 * $failed/$packages source packages failed"
	EOF
}

available_commands+=( ["configure"]="Recreates configuration for target boards" )
configure() {
	report "Creating default configuration"
	[ -n "${TARGET_BOARD:-}" ] || die "No board selected!"
	rm -f .config
	cat "${src_dir}/configs/common"/* "${src_dir}/configs/${TARGET_BOARD}"/* | grep "^CONFIG.*=" | while IFS="=" read -r name value; do
		[ \! -f .config ] || sed -i "/^$name=/ d" .config
		echo "$name=$value" >> .config
	done
	cp .config .config_generated
	{
		[ -n "$DL_MIRROR" ] && echo "CONFIG_DOWNLOAD_FOLDER=$DL_MIRROR"
		echo "CONFIG_VERSION_REPO=\"https://repo.turris.cz/$PUBLISH_BRANCH/$TARGET_BOARD/packages\""
		echo "CONFIG_VERSION_NUMBER=\"$("${src_dir}/helpers/turris-version.sh" version)\""
		echo "CONFIG_VERSION_CODE=\"$(_git_remote_hash)\""
	} >> .config
	_make defconfig diffconfig

	# Verify config
	rm -f .missing_values
	while IFS="=" read -r name value; do
		[ -z "$value" ] && continue
		[ "${value:0:1}" = "#" ] && continue
		if [ "$value" = "n" ]; then
			# No options can be ommited in the result so instead check it
			# they are not enabled by any chance.
			if grep -qE "^$name=y" ./.config; then
				warn "Config option that is forced to 'n' is set to 'y': $name=$value"
				echo "^$name=y" >> .missing_values
			fi
		else
			if ! grep -qE "^$name=$value$" ./.config; then
				warn "Config option is not present in generated config: $name=$value"
				echo "^$name=$value" >> .missing_values
			fi
		fi
	done < .config_generated
	[ \! -f .missing_values ] || [ "$(cat .missing_values | wc -l)" -eq 0 ] || die "Broken configuration"
}

available_commands+=( ["update_mirror"]="Updates all local mirrors" )
update_mirror() {
	use_git_mirror || return 0
	report "Updating local mirrors"
	git_mirror_update_all
}

available_commands+=( ["checkout"]="Start everything from scratch - all changes deleted and fresh copy of OpenWRT gets checked out" )
checkout() {
	local url
	url="$(get_feed "#" "openwrt")"
	report "Starting out: $(feed_ref "$url")"
	report "Checking out OpenWRT repository"
	git_clean_checkout "openwrt" "$url" "." "/turris-build.conf"

	echo "/turris-build.conf" >> ./.gitignore
	_git commit -m 'gitignore: Ignore turris-build related files' ./.gitignore

	cp "$src_dir"/feeds.conf .

	if [ -n "$DL_MIRROR" ]; then
		mkdir -p "$DL_MIRROR"
		rm -rf dl
		ln -s "$DL_MIRROR" dl
	fi
}

available_commands+=( ["clean_ccache"]="Clean persistent ccache paths" )
clean_ccache() {
	if [ "$(which ccache)" ]; then
		report "Cleaning ccache"
		[ -z "$CCACHE_HOST_DIR" ]   || CCACHE_DIR="$CCACHE_HOST_DIR"   ccache -C
		[ -z "$CCACHE_TARGET_DIR" ] || CCACHE_DIR="$CCACHE_TARGET_DIR/$TARGET_ARCH" ccache -C
	else
		report "Not cleaning ccache as you don't have ccache installed"
	fi
}

available_commands+=( ["set_ccache"]="Set persistent ccache paths" )
set_ccache() {
	[ "${CCACHE_SET:-n}" == "y" ] && return 0
	report "Setting ccache paths"
	CCACHE_SET=y
	[ -z "$CCACHE_HOST_DIR" ] || sed -i 's|$(STAGING_DIR_HOST)/ccache|'"$CCACHE_HOST_DIR|" include/host-build.mk
	[ -z "$CCACHE_TARGET_DIR" ] || [ -z "$TARGET_ARCH" ] || sed -i 's|$(STAGING_DIR)/ccache|'"$CCACHE_TARGET_DIR/$TARGET_ARCH|" include/package.mk
	[ -z "$(git diff include/host-build.mk include/package.mk)" ] || _git commit -m "include: ccache settings" include/host-build.mk include/package.mk
}

available_commands+=( ["set_local_feeds"]="Change feed URL to their mirror counterparts" )
set_local_feeds() {
	use_git_mirror || return 0
	report "Setting feeds to their local counterparts"
	while read -r vcs name url rest; do
		if [ "$vcs" = src-git-full ]; then
			git_mirror_update  "$name" "$(feed_url "$url")"
			url="$(git_mirror_url "$name" "$url")"
		fi
		echo "$vcs" "$name" "$url" "$rest" >&3
	done < feeds.conf 3> feeds.conf.new
	mv feeds.conf.new feeds.conf
}

available_commands+=( ["patch_openwrt"]="Patch the main OpenWRT repository" )
patch_openwrt() {
	report "Patching OpenWRT repository"
	patch_repository "openwrt"
	if [ -d "$src_dir/src" ]; then
		cp -r "$src_dir"/src/* .
		find "$src_dir"/src -printf "%P\\0" | xargs --null git add
		_git commit -m 'Add additional files'
	fi
}

_git_remote_hash() {
	local dir="${1:-}"
	git -C "$dir" rev-list HEAD | \
		git -C "$dir" name-rev --annotate-stdin --refs='origin/*' | \
		awk '$2 != "" { print $1; exit }'
}

available_commands+=( ["store_hash"]="Stores hashes of current build" )
store_hash() {
	report "Storing hashes"
	mkdir -p bin
	{
	echo "Project was build from following repositories:"
	echo
	echo " * turris-build: $(_git_remote_hash "${src_dir}")"
	echo " * openwrt: $(_git_remote_hash "${build_dir}")"
	for feed in feeds/*; do
		[ -d "$feed/.git" ] || continue
		echo " * $feed: $(_git_remote_hash "${feed}")"
	done
	echo
	} | tee bin/git-hash
}

available_commands+=( ["repatch_feeds"]="Cleanup feeds, update them and patch them" )
repatch_feeds() {
	set_local_feeds
	report "Getting feeds"
	_perl ./scripts/feeds clean -a
	_perl ./scripts/feeds update -a

	report "Patching feeds"
	while read -r vcs name url rest; do
		if [ "$vcs" = src-git-full ]; then
			patch_repository "$name" "$build_dir/feeds/$name"
		fi
	done < feeds.conf

	report "Installing feeds packages"
	_perl ./scripts/feeds update -a -i
	_perl ./scripts/feeds install -a
}

available_commands+=( ["prefetch"]="Runs make download" )
prefetch() {
	# We do not want to use force here intentionally as we are not building here.
	FORCE="" _openwrt_make download
}

available_commands+=( ["gen_junit"]="Generates junit output from build logs" )
gen_junit() {
	_sh "${src_dir}"/helpers/generate_junit.sh
}

available_commands+=( ["pkgsrepo"]="Extract packages from OpenWRT tree and place them to repository format as they are deployed to Turris servers" )
pkgsrepo() {
	report "Extracting packages to pkgsrepo directory: $OUTPUT_DIR"
	rm -rf "$OUTPUT_DIR"
	mkdir -p "$OUTPUT_DIR"
	mv bin/packages/*/* "$OUTPUT_DIR"
	mv bin/targets/*/*/packages "$OUTPUT_DIR/core"
	mv bin/targets/*/*/config.buildinfo "$OUTPUT_DIR"
	mv bin/git-hash "$OUTPUT_DIR"
}

available_commands+=( ["compile_tools"]="Compile host tools" )
compile_tools() {
	report "Compiling tools"
	_openwrt_make tools/compile toolchain/compile
}

available_commands+=( ["compile_target"]="Compile target specific software (Linux kernel)" )
compile_target() {
	report "Compiling target"
	_openwrt_make target/compile
}

available_commands+=( ["compile_packages"]="Compile packages" )
compile_packages() {
	report "Compiling packages"
	_openwrt_make package/compile
}

available_commands+=( ["compile"]="	Compile tools, target and packages" )
compile() {
	compile_tools
	compile_target
	compile_packages
}

available_commands+=( ["sign"]="	Generate packages index and sign it" )
sign() {
	_openwrt_make package/index BUILD_KEY="$SIGN_KEY"
}

available_commands+=( ["clean"]="	Clean current build directory" )
clean() {
	report "Cleaning up current build directory"
	rm -rf ./build_dir ./tmp ./staging_dir ./logs ./bin
	mkdir -p tmp
}

available_commands+=( ["prepare"]="	Prepare build but don't build it (Implies: checkout clean patch_openwrt repatch_feeds gen_version configure set_ccache)" )
prepare() {
	checkout
	clean
	patch_openwrt
	repatch_feeds
	gen_version
	configure
	set_ccache
}

available_commands+=( ["prepare_tools"]="Prepare build and build tools (Impies: prepare compile_tools compile_target)" )
prepare_tools() {
	prepare
	compile_tools
	compile_target
}

available_commands+=( ["autopkg"]="	Deploy autopkg scripts that can be used for rolling software development" )
autopkg() {
	report "Deploying autopkg scripts"
	cp "$src_dir"/helpers/autopkg/* "$build_dir"/include/
	_git add include/autopkg-*.mk
	_git commit -m 'autopkg: Add autopkg scripts'
}

##################################################################################

print_help() {
	echo "Usage: ${0} [OPTION].. [COMMAND].."
	echo "Compile TurrisOS packages. This script fetches, patches and configures OpenWRT tree."
	echo
	echo "Primary options:"
	echo "	-t BOARD	Set target board to BOARD. This is required for almost all commands."
	echo "	-jNUM		Number of jobs in paralel to be run. In default number of CPUs is used."
	echo "	-a ARG		Add build arguments - passed directly to make. This option can be specified multiple times."
	echo "	-f[NUM]		Try hard to get stuff to compile (optional number specifies number of tries)"
	echo "	-o PATH		Path to output directory where repository is prepared. In default ./pkgsrepo is used."
	echo "	-s KEY		Sign packges with given private key. Key has to be generated using using tool."
	echo "	-h, --help	Show this help text"
	echo
	echo "Advanced options:"
	echo "	-p BRANCH	Name of the target build alias. This is used to set OPKG feeds. If not specified '$PUBLISH_BRANCH' is used."
	echo "	-l		Do not update local git mirrors. Use them as they are to fetch OpenWRT and its feeds."
	echo "	-d		Do not use shallow checkouts"
	echo "	-x		Enable debug mode"
	echo
	echo "Available commands are:"
	for cmd in "${!available_commands[@]}"; do
		echo "	$cmd	${available_commands[$cmd]}"
	done
	echo
	echo "Default commands are: ${DEFAULT_STEPS[*]}"
}

steps=()
# Parse arguments
while [ $# -gt 0 ]; do
	case "$1" in
		-e)
			echo "-e option is obsoleted and execution now always runs as with this option." >&2
			;;
		-j*)
			BUILD_JOBS="${1#-j}"
			;;
		-f*)
			FORCE="${1#-f}"
			[ -n "$FORCE" ] || FORCE="1"
			;;
		-t)
			shift
			set_target "$1"
			;;
		-a)
			shift
			BUILD_ARGS+=( "$1" )
			;;
		-p)
			shift
			PUBLISH_BRANCH="$1"
			;;
		-l)
			set_git_mirrors_updated
			;;
		-o)
			shift
			OUTPUT_DIR="$1"
			;;
		-s)
			shift
			SIGN_KEY="$1"
			;;
		-x)
			debug="y"
			;;
		-h|--help)
			print_help
			exit 0
			;;
		*)
			[[ -v "available_commands[$1]" ]] || \
				die "There is no such option or command: $1"
			steps+=( "$1" )
			;;
	esac
	shift
done

[ "$(id -u)" != 0 ] || die "Build should not be run under root!"

# Set build_dir variable
set_protected_build_dir

# OpenWRT directory for temporally data
export TMPDIR="${build_dir}/tmp"
mkdir -p "$TMPDIR"

[ ${#steps[@]} -gt 0 ] || steps=( "${DEFAULT_STEPS[@]}" )
for step in "${steps[@]}"; do
	"$step"
done