install-custom-kubelet

#!/bin/bash

# Find our helpers
function finddir() {
    local path_to_file
    path_to_file=$(readlink -f "$0")
    if [[ -z $path_to_file ]] ; then
	return 1
    elif [[ -d $path_to_file ]] ; then
	echo "$path_to_file/"
    elif [[ -e $path_to_file ]] ; then
	echo "${path_to_file%/*}/"
    else
	return 1
    fi
    return 0
}

cmddir=$(finddir "$0")
[[ -z $cmddir ]] && fatal "Can't find directory for $0"

PATH="${cmddir}:$PATH"

set -eu

# This was renamed from hyperkube to kubelet between 4.1 and 4.2, evidently.
declare kubelet_bin=kubelet
declare godir=${GOPATH:-$HOME/go}
declare -r srcdir="$godir/src/github.com/openshift/origin"
declare lkubelet="$srcdir/_output/local/bin/linux/amd64/${kubelet_bin}"
declare bastion=bastion-
declare -i rebuild=1
declare -i do_setup_cert=0
declare -i do_uninstall=0
declare -i twidth=1000
declare -i install_all=0
declare -i install_infra=0
declare -i install_master=0
declare -i install_worker=1
declare -i copy_kubelet=1
declare -i parallel_key=1
declare -i parallel_kube=1
declare private_key=

function help() {
    cat <<EOF
Usage: $0 [options] [specific_nodes...]
    Args:
      -A        Don't install on all nodes (default)
      -a        Install on all nodes
      -B        Don't use ssh bastion host (direct ssh)
      -b        Use a bastion ssh host (default)
      -C        Don't setup node certs for ssh (default)
      -c        Set up node certs for ssh (must be done once)
      -I        Don't install on infra nodes (default)
      -i        Install on infra nodes
      -k file   Use the specified kubelet file
      -K file   Use the specified private key file
      -M        Don't install on master nodes (default)
      -m        Install on master nodes
      -N        Don't copy the kubelet to remote (must be there)
      -n        Copy the kubelet to the remote (default)
      -p N      Set up ssh keys N-way parallel
      -P N      Install kubelets N-way paralleo
      -R        Don't attempt to rebuild the kubelet
      -r        Rebuild the kubelet (default)
      -U        Install custom kubelet (default)
      -u        Uninstall (restore original kubelet)
      -W        Don't install on worker nodes
      -w        Install on worker nodes (default)
EOF
    exit 1
}

while getopts "AaBbCchIiK:k:MmNnP:p:RrUuWw" opt ; do
    case "$opt" in
	A) install_all=0		 ;;
	a) install_all=1		 ;;
	B) bastion=                      ;;
	b) bastion=bastion-              ;;
	C) do_setup_cert=0		 ;;
	c) do_setup_cert=1		 ;;
	I) install_infra=0		 ;;
	i) install_infra=1		 ;;
	K) private_key="$OPTARG"	 ;;
	k) lkubelet="$OPTARG"; rebuild=0 ;;
	M) install_master=0		 ;;
	m) install_master=1		 ;;
	N) copy_kubelet=0 		 ;;
	n) copy_kubelet=1		 ;;
	P) parallel_kube="$OPTARG"	 ;;
	p) parallel_key="$OPTARG"	 ;;
	R) rebuild=0			 ;;
	r) rebuild=1			 ;;
	U) do_uninstall=0		 ;;
	u) do_uninstall=1		 ;;
	W) install_worker=0		 ;;
	w) install_worker=1		 ;;
	*) help				 ;;
    esac
done

shift $((OPTIND - 1))

declare -a all_nodes=()

if [[ -n "$*" ]] ; then
    all_nodes=("$@")
else
    # shellcheck disable=SC2034
    while read -r node status class rest ; do
	if [[ ($install_all -eq 1) ||
		  ($install_infra -eq 1 && $class == *infra*) ||
		  ($install_master -eq 1 && $class == *master*) ||
		  ($install_worker -eq 1 && $class == *worker*) ]] ; then
	    all_nodes+=("$node")
	fi
    done < <(oc get nodes --no-headers=true)
fi

function rcmd() {
    "${bastion}ssh" "$@"
}

function rcp() {
    "${bastion}scp" "$@"
}

function build_upload() {
    cd "$srcdir"
    make WHAT=vendor/k8s.io/kubernetes/cmd/${kubelet_bin}
    ls -l _output/local/bin/linux/amd64/${kubelet_bin}
    md5sum _output/local/bin/linux/amd64/${kubelet_bin}
}

if (( rebuild && !do_uninstall && copy_kubelet )) ; then
    (build_upload)
fi

declare expected_md5
expected_md5=$(md5sum "$lkubelet" |awk '{print $1}') || {
    echo "Can't checksum $lkubelet"
    exit 1
}

function generate_grappling_hook_install() {
    cat <<'EOF'
function generate_install_kubelet() {
    cat <<'EOL'
#!/bin/bash

declare lkubelet="$1"
declare -r expected_md5="$2"

declare -r lbindir="/usr/bin"

function die() {
    if [[ $1 = -r ]] ; then
	cd /
	mount -o remount,ro /usr 2>&1 |grep -v 'mount point is busy'
    fi
    echo "$*"
    exit 1
}

if [[ $lkubelet = *.gz ]] ; then
    rm -f "${lkubelet%.gz}"
    gunzip "$lkubelet"
    lkubelet=${lkubelet%.gz}
fi

cd "$lbindir" || die "Can't cd $lbindir!"
[[ -f "$lkubelet" ]] || die "Missing kubelet $lkubelet"
found_md5=$(md5sum "$lkubelet" |awk '{print $1}')
[[ $(stat -c %s "$lkubelet") -ge 1048576 && "$expected_md5" = "$found_md5" ]] || die "Bad kubelet (expected $expected_md5, found $found_md5)"

sudo mount -o remount,rw /usr || die "Can't remount /usr rw"

sudo cp "$lkubelet" @KUBELET@.NEW
cmp "$lkubelet" @KUBELET@.NEW || die -r "$lkubelet and @KUBELET@.NEW failed compare!"
if [[ -f @KUBELET@ && ! -f @KUBELET@.OLD ]] ; then
    sudo mv @KUBELET@ @KUBELET@.OLD
fi
sudo mv @KUBELET@.NEW @KUBELET@ || die -r "Can't rename @KUBELET@.NEW to @KUBELET@!"
cd /
sudo mount -o remount,ro /usr 2>&1 |grep -v 'mount point is busy'
sudo sysctl -w kernel.pid_max=4194304 >/dev/null
sudo systemctl restart kubelet.service
ps aux |grep /usr/bin/@KUBELET@ |grep kubelet |grep -v grep
EOL
}

function install_kubelet_n() {
    local parallel_kube=$1
    shift
    local rotor=$1
    shift
    set -e
    while (( rotor < ${#all_nodes[@]} )) ; do
	local node=${all_nodes[$rotor]}
	echo "$node: " "$(tar cf - install-kubelet @KUBELET@.gz | ssh -o StrictHostKeyChecking=no -i ~/.ssh/temp_install "$node" 'cd /var/tmp; rm -f install-kubelet @KUBELET@.gz; tar xf -; /var/tmp/install-kubelet /var/tmp/@KUBELET@.gz '"$expected_md5")"
	rotor=$((rotor + parallel_kube))
    done
}

set -u
expected_md5="$1"
shift
parallel_kube="$1"
myname="${all_nodes[0]}"
if (( ${#all_nodes[@]} < parallel_kube - 1)) ; then
    parallel_kube=$((${#all_nodes[@]} - 1))
fi

cd /var/tmp
generate_install_kubelet > install-kubelet
chmod +x install-kubelet
chmod +x @KUBELET@
echo "$myname: " "$(./install-kubelet /var/tmp/@KUBELET@ "$expected_md5")"
rm -f /var/tmp/@KUBELET@.gz
gzip -c /var/tmp/@KUBELET@ > /var/tmp/@KUBELET@.gz
chmod +x /var/tmp/@KUBELET@.gz
# Don't try to copy this to ourself.
for i in $(seq 1 $parallel_kube) ; do
    install_kubelet_n "$parallel_kube" "$i" &
done
wait
rm -f /var/tmp/@KUBELET@.gz
ls -l /usr/bin/@KUBELET@
EOF
}

function generate_grappling_hook_uninstall() {
    cat <<'EOF'
function generate_uninstall_kubelet() {
    cat <<'EOL'
#!/bin/bash
declare -r lbindir="/usr/bin"

function die() {
    if [[ $1 = -r ]] ; then
	cd /
	mount -o remount,ro /usr 2>&1 |grep -v 'mount point is busy'
    fi
    echo "$*"
    exit 1
}

cd "$lbindir" || die "Can't cd $lbindir!"

if [[ -f @KUBELET@.OLD ]] ; then
    sudo mount -o remount,rw /usr || die "Can't remount /usr rw"
    sudo mv @KUBELET@.OLD @KUBELET@ || die -r "Can't rename @KUBELET@.OLD to @KUBELET@"
    cd /
    sudo mount -o remount,ro /usr 2>&1 |grep -v 'mount point is busy'
    sudo systemctl restart kubelet.service
    ps aux |grep @KUBELET@ |grep kubelet |grep -v grep
else
    echo "No old @KUBELET@ to restore"
fi
EOL
}

function uninstall_kubelet_n() {
    local parallel_kube=$1
    shift
    local rotor=$1
    shift
    while (( rotor < ${#all_nodes[@]} )) ; do
	local node=${all_nodes[$rotor]}
	echo "$node: " "$(ssh -o StrictHostKeyChecking=no -i ~/.ssh/temp_install "$node" 'cat > /var/tmp/uninstall-kubelet; chmod +x /var/tmp/uninstall-kubelet; /var/tmp/uninstall-kubelet' < uninstall-kubelet)"
	rotor=$((rotor + parallel_kube))
    done
}

set -u
shift
parallel_kube="$1"
myname="${all_nodes[0]}"
if (( ${#all_nodes[@]} < parallel_kube - 1)) ; then
    parallel_kube=$((${#all_nodes[@]} - 1))
fi

cd /var/tmp
generate_uninstall_kubelet > uninstall-kubelet
chmod +x uninstall-kubelet
echo "$myname: " "$(./uninstall-kubelet)"
# Don't try to re-uninstall ourself.
for i in $(seq 1 $parallel_kube) ; do
    uninstall_kubelet_n "$parallel_kube" "$i" &
done
wait
ls -l /usr/bin/@KUBELET@
EOF
}

function generate_grappling_hook_1() {
    echo "#!/bin/bash"
    local -a nodes=()
    readarray -t nodes
    echo "declare -a all_nodes=(${nodes[*]@Q})"
    if (( do_uninstall )) ; then
	generate_grappling_hook_uninstall
    else
	generate_grappling_hook_install
    fi
}

function generate_grappling_hook() {
    generate_grappling_hook_1 | sed "s/@KUBELET@/$kubelet_bin/g"
}

function install_ssh_key_n() {
    local parallel_key=$1
    local rotor=$2
    local pdir=${private_key%/*}
    if [[ -n "$pdir" ]] ; then
	pdir="-C $pdir"
    fi
    local pkey=${private_key##*/}
    set -e
    while (( rotor < ${#all_nodes[@]} )) ; do
	local f=${all_nodes[$rotor]}
	# shellcheck disable=SC2086
	tar c $pdir -f - "$pkey" "${pkey}.pub" | rcmd "$f" "cd .ssh; rm -f '$pkey' '${pkey}.pub'; tar xf -; sudo chown core authorized_keys; sudo chmod 644 authorized_keys; cat '${pkey}.pub' >> authorized_keys"
	rotor=$((rotor + parallel_key))
    done
}

function setup_cert() {
    local tmpdir
    local private_key="$1"
    local i
    if (( ${#all_nodes[@]} < parallel_key )) ; then
	parallel_key=${#all_nodes[@]}
    fi

    if [[ -z "$private_key" ]] ; then
	tmpdir=$(mktemp -d) || {
	    echo "Can't create temporary cert dir"
	    exit 1
	}
	private_key="$tmpdir/temp_install"
	ssh-keygen -q -f "$private_key" -N '' || {
	    echo "Can't create temporary key"
	    rm -rf "$tmpdir"
	    exit 1
	}
    fi
    echo "Install key:"
    for i in $(seq 0 $((parallel_key - 1))) ; do
	install_ssh_key_n "$parallel_key" "$i"&
    done
    wait
    if [[ -n "${tmpdir:-}" && -d "${tmpdir:-}" ]] ; then
	rm -rf "$tmpdir"
    fi
}

function print_node_names() {
    (IFS='
'; echo "${all_nodes[*]}")
}

if [[ -n $TERM ]] ; then
    twidth=$(tput cols)
fi

if (( do_setup_cert )) ; then
    setup_cert "$private_key"
fi

if (( copy_kubelet && ! do_uninstall )) ; then
    echo "Copy $lkubelet to remote"
    rcp -C "$lkubelet" "${all_nodes[0]}:/var/tmp"
fi

echo "Set up kubelet"
generate_grappling_hook < <(print_node_names "$@") | rcmd "${all_nodes[0]}" "cat > /var/tmp/kubelet-grappling-hook; chmod +x /var/tmp/kubelet-grappling-hook; /var/tmp/kubelet-grappling-hook $expected_md5 $parallel_kube" |cut "-c1-$twidth"
oc get nodes