From 2e074671be2371219eac644b2313d646401daf72 Mon Sep 17 00:00:00 2001 From: csears-hg Date: Wed, 27 Oct 2021 10:58:17 -0400 Subject: [PATCH] Refactored 'curl | bash' code in prep.sh to use vendored scripts --- build.sh | 3 + cfn/hd-notebooks-cfn.yaml | 1 + prep.sh | 13 +- scripts/install-helm.sh | 326 ++++++++++++++++++++++++++++++++++++++ scripts/install-k3d.sh | 189 ++++++++++++++++++++++ 5 files changed, 529 insertions(+), 3 deletions(-) create mode 100755 scripts/install-helm.sh create mode 100755 scripts/install-k3d.sh diff --git a/build.sh b/build.sh index b66ffd7..a53201f 100755 --- a/build.sh +++ b/build.sh @@ -21,6 +21,9 @@ k3d cluster create ${CLUSTER_NAME} \ --kubeconfig-update-default=true \ --k3s-arg "--disable=traefik@server:0" +# merge the kubeconfig into the default location +k3d kubeconfig merge -d firefly + # TODO: add check for helm binary. Must be v3 or better. echo "Installing firefly stack..." kubectl create namespace ${FIREFLY_NAMESPACE} diff --git a/cfn/hd-notebooks-cfn.yaml b/cfn/hd-notebooks-cfn.yaml index 591714e..a85863b 100644 --- a/cfn/hd-notebooks-cfn.yaml +++ b/cfn/hd-notebooks-cfn.yaml @@ -174,6 +174,7 @@ Resources: command: /home/ec2-user/bootstrap.sh 02-run-prep-sh: command: /home/ec2-user/firefly/prep.sh + cwd: /home/ec2-user/firefly 03-run-build-sh: command: /home/ec2-user/firefly/build.sh cwd: /home/ec2-user/firefly diff --git a/prep.sh b/prep.sh index 79959ed..b06c920 100755 --- a/prep.sh +++ b/prep.sh @@ -1,9 +1,16 @@ #! /bin/bash set xe -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +# install helm +./scripts/install-helm.sh + +# install kubectl +curl -LO "https://dl.k8s.io/release/v1.22.2/bin/linux/amd64/kubectl" install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl -curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + +# install k3d +./scripts/install-k3d.sh + +# start and enable docker service systemctl start docker systemctl enable docker.service \ No newline at end of file diff --git a/scripts/install-helm.sh b/scripts/install-helm.sh new file mode 100755 index 0000000..832ea66 --- /dev/null +++ b/scripts/install-helm.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash + +# Copyright The Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The install script is based off of the MIT-licensed script from glide, +# the package manager for Go: https://github.com/Masterminds/glide.sh/blob/master/get + +: ${BINARY_NAME:="helm"} +: ${USE_SUDO:="true"} +: ${DEBUG:="false"} +: ${VERIFY_CHECKSUM:="true"} +: ${VERIFY_SIGNATURES:="false"} +: ${HELM_INSTALL_DIR:="/usr/local/bin"} +: ${GPG_PUBRING:="pubring.kbx"} + +HAS_CURL="$(type "curl" &> /dev/null && echo true || echo false)" +HAS_WGET="$(type "wget" &> /dev/null && echo true || echo false)" +HAS_OPENSSL="$(type "openssl" &> /dev/null && echo true || echo false)" +HAS_GPG="$(type "gpg" &> /dev/null && echo true || echo false)" + +# initArch discovers the architecture for this system. +initArch() { + ARCH=$(uname -m) + case $ARCH in + armv5*) ARCH="armv5";; + armv6*) ARCH="armv6";; + armv7*) ARCH="arm";; + aarch64) ARCH="arm64";; + x86) ARCH="386";; + x86_64) ARCH="amd64";; + i686) ARCH="386";; + i386) ARCH="386";; + esac +} + +# initOS discovers the operating system for this system. +initOS() { + OS=$(echo `uname`|tr '[:upper:]' '[:lower:]') + + case "$OS" in + # Minimalist GNU for Windows + mingw*) OS='windows';; + esac +} + +# runs the given command as root (detects if we are root already) +runAsRoot() { + if [ $EUID -ne 0 -a "$USE_SUDO" = "true" ]; then + sudo "${@}" + else + "${@}" + fi +} + +# verifySupported checks that the os/arch combination is supported for +# binary builds, as well whether or not necessary tools are present. +verifySupported() { + local supported="darwin-amd64\ndarwin-arm64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nlinux-ppc64le\nlinux-s390x\nwindows-amd64" + if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then + echo "No prebuilt binary for ${OS}-${ARCH}." + echo "To build from source, go to https://github.com/helm/helm" + exit 1 + fi + + if [ "${HAS_CURL}" != "true" ] && [ "${HAS_WGET}" != "true" ]; then + echo "Either curl or wget is required" + exit 1 + fi + + if [ "${VERIFY_CHECKSUM}" == "true" ] && [ "${HAS_OPENSSL}" != "true" ]; then + echo "In order to verify checksum, openssl must first be installed." + echo "Please install openssl or set VERIFY_CHECKSUM=false in your environment." + exit 1 + fi + + if [ "${VERIFY_SIGNATURES}" == "true" ]; then + if [ "${HAS_GPG}" != "true" ]; then + echo "In order to verify signatures, gpg must first be installed." + echo "Please install gpg or set VERIFY_SIGNATURES=false in your environment." + exit 1 + fi + if [ "${OS}" != "linux" ]; then + echo "Signature verification is currently only supported on Linux." + echo "Please set VERIFY_SIGNATURES=false or verify the signatures manually." + exit 1 + fi + fi +} + +# checkDesiredVersion checks if the desired version is available. +checkDesiredVersion() { + if [ "x$DESIRED_VERSION" == "x" ]; then + # Get tag from release URL + local latest_release_url="https://github.com/helm/helm/releases" + if [ "${HAS_CURL}" == "true" ]; then + TAG=$(curl -Ls $latest_release_url | grep 'href="/helm/helm/releases/tag/v3.[0-9]*.[0-9]*\"' | sed -E 's/.*\/helm\/helm\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1) + elif [ "${HAS_WGET}" == "true" ]; then + TAG=$(wget $latest_release_url -O - 2>&1 | grep 'href="/helm/helm/releases/tag/v3.[0-9]*.[0-9]*\"' | sed -E 's/.*\/helm\/helm\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1) + fi + else + TAG=$DESIRED_VERSION + fi +} + +# checkHelmInstalledVersion checks which version of helm is installed and +# if it needs to be changed. +checkHelmInstalledVersion() { + if [[ -f "${HELM_INSTALL_DIR}/${BINARY_NAME}" ]]; then + local version=$("${HELM_INSTALL_DIR}/${BINARY_NAME}" version --template="{{ .Version }}") + if [[ "$version" == "$TAG" ]]; then + echo "Helm ${version} is already ${DESIRED_VERSION:-latest}" + return 0 + else + echo "Helm ${TAG} is available. Changing from version ${version}." + return 1 + fi + else + return 1 + fi +} + +# downloadFile downloads the latest binary package and also the checksum +# for that binary. +downloadFile() { + HELM_DIST="helm-$TAG-$OS-$ARCH.tar.gz" + DOWNLOAD_URL="https://get.helm.sh/$HELM_DIST" + CHECKSUM_URL="$DOWNLOAD_URL.sha256" + HELM_TMP_ROOT="$(mktemp -dt helm-installer-XXXXXX)" + HELM_TMP_FILE="$HELM_TMP_ROOT/$HELM_DIST" + HELM_SUM_FILE="$HELM_TMP_ROOT/$HELM_DIST.sha256" + echo "Downloading $DOWNLOAD_URL" + if [ "${HAS_CURL}" == "true" ]; then + curl -SsL "$CHECKSUM_URL" -o "$HELM_SUM_FILE" + curl -SsL "$DOWNLOAD_URL" -o "$HELM_TMP_FILE" + elif [ "${HAS_WGET}" == "true" ]; then + wget -q -O "$HELM_SUM_FILE" "$CHECKSUM_URL" + wget -q -O "$HELM_TMP_FILE" "$DOWNLOAD_URL" + fi +} + +# verifyFile verifies the SHA256 checksum of the binary package +# and the GPG signatures for both the package and checksum file +# (depending on settings in environment). +verifyFile() { + if [ "${VERIFY_CHECKSUM}" == "true" ]; then + verifyChecksum + fi + if [ "${VERIFY_SIGNATURES}" == "true" ]; then + verifySignatures + fi +} + +# installFile installs the Helm binary. +installFile() { + HELM_TMP="$HELM_TMP_ROOT/$BINARY_NAME" + mkdir -p "$HELM_TMP" + tar xf "$HELM_TMP_FILE" -C "$HELM_TMP" + HELM_TMP_BIN="$HELM_TMP/$OS-$ARCH/helm" + echo "Preparing to install $BINARY_NAME into ${HELM_INSTALL_DIR}" + runAsRoot cp "$HELM_TMP_BIN" "$HELM_INSTALL_DIR/$BINARY_NAME" + echo "$BINARY_NAME installed into $HELM_INSTALL_DIR/$BINARY_NAME" +} + +# verifyChecksum verifies the SHA256 checksum of the binary package. +verifyChecksum() { + printf "Verifying checksum... " + local sum=$(openssl sha1 -sha256 ${HELM_TMP_FILE} | awk '{print $2}') + local expected_sum=$(cat ${HELM_SUM_FILE}) + if [ "$sum" != "$expected_sum" ]; then + echo "SHA sum of ${HELM_TMP_FILE} does not match. Aborting." + exit 1 + fi + echo "Done." +} + +# verifySignatures obtains the latest KEYS file from GitHub main branch +# as well as the signature .asc files from the specific GitHub release, +# then verifies that the release artifacts were signed by a maintainer's key. +verifySignatures() { + printf "Verifying signatures... " + local keys_filename="KEYS" + local github_keys_url="https://raw.githubusercontent.com/helm/helm/main/${keys_filename}" + if [ "${HAS_CURL}" == "true" ]; then + curl -SsL "${github_keys_url}" -o "${HELM_TMP_ROOT}/${keys_filename}" + elif [ "${HAS_WGET}" == "true" ]; then + wget -q -O "${HELM_TMP_ROOT}/${keys_filename}" "${github_keys_url}" + fi + local gpg_keyring="${HELM_TMP_ROOT}/keyring.gpg" + local gpg_homedir="${HELM_TMP_ROOT}/gnupg" + mkdir -p -m 0700 "${gpg_homedir}" + local gpg_stderr_device="/dev/null" + if [ "${DEBUG}" == "true" ]; then + gpg_stderr_device="/dev/stderr" + fi + gpg --batch --quiet --homedir="${gpg_homedir}" --import "${HELM_TMP_ROOT}/${keys_filename}" 2> "${gpg_stderr_device}" + gpg --batch --no-default-keyring --keyring "${gpg_homedir}/${GPG_PUBRING}" --export > "${gpg_keyring}" + local github_release_url="https://github.com/helm/helm/releases/download/${TAG}" + if [ "${HAS_CURL}" == "true" ]; then + curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" + curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" + elif [ "${HAS_WGET}" == "true" ]; then + wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" + wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" + fi + local error_text="If you think this might be a potential security issue," + error_text="${error_text}\nplease see here: https://github.com/helm/community/blob/master/SECURITY.md" + local num_goodlines_sha=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)') + if [[ ${num_goodlines_sha} -lt 2 ]]; then + echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256!" + echo -e "${error_text}" + exit 1 + fi + local num_goodlines_tar=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)') + if [[ ${num_goodlines_tar} -lt 2 ]]; then + echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz!" + echo -e "${error_text}" + exit 1 + fi + echo "Done." +} + +# fail_trap is executed if an error occurs. +fail_trap() { + result=$? + if [ "$result" != "0" ]; then + if [[ -n "$INPUT_ARGUMENTS" ]]; then + echo "Failed to install $BINARY_NAME with the arguments provided: $INPUT_ARGUMENTS" + help + else + echo "Failed to install $BINARY_NAME" + fi + echo -e "\tFor support, go to https://github.com/helm/helm." + fi + cleanup + exit $result +} + +# testVersion tests the installed client to make sure it is working. +testVersion() { + set +e + HELM="$(command -v $BINARY_NAME)" + if [ "$?" = "1" ]; then + echo "$BINARY_NAME not found. Is $HELM_INSTALL_DIR on your "'$PATH?' + exit 1 + fi + set -e +} + +# help provides possible cli installation arguments +help () { + echo "Accepted cli arguments are:" + echo -e "\t[--help|-h ] ->> prints this help" + echo -e "\t[--version|-v ] . When not defined it fetches the latest release from GitHub" + echo -e "\te.g. --version v3.0.0 or -v canary" + echo -e "\t[--no-sudo] ->> install without sudo" +} + +# cleanup temporary files to avoid https://github.com/helm/helm/issues/2977 +cleanup() { + if [[ -d "${HELM_TMP_ROOT:-}" ]]; then + rm -rf "$HELM_TMP_ROOT" + fi +} + +# Execution + +#Stop execution on any error +trap "fail_trap" EXIT +set -e + +# Set debug if desired +if [ "${DEBUG}" == "true" ]; then + set -x +fi + +# Parsing input arguments (if any) +export INPUT_ARGUMENTS="${@}" +set -u +while [[ $# -gt 0 ]]; do + case $1 in + '--version'|-v) + shift + if [[ $# -ne 0 ]]; then + export DESIRED_VERSION="${1}" + else + echo -e "Please provide the desired version. e.g. --version v3.0.0 or -v canary" + exit 0 + fi + ;; + '--no-sudo') + USE_SUDO="false" + ;; + '--help'|-h) + help + exit 0 + ;; + *) exit 1 + ;; + esac + shift +done +set +u + +initArch +initOS +verifySupported +checkDesiredVersion +if ! checkHelmInstalledVersion; then + downloadFile + verifyFile + installFile +fi +testVersion +cleanup diff --git a/scripts/install-k3d.sh b/scripts/install-k3d.sh new file mode 100755 index 0000000..8bfec8d --- /dev/null +++ b/scripts/install-k3d.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +APP_NAME="k3d" +REPO_URL="https://github.com/rancher/k3d" + +: ${USE_SUDO:="true"} +: ${K3D_INSTALL_DIR:="/usr/local/bin"} + +# initArch discovers the architecture for this system. +initArch() { + ARCH=$(uname -m) + case $ARCH in + armv5*) ARCH="armv5";; + armv6*) ARCH="armv6";; + armv7*) ARCH="arm";; + aarch64) ARCH="arm64";; + x86) ARCH="386";; + x86_64) ARCH="amd64";; + i686) ARCH="386";; + i386) ARCH="386";; + esac +} + +# initOS discovers the operating system for this system. +initOS() { + OS=$(uname|tr '[:upper:]' '[:lower:]') + + case "$OS" in + # Minimalist GNU for Windows + mingw*) OS='windows';; + esac +} + +# runs the given command as root (detects if we are root already) +runAsRoot() { + local CMD="$*" + + if [ $EUID -ne 0 -a $USE_SUDO = "true" ]; then + CMD="sudo $CMD" + fi + + $CMD +} + +# verifySupported checks that the os/arch combination is supported for +# binary builds. +verifySupported() { + local supported="darwin-386\ndarwin-amd64\ndarwin-arm64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nwindows-386\nwindows-amd64" + if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then + echo "No prebuilt binary for ${OS}-${ARCH}." + echo "To build from source, go to $REPO_URL" + exit 1 + fi + + if ! type "curl" > /dev/null && ! type "wget" > /dev/null; then + echo "Either curl or wget is required" + exit 1 + fi +} + +# checkK3dInstalledVersion checks which version of k3d is installed and +# if it needs to be changed. +checkK3dInstalledVersion() { + if [[ -f "${K3D_INSTALL_DIR}/${APP_NAME}" ]]; then + local version=$(k3d version | grep 'k3d version' | cut -d " " -f3) + if [[ "$version" == "$TAG" ]]; then + echo "k3d ${version} is already ${DESIRED_VERSION:-latest}" + return 0 + else + echo "k3d ${TAG} is available. Changing from version ${version}." + return 1 + fi + else + return 1 + fi +} + +# checkTagProvided checks whether TAG has provided as an environment variable so we can skip checkLatestVersion. +checkTagProvided() { + [[ ! -z "$TAG" ]] +} + +# checkLatestVersion grabs the latest version string from the releases +checkLatestVersion() { + local latest_release_url="$REPO_URL/releases/latest" + if type "curl" > /dev/null; then + TAG=$(curl -Ls -o /dev/null -w %{url_effective} $latest_release_url | grep -oE "[^/]+$" ) + elif type "wget" > /dev/null; then + TAG=$(wget $latest_release_url --server-response -O /dev/null 2>&1 | awk '/^\s*Location: /{DEST=$2} END{ print DEST}' | grep -oE "[^/]+$") + fi +} + +# downloadFile downloads the latest binary package and also the checksum +# for that binary. +downloadFile() { + K3D_DIST="k3d-$OS-$ARCH" + DOWNLOAD_URL="$REPO_URL/releases/download/$TAG/$K3D_DIST" + K3D_TMP_ROOT="$(mktemp -dt k3d-binary-XXXXXX)" + K3D_TMP_FILE="$K3D_TMP_ROOT/$K3D_DIST" + if type "curl" > /dev/null; then + curl -SsL "$DOWNLOAD_URL" -o "$K3D_TMP_FILE" + elif type "wget" > /dev/null; then + wget -q -O "$K3D_TMP_FILE" "$DOWNLOAD_URL" + fi +} + +# installFile verifies the SHA256 for the file, then unpacks and +# installs it. +installFile() { + echo "Preparing to install $APP_NAME into ${K3D_INSTALL_DIR}" + runAsRoot chmod +x "$K3D_TMP_FILE" + runAsRoot cp "$K3D_TMP_FILE" "$K3D_INSTALL_DIR/$APP_NAME" + echo "$APP_NAME installed into $K3D_INSTALL_DIR/$APP_NAME" +} + +# fail_trap is executed if an error occurs. +fail_trap() { + result=$? + if [ "$result" != "0" ]; then + if [[ -n "$INPUT_ARGUMENTS" ]]; then + echo "Failed to install $APP_NAME with the arguments provided: $INPUT_ARGUMENTS" + help + else + echo "Failed to install $APP_NAME" + fi + echo -e "\tFor support, go to $REPO_URL." + fi + cleanup + exit $result +} + +# testVersion tests the installed client to make sure it is working. +testVersion() { + if ! command -v $APP_NAME &> /dev/null; then + echo "$APP_NAME not found. Is $K3D_INSTALL_DIR on your "'$PATH?' + exit 1 + fi + echo "Run '$APP_NAME --help' to see what you can do with it." +} + +# help provides possible cli installation arguments +help () { + echo "Accepted cli arguments are:" + echo -e "\t[--help|-h ] ->> prints this help" + echo -e "\t[--no-sudo] ->> install without sudo" +} + +# cleanup temporary files +cleanup() { + if [[ -d "${K3D_TMP_ROOT:-}" ]]; then + rm -rf "$K3D_TMP_ROOT" + fi +} + +# Execution + +#Stop execution on any error +trap "fail_trap" EXIT +set -e + +# Parsing input arguments (if any) +export INPUT_ARGUMENTS="${@}" +set -u +while [[ $# -gt 0 ]]; do + case $1 in + '--no-sudo') + USE_SUDO="false" + ;; + '--help'|-h) + help + exit 0 + ;; + *) exit 1 + ;; + esac + shift +done +set +u + +initArch +initOS +verifySupported +checkTagProvided || checkLatestVersion +if ! checkK3dInstalledVersion; then + downloadFile + installFile +fi +testVersion +cleanup