cc-ansible

#!/usr/bin/env bash

set -o errexit
set -o nounset
set -o pipefail
# set -o xtrace

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
VIRTUALENV="$(realpath "${VIRTUALENV:-"$DIR"/venv}")"

FORCE_UPDATES="${FORCE_UPDATES:-no}"
if [[ ! -d "$VIRTUALENV" ]]; then
  echo "Creating virtualenv at $VIRTUALENV ..."
  python3 -m venv "$VIRTUALENV" --system-site-packages
  "$VIRTUALENV/bin/pip" install --upgrade pip
  FORCE_UPDATES=yes
fi

set +o nounset
# shellcheck disable=1090 # TODO use standard VIRTUAL_ENV and PATH method
source "$(realpath "$VIRTUALENV/bin/activate")"
set -o nounset

CHECK_UPDATES=yes
declare -a POSARGS=()
declare -a TRAPS=()
cleanup() {
  for exit_trap in "${TRAPS[@]}"; do "$exit_trap"; done
}
trap cleanup EXIT

while [[ $# -gt 0 ]]; do
  key="$1"

  case $key in
    -p|--playbook)
      CC_ANSIBLE_PLAYBOOK="$2"
      shift # Past arg
      ;;
    -s|--site)
      CC_ANSIBLE_SITE="$(realpath "$2")"
      shift # Past arg
      ;;
    --check)
      # Add proper flag support for --check as an option for dry-runs
      export EXTRA_OPTS="${EXTRA_OPTS:-} --check"
      ;;
    decrypt_passwords|edit_passwords|help|init|view_passwords)
      # Special subcommand!
      command="$key"
      ;;
    -h|--help)
      command=help
      ;;
    --no-update)
      CHECK_UPDATES=no
      ;;
    --force-update)
      FORCE_UPDATES=yes
      ;;
    *)
      POSARGS+=("$key")
      ;;
  esac

  shift
done

#
# Subcommands
#

init() {
  local site_config="$CC_ANSIBLE_SITE"

  if [[ -d "$site_config" ]]; then
    echo "Site config already exists at $site_config! Aborting."
    return 1
  fi

  # Copy files from example
  mkdir -p "$(dirname "$site_config")"
  cp -a "$DIR/site-config.example/" "$site_config"

  local inventory="$site_config/inventory"

  # Do some simple hostname guessing
  this_host=$(hostname -s)
  sed -i.bak "s/<host>/$this_host/g" "$inventory/hosts" \
    && rm -f "$inventory/hosts.bak"
  mv "$inventory/host_vars/<host>" "$inventory/host_vars/$this_host"

  # Generate vault password
  openssl rand -base64 2048 >"$site_config/vault_password" \
    && chmod 400 "$site_config/vault_password"

  # Generate passwords
  kolla-genpwd --passwords "$site_config/passwords.yml"
  ansible-vault --vault-password-file "$site_config/vault_password" \
    encrypt "$site_config/passwords.yml"

  echo "The beginnings of your site configuration were installed to $site_config."
  echo "To use this site configuration automatically for future commands, you can"
  echo "set an environment variable:"
  echo
  echo -e "\texport CC_ANSIBLE_SITE=$site_config"
  echo
}

edit_passwords() {
  local tmpfile
  local passwords_file="$CC_ANSIBLE_SITE/passwords.yml"
  local passwords_file_chksum="${passwords_file}.sha256"

  tmpfile="$(mktemp)"
  _edit_passwords_cleanup() {
    rm -f "$tmpfile"
    rm -f "$passwords_file_chksum"
  }
  TRAPS+=(_edit_passwords_cleanup)

  echo "Decrypting passwords..."
  ansible-vault view \
    --vault-password-file "$CC_ANSIBLE_VAULT_PASSWORD" \
    "$CC_ANSIBLE_SITE/passwords.yml" >"$tmpfile"

  if [[ ! -s "$tmpfile" ]]; then
    echo "Failed to decrypt $passwords_file with vault token."
    exit 1
  fi

  sha256sum "$tmpfile" >"$passwords_file_chksum"
  ${EDITOR:-vi} "$tmpfile"
  local ret=$?
  if [[ $ret -gt 0 ]]; then exit $ret; fi
  if sha256sum --quiet --check "$passwords_file_chksum"; then
    echo "No passwords were changed."
    exit 0
  fi

  echo "Generating placeholder passwords for any missing values..."
  kolla-genpwd --passwords "$tmpfile"
  echo "Encrypting passwords..."
  ansible-vault encrypt \
    --vault-password-file "$CC_ANSIBLE_VAULT_PASSWORD" \
    "$tmpfile"
  cp "$tmpfile" "$CC_ANSIBLE_SITE/passwords.yml"
}

# Deprecated function name
decrypt_passwords() {
  view_passwords
}

view_passwords() {
  ansible-vault view \
    --vault-password-file "$CC_ANSIBLE_VAULT_PASSWORD" \
    "$CC_ANSIBLE_SITE/passwords.yml"
}

help() {
  cat <<USAGE
Usage: cc-ansible [-s|--site SITE]
                  [subcommand|--playbook PLAYBOOK]
                  ARGS

Subcommands:
  view_passwords: View the contents of the encrypted password file.
  edit_passwords: Update an encrypted passwords file for the given environment.
                  Opens an interactive editor and saves the results back out as
                  en encrypted file.

Examples:
  # Run the 'deploy' step for Kolla-Ansible in a 'production' site
  cc-ansible -s /path/to/sites/production deploy

  # Run the 'upgrade' step for only Ironic tags
  cc-ansible upgrade --tags ironic

  # Run an external playbook
  cc-ansible --playbook path/to/playbook.yml

  # Update the passwords file for the environment
  cc-ansible edit_passwords
USAGE
  exit 1
}


# On init, there is no requirement that the site config directory be defined,
# because one of init's tasks is creating this directory. Set a default.
if [[ "${command:-}" == "init" ]]; then
  CC_ANSIBLE_SITE="${CC_ANSIBLE_SITE:-$DIR/site-config}"
elif [[ -z "${CC_ANSIBLE_SITE:-}" ]]; then
  cat <<ERRMSG
Error: no site specified! Please specify which site to execute under with either
the --site <dir> flag or by setting the CC_ANSIBLE_SITE environment variable.

Example:
  cc-ansible --site /etc/sites/production

  CC_ANSIBLE_SITE=/etc/sites/production "$(basename "$0")"
ERRMSG
  exit 1
fi

CC_ANSIBLE_VAULT_PASSWORD="${CC_ANSIBLE_VAULT_PASSWORD:-$CC_ANSIBLE_SITE/vault_password}"
CC_ANSIBLE_ENV="$CC_ANSIBLE_SITE/.env"
# shellcheck disable=1090 # TODO add sample .env file for testing
if [[ -f "$CC_ANSIBLE_ENV" ]]; then
  set -a; source "$(realpath "$CC_ANSIBLE_ENV")"; set +a
fi

# Automatically update dependencies
if [[ "$CHECK_UPDATES" == "yes" || "$FORCE_UPDATES" == "yes" ]]; then
  # Update base pip packages
  pip_requirements="$DIR/requirements.txt"
  pip_requirements_chksum="$VIRTUALENV/requirements.txt.sha256"
  if [[ "$FORCE_UPDATES" == "yes" || ! -f "$pip_requirements_chksum" ]] || \
        ! sha256sum --quiet --check "$pip_requirements_chksum"; then
    pip install --upgrade --force-reinstall -r "$pip_requirements"
    sha256sum "$pip_requirements" > "$pip_requirements_chksum"
  fi

  # Update Ansible Galaxy roles
  galaxy_role_path="$DIR/roles/galaxy.ansible.com"
  galaxy_role_requirements="$DIR/requirements.yml"
  galaxy_role_requirements_chksum="$VIRTUALENV/requirements.yml.sha256"
  if [[ "$FORCE_UPDATES" == "yes" || ! -f "$galaxy_role_requirements_chksum" ]] || \
        ! sha256sum --quiet --check "$galaxy_role_requirements_chksum"; then
    ansible-galaxy install --force -p "$galaxy_role_path" -r "$galaxy_role_requirements"
    sha256sum "$galaxy_role_requirements" > "$galaxy_role_requirements_chksum"
  fi

  kolla_ansible_remote=https://github.com/chameleoncloud/kolla-ansible.git
  kolla_ansible_checkout=${KOLLA_ANSIBLE_BRANCH:-chameleoncloud/train}
  kolla_ansible_gitref="$VIRTUALENV/kolla-ansible.gitref"
  kolla_ansible_egglink="$VIRTUALENV/src/kolla-ansible"
  if [[ "$FORCE_UPDATES" == "yes" || ! -f "$kolla_ansible_gitref" || ! -d "$kolla_ansible_egglink" ]] || \
        ! diff -q >/dev/null \
          "$kolla_ansible_gitref" \
          <(cd "$kolla_ansible_egglink"; git fetch; git show-ref -s -d origin/"$kolla_ansible_checkout"); then
    pushd "$VIRTUALENV" || ( echo "pushd error!" && exit 1 )
    pip install -e git+"$kolla_ansible_remote"@"$kolla_ansible_checkout"#egg=kolla-ansible
    popd || ( echo "popd error!" && exit 1 )
    # [jca 2020-01-30] TODO:
    # Ensure the /share folder is placed; this is not copied when using the "develop" setup.py method.
    # This is a bit weird, perhaps there is some way to pass an additional flag to pip to make it
    # copy this even though it's installing as source. We use source install to keep track of the Git revision.
    mkdir -p "$VIRTUALENV/share" && ln -sf "$kolla_ansible_egglink" "$VIRTUALENV/share/kolla-ansible"
    (cd "$kolla_ansible_egglink"; git rev-parse HEAD > "$kolla_ansible_gitref")
  fi

  # Update Mitogen
  MITOGEN_VERSION=0.2.8
  MITOGEN_TARBALL=/tmp/mitogen.tar.gz
  MITOGEN_INSTALL_DIR="$VIRTUALENV/lib/mitogen-$MITOGEN_VERSION"
  if [[ ! -d "$MITOGEN_INSTALL_DIR" ]]; then
    curl -L -o "$MITOGEN_TARBALL" "https://github.com/dw/mitogen/archive/v$MITOGEN_VERSION.tar.gz" \
      && tar -xf "$MITOGEN_TARBALL" -C "$(dirname "$MITOGEN_INSTALL_DIR")" \
      && rm -f "$MITOGEN_TARBALL"
  fi
  ln -sf "$MITOGEN_INSTALL_DIR" "$VIRTUALENV/lib/mitogen-latest"

  # Update/install yq
  YQ_VERSION=4.9.6
  if [[ "$(type -t yq)" != "file" ]]; then
    YQ_BINARY="yq_linux_amd64"
    wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz -O - \
      | tar xz && mv ${YQ_BINARY} "$VIRTUALENV/bin/yq"
  fi
fi

ansible_path="$VIRTUALENV/share/kolla-ansible/ansible"

# Handle subcommands
if [[ -n "${command:-}" ]]; then
  $command "$@"
  exit $?
fi

if [[ -n "${CC_ANSIBLE_PLAYBOOK:-}" ]]; then
  echo "**********************************************************************"
  echo "* Playbook override detected! This playbook will be executed within  *"
  echo "* Kolla-Ansible's playbook context.                                  *"
  echo "**********************************************************************"
  echo

  playbook_file="$ansible_path/$(basename "$CC_ANSIBLE_PLAYBOOK")"

  # Because we execute within Kolla-Ansible's playbook directory, "magic"
  # paths will not work any more--add our plugins/libraries to Ansible's
  # search path directly.
  export ANSIBLE_ACTION_PLUGINS="$DIR/playbooks/action_plugins"
  export ANSIBLE_LIBRARY="$DIR/playbooks/library"
  export ANSIBLE_TEMPLATES="$DIR/playbooks/templates"

  # Copy the playbook to a new location relative to the Kolla-Ansible installation
  # to allow the group_vars/ in the playbook directory to take effect.
  cp "$(realpath "$CC_ANSIBLE_PLAYBOOK")" "$playbook_file"
  _playbook_override_cleanup() {
    rm -f "$playbook_file"
  }
  TRAPS+=(_playbook_override_cleanup)

  # Prepare an invocation of Kolla-Ansible targeting this playbook
  POSARGS+=(deploy --playbook "$playbook_file")
fi

lockfile="$CC_ANSIBLE_SITE/.lock"
if [[ -f "$lockfile" ]]; then
  cat <<ERRMSG
ERROR: Lockfile $lockfile exists! Another process may be updating this site.
Wait until the other process finishes, or optionally stop the running process
in order to continue.
ERRMSG
  exit 1
fi

touch "$lockfile"
_lock_cleanup() {
  rm -f "$lockfile"
}
TRAPS+=(_lock_cleanup)

# Prepare the configuration directory for the Kolla-Ansible invocation:
CONFIG_DIR="$(mktemp -d)"
_config_dir_cleanup() {
  rm -rf "$CONFIG_DIR"
}
TRAPS+=(_config_dir_cleanup)
# 1. Copy base node_custom_config
rsync -a "$DIR/kolla/node_custom_config/" "$CONFIG_DIR/node_custom_config/"
# 2. Copy anything declared in site-config (can override files in node_custom_config)
rsync -a "$CC_ANSIBLE_SITE/" "$CONFIG_DIR/"
# 3. Ensure a defaults.yml is defined, if the site-config didn't define it.
if [[ ! -f "$CONFIG_DIR/defaults.yml" ]]; then
  echo -e "---\n_placeholder_for_valid_yaml:" >"$CONFIG_DIR/defaults.yml"
fi
if [[ ! -f "$CONFIG_DIR/globals.yml" ]]; then
  echo -e "---\n_placeholder_for_valid_yaml:" >"$CONFIG_DIR/globals.yml"
fi
# 4. Ensure inventory/group_vars directory exists
mkdir -p "${CONFIG_DIR}/inventory/group_vars/"
# 5. Stamp out a group vars file to set defaults for any host in baremetal group
#    (should be all nodes enrolled with Kolla-Ansible). Doing it this way allows
#    the site-config to override anything here in host_vars or more specific
#    group_vars files.
# shellcheck disable=2016 # yq is directly evaluating this string
yq eval-all '. as $item ireduce ({}; . * $item )' \
  "$DIR/kolla/defaults.yml" "$CONFIG_DIR/defaults.yml" \
  >"$CONFIG_DIR/inventory/group_vars/baremetal.yml"

declare -a kolla_args=()
kolla_args+=(--configdir="$CONFIG_DIR")
kolla_args+=(--inventory="$CONFIG_DIR/inventory")
kolla_args+=(--key="$CC_ANSIBLE_VAULT_PASSWORD")
kolla_args+=(--globals="$CONFIG_DIR/globals.yml")
kolla_args+=(--passwords="$CONFIG_DIR/passwords.yml")
kolla_args+=(--extra node_custom_config="$CONFIG_DIR/node_custom_config")
kolla_args+=(--extra cc_ansible_site_dir="$CONFIG_DIR")
kolla_args+=(--extra deployment_dir="$DIR")
kolla_args+=(--extra site_config_dir="$CC_ANSIBLE_SITE")

# We set ansible ansible_python_interpreter to the virtualenv inside site-config, and override here during bootstrap.
# Overriding at the CLI here applies to all hosts, including the deploy host.
# Bootstrap-servers creates a virtualenv at $VIRTUALENV, on all hosts except the deploy host, requiring a special case.
# The old behavior was to set it here if NOT bootstrapping, but that fails for the deploy-host case.

if [[ "${POSARGS[*]}" =~ bootstrap-servers ]]; then
  # Assume python3 can be found in $PATH, only used for creating the virtualenv.
  kolla_args+=(--extra ansible_python_interpreter="python3")
fi

if [[ "${POSARGS[*]}" =~ post-deploy ]]; then
  cp -f "$DIR/playbooks/post_deploy.yml" "$ansible_path/chi_in_a_box_post_deploy.yml"
  kolla_args+=(--extra post_deploy_extra_play="chi_in_a_box_post_deploy.yml")
fi
kolla_args+=("${POSARGS[@]}")

#load tags to skip
ANSIBLE_SKIP_TAGS="$(grep -v "^#" ./kolla-skip-tags | paste -sd ',' || true )"
if [[ -n "${ANSIBLE_SKIP_TAGS}" ]]; then
  kolla_args+=("--skip-tags")
  kolla_args+=("${ANSIBLE_SKIP_TAGS}")
fi

"$VIRTUALENV"/bin/kolla-ansible "${kolla_args[@]}"
# set +o xtrace