Skip to content

Commit

Permalink
Stop using dev-env in E2E setup
Browse files Browse the repository at this point in the history
Signed-off-by: Huy Mai <[email protected]>
  • Loading branch information
mquhuy committed Jan 16, 2025
1 parent c8eb5fc commit beb15b0
Show file tree
Hide file tree
Showing 24 changed files with 793 additions and 160 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ e2e-tests: $(GINKGO) e2e-substitutions cluster-templates # This target should be
--focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) "$(ROOT_DIR)/$(TEST_DIR)/e2e/" -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.bmcsConfig="$(E2E_BMCS_CONF_FILE)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \
-e2e.trigger-ephemeral-test=$(EPHEMERAL_TEST) \
-e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)
Expand Down
34 changes: 34 additions & 0 deletions hack/e2e/fake-ipa.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

set -euxo pipefail

launch_fake_ipa()
{
# Create a folder to host fakeIPA config and certs
mkdir -p "${WORKING_DIR}/fake-ipa"
if [[ "${EPHEMERAL_CLUSTER}" = "kind" ]] && [[ "${IRONIC_TLS_SETUP}" = "true" ]]; then
cp "${IRONIC_CACERT_FILE}" "${WORKING_DIR}/fake-ipa/ironic-ca.crt"
elif [[ "${IRONIC_TLS_SETUP}" = "true" ]]; then
# wait for ironic to be running to ensure ironic-cert is created
kubectl -n baremetal-operator-system wait --for=condition=available deployment/baremetal-operator-ironic --timeout=900s
# Extract ironic-cert to be used inside fakeIPA for TLS
kubectl get secret -n baremetal-operator-system ironic-cert -o json -o=jsonpath="{.data.ca\.crt}" | base64 -d > "${WORKING_DIR}/fake-ipa/ironic-ca.crt"
fi

# Create fake IPA custom config
cat <<EOF > "${WORKING_DIR}/fake-ipa/config.py"
FAKE_IPA_API_URL = "https://${CLUSTER_BARE_METAL_PROVISIONER_IP}:${IRONIC_API_PORT}"
FAKE_IPA_INSPECTION_CALLBACK_URL = "${IRONIC_URL}/continue_inspection"
FAKE_IPA_ADVERTISE_ADDRESS_IP = "${EXTERNAL_SUBNET_V4_HOST}"
FAKE_IPA_INSECURE = ${FAKE_IPA_INSECURE:-False}
FAKE_IPA_CAFILE = "${FAKE_IPA_CAFILE:-/root/cert/ironic-ca.crt}"
FAKE_IPA_MIN_BOOT_TIME = ${FAKE_IPA_MIN_BOOT_TIME:-20}
FAKE_IPA_MAX_BOOT_TIME = ${FAKE_IPA_MAX_BOOT_TIME:-30}
EOF

# shellcheck disable=SC2086
sudo "${CONTAINER_RUNTIME}" run -d --net host --name fake-ipa ${POD_NAME_INFRA} \
-v "/opt/metal3-dev-env/fake-ipa":/root/cert -v "/root/.ssh":/root/ssh \
-e CONFIG='/root/cert/config.py' \
"${FAKE_IPA_IMAGE}"
}
16 changes: 16 additions & 0 deletions hack/e2e/net.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<network>
<name>baremetal-e2e</name>
<forward mode='nat'>
<nat>
<port start='1024' end='65535'/>
</nat>
</forward>
<bridge name='metal3'/>
<ip address='192.168.222.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.222.2' end='192.168.222.199'/>
<host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
<bootp file='http://192.168.222.199:6180/boot.ipxe'/>
</dhcp>
</ip>
</network>
49 changes: 49 additions & 0 deletions hack/ensure_kubectl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

USR_LOCAL_BIN="/usr/local/bin"
MINIMUM_KUBECTL_VERSION=v1.32.0

# Ensure the kubectl tool exists and is a viable version, or installs it
verify_kubectl_version()
{
# If kubectl is not available on the path, get it
if ! [ -x "$(command -v kubectl)" ]; then
if [[ "${OSTYPE}" == "linux-gnu" ]]; then
echo "kubectl not found, installing"
curl -LO "https://dl.k8s.io/release/${MINIMUM_KUBECTL_VERSION}/bin/linux/amd64/kubectl"
sudo install kubectl "${USR_LOCAL_BIN}/kubectl"
else
echo "Missing required binary in path: kubectl"
return 2
fi
fi

local kubectl_version
IFS=" " read -ra kubectl_version <<< "$(kubectl version --client)"
if [[ "${MINIMUM_KUBECTL_VERSION}" != $(echo -e "${MINIMUM_KUBECTL_VERSION}\n${kubectl_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then
cat << EOF
Detected kubectl version: ${kubectl_version[2]}.
Requires ${MINIMUM_KUBECTL_VERSION} or greater.
Please install ${MINIMUM_KUBECTL_VERSION} or later.
EOF
return 2
fi
}

verify_kubectl_version
33 changes: 33 additions & 0 deletions hack/ensure_minikube.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash

set -eux

USR_LOCAL_BIN="/usr/local/bin"
OS=$(go env GOOS)
ARCH=$(go env GOARCH)
MINIMUM_MINIKUBE_VERSION=v1.33.0

verify_minikube_version() {
if ! [ -x "$(command -v minikube)" ]; then
if [[ "${OSTYPE}" == "linux-gnu" ]]; then
echo "minikube not found, installing"
curl -LO "https://storage.googleapis.com/minikube/releases/${MINIMUM_MINIKUBE_VERSION}/minikube-${OS}-${ARCH}"
sudo install minikube-linux-amd64 "${USR_LOCAL_BIN}/minikube"
else
echo "Missing required binary in path: minikube"
return 2
fi
fi
local minikube_version
minikube_version="$(minikube version --short)"
if [[ "${MINIMUM_MINIKUBE_VERSION}" != $(echo -e "${MINIMUM_MINIKUBE_VERSION}\n${minikube_version}" | sort -V | head -n1) ]]; then
cat << EOF
Detected minikube version: ${minikube_version}.
Requires ${MINIMUM_MINIKUBE_VERSION} or greater.
Please install ${MINIMUM_MINIKUBE_VERSION} or later.
EOF
return 2
fi
}

verify_minikube_version
157 changes: 89 additions & 68 deletions scripts/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,37 @@
set -euxo pipefail

REPO_ROOT=$(realpath "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/..)
cd "${REPO_ROOT}"
cd "${REPO_ROOT}" || exit 1
export CAPM3PATH="${REPO_ROOT}"
export WORKING_DIR=/opt/metal3-dev-env
export WORKING_DIR=/tmp/metal3-dev-env
FORCE_REPO_UPDATE="${FORCE_REPO_UPDATE:-false}"

export CAPM3RELEASEBRANCH="${CAPM3RELEASEBRANCH:-main}"

# Verify they are available and have correct versions.
PATH=$PATH:/usr/local/go/bin
PATH=$PATH:$(go env GOPATH)/bin

# shellcheck source=./hack/ensure-go.sh
source "${REPO_ROOT}/hack/ensure-go.sh"
# shellcheck source=./hack/ensure-kind.sh
source "${REPO_ROOT}/hack/ensure-kind.sh"
# shellcheck source=./hack/ensure-kubectl.sh
source "${REPO_ROOT}/hack/ensure_kubectl.sh"
# shellcheck source=./hack/e2e/fake-ipa.sh
source "${REPO_ROOT}/hack/e2e/fake-ipa.sh"

"${REPO_ROOT}/hack/ensure_minikube.sh"
# Ensure kustomize
make kustomize

# Extract release version from release-branch name
export CAPM3RELEASE="v1.10.99"
export CAPI_RELEASE_PREFIX="v1.9."
if [[ "${CAPM3RELEASEBRANCH}" == release-* ]]; then
CAPM3_RELEASE_PREFIX="${CAPM3RELEASEBRANCH#release-}"
export CAPM3RELEASE="v${CAPM3_RELEASE_PREFIX}.99"
export CAPI_RELEASE_PREFIX="v${CAPM3_RELEASE_PREFIX}."
else
export CAPM3RELEASE="v1.10.99"
export CAPI_RELEASE_PREFIX="v1.9."
fi

# Default CAPI_CONFIG_FOLDER to $HOME/.config folder if XDG_CONFIG_HOME not set
Expand All @@ -27,72 +43,77 @@ export CAPI_CONFIG_FOLDER="${CONFIG_FOLDER}/cluster-api"
# shellcheck source=./scripts/environment.sh
source "${REPO_ROOT}/scripts/environment.sh"

# Clone dev-env repo
sudo mkdir -p ${WORKING_DIR}
sudo chown "${USER}":"${USER}" ${WORKING_DIR}
M3_DEV_ENV_REPO="https://github.com/metal3-io/metal3-dev-env.git"
M3_DEV_ENV_BRANCH=main
M3_DEV_ENV_PATH="${M3_DEV_ENV_PATH:-${WORKING_DIR}/metal3-dev-env}"
clone_repo "${M3_DEV_ENV_REPO}" "${M3_DEV_ENV_BRANCH}" "${M3_DEV_ENV_PATH}"

# Config devenv
cat <<-EOF >"${M3_DEV_ENV_PATH}/config_${USER}.sh"
export CAPI_VERSION=${CAPI_VERSION:-"v1beta1"}
export CAPM3_VERSION=${CAPM3_VERSION:-"v1beta1"}
export NUM_NODES=${NUM_NODES:-"4"}
export KUBERNETES_VERSION=${KUBERNETES_VERSION}
export IMAGE_OS=${IMAGE_OS}
export FORCE_REPO_UPDATE="false"
export USE_IRSO="${USE_IRSO:-false}"
EOF
# if running a scalability test skip apply bmhs in dev-env and run fakeIPA
if [[ ${GINKGO_FOCUS:-} == "clusterctl-upgrade" ]]; then
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
fi
if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
mkdir -p "$CAPI_CONFIG_FOLDER"
echo "ENABLE_BMH_NAME_BASED_PREALLOCATION: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
fi
# if running a scalability tests, configure dev-env with fakeIPA
if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
export NUM_NODES="${NUM_NODES:-100}"
echo 'export NODES_PLATFORM="fake"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
sed -i "s/^export NUM_NODES=.*/export NUM_NODES=${NUM_NODES:-100}/" "${M3_DEV_ENV_PATH}/config_${USER}.sh"
mkdir -p "$CAPI_CONFIG_FOLDER"
echo 'CLUSTER_TOPOLOGY: true' >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
echo 'export EPHEMERAL_CLUSTER="minikube"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
BMC_PROTOCOL="${BMC_PROTOCOL:-"redfish-virtualmedia"}"
if [[ "${BMC_PROTOCOL}" == "redfish" ]] || [[ "${BMC_PROTOCOL}" == "redfish-virtualmedia" ]]; then
BMO_E2E_EMULATOR="sushy-tools"
elif [[ "${BMC_PROTOCOL}" == "ipmi" ]]; then
BMO_E2E_EMULATOR="vbmc"
else
# Don't run scalability tests if not asked for.
export GINKGO_SKIP="${GINKGO_SKIP:-} scalability"
echo "FATAL: Invalid BMC protocol specified: ${BMC_PROTOCOL}"
exit 1
fi
# Run make devenv to boot the source cluster
pushd "${M3_DEV_ENV_PATH}" || exit 1
make
popd || exit 1

# Binaries checked below should have been installed by metal3-dev-env make.
# Verify they are available and have correct versions.
PATH=$PATH:/usr/local/go/bin
PATH=$PATH:$(go env GOPATH)/bin
echo "BMC_PROTOCOL=${BMC_PROTOCOL}"
echo "BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR}"
export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMC_PROTOCOL}.yaml"

# Clone BMO repo and install vbmctl
if ! command -v vbmctl >/dev/null 2>&1; then
clone_repo "https://github.com/metal3-io/baremetal-operator.git" "main" "${WORKING_DIR}/baremetal-operator"
pushd "${WORKING_DIR}/baremetal-operator/test/vbmctl/"
go build -tags=e2e,integration -o vbmctl ./main.go
sudo install vbmctl /usr/local/bin/vbmctl
popd
fi

# shellcheck source=./hack/ensure-go.sh
source "${REPO_ROOT}/hack/ensure-go.sh"
# shellcheck source=./hack/ensure-kind.sh
source "${REPO_ROOT}/hack/ensure-kind.sh"
# shellcheck source=./hack/ensure-kubectl.sh
source "${REPO_ROOT}/hack/ensure-kubectl.sh"
# Ensure kustomize
make kustomize
# Set up minikube
minikube start --driver=kvm2

virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
virsh -c qemu:///system net-start baremetal-e2e
# Attach baremetal-e2e interface to minikube with specific mac.
# This will give minikube a known reserved IP address that we can use for Ironic
virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
--model virtio --source baremetal-e2e --type network --config

# Restart minikube to apply the changes
minikube stop
## Following loop is to avoid minikube restart issue
## https://github.com/kubernetes/minikube/issues/14456
while ! minikube start; do sleep 30; done

# This IP is defined by the network we created above.
IP_ADDRESS="192.168.222.1"
vbmctl --yaml-source-file "${E2E_BMCS_CONF_FILE}"
if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
# Start VBMC
docker run --name vbmc --network host -d \
-v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
-v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
quay.io/metal3-io/vbmc

readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${E2E_BMCS_CONF_FILE}")
for bmc in "${BMCS[@]}"; do
address=$(echo "${bmc}" | jq -r '.address')
hostName=$(echo "${bmc}" | jq -r '.hostName')
vbmc_port="${address##*:}"
"${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
done

elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
# Sushy-tools variables
SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/data/sushy-tools/sushy-emulator.conf
# Start sushy-tools
docker run --name sushy-tools -d --network host \
-v "${SUSHY_EMULATOR_FILE}":/etc/sushy/sushy-emulator.conf:Z \
-v /var/run/libvirt:/var/run/libvirt:Z \
-e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \
quay.io/metal3-io/sushy-tools:latest sushy-emulator

# shellcheck disable=SC1091,SC1090
source "${M3_DEV_ENV_PATH}/lib/images.sh"
# shellcheck disable=SC1091,SC1090
source "${M3_DEV_ENV_PATH}/lib/releases.sh"
# shellcheck disable=SC1091,SC1090
source "${M3_DEV_ENV_PATH}/lib/ironic_basic_auth.sh"
# shellcheck disable=SC1091,SC1090
source "${M3_DEV_ENV_PATH}/lib/ironic_tls_setup.sh"
else
echo "FATAL: Invalid e2e emulator specified: ${BMO_E2E_EMULATOR}"
exit 1
fi

# Generate credentials
BMO_OVERLAYS=(
Expand All @@ -115,15 +136,15 @@ if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
# If usernames and passwords are unset, read them from file or generate them
if [[ -z "${IRONIC_USERNAME:-}" ]]; then
if [[ ! -f "${IRONIC_AUTH_DIR}/ironic-username" ]]; then
IRONIC_USERNAME="$(uuid-gen)"
IRONIC_USERNAME="$(uuidgen)"
echo "${IRONIC_USERNAME}" > "${IRONIC_AUTH_DIR}/ironic-username"
else
IRONIC_USERNAME="$(cat "${IRONIC_AUTH_DIR}/ironic-username")"
fi
fi
if [[ -z "${IRONIC_PASSWORD:-}" ]]; then
if [ ! -f "${IRONIC_AUTH_DIR}/ironic-password" ]; then
IRONIC_PASSWORD="$(uuid-gen)"
IRONIC_PASSWORD="$(uuidgen)"
echo "${IRONIC_PASSWORD}" > "${IRONIC_AUTH_DIR}/ironic-password"
else
IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
Expand Down
27 changes: 27 additions & 0 deletions scripts/clean-e2e.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env bash

REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
cd "${REPO_ROOT}" || exit 1

minikube delete
docker rm -f vbmc
docker rm -f image-server-e2e
docker rm -f sushy-tools

"${REPO_ROOT}/hack/e2e/data/clean_local_bmh_test_setup.sh" "^node-"

rm -rf "${REPO_ROOT}/test/e2e/_artifacts"
rm -rf "${REPO_ROOT}"/artifacts-*
rm -rf "${REPO_ROOT}/test/e2e/images"

# Clear network
virsh -c qemu:///system net-destroy baremetal-e2e
virsh -c qemu:///system net-undefine baremetal-e2e

# Clean volume pool directory
rm -rf /tmp/pool_oo/*

# Clean volume pool
virsh pool-destroy default || true
virsh pool-delete default || true
virsh pool-undefine default || true
Loading

0 comments on commit beb15b0

Please sign in to comment.