From df8ce91ae74cce8fe052bc3b5c29b9492b62e605 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Wed, 20 Dec 2023 23:44:08 +0100 Subject: [PATCH 01/10] Parameterize build branches If I want to merge a branch into the two-node brach, I should be able to test that branch, hence I should build from that branch. Signed-off-by: Oz Tiram --- test/test-two-node.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test-two-node.sh b/test/test-two-node.sh index d2c8f1f..5870cc7 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -57,7 +57,9 @@ export CLUSTER_VIP= # choose an unassigned VIP # image vars export EARTHLY_BUILDKIT_CACHE_SIZE_MB=100000 -export OCI_REGISTRY=ttl.sh +export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh} +export STYLUS_BRANCH=${STYLUS_BRANCH:-two-node} +export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-2-node-health-checks} # cluster vars export BACKEND=postgres # postgres or sqlite @@ -408,7 +410,7 @@ function build_canvos() { function build_all() { # optionally build/rebuild provider-k3s - test -d ../provider-k3s || ( cd .. && git clone https://github.com/kairos-io/provider-k3s -b two-node ) + test -d ../provider-k3s || ( cd .. && git clone https://github.com/kairos-io/provider-k3s -b ${PROVIDER_K3S_BRANCH}}) cd ../provider-k3s export PROVIDER_K3S_HASH=$(git describe --always) ( @@ -417,7 +419,7 @@ function build_all() { ) || ( build_provider_k3s ) # optionally build/rebuild stylus images - test -d ../stylus || ( cd .. && git clone https://github.com/spectrocloud/stylus -b 2-node-health-checks ) + test -d ../stylus || ( cd .. && git clone https://github.com/spectrocloud/stylus -b ${STYLUS_BRANCH} ) cd ../stylus export STYLUS_HASH=$(git describe --always) ( From 74eedb8058e130f6ca95080d127d66c70fd4f5f6 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Wed, 20 Dec 2023 23:50:57 +0100 Subject: [PATCH 02/10] Move the variables for the test script to own file This allows merging changes to the test script without risking git merge havoc if these are overriden in the script. Also, it prevents people from accidentally commiting secrets. Signed-off-by: Oz Tiram --- .gitignore | 1 + test/env.example | 36 +++++++++++++++++++++++++++++++++ test/test-two-node.sh | 47 ++++++++----------------------------------- 3 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 test/env.example diff --git a/.gitignore b/.gitignore index e600fa6..245dd93 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ content-*/* .idea hack/*.img .DS_Store +test/.env diff --git a/test/env.example b/test/env.example new file mode 100644 index 0000000..0301579 --- /dev/null +++ b/test/env.example @@ -0,0 +1,36 @@ +export GOVC_USERNAME=@vsphere.local +export GOVC_PASSWORD= +# this is the IP address of US VMWare lab +# you can set this to India lab +export GOVC_URL=10.10.128.10 +export GOVC_INSECURE=true +export GOVC_DATACENTER=Datacenter +export GOVC_DATASTORE=vsanDatastore2 +export GOVC_NETWORK=VM-NETWORK +export GOVC_RESOURCE_POOL= +export GOVC_FOLDER= + +# vSphere vars +export HOST_SUFFIX= # required to ensure unique edge host IDs +export ISO_FOLDER= e.g. "ISO/01-tyler" +export STYLUS_ISO="${ISO_FOLDER}/stylus-dev-amd64.iso" +export NIC_NAME=ens160 + +# palette vars +export API_KEY= +export PROJECT_UID= +export EDGE_REGISTRATION_TOKEN= +export DOMAIN=dev.spectrocloud.com +export PUBLIC_PACK_REPO_UID= # this varies per Palette tenant +export CLUSTER_NAME=two-node +export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created +export CLUSTER_VIP= # choose an unassigned VIP + +# image vars +export EARTHLY_BUILDKIT_CACHE_SIZE_MB=100000 +export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh} +export STYLUS_BRANCH=${STYLUS_BRANCH:-two-node} +export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-2-node-health-checks} + +# cluster vars +export BACKEND=postgres # postgres or sqlite diff --git a/test/test-two-node.sh b/test/test-two-node.sh index 5870cc7..d6e2a2f 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -19,51 +19,20 @@ set -e # credentials capable of cloning Spectro Cloud internal repos # (required for building stylus). # -# 4. Edit the global variables below as needed. +# 4. Copy the test/en.example file to test/.env and edit test/.env +# as required. # # 5. Source and execute this script: # # source ./test/test-two-node.sh # ./test/test-two-node.sh -# Edit these variables - -# govc vars -export GOVC_USERNAME=@vsphere.local -export GOVC_PASSWORD= -export GOVC_URL=10.10.128.10 -export GOVC_INSECURE=true -export GOVC_DATACENTER=Datacenter -export GOVC_DATASTORE=vsanDatastore2 -export GOVC_NETWORK=VM-NETWORK -export GOVC_RESOURCE_POOL= -export GOVC_FOLDER= - -# vSphere vars -export HOST_SUFFIX= # required to ensure unique edge host IDs -export ISO_FOLDER= e.g. "ISO/01-tyler" -export STYLUS_ISO="${ISO_FOLDER}/stylus-dev-amd64.iso" -export NIC_NAME=ens160 - -# palette vars -export API_KEY= -export PROJECT_UID= -export EDGE_REGISTRATION_TOKEN= -export DOMAIN=dev.spectrocloud.com -export PUBLIC_PACK_REPO_UID= # this varies per Palette tenant -export CLUSTER_NAME=two-node -export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created -export CLUSTER_VIP= # choose an unassigned VIP - -# image vars -export EARTHLY_BUILDKIT_CACHE_SIZE_MB=100000 -export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh} -export STYLUS_BRANCH=${STYLUS_BRANCH:-two-node} -export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-2-node-health-checks} - -# cluster vars -export BACKEND=postgres # postgres or sqlite - +if [ -f ./test/.env ]; then + source ./test/.env +else + echo "Please create a .env file in the test directory and populate it with the required variables." + exit 1 +fi # Do not edit anything below From 98a47b307e6c73087000e37fa59c47b94e841a7f Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Thu, 21 Dec 2023 10:40:16 +0100 Subject: [PATCH 03/10] Make use of absolute path to env file This allows running the script from the test dir and also any other path (e.g. top dir of the repo) Signed-off-by: Oz Tiram --- test/test-two-node.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test-two-node.sh b/test/test-two-node.sh index d6e2a2f..f1d153f 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -26,9 +26,9 @@ set -e # # source ./test/test-two-node.sh # ./test/test-two-node.sh - -if [ -f ./test/.env ]; then - source ./test/.env +envfile=$(dirname $0)/.env +if [ -f ${envfile} ]; then + source ${envfile} else echo "Please create a .env file in the test directory and populate it with the required variables." exit 1 From 3d65bea3aa652d36ae685c81636499e11ef1a55f Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Thu, 21 Dec 2023 10:41:07 +0100 Subject: [PATCH 04/10] Remove extra curly brace --- test/test-two-node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-two-node.sh b/test/test-two-node.sh index f1d153f..1ba326f 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -379,7 +379,7 @@ function build_canvos() { function build_all() { # optionally build/rebuild provider-k3s - test -d ../provider-k3s || ( cd .. && git clone https://github.com/kairos-io/provider-k3s -b ${PROVIDER_K3S_BRANCH}}) + test -d ../provider-k3s || ( cd .. && git clone https://github.com/kairos-io/provider-k3s -b ${PROVIDER_K3S_BRANCH}) cd ../provider-k3s export PROVIDER_K3S_HASH=$(git describe --always) ( From 2196fe5eb9b47c7a2e5152002ca98cca7db56e69 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Thu, 21 Dec 2023 17:03:28 +0100 Subject: [PATCH 05/10] Fix how PG_CONF_DIR is referenced Also, used explicit RUN command for each shell line Signed-off-by: Oz Tiram --- Earthfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Earthfile b/Earthfile index b1ee76d..309cc06 100644 --- a/Earthfile +++ b/Earthfile @@ -360,11 +360,11 @@ base-image: if grep "selinux=1" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/selinux=1/selinux=0/g' /etc/cos/bootargs.cfg; fi IF $TWO_NODE - RUN sed -i '/^#wal_level = replica/ s/#wal_level = replica/wal_level = logical/' $PG_CONF_DIR/postgresql.conf && \ - sed -i '/^#max_worker_processes = 8/ s/#max_worker_processes = 8/max_worker_processes = 16/' $PG_CONF_DIR/postgresql.conf && \ - sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" $PG_CONF_DIR/postgresql.conf && \ - echo "host all all 0.0.0.0/0 md5" | sudo tee -a PG_CONF_DIR/pg_hba.conf && \ - systemctl enable postresql + RUN sed -i '/^#wal_level = replica/ s/#wal_level = replica/wal_level = logical/' ${PG_CONF_DIR}/postgresql.conf + RUN sed -i '/^#max_worker_processes = 8/ s/#max_worker_processes = 8/max_worker_processes = 16/' ${PG_CONF_DIR}/postgresql.conf + RUN sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" ${PG_CONF_DIR}/postgresql.conf + RUN echo "host all all 0.0.0.0/0 md5" | tee -a ${PG_CONF_DIR}/pg_hba.conf + RUN systemctl enable postresql END # Used to build the installer image. The installer ISO will be created from this. From 0154e9c7ddf5649bc377451ada41ad2672c30882 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Thu, 21 Dec 2023 21:34:23 +0000 Subject: [PATCH 06/10] Fix IF ELSE for PG_CONF_FILE --- Earthfile | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Earthfile b/Earthfile index b1ee76d..f2b9ec8 100644 --- a/Earthfile +++ b/Earthfile @@ -49,6 +49,17 @@ ELSE IF [ "$OS_DISTRIBUTION" = "rhel" ] || [ "$OS_DISTRIBUTION" = "sles" ] ARG BASE_IMAGE END +IF [ "$OS_DISTRIBUTION" = "ubuntu" ] + IF [ "$TWO_NODE_BACKEND" = "postgres" ] + ARG PG_CONF_DIR=/etc/postgresql/16/main + END +ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] + IF [ "$TWO_NODE_BACKEND" = "postgres" ] + ARG PG_CONF_DIR=/var/lib/pgsql/data + END +END + + IF [[ "$BASE_IMAGE" =~ "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] ARG IS_JETSON=true END @@ -242,15 +253,8 @@ base-image: RUN mkdir -p /opt/spectrocloud/bin && \ curl -sL https://github.com/maxpert/marmot/releases/download/v"${MARMOT_VERSION}"/marmot-v"${MARMOT_VERSION}"-linux-amd64-static.tar.gz | tar -zxv marmot && \ install marmot -o root -g root -m 755 /opt/spectrocloud/bin/ && \ - rm -f marmot \ + rm -f marmot && \ curl -sL https://github.com/k3s-io/kine/releases/download/v${KINE_VERSION}/kine-amd64 | install -m 755 /dev/stdin /opt/spectrocloud/bin/kine - IF [ TWO_NODE_BACKEND = "postgres" ] - IF [ $OS_DISTRIBUTION = "ubuntu" ] - ARG PG_CONF_DIR=/etc/postgresql/16/main/ - ELSE IF [ $OS_DISTRIBUTION = "opensuse-leap" ] - ARG PG_CONF_DIR=/var/lib/pgsql/data/ - END - END END IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$ARCH" = "amd64" ] @@ -281,7 +285,7 @@ base-image: apt clean IF $TWO_NODE - IF [ $TWO_NODE_BACKEND = "sqlite" ] + IF [ "$TWO_NODE_BACKEND" = "sqlite" ] RUN apt install -y sqlite3 iputils-ping ELSE RUN apt install -y apt-transport-https ca-certificates curl && \ @@ -360,11 +364,11 @@ base-image: if grep "selinux=1" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/selinux=1/selinux=0/g' /etc/cos/bootargs.cfg; fi IF $TWO_NODE - RUN sed -i '/^#wal_level = replica/ s/#wal_level = replica/wal_level = logical/' $PG_CONF_DIR/postgresql.conf && \ - sed -i '/^#max_worker_processes = 8/ s/#max_worker_processes = 8/max_worker_processes = 16/' $PG_CONF_DIR/postgresql.conf && \ - sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" $PG_CONF_DIR/postgresql.conf && \ - echo "host all all 0.0.0.0/0 md5" | sudo tee -a PG_CONF_DIR/pg_hba.conf && \ - systemctl enable postresql + RUN sed -i '/^#wal_level = replica/ s/#wal_level = replica/wal_level = logical/' "${PG_CONF_DIR}"/postgresql.conf && \ + sed -i '/^#max_worker_processes = 8/ s/#max_worker_processes = 8/max_worker_processes = 16/' ${PG_CONF_DIR}/postgresql.conf && \ + sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" ${PG_CONF_DIR}/postgresql.conf && \ + echo "host all all 0.0.0.0/0 md5" | tee -a ${PG_CONF_DIR}/pg_hba.conf && \ + systemctl enable postgresql END # Used to build the installer image. The installer ISO will be created from this. From 12f1a761deab73f658892bc5ff2a7a5e37d4d576 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Thu, 21 Dec 2023 15:12:52 -0700 Subject: [PATCH 07/10] Update test-two-node.sh --- test/test-two-node.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test-two-node.sh b/test/test-two-node.sh index 1ba326f..371811f 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -19,13 +19,16 @@ set -e # credentials capable of cloning Spectro Cloud internal repos # (required for building stylus). # -# 4. Copy the test/en.example file to test/.env and edit test/.env +# 4. Copy the test/env.example file to test/.env and edit test/.env # as required. # # 5. Source and execute this script: # # source ./test/test-two-node.sh # ./test/test-two-node.sh + +# Do not edit anything below + envfile=$(dirname $0)/.env if [ -f ${envfile} ]; then source ${envfile} @@ -34,8 +37,6 @@ else exit 1 fi -# Do not edit anything below - declare -a vm_array=("2n1-$HOST_SUFFIX" "2n2-$HOST_SUFFIX") export HOST_1="${vm_array[0]}-$HOST_SUFFIX" export HOST_2="${vm_array[1]}-$HOST_SUFFIX" From 755b902026100c193504093243d2868a77dbc6ad Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Thu, 21 Dec 2023 15:13:13 -0700 Subject: [PATCH 08/10] Update Earthfile --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index f2b9ec8..2398db9 100644 --- a/Earthfile +++ b/Earthfile @@ -49,7 +49,7 @@ ELSE IF [ "$OS_DISTRIBUTION" = "rhel" ] || [ "$OS_DISTRIBUTION" = "sles" ] ARG BASE_IMAGE END -IF [ "$OS_DISTRIBUTION" = "ubuntu" ] +IF [ "$OS_DISTRIBUTION" = "ubuntu" ] IF [ "$TWO_NODE_BACKEND" = "postgres" ] ARG PG_CONF_DIR=/etc/postgresql/16/main END From e9e97297c037262255f9a5c6e27d75441bfa1cf9 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Thu, 21 Dec 2023 15:15:47 -0700 Subject: [PATCH 09/10] Update env.example --- test/env.example | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/env.example b/test/env.example index 0301579..23f7e4c 100644 --- a/test/env.example +++ b/test/env.example @@ -1,8 +1,7 @@ +# govc vars export GOVC_USERNAME=@vsphere.local export GOVC_PASSWORD= -# this is the IP address of US VMWare lab -# you can set this to India lab -export GOVC_URL=10.10.128.10 +export GOVC_URL=10.10.128.10 # IP address of USDC; edit as needed export GOVC_INSECURE=true export GOVC_DATACENTER=Datacenter export GOVC_DATASTORE=vsanDatastore2 @@ -21,7 +20,7 @@ export API_KEY= export PROJECT_UID= export EDGE_REGISTRATION_TOKEN= export DOMAIN=dev.spectrocloud.com -export PUBLIC_PACK_REPO_UID= # this varies per Palette tenant +export PUBLIC_PACK_REPO_UID= # this varies per Palette tenant, identify via Chrome inspector on Tenant Admin -> Pack Registries page export CLUSTER_NAME=two-node export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created export CLUSTER_VIP= # choose an unassigned VIP @@ -32,5 +31,5 @@ export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh} export STYLUS_BRANCH=${STYLUS_BRANCH:-two-node} export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-2-node-health-checks} -# cluster vars -export BACKEND=postgres # postgres or sqlite +# two node vars +export BACKEND=postgres # one of: [ postgres | sqlite ] From 3e2b06540c4415ebf88b48eabf3cb0baec3b5892 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Thu, 21 Dec 2023 15:18:02 -0700 Subject: [PATCH 10/10] Update Earthfile --- Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Earthfile b/Earthfile index 2398db9..9fcb41f 100644 --- a/Earthfile +++ b/Earthfile @@ -30,7 +30,7 @@ ARG no_proxy=${NO_PROXY} ARG PROXY_CERT_PATH ARG UPDATE_KERNEL=false ARG TWO_NODE=false -# You can use either sqlite or postgres as backend +# You can use either sqlite or postgres ARG TWO_NODE_BACKEND=sqlite ARG MARMOT_VERSION=0.8.7-beta.1 @@ -49,6 +49,7 @@ ELSE IF [ "$OS_DISTRIBUTION" = "rhel" ] || [ "$OS_DISTRIBUTION" = "sles" ] ARG BASE_IMAGE END +# Determine PG config dir (only relevant if TWO_NODE=true) IF [ "$OS_DISTRIBUTION" = "ubuntu" ] IF [ "$TWO_NODE_BACKEND" = "postgres" ] ARG PG_CONF_DIR=/etc/postgresql/16/main @@ -59,7 +60,6 @@ ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] END END - IF [[ "$BASE_IMAGE" =~ "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] ARG IS_JETSON=true END