feat: Two node edge clusters

.gitignore

@@ -8,5 +8,6 @@ config.yaml
 content-*/*
 *.arg
 .idea
-
-.DS_Store
+hack/*.img
+.DS_Store
+test/.env

Dockerfile

@@ -17,15 +17,18 @@ RUN if [ "${OS_DISTRIBUTION}" = "opensuse-leap" ] && [ "${PROXY_CERT_PATH}" != "
  update-ca-certificates; \
  fi
 
-###########################Add any other image customizations here #######################
+########################### Add any other image customizations here #######################
 
 #### Examples ####
 
 ### To install the nginx package for Ubuntu ###
 
 # RUN apt-get update && apt-get install nginx -y
-### or
 
 ### To install the nginx package for opensuse ###
 
 # RUN zypper refresh && zypper install nginx -y
+
+### To add a custom health script for two-node liveness checks ###
+
+# ADD overlay/files/opt/spectrocloud/bin/check-disk-size.sh /opt/spectrocloud/bin/

Earthfile

@@ -32,7 +32,8 @@ ARG https_proxy=${HTTPS_PROXY}
 ARG no_proxy=${NO_PROXY}
 ARG PROXY_CERT_PATH
 ARG UPDATE_KERNEL=false
-
+ARG TWO_NODE=false
+ARG KINE_VERSION=0.10.3
 ARG ETCD_VERSION="v3.5.5"
 
 IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$BASE_IMAGE" = "" ]
@@ -73,20 +74,18 @@ build-all-images:
  END
 
 build-provider-images:
- BUILD +provider-image --K8S_VERSION=1.24.6
- BUILD +provider-image --K8S_VERSION=1.25.2
- BUILD +provider-image --K8S_VERSION=1.26.4
- BUILD +provider-image --K8S_VERSION=1.27.2
- BUILD +provider-image --K8S_VERSION=1.25.13
- BUILD +provider-image --K8S_VERSION=1.26.8
- BUILD +provider-image --K8S_VERSION=1.27.5
- BUILD +provider-image --K8S_VERSION=1.27.7
- BUILD +provider-image --K8S_VERSION=1.26.10
- BUILD +provider-image --K8S_VERSION=1.25.15
+ # BUILD +provider-image --K8S_VERSION=1.24.6
+ # BUILD +provider-image --K8S_VERSION=1.25.2
+ # BUILD +provider-image --K8S_VERSION=1.26.4
+ # BUILD +provider-image --K8S_VERSION=1.27.2
+ # BUILD +provider-image --K8S_VERSION=1.25.13
+ # BUILD +provider-image --K8S_VERSION=1.26.8
+ # BUILD +provider-image --K8S_VERSION=1.27.5
+ # BUILD +provider-image --K8S_VERSION=1.27.7
+ # BUILD +provider-image --K8S_VERSION=1.26.10
+ # BUILD +provider-image --K8S_VERSION=1.25.15
  BUILD +provider-image --K8S_VERSION=1.28.2
 
-
-
 build-provider-images-fips:
  IF [ "$K8S_DISTRIBUTION" = "kubeadm-fips" ]
  BUILD +provider-image --K8S_VERSION=1.24.13
@@ -247,6 +246,7 @@ base-image:
  ARG BASE_K8S_VERSION=$K8S_VERSION-$K8S_DISTRIBUTION_TAG
  END
 
+ # OS == Ubuntu
  IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$ARCH" = "amd64" ]
  # Add proxy certificate if present
  IF [ ! -z $PROXY_CERT_PATH ]
@@ -275,30 +275,46 @@ base-image:
 
  RUN rm -rf /var/cache/* && \
  apt clean
+
+ IF $TWO_NODE
+ RUN apt install -y apt-transport-https ca-certificates curl && \
+ echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
+ curl -fsSL -o postgresql.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
+ gpg --batch --yes --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg postgresql.asc && \
+ rm postgresql.asc && \
+ apt update && \
+ apt install -y postgresql-16 postgresql-contrib-16 iputils-ping
+ END
 
- # IF OS Type is Opensuse
+ # OS == Opensuse
  ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$ARCH" = "amd64" ]
  # Add proxy certificate if present
  IF [ ! -z $PROXY_CERT_PATH ]
  COPY sc.crt /usr/share/pki/trust/anchors
- RUN  update-ca-certificates
+ RUN update-ca-certificates
  END
 
  IF [ "$UPDATE_KERNEL" = "false" ]
  RUN zypper al kernel-de*
  END
 
  RUN zypper refresh && \
- zypper update -y
-
- IF [ -e "/usr/bin/dracut" ]
- RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
- RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
- END
- # zypper up kernel-default && \
- # zypper purge-kernels && \
- RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump
- RUN zypper cc && \
+ zypper update -y
+
+ IF [ -e "/usr/bin/dracut" ]
+ RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
+ RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
+ END
+ # zypper up kernel-default && \
+ # zypper purge-kernels && \
+
+ IF $TWO_NODE
+ RUN zypper --non-interactive --quiet addrepo --refresh -p 90 http://download.opensuse.org/repositories/server:database:postgresql/openSUSE_Tumbleweed/ PostgreSQL && \
+ zypper --gpg-auto-import-keys ref && \
+ zypper install -y postgresql-16 postgresql-server-16 postgresql-contrib iputils
+ END
+ RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump && \
+ zypper cc && \
  zypper clean
  END
 
@@ -337,6 +353,14 @@ base-image:
  RUN if grep "security=selinux" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/security=selinux //g' /etc/cos/bootargs.cfg; fi &&\
  if grep "selinux=1" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/selinux=1/selinux=0/g' /etc/cos/bootargs.cfg; fi
 
+ IF $TWO_NODE
+ RUN mkdir -p /opt/spectrocloud/bin && \
+ curl -L https://github.com/k3s-io/kine/releases/download/v${KINE_VERSION}/kine-amd64 | install -m 755 /dev/stdin /opt/spectrocloud/bin/kine
+
+ # ensure psql works ootb for the postgres user
+ RUN su postgres -c 'echo "export PERL5LIB=/usr/share/perl/5.34:/usr/share/perl5:/usr/lib/x86_64-linux-gnu/perl/5.34" > ~/.bash_profile'
+ END
+
 # Used to build the installer image. The installer ISO will be created from this.
 iso-image:
  FROM --platform=linux/${ARCH} +base-image

hack/Earthfile

@@ -0,0 +1,16 @@
+VERSION 0.6
+
+ARG OSBUILDER_VERSION=v0.7.11
+ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
+ARG ISO_NAME=debug
+
+# replace with your CanvOS provider image
+ARG PROVIDER_IMAGE=oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+
+build:
+ FROM $OSBUILDER_IMAGE
+ WORKDIR /build
+ COPY . ./
+
+ RUN /entrypoint.sh --name $ISO_NAME --debug build-iso --squash-no-compression --date=false $PROVIDER_IMAGE --output /build/
+ SAVE ARTIFACT /build/$ISO_NAME.iso kairos.iso AS LOCAL build/$ISO_NAME.iso

hack/README.md

@@ -0,0 +1,19 @@
+# Debugging Kairos
+
+If you're facing hard-to-diagnose issues with your custom provider image, you can use the scripts in this directory to obtain verbose Kairos output.
+
+## Steps
+1. Use earthly to generate an ISO from your CanvOS provider image:
+ ```
+ earthly +build --PROVIDER_IMAGE=<your_provider_image> # e.g., oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+ ```
+ If successful, `build/debug.iso` will be created.
+
+2. Launch a local VM based on the debug ISO using QEMU and pipe all output to a log file:
+ ```
+ ./launch-qemu.sh build/debug.iso | tee out.log
+ ```
+
+3. Boot the VM in `Kairos (manual)` mode. Once booted, create `userdata.yaml` with your desired Kairos config and execute a manual Kairos installation: `kairos-agent --debug manual-install --device auto userdata.yaml`.
+
+4. The VM should eventually reboot itself once the installation completes. Rather than waiting, execute `reboot` to return to the GRUB menu, select `Palette eXtended Kubernetes Edge` and hit `e` to edit it. Add `rd.debug rd.immucore.debug` to the end of the `linux` line, then hit `CTRL+x` to boot with your edits. You should see verbose Kairos debug logs and they will be persisted to `out.log`.

hack/launch-qemu.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Screenshot capability:
+# https://unix.stackexchange.com/a/476617
+
+if [ ! -e disk.img ]; then
+ qemu-img create -f qcow2 disk.img 60g
+fi
+
+# -nic bridge,br=br0,model=virtio-net-pci \
+qemu-system-x86_64 \
+ -enable-kvm \
+ -cpu "${CPU:=host}" \
+ -nographic \
+ -spice port=9000,addr=127.0.0.1,disable-ticketing=yes \
+ -m ${MEMORY:=10096} \
+ -smp ${CORES:=5} \
+ -monitor unix:/tmp/qemu-monitor.sock,server=on,wait=off \
+ -serial mon:stdio \
+ -rtc base=utc,clock=rt \
+ -chardev socket,path=qga.sock,server=on,wait=off,id=qga0 \
+ -device virtio-serial \
+ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+ -drive if=virtio,media=disk,file=disk.img \
+ -drive if=ide,media=cdrom,file="${1}"

overlay/files/opt/spectrocloud/bin/check-disk-size.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+REQUIRED_FREE_DISK=$1
+
+FREE=$(df -h --output=pcent /var/ | tail -n 1 | tr -d '\% ')
+
+if (( $FREE < $REQUIRED_FREE_DISK )); then
+ echo "Not enough free disk, required: $1. Free: $FREE"
+ exit 1
+fi
+
+echo "Free disk ok, required: $1. Free: $FREE"
+exit 0

test/env.example

@@ -0,0 +1,37 @@
+# govc vars
+export GOVC_USERNAME=<YOUR_NAME>@vsphere.local
+export GOVC_PASSWORD=<YOUR_VSPHERE_PASSWORD>
+export GOVC_URL=10.10.128.10 # IP address of USDC; edit as needed
+export GOVC_INSECURE=true
+export GOVC_DATACENTER=Datacenter
+export GOVC_DATASTORE=vsanDatastore2
+export GOVC_NETWORK=VM-NETWORK
+export GOVC_RESOURCE_POOL=<YOUR_RESOURCE_POOL>
+export GOVC_FOLDER=<YOUR_FOLDER>
+
+# vSphere vars
+export HOST_SUFFIX=<YOUR_NAME>-$(git -C ../stylus describe --always) # required to ensure unique edge host IDs
+export ISO_FOLDER=<YOUR_FOLDER> e.g. "ISO/01-tyler"
+export STYLUS_ISO="${ISO_FOLDER}/stylus-dev-amd64.iso"
+export NIC_NAME=ens160
+
+# palette vars
+export API_KEY=<YOUR_PALETTE_API_KEY>
+export PROJECT_UID=<YOUR_PROJECT_ID>
+export EDGE_REGISTRATION_TOKEN=<YOUR_REGISTRATION_TOKEN>
+export DOMAIN=dev.spectrocloud.com
+export PUBLIC_PACK_REPO_UID=<YOUR_PUBLIC_PACK_REPO_UID> # this varies per Palette tenant, identify via Chrome inspector on Tenant Admin -> Pack Registries page
+export CLUSTER_NAME=two-node-<YOUR_NAME>-$(git -C ../stylus describe --always)
+export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created
+export CLUSTER_VIP= # choose an unassigned VIP
+
+# image vars
+export EARTHLY_BUILDKIT_CACHE_SIZE_MB=500000
+export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh}
+export STYLUS_BRANCH=${STYLUS_BRANCH:-PAD-178-kine-validator}
+export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-two-node}
+export K3S_VERSION="1.28.2"
+export PE_VERSION="4.1.2"
+
+# two node vars
+export TWO_NODE_BACKEND=postgres