Merge branch 'master' into fix_binds

.github/workflows/image.yaml

@@ -210,13 +210,36 @@ jobs:
       uses: actions/download-artifact@v3
       with:
         name: kairos-${{ matrix.flavor }}.iso.zip
-    - run: |
+    - env:
+        KVM: true
+      run: |
             ls -liah
             export ISO=$PWD/$(ls *.iso)
             mkdir build
             mv $ISO build/kairos.iso
-            ./earthly.sh +prepare-bundles-tests
-            ./earthly.sh +run-qemu-bundles-tests --FLAVOR=${{ matrix.flavor }}
+
+            # Get earthly with luet
+            curl -L  https://github.com/mudler/luet/releases/download/0.33.0/luet-0.33.0-linux-amd64 -o luet
+            chmod +x luet
+            sudo mv luet /usr/bin/luet
+            sudo mkdir -p /etc/luet/repos.conf.d/
+            sudo luet repo add -y kairos --url quay.io/kairos/packages --type docker
+            LUET_NOLOCK=true sudo -E luet install -y utils/earthly
+
+            # Configure earthly to use the docker mirror in CI
+            # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache
+            mkdir -p ~/.earthly/
+            cat << EOF > ~/.earthly/config.yml
+            global:
+              buildkit_additional_config: |
+                [registry."docker.io"]
+                  mirrors = ["registry.docker-mirror.svc.cluster.local:5000"]
+                [registry."registry.docker-mirror.svc.cluster.local:5000"]
+                  insecure = true
+            EOF
+
+            earthly -P +prepare-bundles-tests
+            earthly -P +run-qemu-bundles-tests --FLAVOR=${{ matrix.flavor }}
 
   qemu-reset-tests:
     needs:
@@ -237,14 +260,37 @@ jobs:
       uses: actions/download-artifact@v3
       with:
         name: kairos-${{ matrix.flavor }}.iso.zip
-    - run: |
+    - env:
+        KVM: true
+      run: |
             ls -liah
             sed -i '/build.*/d' .earthlyignore
             export ISO=$PWD/$(ls *.iso)
             mkdir build
             mv $ISO build/kairos.iso
-            ./earthly.sh +datasource-iso --CLOUD_CONFIG=tests/assets/autoinstall.yaml
-            ./earthly.sh +run-qemu-datasource-tests --TEST_SUITE=reset-test --FLAVOR=${{ matrix.flavor }}
+
+            # Get earthly with luet
+            curl -L  https://github.com/mudler/luet/releases/download/0.33.0/luet-0.33.0-linux-amd64 -o luet
+            chmod +x luet
+            sudo mv luet /usr/bin/luet
+            sudo mkdir -p /etc/luet/repos.conf.d/
+            sudo luet repo add -y kairos --url quay.io/kairos/packages --type docker
+            LUET_NOLOCK=true sudo -E luet install -y utils/earthly
+
+            # Configure earthly to use the docker mirror in CI
+            # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache
+            mkdir -p ~/.earthly/
+            cat << EOF > ~/.earthly/config.yml
+            global:
+              buildkit_additional_config: |
+                [registry."docker.io"]
+                  mirrors = ["registry.docker-mirror.svc.cluster.local:5000"]
+                [registry."registry.docker-mirror.svc.cluster.local:5000"]
+                  insecure = true
+            EOF
+
+            earthly -P +datasource-iso --CLOUD_CONFIG=tests/assets/autoinstall.yaml
+            earthly -P +run-qemu-datasource-tests --TEST_SUITE=reset-test --FLAVOR=${{ matrix.flavor }}
 
   qemu-netboot-tests:
     needs:

Earthfile

@@ -219,6 +219,8 @@ framework:
 
     RUN go run -ldflags "${LDFLAGS}" ./cmd/profile-build/main.go ${FLAVOR} $REPOSITORIES_FILE /framework
 
+    # Copy kairos binaries
+    COPY +build-kairos-agent/kairos-agent /framework/usr/bin/kairos-agent
     COPY +luet/luet /framework/usr/bin/luet
 
     RUN luet cleanup --system-target /framework
@@ -283,8 +285,6 @@ docker:
         COPY overlay/files-ubuntu/ /
     END
 
-    # Copy kairos binaries
-    COPY +build-kairos-agent/kairos-agent /usr/bin/kairos-agent
     # Enable services
     IF [ -f /sbin/openrc ]
      RUN mkdir -p /etc/runlevels/default && \

docs/content/en/docs/Architecture/meta.md

@@ -12,7 +12,7 @@ We like to define Kairos as a meta-Linux Distribution, as its goal is to convert
 
 The Kairos stack is composed of the following:
 
-- A core OS image release for each flavor in ISO, qcow2, and other similar formats (currently can pick from openSUSE and Alpine based) provided for user convenience
+- A core OS image release for each flavor in ISO, qcow2, and other similar formats (see [the list of supported distributions](/docs/reference/image_matrix)) provided for user convenience
 - A release with K3s embedded.
 - A set of Kubernetes Native API components (CRDs) to install into the control-plane node, to manage deployment, artifacts creation, and lifecycle (WIP).
 - A set of Kubernetes Native API components (CRDs) to install into the target nodes to manage and control the node after deployment (WIP).

docs/content/en/docs/Architecture/network.md

@@ -15,6 +15,10 @@ To address these challenges, Kairos provides an easy and robust solution for dep
 
 In this document, we will examine the advantages of using Kairos to deploy Kubernetes clusters at the edge, and how p2p technology facilitates self-coordination for a zero-touch configuration experience. We will also explore how Kairos' highly adaptable and container-based approach, combined with an immutable OS and meta-distribution, makes it an excellent choice for edge deployments.
 
+{{% alert title="Note" %}}
+You can also watch our [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media")
+{{% /alert %}}
+
 ## Overview: P2P for self-coordination
 
 <img align="right" width="200" src="https://user-images.githubusercontent.com/2420543/219048504-986da0e9-aca3-4c9e-b980-ba2a6dc03bf7.png">

docs/content/en/docs/Examples/p2p_e2e.md

@@ -18,6 +18,10 @@ Deploying Kubernetes at the Edge can be a complex and time-consuming process, es
 
 To leverage p2p self-coordination capabilities of Kairos, you will need to configure the `network_token` under the `p2p` configuration block in your cloud-config file. Once you have set this, Kairos will handle the configuration of each node.
 
+{{% alert title="Note" %}}
+You can see this example live in the [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media")
+{{% /alert %}}
+
 ## Description
 
 In the following example we are going to bootstrap a new multi-node, single cluster with Kairos. We will use at least 2 nodes, one as a master and one as a worker. Note how we don't specify any role, or either pin any IP in the following configurations.

docs/content/en/docs/Installation/automated.md

@@ -96,6 +96,7 @@ $ docker pull $IMAGE
 # Build the ISO
 $ docker run -v $PWD/cloud_init.yaml:/cloud_init.yaml \
                     -v $PWD/build:/tmp/auroraboot \
+                    -v /var/run/docker.sock:/var/run/docker.sock \
                     --rm -ti quay.io/kairos/auroraboot \
                     --set container_image=docker://$IMAGE \
                     --set "disable_http_server=true" \

docs/content/en/docs/Media/_index.md

@@ -27,3 +27,7 @@ description: >
 ### How we build and maintain Kairos
 
 {{< youtube id="XD5nfMf59v4" title="How we build and maintain Kairos" >}}
+
+### How Kairos uses libp2p
+
+{{< youtube id="7Vym18wz9Uw" title="Kairos and libp2p" >}}

docs/content/en/docs/Reference/build.md

@@ -171,11 +171,42 @@ RUN luet install -y distro-kernels/opensuse-leap distro-initrd/opensuse-leap
 # RUN cp cloud.yaml /system/oem/configuration.yaml 
 ```
 
-After building the container image, use the `osbuilder-tools` image to create an ISO:
+After building the container image, we can create an ISO:
 
+{{< tabpane text=true  >}}
+{{% tab header="AuroraBoot" %}}
+
+We can use [AuroraBoot](/docs/reference/auroraboot) to handle the the ISO build process and optionally attach it a default cloud config, for example:
+
+```bash
+$ IMAGE=<source/image>
+$ docker run -v $PWD/cloud_init.yaml:/cloud_init.yaml \
+                    -v $PWD/build:/tmp/auroraboot \
+                    -v /var/run/docker.sock:/var/run/docker.sock \
+                    --rm -ti quay.io/kairos/auroraboot \
+                    --set container_image=docker://$IMAGE \
+                    --set "disable_http_server=true" \
+                    --set "disable_netboot=true" \
+                    --cloud-config /cloud_init.yaml \
+                    --set "state_dir=/tmp/auroraboot"
+# Artifacts are under build/
+$ sudo ls -liah build/iso
+total 778M
+34648528 drwx------ 2 root root 4.0K Feb  8 16:39 .
+34648526 drwxr-xr-x 5 root root 4.0K Feb  8 16:38 ..
+34648529 -rw-r--r-- 1 root root  253 Feb  8 16:38 config.yaml
+34649370 -rw-r--r-- 1 root root 389M Feb  8 16:38 kairos.iso
+34649372 -rw-r--r-- 1 root root 389M Feb  8 16:39 kairos.iso.custom.iso
+34649371 -rw-r--r-- 1 root root   76 Feb  8 16:39 kairos.iso.sha256
+```
+{{% /tab %}}
+{{% tab header="Manually" %}}
+Use `osbuilder-tools`:
 ```bash
 docker run -v $PWD:/cOS -v /var/run/docker.sock:/var/run/docker.sock -i --rm quay.io/kairos/osbuilder-tools:latest --name "custom-iso" --debug build-iso --date=false --local $IMAGE --output /cOS/
 ```
+{{% /tab %}}
+{{< /tabpane >}}
 
 Use QEMU to test the ISO:
 

docs/content/en/docs/Reference/image_matrix.md

@@ -37,7 +37,7 @@ Base images are tagged with specific upstream versions (e.g. Ubuntu 20 LTS is pi
 | **Ubuntu based (rolling)** **                                   	       | https://quay.io/repository/kairos/core-ubuntu           	               | https://quay.io/repository/kairos/kairos-ubuntu           	               |
 | **Ubuntu based (22 LTS)** **                                    	       | https://quay.io/repository/kairos/core-ubuntu-22-lts           	       | https://quay.io/repository/kairos/kairos-ubuntu-22-lts           	       |
 | **Ubuntu based (20 LTS)** **                                   	         | https://quay.io/repository/kairos/core-ubuntu-20-lts           	       | https://quay.io/repository/kairos/kairos-ubuntu-20-lts           	       |
-| **Rocky Linux based**                                                    | https://quay.io/repository/kairos/core-rockylinux                       | N/A                                                                       |
+| **Rocky Linux based**                                                    | https://quay.io/repository/kairos/core-rockylinux                       | https://quay.io/repository/kairos/kairos-rockylinux                       |
 
 {{% alert title="Note" color="info" %}}
 

earthly.ps1

@@ -1 +1 @@
-docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.6.30 --allow-privileged @args
+docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.0 --allow-privileged @args

earthly.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.6.30 --allow-privileged $@
+docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.0 --allow-privileged $@

framework-profile.yaml

@@ -89,9 +89,9 @@ repositories:
     priority: 2
     urls:
       - "quay.io/kairos/packages"
-    reference: 20230210110206-repository.yaml
+    reference: 20230219230039-repository.yaml
   - !!merge <<: *kairos
     arch: arm64
     urls:
       - "quay.io/kairos/packages-arm64"
-    reference: 20230210110549-repository.yaml
+    reference: 20230219220016-repository.yaml

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/jaypipes/ghw v0.9.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kairos-io/kcrypt v0.5.1
-	github.com/labstack/echo/v4 v4.10.1
+	github.com/labstack/echo/v4 v4.10.2
 	github.com/mudler/go-nodepair v0.0.0-20221223092639-ba399a66fdfb
 	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5
 	github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d
@@ -28,7 +28,7 @@ require (
 	github.com/spectrocloud/peg v0.0.0-20230210110137-1780ceedb1e5
 	github.com/swaggest/jsonschema-go v0.3.47
 	github.com/twpayne/go-vfs v1.7.2
-	github.com/urfave/cli/v2 v2.24.3
+	github.com/urfave/cli/v2 v2.24.4
 	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.5.0
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0

go.sum

@@ -283,6 +283,8 @@ github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280Z
 github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ=
 github.com/labstack/echo/v4 v4.10.1 h1:rB+D8In9PWjsp1OpHaqK+t04nQv/SBD1IoIcXCg0lpY=
 github.com/labstack/echo/v4 v4.10.1/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k=
+github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M=
+github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k=
 github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
 github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
 github.com/lithammer/fuzzysearch v1.1.5 h1:Ag7aKU08wp0R9QCfF4GoGST9HbmAIeLP7xwMrOBEp1c=
@@ -537,6 +539,8 @@ github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk=
 github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0=
 github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU=
+github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=

internal/agent/validate.go

@@ -23,11 +23,11 @@ func JSONSchema(version string) (string, error) {
 }
 
 // Validate ensures that a given schema is Valid according to the Root Schema from the agent.
-func Validate(file string) error {
+func Validate(source string) error {
 	var yaml string
 
-	if strings.HasPrefix(file, "http") {
-		resp, err := http.Get(file)
+	if strings.HasPrefix(source, "http") {
+		resp, err := http.Get(source)
 		if err != nil {
 			return err
 		}
@@ -38,11 +38,16 @@ func Validate(file string) error {
 		//Convert the body to type string
 		yaml = string(body)
 	} else {
-		dat, err := os.ReadFile(file)
+		dat, err := os.ReadFile(source)
 		if err != nil {
-			return err
+			if strings.Contains(err.Error(), "no such file or directory") {
+				yaml = source
+			} else {
+				return err
+			}
+		} else {
+			yaml = string(dat)
 		}
-		yaml = string(dat)
 	}
 
 	config, err := schema.NewConfigFromYAML(yaml, schema.RootSchema{})

internal/webui/public/index.html

@@ -69,7 +69,22 @@ <h1>Welcome to the Installer!</h1>
                   try {
                     // Parse the YAML
                     const yaml = YAML.parse(this.content);
-                    this.valid=true
+                    const formData = new FormData()
+                    formData.append('cloud-config', this.content)
+                    const settings = {
+                        method: 'POST',
+                        body: formData,
+                    };
+                    fetch(`/validate`, settings)
+                      .then((response) => response.text())
+                      .then((text) => {
+                        if (text === '') {
+                          this.valid = true
+                        } else {
+                          this.error = text
+                          this.valid = false
+                        }
+                      })
                   } catch(error) {
                     this.error = error
                     this.valid = false

internal/webui/webui.go

@@ -161,6 +161,21 @@ func Start(ctx context.Context) error {
 
 	ec.GET("/*", echo.WrapHandler(http.StripPrefix("/", assetHandler)))
 
+	ec.POST("/validate", func(c echo.Context) error {
+		formData := new(FormData)
+		if err := c.Bind(formData); err != nil {
+			return err
+		}
+		cloudConfig := formData.CloudConfig
+
+		err := agent.Validate(cloudConfig)
+		if err != nil {
+			return c.String(http.StatusOK, err.Error())
+		}
+
+		return c.String(http.StatusOK, "")
+	})
+
 	ec.POST("/install", func(c echo.Context) error {
 
 		s.Lock()

overlay/files/system/oem/31_hosts.yaml

@@ -0,0 +1,11 @@
+stages:
+  initramfs.before:
+    # For debian based distributions, /etc/hosts is present but empty. This is because the file
+    # is populated when running the configuration. For those cases we insert a record so it can be
+    # manipulated later on by yip's hostname plugin
+    # Read more: https://wiki.debian.org/ConfigPackages
+    - name: "Make sure hosts file is present and includes a record for 127.0.0.1"
+      if: |
+        ! [[ -f /etc/hosts ]] || ! [[ $(grep '127.0.0.1' /etc/hosts) ]]
+      commands:
+        - echo '127.0.0.1\tlocalhost' >> /etc/hosts