diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 00ffc03e4..eff6143a6 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -210,13 +210,36 @@ jobs: uses: actions/download-artifact@v3 with: name: kairos-${{ matrix.flavor }}.iso.zip - - run: | + - env: + KVM: true + run: | ls -liah export ISO=$PWD/$(ls *.iso) mkdir build mv $ISO build/kairos.iso - ./earthly.sh +prepare-bundles-tests - ./earthly.sh +run-qemu-bundles-tests --FLAVOR=${{ matrix.flavor }} + + # Get earthly with luet + curl -L https://github.com/mudler/luet/releases/download/0.33.0/luet-0.33.0-linux-amd64 -o luet + chmod +x luet + sudo mv luet /usr/bin/luet + sudo mkdir -p /etc/luet/repos.conf.d/ + sudo luet repo add -y kairos --url quay.io/kairos/packages --type docker + LUET_NOLOCK=true sudo -E luet install -y utils/earthly + + # Configure earthly to use the docker mirror in CI + # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache + mkdir -p ~/.earthly/ + cat << EOF > ~/.earthly/config.yml + global: + buildkit_additional_config: | + [registry."docker.io"] + mirrors = ["registry.docker-mirror.svc.cluster.local:5000"] + [registry."registry.docker-mirror.svc.cluster.local:5000"] + insecure = true + EOF + + earthly -P +prepare-bundles-tests + earthly -P +run-qemu-bundles-tests --FLAVOR=${{ matrix.flavor }} qemu-reset-tests: needs: @@ -237,14 +260,37 @@ jobs: uses: actions/download-artifact@v3 with: name: kairos-${{ matrix.flavor }}.iso.zip - - run: | + - env: + KVM: true + run: | ls -liah sed -i '/build.*/d' .earthlyignore export ISO=$PWD/$(ls *.iso) mkdir build mv $ISO build/kairos.iso - ./earthly.sh +datasource-iso --CLOUD_CONFIG=tests/assets/autoinstall.yaml - ./earthly.sh +run-qemu-datasource-tests --TEST_SUITE=reset-test --FLAVOR=${{ matrix.flavor }} + + # Get earthly with luet + curl -L https://github.com/mudler/luet/releases/download/0.33.0/luet-0.33.0-linux-amd64 -o luet + chmod +x luet + sudo mv luet /usr/bin/luet + sudo mkdir -p /etc/luet/repos.conf.d/ + sudo luet repo add -y kairos --url quay.io/kairos/packages --type docker + LUET_NOLOCK=true sudo -E luet install -y utils/earthly + + # Configure earthly to use the docker mirror in CI + # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache + mkdir -p ~/.earthly/ + cat << EOF > ~/.earthly/config.yml + global: + buildkit_additional_config: | + [registry."docker.io"] + mirrors = ["registry.docker-mirror.svc.cluster.local:5000"] + [registry."registry.docker-mirror.svc.cluster.local:5000"] + insecure = true + EOF + + earthly -P +datasource-iso --CLOUD_CONFIG=tests/assets/autoinstall.yaml + earthly -P +run-qemu-datasource-tests --TEST_SUITE=reset-test --FLAVOR=${{ matrix.flavor }} qemu-netboot-tests: needs: diff --git a/Earthfile b/Earthfile index b1f6492de..9303b402c 100644 --- a/Earthfile +++ b/Earthfile @@ -219,6 +219,8 @@ framework: RUN go run -ldflags "${LDFLAGS}" ./cmd/profile-build/main.go ${FLAVOR} $REPOSITORIES_FILE /framework + # Copy kairos binaries + COPY +build-kairos-agent/kairos-agent /framework/usr/bin/kairos-agent COPY +luet/luet /framework/usr/bin/luet RUN luet cleanup --system-target /framework @@ -283,8 +285,6 @@ docker: COPY overlay/files-ubuntu/ / END - # Copy kairos binaries - COPY +build-kairos-agent/kairos-agent /usr/bin/kairos-agent # Enable services IF [ -f /sbin/openrc ] RUN mkdir -p /etc/runlevels/default && \ diff --git a/docs/content/en/docs/Architecture/meta.md b/docs/content/en/docs/Architecture/meta.md index e124e9354..f294b8a7f 100644 --- a/docs/content/en/docs/Architecture/meta.md +++ b/docs/content/en/docs/Architecture/meta.md @@ -12,7 +12,7 @@ We like to define Kairos as a meta-Linux Distribution, as its goal is to convert The Kairos stack is composed of the following: -- A core OS image release for each flavor in ISO, qcow2, and other similar formats (currently can pick from openSUSE and Alpine based) provided for user convenience +- A core OS image release for each flavor in ISO, qcow2, and other similar formats (see [the list of supported distributions](/docs/reference/image_matrix)) provided for user convenience - A release with K3s embedded. - A set of Kubernetes Native API components (CRDs) to install into the control-plane node, to manage deployment, artifacts creation, and lifecycle (WIP). - A set of Kubernetes Native API components (CRDs) to install into the target nodes to manage and control the node after deployment (WIP). diff --git a/docs/content/en/docs/Architecture/network.md b/docs/content/en/docs/Architecture/network.md index 791c534b0..a58bb2849 100644 --- a/docs/content/en/docs/Architecture/network.md +++ b/docs/content/en/docs/Architecture/network.md @@ -15,6 +15,10 @@ To address these challenges, Kairos provides an easy and robust solution for dep In this document, we will examine the advantages of using Kairos to deploy Kubernetes clusters at the edge, and how p2p technology facilitates self-coordination for a zero-touch configuration experience. We will also explore how Kairos' highly adaptable and container-based approach, combined with an immutable OS and meta-distribution, makes it an excellent choice for edge deployments. +{{% alert title="Note" %}} +You can also watch our [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media") +{{% /alert %}} + ## Overview: P2P for self-coordination diff --git a/docs/content/en/docs/Examples/p2p_e2e.md b/docs/content/en/docs/Examples/p2p_e2e.md index 692d6423e..dd70e20c1 100644 --- a/docs/content/en/docs/Examples/p2p_e2e.md +++ b/docs/content/en/docs/Examples/p2p_e2e.md @@ -18,6 +18,10 @@ Deploying Kubernetes at the Edge can be a complex and time-consuming process, es To leverage p2p self-coordination capabilities of Kairos, you will need to configure the `network_token` under the `p2p` configuration block in your cloud-config file. Once you have set this, Kairos will handle the configuration of each node. +{{% alert title="Note" %}} +You can see this example live in the [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media") +{{% /alert %}} + ## Description In the following example we are going to bootstrap a new multi-node, single cluster with Kairos. We will use at least 2 nodes, one as a master and one as a worker. Note how we don't specify any role, or either pin any IP in the following configurations. diff --git a/docs/content/en/docs/Installation/automated.md b/docs/content/en/docs/Installation/automated.md index 9bbcb4765..d51a06560 100644 --- a/docs/content/en/docs/Installation/automated.md +++ b/docs/content/en/docs/Installation/automated.md @@ -96,6 +96,7 @@ $ docker pull $IMAGE # Build the ISO $ docker run -v $PWD/cloud_init.yaml:/cloud_init.yaml \ -v $PWD/build:/tmp/auroraboot \ + -v /var/run/docker.sock:/var/run/docker.sock \ --rm -ti quay.io/kairos/auroraboot \ --set container_image=docker://$IMAGE \ --set "disable_http_server=true" \ diff --git a/docs/content/en/docs/Media/_index.md b/docs/content/en/docs/Media/_index.md index b714f08b3..73f11a253 100644 --- a/docs/content/en/docs/Media/_index.md +++ b/docs/content/en/docs/Media/_index.md @@ -27,3 +27,7 @@ description: > ### How we build and maintain Kairos {{< youtube id="XD5nfMf59v4" title="How we build and maintain Kairos" >}} + +### How Kairos uses libp2p + +{{< youtube id="7Vym18wz9Uw" title="Kairos and libp2p" >}} diff --git a/docs/content/en/docs/Reference/build.md b/docs/content/en/docs/Reference/build.md index 93ecd5a22..7f7f3c634 100644 --- a/docs/content/en/docs/Reference/build.md +++ b/docs/content/en/docs/Reference/build.md @@ -171,11 +171,42 @@ RUN luet install -y distro-kernels/opensuse-leap distro-initrd/opensuse-leap # RUN cp cloud.yaml /system/oem/configuration.yaml ``` -After building the container image, use the `osbuilder-tools` image to create an ISO: +After building the container image, we can create an ISO: +{{< tabpane text=true >}} +{{% tab header="AuroraBoot" %}} + +We can use [AuroraBoot](/docs/reference/auroraboot) to handle the the ISO build process and optionally attach it a default cloud config, for example: + +```bash +$ IMAGE= +$ docker run -v $PWD/cloud_init.yaml:/cloud_init.yaml \ + -v $PWD/build:/tmp/auroraboot \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --rm -ti quay.io/kairos/auroraboot \ + --set container_image=docker://$IMAGE \ + --set "disable_http_server=true" \ + --set "disable_netboot=true" \ + --cloud-config /cloud_init.yaml \ + --set "state_dir=/tmp/auroraboot" +# Artifacts are under build/ +$ sudo ls -liah build/iso +total 778M +34648528 drwx------ 2 root root 4.0K Feb 8 16:39 . +34648526 drwxr-xr-x 5 root root 4.0K Feb 8 16:38 .. +34648529 -rw-r--r-- 1 root root 253 Feb 8 16:38 config.yaml +34649370 -rw-r--r-- 1 root root 389M Feb 8 16:38 kairos.iso +34649372 -rw-r--r-- 1 root root 389M Feb 8 16:39 kairos.iso.custom.iso +34649371 -rw-r--r-- 1 root root 76 Feb 8 16:39 kairos.iso.sha256 +``` +{{% /tab %}} +{{% tab header="Manually" %}} +Use `osbuilder-tools`: ```bash docker run -v $PWD:/cOS -v /var/run/docker.sock:/var/run/docker.sock -i --rm quay.io/kairos/osbuilder-tools:latest --name "custom-iso" --debug build-iso --date=false --local $IMAGE --output /cOS/ ``` +{{% /tab %}} +{{< /tabpane >}} Use QEMU to test the ISO: diff --git a/docs/content/en/docs/Reference/image_matrix.md b/docs/content/en/docs/Reference/image_matrix.md index 5509b5f0a..e6cd45531 100644 --- a/docs/content/en/docs/Reference/image_matrix.md +++ b/docs/content/en/docs/Reference/image_matrix.md @@ -37,7 +37,7 @@ Base images are tagged with specific upstream versions (e.g. Ubuntu 20 LTS is pi | **Ubuntu based (rolling)** ** | https://quay.io/repository/kairos/core-ubuntu | https://quay.io/repository/kairos/kairos-ubuntu | | **Ubuntu based (22 LTS)** ** | https://quay.io/repository/kairos/core-ubuntu-22-lts | https://quay.io/repository/kairos/kairos-ubuntu-22-lts | | **Ubuntu based (20 LTS)** ** | https://quay.io/repository/kairos/core-ubuntu-20-lts | https://quay.io/repository/kairos/kairos-ubuntu-20-lts | -| **Rocky Linux based** | https://quay.io/repository/kairos/core-rockylinux | N/A | +| **Rocky Linux based** | https://quay.io/repository/kairos/core-rockylinux | https://quay.io/repository/kairos/kairos-rockylinux | {{% alert title="Note" color="info" %}} diff --git a/earthly.ps1 b/earthly.ps1 index 25c65300f..c02c78e22 100644 --- a/earthly.ps1 +++ b/earthly.ps1 @@ -1 +1 @@ -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.6.30 --allow-privileged @args \ No newline at end of file +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.0 --allow-privileged @args \ No newline at end of file diff --git a/earthly.sh b/earthly.sh index 2679aa870..2793a1bd5 100755 --- a/earthly.sh +++ b/earthly.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.6.30 --allow-privileged $@ \ No newline at end of file +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.0 --allow-privileged $@ \ No newline at end of file diff --git a/framework-profile.yaml b/framework-profile.yaml index 1c6f6cacc..cff72034b 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -89,9 +89,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230210110206-repository.yaml + reference: 20230219230039-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230210110549-repository.yaml + reference: 20230219220016-repository.yaml diff --git a/go.mod b/go.mod index d22a56992..4f89ef844 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/jaypipes/ghw v0.9.0 github.com/joho/godotenv v1.5.1 github.com/kairos-io/kcrypt v0.5.1 - github.com/labstack/echo/v4 v4.10.1 + github.com/labstack/echo/v4 v4.10.2 github.com/mudler/go-nodepair v0.0.0-20221223092639-ba399a66fdfb github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d @@ -28,7 +28,7 @@ require ( github.com/spectrocloud/peg v0.0.0-20230210110137-1780ceedb1e5 github.com/swaggest/jsonschema-go v0.3.47 github.com/twpayne/go-vfs v1.7.2 - github.com/urfave/cli/v2 v2.24.3 + github.com/urfave/cli/v2 v2.24.4 golang.org/x/net v0.7.0 golang.org/x/oauth2 v0.5.0 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 diff --git a/go.sum b/go.sum index ae2fcdf13..1d172f9e6 100644 --- a/go.sum +++ b/go.sum @@ -283,6 +283,8 @@ github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280Z github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ= github.com/labstack/echo/v4 v4.10.1 h1:rB+D8In9PWjsp1OpHaqK+t04nQv/SBD1IoIcXCg0lpY= github.com/labstack/echo/v4 v4.10.1/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= +github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= +github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lithammer/fuzzysearch v1.1.5 h1:Ag7aKU08wp0R9QCfF4GoGST9HbmAIeLP7xwMrOBEp1c= @@ -537,6 +539,8 @@ github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk= github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0= github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= +github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= diff --git a/internal/agent/validate.go b/internal/agent/validate.go index d168147a4..ae6a7388c 100644 --- a/internal/agent/validate.go +++ b/internal/agent/validate.go @@ -23,11 +23,11 @@ func JSONSchema(version string) (string, error) { } // Validate ensures that a given schema is Valid according to the Root Schema from the agent. -func Validate(file string) error { +func Validate(source string) error { var yaml string - if strings.HasPrefix(file, "http") { - resp, err := http.Get(file) + if strings.HasPrefix(source, "http") { + resp, err := http.Get(source) if err != nil { return err } @@ -38,11 +38,16 @@ func Validate(file string) error { //Convert the body to type string yaml = string(body) } else { - dat, err := os.ReadFile(file) + dat, err := os.ReadFile(source) if err != nil { - return err + if strings.Contains(err.Error(), "no such file or directory") { + yaml = source + } else { + return err + } + } else { + yaml = string(dat) } - yaml = string(dat) } config, err := schema.NewConfigFromYAML(yaml, schema.RootSchema{}) diff --git a/internal/webui/public/index.html b/internal/webui/public/index.html index be7659f9f..9bb4676e4 100644 --- a/internal/webui/public/index.html +++ b/internal/webui/public/index.html @@ -69,7 +69,22 @@

Welcome to the Installer!

try { // Parse the YAML const yaml = YAML.parse(this.content); - this.valid=true + const formData = new FormData() + formData.append('cloud-config', this.content) + const settings = { + method: 'POST', + body: formData, + }; + fetch(`/validate`, settings) + .then((response) => response.text()) + .then((text) => { + if (text === '') { + this.valid = true + } else { + this.error = text + this.valid = false + } + }) } catch(error) { this.error = error this.valid = false diff --git a/internal/webui/webui.go b/internal/webui/webui.go index c9cc53705..ee5fb7b43 100644 --- a/internal/webui/webui.go +++ b/internal/webui/webui.go @@ -161,6 +161,21 @@ func Start(ctx context.Context) error { ec.GET("/*", echo.WrapHandler(http.StripPrefix("/", assetHandler))) + ec.POST("/validate", func(c echo.Context) error { + formData := new(FormData) + if err := c.Bind(formData); err != nil { + return err + } + cloudConfig := formData.CloudConfig + + err := agent.Validate(cloudConfig) + if err != nil { + return c.String(http.StatusOK, err.Error()) + } + + return c.String(http.StatusOK, "") + }) + ec.POST("/install", func(c echo.Context) error { s.Lock() diff --git a/overlay/files/system/oem/31_hosts.yaml b/overlay/files/system/oem/31_hosts.yaml new file mode 100644 index 000000000..3cbddaa2c --- /dev/null +++ b/overlay/files/system/oem/31_hosts.yaml @@ -0,0 +1,11 @@ +stages: + initramfs.before: + # For debian based distributions, /etc/hosts is present but empty. This is because the file + # is populated when running the configuration. For those cases we insert a record so it can be + # manipulated later on by yip's hostname plugin + # Read more: https://wiki.debian.org/ConfigPackages + - name: "Make sure hosts file is present and includes a record for 127.0.0.1" + if: | + ! [[ -f /etc/hosts ]] || ! [[ $(grep '127.0.0.1' /etc/hosts) ]] + commands: + - echo '127.0.0.1\tlocalhost' >> /etc/hosts