more info about how to scrape k0s internals

.github/dependabot.yml

@@ -51,7 +51,6 @@ updates:
 
       # runc/containerd
       - dependency-name: github.com/containerd/containerd
-      - dependency-name: github.com/opencontainers/runc
 
   - package-ecosystem: github-actions
     directory: /

.github/workflows/build-airgap-image-bundle.yml

@@ -20,9 +20,6 @@ jobs:
     name: "${{ inputs.target-os }}-${{ inputs.target-arch }}"
     runs-on: ubuntu-22.04
 
-    outputs:
-      cache-key: ${{ steps.cache-airgap-image-bundle-calc-key.outputs.cache-key }}
-
     env:
       TARGET_OS: ${{ inputs.target-os }}
       TARGET_ARCH: ${{ inputs.target-arch }}
@@ -38,11 +35,9 @@ jobs:
         with:
           name: airgap-image-list-${{ inputs.target-os }}-${{ inputs.target-arch }}
 
-        # Capture the calculated image bundle source hash in a build output, so
-        # it can be shared between the cache actions in this job and in the
-        # smoketests. Do this in a separate step, as the hashFiles function is
-        # evaluated before the step execution. So all the required files need to
-        # exist before that.
+        # Capture the calculated image bundle source hash in a separate step, as
+        # the hashFiles function is evaluated before the step execution. So all
+        # the required files need to exist before that.
       - name: "Cache :: Airgap image bundle :: Calculate cache key"
         id: cache-airgap-image-bundle-calc-key
         env:
@@ -56,11 +51,17 @@ jobs:
         with:
           key: ${{ steps.cache-airgap-image-bundle-calc-key.outputs.cache-key }}
           path: airgap-image-bundle-${{ inputs.target-os }}-${{ inputs.target-arch }}.tar
-          lookup-only: true
 
       - name: "Build :: Airgap image bundle"
         if: steps.cache-airgap-image-bundle.outputs.cache-hit != 'true'
         run: |
           mkdir -p "embedded-bins/staging/$TARGET_OS/bin"
           make --touch airgap-images.txt
           make "airgap-image-bundle-$TARGET_OS-$TARGET_ARCH.tar"
+
+      - name: "Upload :: Airgap image bundle"
+        uses: actions/upload-artifact@v4
+        with:
+          name: airgap-image-bundle-${{ inputs.target-os }}-${{ inputs.target-arch }}
+          path: airgap-image-bundle-${{ inputs.target-os }}-${{ inputs.target-arch }}.tar
+          compression: 0

.github/workflows/check-network.yaml

@@ -15,7 +15,6 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_TERRAFORM_KEY }}
       AWS_DEFAULT_REGION: eu-west-1
       TF_VERSION: 1.2.2
-      K0SCTL_VERSION: 0.17.2
       KUBECONFIG: ${{ github.workspace }}/kubeconfig
 
     name: "K8s Network Conformance Testing"
@@ -131,11 +130,16 @@ jobs:
 
           terraform apply -auto-approve
 
+      - name: Set k0sctl version
+        run: |
+          version=$(cd hack/tool; go list -m -f '{{.Version}}' github.com/k0sproject/k0sctl)
+          echo "K0SCTL_VERSION=${version}" >> $GITHUB_ENV
+
       - name: Create k0s Cluster using k0sctl
         id: k0sctl
         run: |
           # download k0sctl
-          curl --silent -L "https://github.com/k0sproject/k0sctl/releases/download/v${K0SCTL_VERSION}/k0sctl-linux-x64" -o k0sctl
+          curl --silent -L "https://github.com/k0sproject/k0sctl/releases/download/${K0SCTL_VERSION}/k0sctl-linux-x64" -o k0sctl
           chmod +x ./k0sctl
           ./k0sctl apply -c k0sctl.yaml
 

.github/workflows/go.yml

@@ -139,13 +139,7 @@ jobs:
       - name: Run unit tests
         env:
           EMBEDDED_BINS_BUILDMODE: none
-        run: |
-          make bindata
-          make --touch codegen
-          make check-unit
-
-      - name: Validate OCI images manifests
-        run: make check-image-validity
+        run: make check-unit
 
   unittests-k0s-windows-amd64:
     name: "Unit tests :: windows-amd64"
@@ -194,9 +188,7 @@ jobs:
           GO: go
           GO_ENV: ''
         run: |
-          make --touch .k0sbuild.docker-image.k0s go.sum
-          make bindata
-          make --touch codegen
+          make --touch .k0sbuild.docker-image.k0s
           make check-unit
 
   smoketests:
@@ -238,23 +230,14 @@ jobs:
           chmod +x k0s
           ./k0s sysinfo
 
-      - name: Cache airgap image bundle
-        id: cache-airgap-image-bundle
+      - name: Download airgap image bundle
         if: contains(matrix.smoke-suite, 'airgap')
-        uses: actions/cache@v3
+        uses: actions/download-artifact@v4
         with:
-          key: ${{ needs.build-airgap-image-bundle.outputs.cache-key }}
-          path: airgap-image-bundle-linux-amd64.tar
+          name: airgap-image-bundle-linux-amd64
 
       - name: Run inttest
-        env:
-          NEEDS_AIRGAP_IMAGE_BUNDLE: ${{ contains(matrix.smoke-suite, 'airgap') }}
-        run: |
-          [ "$NEEDS_AIRGAP_IMAGE_BUNDLE" != true ] || [ -f airgap-image-bundle-linux-amd64.tar ] || {
-            echo Airgap image bundle file missing!
-            exit 1
-          }
-          make -C inttest ${{ matrix.smoke-suite }}
+        run: make -C inttest ${{ matrix.smoke-suite }}
 
       - name: Collect k0s logs and support bundle
         if: failure()

.github/workflows/ostests-e2e.yaml

@@ -35,7 +35,6 @@ on:
       k0sctl-version:
         type: string
         description: The k0sctl version to use when bootstrapping the test cluster.
-        default: 0.17.2
     secrets:
       aws-access-key-id:
         description: The AWS access key ID to use when provisioning test resources.
@@ -93,17 +92,27 @@ jobs:
           name: k0s-linux-amd64
           path: ${{ github.workspace }}/.cache
 
+      - name: Set k0sctl version
+        run: |
+          if [ -z "${{ inputs.k0sctl-version }}" ]; then
+            version=$(grep k0sproject/k0sctl hack/tool/go.mod|cut -d" " -f2)
+            echo "Detected k0sctl dependency version ${version}"
+          else
+            version="${{ inputs.k0sctl-version }}"
+            echo "Using given k0sctl version ${version}"
+          fi
+          echo "K0SCTL_VERSION=${version}" >> $GITHUB_ENV
+
       - name: "Terraform :: Requisites :: Prepare"
         env:
-          K0SCTL_VERSION: ${{ inputs.k0sctl-version }}
           K0S_VERSION: ${{ inputs.k0s-version }}
           K0S_EXECUTABLE_PATH: ${{ github.workspace }}/.cache/k0s
         run: |
           kubectl version --client
           jq --version
 
           mkdir -p "$(dirname -- "$TF_VAR_k0sctl_executable_path")"
-          curl -sSLo "$TF_VAR_k0sctl_executable_path" "https://github.com/k0sproject/k0sctl/releases/download/v${K0SCTL_VERSION}/k0sctl-linux-x64"
+          curl -sSLo "$TF_VAR_k0sctl_executable_path" "https://github.com/k0sproject/k0sctl/releases/download/${K0SCTL_VERSION}/k0sctl-linux-x64"
           chmod +x -- "$TF_VAR_k0sctl_executable_path"
           "$TF_VAR_k0sctl_executable_path" version
 

.github/workflows/release.yml

@@ -5,9 +5,6 @@ on:
     tags:
       - v* # Push events to matching v*, i.e. v1.0, v20.15.10
 
-env:
-  K0SCTL_VERSION: 0.13.2
-
 jobs:
   release:
     env:
@@ -612,11 +609,16 @@ jobs:
 
           terraform apply -auto-approve
 
+      - name: Set k0sctl version
+        run: |
+          version=$(cd hack/tool; go list -m -f '{{.Version}}' github.com/k0sproject/k0sctl)
+          echo "K0SCTL_VERSION=${version}" >> $GITHUB_ENV
+
       - name: Create k0s Cluster using k0sctl
         id: k0sctl
         run: |
           # download k0sctl
-          curl --silent -L "https://github.com/k0sproject/k0sctl/releases/download/v${K0SCTL_VERSION}/k0sctl-linux-x64" -o k0sctl
+          curl --silent -L "https://github.com/k0sproject/k0sctl/releases/download/${K0SCTL_VERSION}/k0sctl-linux-x64" -o k0sctl
           chmod +x ./k0sctl
           ./k0sctl apply -c k0sctl.yaml
 

Makefile

@@ -30,7 +30,8 @@ EMBEDDED_BINS_BUILDMODE ?= docker
 TARGET_OS ?= linux
 BUILD_UID ?= $(shell id -u)
 BUILD_GID ?= $(shell id -g)
-BUILD_GO_FLAGS := -tags osusergo -buildvcs=false -trimpath
+BUILD_GO_TAGS ?= osusergo
+BUILD_GO_FLAGS = -tags=$(subst $(space),$(comma),$(BUILD_GO_TAGS)) -buildvcs=false -trimpath
 BUILD_CGO_CFLAGS :=
 BUILD_GO_LDFLAGS_EXTRA :=
 DEBUG ?= false
@@ -126,8 +127,7 @@ pkg/apis/autopilot/v1beta2/.controller-gen.stamp: gen_output_dir = autopilot
 pkg/apis/%/.controller-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerplate.go.txt hack/tools/Makefile.variables
 	rm -rf 'static/manifests/$(gen_output_dir)/CustomResourceDefinition'
 	rm -f -- '$(dir $@)'zz_*.go
-	CGO_ENABLED=0 $(GO) install sigs.k8s.io/controller-tools/cmd/controller-gen@v$(controller-gen_version)
-	$(GO_ENV) controller-gen \
+	CGO_ENABLED=0 $(GO) run sigs.k8s.io/controller-tools/cmd/controller-gen@v$(controller-gen_version) \
 	  crd \
 	  paths="./$(dir $@)..." \
 	  output:crd:artifacts:config=./static/manifests/$(gen_output_dir)/CustomResourceDefinition \
@@ -157,17 +157,11 @@ static/zz_generated_assets.go: $(shell find $(static_asset_dirs) -type f)
 	CGO_ENABLED=0 $(GO) install github.com/kevinburke/go-bindata/go-bindata@v$(go-bindata_version)
 	$(GO_ENV) go-bindata -o '$@' -pkg static -prefix static $(patsubst %,%/...,$(static_asset_dirs))
 
-codegen_targets += pkg/assets/zz_generated_offsets_$(TARGET_OS).go
-zz_os = $(patsubst pkg/assets/zz_generated_offsets_%.go,%,$@)
-print_empty_generated_offsets = printf "%s\n\n%s\n%s\n" \
-			"package assets" \
-			"var BinData = map[string]struct{ offset, size, originalSize int64 }{}" \
-			"var BinDataSize int64"
 ifeq ($(EMBEDDED_BINS_BUILDMODE),none)
-pkg/assets/zz_generated_offsets_linux.go pkg/assets/zz_generated_offsets_windows.go:
-	rm -f bindata_$(zz_os) && touch bindata_$(zz_os)
-	$(print_empty_generated_offsets) > $@
+BUILD_GO_TAGS += noembedbins
 else
+codegen_targets += pkg/assets/zz_generated_offsets_$(TARGET_OS).go
+zz_os = $(patsubst pkg/assets/zz_generated_offsets_%.go,%,$@)
 pkg/assets/zz_generated_offsets_linux.go: .bins.linux.stamp
 pkg/assets/zz_generated_offsets_windows.go: .bins.windows.stamp
 pkg/assets/zz_generated_offsets_linux.go pkg/assets/zz_generated_offsets_windows.go: .k0sbuild.docker-image.k0s go.sum
@@ -176,10 +170,6 @@ pkg/assets/zz_generated_offsets_linux.go pkg/assets/zz_generated_offsets_windows
 	     -prefix embedded-bins/staging/$(zz_os)/ embedded-bins/staging/$(zz_os)/bin
 endif
 
-# needed for unit tests on macos
-pkg/assets/zz_generated_offsets_darwin.go:
-	$(print_empty_generated_offsets) > $@
-
 k0s: TARGET_OS = linux
 k0s: BUILD_GO_CGO_ENABLED = 1
 k0s: .k0sbuild.docker-image.k0s
@@ -188,12 +178,11 @@ k0s.exe: TARGET_OS = windows
 k0s.exe: BUILD_GO_CGO_ENABLED = 0
 
 k0s.exe k0s: $(GO_SRCS) $(codegen_targets) go.sum
-	CGO_ENABLED=$(BUILD_GO_CGO_ENABLED) CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' GOOS=$(TARGET_OS) $(GO) build $(BUILD_GO_FLAGS) -ldflags='$(LD_FLAGS)' -o [email protected] main.go
-	cat [email protected] bindata_$(TARGET_OS) > [email protected] \
-		&& rm -f [email protected] \
-		&& printf "\nk0s size: %s\n\n" "$$(du -sh [email protected] | cut -f1)" \
-		&& chmod +x [email protected] \
-		&& mv [email protected] $@
+	CGO_ENABLED=$(BUILD_GO_CGO_ENABLED) CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' GOOS=$(TARGET_OS) $(GO) build $(BUILD_GO_FLAGS) -ldflags='$(LD_FLAGS)' -o '$@' main.go
+ifneq ($(EMBEDDED_BINS_BUILDMODE),none)
+	cat -- bindata_$(TARGET_OS) >>$@
+endif
+	@printf '\n%s size: %s\n\n' '$@' "$$(du -sh -- $@ | cut -f1)"
 
 .bins.windows.stamp .bins.linux.stamp: embedded-bins/Makefile.variables
 	$(MAKE) -C embedded-bins \
@@ -206,17 +195,20 @@ codegen: $(codegen_targets)
 
 # bindata contains the parts of codegen which aren't version controlled.
 .PHONY: bindata
-bindata: static/zz_generated_assets.go pkg/assets/zz_generated_offsets_$(TARGET_OS).go
+bindata: static/zz_generated_assets.go
+ifneq ($(EMBEDDED_BINS_BUILDMODE),none)
+bindata: pkg/assets/zz_generated_offsets_$(TARGET_OS).go
+endif
 
 .PHONY: lint-copyright
 lint-copyright:
 	hack/copyright.sh
 
 .PHONY: lint-go
 lint-go: GOLANGCI_LINT_FLAGS ?=
-lint-go: .k0sbuild.docker-image.k0s go.sum codegen
+lint-go: .k0sbuild.docker-image.k0s go.sum bindata
 	CGO_ENABLED=0 $(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(golangci-lint_version)
-	CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' $(GO_ENV) golangci-lint run --verbose $(GOLANGCI_LINT_FLAGS) $(GO_LINT_DIRS)
+	CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' $(GO_ENV) golangci-lint run --verbose --build-tags=$(subst $(space),$(comma),$(BUILD_GO_TAGS)) $(GOLANGCI_LINT_FLAGS) $(GO_LINT_DIRS)
 
 .PHONY: lint
 lint: lint-copyright lint-go
@@ -253,12 +245,9 @@ check-unit: GO_TEST_RACE ?=
 else
 check-unit: GO_TEST_RACE ?= -race
 endif
-check-unit: go.sum codegen
-	CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' $(GO) test -tags=hack $(GO_TEST_RACE) -ldflags='$(LD_FLAGS)' `$(GO) list -tags=hack $(GO_CHECK_UNIT_DIRS)`
-
-.PHONY: check-image-validity
-check-image-validity: go.sum
-	$(GO) run -tags=hack hack/validate-images/main.go -architectures amd64,arm64,arm
+check-unit: BUILD_GO_TAGS += hack
+check-unit: .k0sbuild.docker-image.k0s go.sum bindata
+	CGO_CFLAGS='$(BUILD_CGO_CFLAGS)' $(GO) test -tags=$(subst $(space),$(comma),$(BUILD_GO_TAGS)) $(GO_TEST_RACE) -ldflags='$(LD_FLAGS)' `$(GO) list -tags=$(subst $(space),$(comma),$(BUILD_GO_TAGS)) $(GO_CHECK_UNIT_DIRS)`
 
 .PHONY: clean-gocache
 clean-gocache:

docs/configuration.md

@@ -91,6 +91,7 @@ spec:
       mtu: 0
       peerRouterASNs: ""
       peerRouterIPs: ""
+      extraArgs:
     nodeLocalLoadBalancing:
       enabled: false
       envoyProxy:
@@ -213,15 +214,16 @@ CALICO_IPV6POOL_CIDR: "{{ spec.network.dualStack.IPv6podCIDR }}"
 #### `spec.network.kuberouter`
 
 | Element          | Description                                                                                                                                                                                                                                                               |
-| ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `autoMTU`        | Autodetection of used MTU (default: `true`).                                                                                                                                                                                                                              |
 | `mtu`            | Override MTU setting, if `autoMTU` must be set to `false`).                                                                                                                                                                                                               |
 | `metricsPort`    | Kube-router metrics server port. Set to 0 to disable metrics  (default: `8080`).                                                                                                                                                                                          |
-| `peerRouterIPs`  | Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                        |
-| `peerRouterASNs` | Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                             |
+| `peerRouterIPs`  | DEPRECATED: Use extraArgs with peerRouterIPs instead. Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                  |
+| `peerRouterASNs` | DEPRECATED: Use extraArgs with peerRouterASNs instead. Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                      |
 | `hairpin`        | Hairpin mode, supported modes `Enabled`: enabled cluster wide, `Allowed`: must be allowed per service [using annotations](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode), `Disabled`: doesn't work at all (default: Enabled) |
 | `hairpinMode`    | **Deprecated** Use `hairpin` instead. If both `hairpin` and `hairpinMode` are defined, this is ignored. If only hairpinMode is configured explicitly activates hairpinMode (https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode).  |
-| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false) |
+| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false)                                                                                                                                                                   |
+| `extraArgs`      | Extra arguments to pass to kube-router. Can be also used to override any k0s managed args. For reference, see kube-router [documentation](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#command-line-options). (default: empty)           |
 
 **Note**: Kube-router allows many networking aspects to be configured per node, service, and pod (for more information, refer to the [Kube-router user guide](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md)).
 

docs/external-runtime-deps.md

@@ -169,7 +169,7 @@ In order to use containerd in conjunction with [AppArmor], it must be enabled in
 the kernel and the `/sbin/apparmor_parser` executable must be installed on the
 host, otherwise containerd will [disable][cd-aa] AppArmor support.
 
-[cd-aa]: https://github.com/containerd/containerd/blob/v1.7.11/pkg/apparmor/apparmor_linux.go#L34-L45
+[cd-aa]: https://github.com/containerd/containerd/blob/v1.7.12/pkg/apparmor/apparmor_linux.go#L34-L45
 [AppArmor]: https://wiki.ubuntu.com/AppArmor
 
 #### iptables

docs/k0s-in-docker.md

@@ -54,7 +54,7 @@ For each required worker:
 Access your cluster using kubectl:
 
 ```sh
-docker exec k0s kubectl get nodes
+docker exec k0s k0s kubectl get nodes
 ```
 
 Alternatively, grab the kubeconfig file with `docker exec k0s cat /var/lib/k0s/pki/admin.conf` and paste it into [Lens](https://github.com/lensapp/lens/).