Merge pull request #643 from carreter/server-counter-e2e #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e | |
| on: | |
| push: | |
| branches: | |
| - 'master' | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| branches: [ master ] | |
| workflow_dispatch: | |
| env: | |
| GO_TOOLCHAIN: "golang" | |
| # TODO: match BASEIMAGE with Makefile default (nonroot variant) | |
| BASEIMAGE: "gcr.io/distroless/static-debian11" | |
| KIND_CLUSTER_NAME: "kind" | |
| jobs: | |
| build: | |
| name: build | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: go.mod | |
| id: go | |
| - name: Build | |
| run: | | |
| mkdir _output | |
| make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} | |
| docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master | |
| docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar | |
| make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} | |
| docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-server:master | |
| docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: konnectivity-server | |
| path: _output/konnectivity-server.tar | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: konnectivity-agent | |
| path: _output/konnectivity-agent.tar | |
| kind-e2e: | |
| name: kind-e2e | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 100 | |
| needs: | |
| - build | |
| env: | |
| REGISTRY: gcr.io/k8s-staging-kas-network-proxy | |
| KIND_IMAGE: kindest/node${{ matrix.k8s }} | |
| TAG: master | |
| CONNECTION_MODE: ${{ matrix.connection-mode }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| k8s: [ v1.27.11, v1.28.7, v1.29.2 ] | |
| connection-mode: [ grpc, http-connect ] | |
| steps: | |
| - name: Install kind | |
| run: | | |
| curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 | |
| chmod +x ./kind | |
| sudo mv ./kind /usr/local/bin/kind | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: go.mod | |
| id: go | |
| - name: Download prebuilt konnectivity-server image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: konnectivity-server | |
| - name: Download prebuilt konnectivity-agent image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: konnectivity-agent | |
| - name: Load prebuilt konnectivity images | |
| run: | | |
| docker load --input konnectivity-server.tar | |
| docker load --input konnectivity-agent.tar | |
| - name: Fix konnectivity docker image tags | |
| run: | | |
| docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master | |
| docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master | |
| - name: Run e2e tests | |
| run: make test-e2e-ci | |
| e2e: | |
| name: e2e | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 100 | |
| needs: | |
| - build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ipFamily: ["ipv4", "ipv6", "dual"] | |
| k8s: [ v1.27.11, v1.28.7, v1.29.2 ] | |
| env: | |
| JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}" | |
| IP_FAMILY: ${{ matrix.ipFamily }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Enable ipv4 and ipv6 forwarding | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.forwarding=1 | |
| sudo sysctl -w net.ipv4.ip_forward=1 | |
| - name: Set up environment (download dependencies) | |
| run: | | |
| TMP_DIR=$(mktemp -d) | |
| # Test binaries | |
| curl -L https://dl.k8s.io/${{ matrix.k8s }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz | |
| tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \ | |
| --directory ${TMP_DIR} \ | |
| --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test | |
| # kubectl | |
| curl -L https://dl.k8s.io/${{ matrix.k8s }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl | |
| # kind | |
| curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 | |
| # Install | |
| sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo | |
| sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test | |
| sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl | |
| sudo cp ${TMP_DIR}/kind /usr/local/bin/kind | |
| sudo chmod +x /usr/local/bin/* | |
| - name: Create multi node cluster | |
| run: | | |
| # output_dir | |
| mkdir -p _artifacts | |
| # create cluster | |
| cat <<EOF | /usr/local/bin/kind create cluster \ | |
| --name ${{ env.KIND_CLUSTER_NAME}} \ | |
| --image kindest/node:${{ matrix.k8s }} \ | |
| -v7 --wait 1m --retain --config=- | |
| kind: Cluster | |
| apiVersion: kind.x-k8s.io/v1alpha4 | |
| networking: | |
| ipFamily: ${IP_FAMILY} | |
| nodes: | |
| - role: control-plane | |
| image: ${{ matrix.k8s }} | |
| kubeadmConfigPatchesJSON6902: | |
| - kind: ClusterConfiguration | |
| patch: | | |
| - op: add | |
| path: /apiServer/certSANs/- | |
| value: konnectivity-server.kube-system.svc.cluster.local | |
| kubeadmConfigPatches: | |
| - | | |
| kind: ClusterConfiguration | |
| apiServer: | |
| extraArgs: | |
| "egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" | |
| extraVolumes: | |
| - name: egress-selector-config-file | |
| hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" | |
| mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml" | |
| readOnly: true | |
| - name: konnectivity-server | |
| hostPath: "/etc/kubernetes/konnectivity-server" | |
| mountPath: "/etc/kubernetes/konnectivity-server" | |
| readOnly: true | |
| extraMounts: | |
| - hostPath: ./examples/kind/egress_selector_configuration.yaml | |
| containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml | |
| - role: worker | |
| image: ${{ matrix.k8s }} | |
| - role: worker | |
| image: ${{ matrix.k8s }} | |
| EOF | |
| # dump the kubeconfig for later | |
| /usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: konnectivity-server | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: konnectivity-agent | |
| - name: Install konnectivity | |
| run: | | |
| # preload konnectivity images | |
| docker load --input konnectivity-server.tar | |
| docker load --input konnectivity-agent.tar | |
| /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}} | |
| /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}} | |
| kubectl apply -f examples/kind/konnectivity-server.yaml | |
| kubectl apply -f examples/kind/konnectivity-agent-ds.yaml | |
| - name: Get Cluster status | |
| run: | | |
| # wait network is ready | |
| sleep 5 | |
| /usr/local/bin/kubectl get nodes -o wide | |
| /usr/local/bin/kubectl get pods -A | |
| /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns | |
| # smoke test | |
| /usr/local/bin/kubectl run test --image httpd:2 | |
| /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test | |
| /usr/local/bin/kubectl get pods -A -owide | |
| /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent | |
| /usr/local/bin/kubectl logs test | |
| - name: Workaround CoreDNS for IPv6 airgapped | |
| if: ${{ matrix.ipFamily == 'ipv6' }} | |
| run: | | |
| # Patch CoreDNS to work in Github CI | |
| # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured | |
| # to work in an offline environment: | |
| # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452 | |
| # 2. Github CI adds following domains to resolv.conf search field: | |
| # .net. | |
| # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL | |
| # otherwise pods stops trying to resolve the domain. | |
| # Get the current config | |
| original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns) | |
| echo "Original CoreDNS config:" | |
| echo "${original_coredns}" | |
| # Patch it | |
| fixed_coredns=$( | |
| printf '%s' "${original_coredns}" | sed \ | |
| -e 's/^.*kubernetes cluster\.local/& net/' \ | |
| -e '/^.*upstream$/d' \ | |
| -e '/^.*fallthrough.*$/d' \ | |
| -e '/^.*forward . \/etc\/resolv.conf$/d' \ | |
| -e '/^.*loop$/d' \ | |
| ) | |
| echo "Patched CoreDNS config:" | |
| echo "${fixed_coredns}" | |
| printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f - | |
| - name: Run tests | |
| run: | | |
| export KUBERNETES_CONFORMANCE_TEST='y' | |
| export E2E_REPORT_DIR=${PWD}/_artifacts | |
| # Run tests | |
| /usr/local/bin/ginkgo --nodes=25 \ | |
| --focus="\[Conformance\]" \ | |
| --skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \ | |
| /usr/local/bin/e2e.test \ | |
| -- \ | |
| --kubeconfig=${PWD}/_artifacts/kubeconfig.conf \ | |
| --provider=local \ | |
| --dump-logs-on-failure=false \ | |
| --report-dir=${E2E_REPORT_DIR} \ | |
| --disable-log-dump=true | |
| - name: Upload Junit Reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kind-junit-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }} | |
| path: './_artifacts/*.xml' | |
| - name: Export logs | |
| if: always() | |
| run: | | |
| /usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kind-logs-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }} | |
| path: ./_artifacts/logs |