Merge pull request #160 from cybozu-go/k8s-matrix-e2e-test

test: Add k8s matrix e2e test.

.github/workflows/ci.yaml

@@ -43,6 +43,7 @@ jobs:
  strategy:
  matrix:
  mysql-version: ["8.0.18", "8.0.20"]
+ k8s-version: ["1.19.4", "1.20.2"]
  runs-on: ubuntu-20.04
  steps:
  - uses: actions/checkout@v2
@@ -59,7 +60,7 @@ jobs:
  ./e2e/script/watch_mysql.sh e2e-test-external mysqlcluster $VARIABLES > watch_mysql_e2e-external.txt &
  ./e2e/script/watch_pod.sh e2e-test > watch_pod_e2e.txt &
  ./e2e/script/watch_pod.sh e2e-test-external > watch_pod_e2e-external.txt &
- - run: cd e2e && make test KIND_CLUSTER_CONFIG_SUFFIX=-actions MYSQL_VERSION=${{ matrix.mysql-version }}
+ - run: cd e2e && make test KIND_CLUSTER_CONFIG_SUFFIX=-actions MYSQL_VERSION=${{ matrix.mysql-version }} KUBERNETES_VERSION=${{ matrix.k8s-version }}
 
  # Print status and log of each component
  - name: Watch MySQL status (e2e-test)

Makefile

@@ -22,7 +22,6 @@ endif
 
 KUBEBUILDER_VERSION := 2.3.1
 CTRLTOOLS_VERSION := 0.4.0
-MYSQL_VERSION ?= 8.0.20
 
 .PHONY: all
 all: build/moco-controller

README.md

@@ -25,6 +25,10 @@ TBD
 
 8.0.18 and 8.0.20
 
+## Supported Kubernetes version
+
+1.20 and 1.19
+
 ## Documentation
 
 [docs](docs/) directory contains documents about designs and specifications.

common.mk

@@ -1 +1,8 @@
+# Kubernetes version for testing
+# This variable is used for getting kube-apiserver (unit tests) and kindest/node (e2e test).
+# Please specify a version that can be used with both (especially kindest/node).
+# ref: https://hub.docker.com/r/kindest/node/tags
 KUBERNETES_VERSION=1.19.4
+
+# MySQL version for testing
+MYSQL_VERSION = 8.0.20

config/rbac/leader_election_role.yaml

@@ -6,8 +6,10 @@ metadata:
 rules:
 - apiGroups:
  - ""
+ - coordination.k8s.io
  resources:
  - configmaps
+ - leases
  verbs:
  - get
  - list
@@ -16,17 +18,10 @@ rules:
  - update
  - patch
  - delete
-- apiGroups:
- - ""
- resources:
- - configmaps/status
- verbs:
- - get
- - update
- - patch
 - apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
+ - patch

config/rbac/role.yaml

@@ -139,17 +139,6 @@ rules:
  - cronjobs/status
  verbs:
  - get
-- apiGroups:
- - coordination.k8s.io
- resources:
- - leases
- verbs:
- - create
- - delete
- - get
- - list
- - update
- - watch
 - apiGroups:
  - moco.cybozu.com
  resources:

controllers/mysqlcluster_controller.go

@@ -91,7 +91,6 @@ type MySQLClusterReconciler struct {
 // +kubebuilder:rbac:groups="batch",resources=cronjobs/status,verbs=get
 // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;patch
 // +kubebuilder:rbac:groups="policy",resources=poddisruptionbudgets,verbs=get;list;watch;create;update
-// +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;list;watch;create;update;delete
 
 // Reconcile reconciles MySQLCluster.
 func (r *MySQLClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

e2e/Makefile

@@ -1,23 +1,19 @@
 include ../common.mk
 
 KUBECTL := $(PWD)/bin/kubectl
-KIND_VERSION=0.9.0
+KIND_VERSION=0.10.0
 KUSTOMIZE_VERSION=3.7.0
 KUSTOMIZE := $(PWD)/bin/kustomize
 KIND_CLUSTER_NAME=moco-e2e
 KUBECTL_MOCO := $(PWD)/bin/kubectl-moco
 
 MOCO_CONF_GEN_VERSION=0.3.0
-KUBEADM_APIVERSION=kubeadm.k8s.io/v1beta2
-MYSQL_VERSION ?= 8.0.20
 
 GO_FILES := $(shell find .. -path ../e2e -prune -o -name '*.go' -print)
 
 launch-kind: kind
  if [ ! "$(shell kind get clusters | grep $(KIND_CLUSTER_NAME))" ]; then \
- sed -e "s|@KUBERNETES_VERSION@|$(KUBERNETES_VERSION)|" \
- -e "s|@KUBEADM_APIVERSION@|$(KUBEADM_APIVERSION)|" kind-cluster$(KIND_CLUSTER_CONFIG_SUFFIX).yaml > /tmp/kind-cluster.yaml; \
- kind create cluster --name=$(KIND_CLUSTER_NAME) --config /tmp/kind-cluster.yaml --image kindest/node:v$(KUBERNETES_VERSION); \
+ kind create cluster --name=$(KIND_CLUSTER_NAME) --config kind-cluster$(KIND_CLUSTER_CONFIG_SUFFIX).yaml --image kindest/node:v$(KUBERNETES_VERSION); \
  fi
 
 shutdown-kind: kind

e2e/gc_test.go

@@ -20,18 +20,28 @@ func testGarbageCollector() {
  stdout, stderr, err := kubectl("delete", "-n", "e2e-test", "-f", "manifests/mysql_cluster.yaml")
  Expect(err).ShouldNot(HaveOccurred(), "stdout=%s, stderr=%s", stdout, stderr)
 
- for _, kind := range []string{"configmaps", "service", "cronjobs", "jobs", "statefulsets", "pods"} {
- Eventually(func() error {
- stdout, stderr, err := kubectl("get", "-n", "e2e-test", kind)
- if err != nil {
- return fmt.Errorf("failed to get resource. stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
- }
- if !strings.Contains(string(stderr), "No resources found in e2e-test namespace.") {
- return fmt.Errorf("resources remain: %s, stdout: %s, stderr: %s", kind, stdout, stderr)
- }
+ kinds := strings.Join([]string{"configmaps", "services", "cronjobs", "jobs", "statefulsets", "pods"}, ",")
+ Eventually(func() error {
+ stdout, stderr, err := kubectl("get", "-n", "e2e-test", kinds, "-o", "name")
+ if err != nil {
+ return fmt.Errorf("failed to get resource. stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
+ }
+ resources := strings.Split(strings.TrimSuffix(string(stdout), "\n"), "\n")
+
+ // k8s >=v1.20 does not need this condition,
+ // because kube-root-ca.crt cm is created in every namespace.
+ // Please remove this line when the support for k8s <=v1.19 is dropped
+ if len(resources) == 1 && resources[0] == "" {
  return nil
- }, 2*time.Minute).Should(Succeed())
- }
+ }
+ if len(resources) == 1 && resources[0] == "configmap/kube-root-ca.crt" {
+
+ fmt.Println("only remain configmap/kube-root-ca.crt")
+ return nil
+ }
+
+ return fmt.Errorf("resources remain: %s", resources)
+ }, 2*time.Minute).Should(Succeed())
 
  for _, resource := range []string{
  "serviceaccount/mysqld-sa-" + moco.UniqueName(cluster),

e2e/kind-cluster-actions.yaml

@@ -1,26 +1,10 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-# patch the generated kubeadm config with some extra settings
-kubeadmConfigPatches:
-- |
- # When update to Kubernetes 1.15, make sure to change the
- # apiVersion to "kubeadm.k8s.io/v1beta2". Also make sure
- # to change the node image in Makefile (--image kindest/node)
- apiVersion: "@KUBEADM_APIVERSION@"
- kind: ClusterConfiguration
- metadata:
- name: config
- kubernetesVersion: "v@KUBERNETES_VERSION@"
- networking:
- serviceSubnet: 10.0.0.0/16
- controllerManager:
- extraArgs:
- "v": "5"
+networking:
+ serviceSubnet: "10.0.0.0/16"
 nodes:
 - role: control-plane
 - role: worker
  extraMounts:
  - hostPath: /mnt/local-path-provisioner
  containerPath: /var/local-path-provisioner
-# - role: worker
-# - role: worker

e2e/kind-cluster.yaml

@@ -1,23 +1,7 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-# patch the generated kubeadm config with some extra settings
-kubeadmConfigPatches:
-- |
- # When update to Kubernetes 1.15, make sure to change the
- # apiVersion to "kubeadm.k8s.io/v1beta2". Also make sure
- # to change the node image in Makefile (--image kindest/node)
- apiVersion: "@KUBEADM_APIVERSION@"
- kind: ClusterConfiguration
- metadata:
- name: config
- kubernetesVersion: "v@KUBERNETES_VERSION@"
- networking:
- serviceSubnet: 10.0.0.0/16
- controllerManager:
- extraArgs:
- "v": "5"
+networking:
+ serviceSubnet: "10.0.0.0/16"
 nodes:
 - role: control-plane
 - role: worker
-# - role: worker
-# - role: worker

go.mod

@@ -25,7 +25,7 @@ require (
  k8s.io/client-go v0.20.2
  k8s.io/klog v1.0.0
  k8s.io/kubectl v0.20.2
- sigs.k8s.io/controller-runtime v0.8.0
+ sigs.k8s.io/controller-runtime v0.8.1
  sigs.k8s.io/controller-tools v0.4.1
  sigs.k8s.io/yaml v1.2.0
 )