Add kubeadm cluster support to kne deploy (#504)

* add kubeadm cluster support to kne deploy

* fix issues

* add test skeleton

* add tests

* address feedback

cloudbuild/kne_test.sh

@@ -83,13 +83,21 @@ fi
 # Cleanup the cluster
 $cli teardown kne/deploy/kne/kind-bridge.yaml
 
+# Create a kubeadm cluster
+$cli deploy kne/deploy/kne/kubeadm.yaml --report_usage=false
+
+kubectl get pods -A
+
+# Cleanup the kubeadm cluster
+$cli teardown kne/deploy/kne/kubeadm.yaml
+
 ## Create a kubeadm single node cluster
 sudo kubeadm init --cri-socket unix:///var/run/cri-dockerd.sock --pod-network-cidr 10.244.0.0/16
 mkdir -p "$HOME"/.kube
 sudo cp /etc/kubernetes/admin.conf "$HOME"/.kube/config
 sudo chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config
 kubectl taint nodes --all node-role.kubernetes.io/control-plane-  # allows pods to be scheduled on control plane node
-kubectl apply -f "$HOME"/flannel/Documentation/kube-flannel.yml
+kubectl apply -f kne/manifests/flannel/manifest.yaml
 docker network create multinode
 
 # Deploy an external cluster

deploy/deploy.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openconfig/kne/metrics"
 	"github.com/openconfig/kne/pods"
 	epb "github.com/openconfig/kne/proto/event"
+	"github.com/pborman/uuid"
 	metallbv1 "go.universe.tf/metallb/api/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -48,6 +49,7 @@ var (
 	// Stubs for testing.
 	execLookPath       = exec.LookPath
 	kindSetupGARAccess = kind.SetupGARAccess
+	homeDir            = homedir.HomeDir
 )
 
 type Cluster interface {
@@ -402,6 +404,132 @@ func kubectlApply(cfg []byte) error {
 	return run.LogCommandWithInput(cfg, "kubectl", "apply", "-f", "-")
 }
 
+func init() {
+	load.Register("Kubeadm", &load.Spec{
+		Type: KubeadmSpec{},
+		Tag:  "cluster",
+	})
+}
+
+type KubeadmSpec struct {
+	CRISocket                   string `yaml:"criSocket"`
+	PodNetworkCIDR              string `yaml:"podNetworkCIDR"`
+	PodNetworkAddOnManifest     string `yaml:"podNetworkAddOnManifest" kne:"yaml"`
+	PodNetworkAddOnManifestData []byte
+	TokenTTL                    string `yaml:"tokenTTL"`
+	Network                     string `yaml:"network"`
+	AllowControlPlaneScheduling bool   `yaml:"allowControlPlaneScheduling"`
+}
+
+func (k *KubeadmSpec) checkDependencies() error {
+	var errs errlist.List
+	bins := []string{"kubeadm"}
+	for _, bin := range bins {
+		if _, err := execLookPath(bin); err != nil {
+			errs.Add(fmt.Errorf("install dependency %q to deploy", bin))
+		}
+	}
+	return errs.Err()
+}
+
+func (k *KubeadmSpec) Deploy(ctx context.Context) error {
+	if err := k.checkDependencies(); err != nil {
+		return fmt.Errorf("failed to check for dependencies: %w", err)
+	}
+	args := []string{"kubeadm", "init"}
+	if k.CRISocket != "" {
+		args = append(args, "--cri-socket", k.CRISocket)
+	}
+	if k.PodNetworkCIDR != "" {
+		args = append(args, "--pod-network-cidr", k.PodNetworkCIDR)
+	}
+	if k.TokenTTL != "" {
+		args = append(args, "--token-ttl", k.TokenTTL)
+	}
+	if err := run.LogCommand("sudo", args...); err != nil {
+		return err
+	}
+	kubeDir := filepath.Join(homeDir(), ".kube")
+	if err := os.MkdirAll(kubeDir, 0750); err != nil {
+		return err
+	}
+	b, err := run.OutCommand("sudo", "cat", "/etc/kubernetes/admin.conf")
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(kubeDir, "config"), b, 0640); err != nil {
+		return err
+	}
+	if k.AllowControlPlaneScheduling {
+		if err := run.LogCommand("kubectl", "taint", "nodes", "--all", "node-role.kubernetes.io/control-plane:NoSchedule-"); err != nil {
+			return err
+		}
+	}
+	// If add on is provided, apply it.
+	if k.PodNetworkAddOnManifestData != nil {
+		f, err := os.CreateTemp("", "kubeadm-pod-network-add-on-manifest-*.yaml")
+		if err != nil {
+			return err
+		}
+		defer os.Remove(f.Name())
+		if _, err := f.Write(k.PodNetworkAddOnManifestData); err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+		k.PodNetworkAddOnManifest = f.Name()
+	}
+	if k.PodNetworkAddOnManifest != "" {
+		b, err := os.ReadFile(k.PodNetworkAddOnManifest)
+		if err != nil {
+			return err
+		}
+		if err := kubectlApply(b); err != nil {
+			return err
+		}
+	}
+
+	// Create a new docker network if not specified.
+	if k.Network == "" {
+		k.Network = "kne-kubeadm-" + uuid.New()
+		if err := run.LogCommand("docker", "network", "create", k.Network); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *KubeadmSpec) Delete() error {
+	args := []string{"kubeadm", "reset", "--force"}
+	if k.CRISocket != "" {
+		args = append(args, "--cri-socket", k.CRISocket)
+	}
+	if err := run.LogCommand("sudo", args...); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (k *KubeadmSpec) Healthy() error {
+	if err := run.LogCommand("kubectl", "cluster-info"); err != nil {
+		return fmt.Errorf("cluster not healthy: %w", err)
+	}
+	return nil
+}
+
+func (k *KubeadmSpec) GetName() string {
+	return "kubeadm"
+}
+
+func (k *KubeadmSpec) GetDockerNetworkResourceName() string {
+	return k.Network
+}
+
+func (k *KubeadmSpec) Apply(cfg []byte) error {
+	return kubectlApply(cfg)
+}
+
 func init() {
 	load.Register("Kind", &load.Spec{
 		Type: KindSpec{},

deploy/deploy_test.go

@@ -36,6 +36,92 @@ func init() {
 	klog.LogToStderr(false)
 }
 
+func TestKubeadmSpec(t *testing.T) {
+	ctx := context.Background()
+
+	origHomeDir := homeDir
+	defer func() {
+		homeDir = origHomeDir
+	}()
+	homeDir = func() string { return t.TempDir() }
+
+	tests := []struct {
+		desc        string
+		k           *KubeadmSpec
+		resp        []fexec.Response
+		execPathErr bool
+		wantErr     string
+	}{{
+		desc:        "kubeadm not found",
+		k:           &KubeadmSpec{},
+		execPathErr: true,
+		wantErr:     `install dependency "kubeadm" to deploy`,
+	}, {
+		desc: "create cluster",
+		k:    &KubeadmSpec{},
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "init"}},
+			{Cmd: "sudo", Args: []string{"cat", "/etc/kubernetes/admin.conf"}},
+			{Cmd: "docker", Args: []string{"network", "create", "kne-kubeadm-.*"}},
+		},
+	}, {
+		desc: "allow control plane scheduling",
+		k: &KubeadmSpec{
+			AllowControlPlaneScheduling: true,
+		},
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "init"}},
+			{Cmd: "sudo", Args: []string{"cat", "/etc/kubernetes/admin.conf"}},
+			{Cmd: "kubectl", Args: []string{"taint", "nodes", "--all", "node-role.kubernetes.io/control-plane:NoSchedule-"}},
+			{Cmd: "docker", Args: []string{"network", "create", "kne-kubeadm-.*"}},
+		},
+	}, {
+		desc: "pod manifest add on data",
+		k: &KubeadmSpec{
+			PodNetworkAddOnManifestData: []byte("manifest yaml"),
+		},
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "init"}},
+			{Cmd: "sudo", Args: []string{"cat", "/etc/kubernetes/admin.conf"}},
+			{Cmd: "kubectl", Args: []string{"apply", "-f", "-"}},
+			{Cmd: "docker", Args: []string{"network", "create", "kne-kubeadm-.*"}},
+		},
+	}, {
+		desc: "provided network",
+		k: &KubeadmSpec{
+			Network: "my-network",
+		},
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "init"}},
+			{Cmd: "sudo", Args: []string{"cat", "/etc/kubernetes/admin.conf"}},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			if verbose {
+				fexec.LogCommand = func(s string) {
+					t.Logf("%s: %s", tt.desc, s)
+				}
+			}
+			cmds := fexec.Commands(tt.resp)
+			kexec.Command = cmds.Command
+			defer checkCmds(t, cmds)
+
+			execLookPath = func(_ string) (string, error) {
+				if tt.execPathErr {
+					return "", errors.New("unable to find on path")
+				}
+				return "fakePath", nil
+			}
+
+			err := tt.k.Deploy(ctx)
+			if s := errdiff.Substring(err, tt.wantErr); s != "" {
+				t.Fatalf("unexpected error: %s", s)
+			}
+		})
+	}
+}
+
 func TestKubectlApply(t *testing.T) {
 	tests := []struct {
 		desc    string

deploy/kne/kubeadm.yaml

@@ -0,0 +1,34 @@
+# kubeadm.yaml cluster config file sets up ingress, cni, and controllers in a new k8 cluster
+# created using kubeadm. The kubeadm cluster starts as a single node cluster but can be joined
+# from other hosts to create a multinode cluster.
+cluster:
+  kind: Kubeadm
+  spec:
+    criSocket: unix:///var/run/cri-dockerd.sock
+    podNetworkCIDR: 10.244.0.0/16
+    tokenTTL: 0 # no timeout
+    podNetworkAddOnManifest: ../../manifests/flannel/manifest.yaml
+    allowControlPlaneScheduling: true
+ingress:
+  kind: MetalLB
+  spec:
+    manifest: ../../manifests/metallb/manifest.yaml
+    ip_count: 200
+cni:
+  kind: Meshnet
+  spec:
+    manifest: ../../manifests/meshnet/grpc/manifest.yaml
+controllers:
+  - kind: IxiaTG
+    spec:
+      operator: ../../manifests/keysight/ixiatg-operator.yaml
+      configMap: ../../manifests/keysight/ixiatg-configmap.yaml
+  - kind: SRLinux
+    spec:
+      operator: ../../manifests/controllers/srlinux/manifest.yaml
+  - kind: CEOSLab
+    spec:
+      operator: ../../manifests/controllers/ceoslab/manifest.yaml
+  - kind: Lemming
+    spec:
+      operator: ../../manifests/controllers/lemming/manifest.yaml