cluster: add registry mirrors for the cluster network (#51)

Fixes #49

.circleci/config.yml

@@ -1,17 +1,49 @@
 version: 2.1
 orbs:
   slack: circleci/[email protected]
+  kubernetes: circleci/[email protected]
 jobs:
   build:
     docker:
       - image: circleci/golang:1.14
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    working_directory: /go/src/github.com/tilt-dev/ctlptl
     steps:
       - checkout
       - run: go get -v -t -d ./...
       - run: go test -v ./...
       - slack/notify-on-failure:
           only_for_branches: master
+
+  e2e:
+    machine:
+      image: ubuntu-1604:201903-01
+    working_directory: /home/circleci/.go_workspace/src/github.com/tilt-dev/ctlptl
+    steps:
+      - checkout
+      - kubernetes/install-kubectl
+      - run: |
+          set -ex
+          wget https://golang.org/dl/go1.15.5.linux-amd64.tar.gz
+          sudo rm -fR /usr/local/go
+          sudo tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz
+      - run: |
+          set -ex
+          export MINIKUBE_VERSION=v1.15.0
+          curl -fLo ./minikube-linux-amd64 "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64"
+          chmod +x ./minikube-linux-amd64
+          sudo mv ./minikube-linux-amd64 /usr/local/bin/minikube
+      - run: |
+          set -ex
+          export KIND_VERSION=v0.9.0
+          curl -fLo ./kind-linux-amd64 "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+          chmod +x ./kind-linux-amd64
+          sudo mv ./kind-linux-amd64 /usr/local/bin/kind
+      - run: |
+          set -ex
+          go get -v -t -d ./...
+          test/e2e.sh
+      - slack/notify-on-failure:
+          only_for_branches: master
 
   release-dry-run:
     docker:
@@ -43,6 +75,10 @@ workflows:
     jobs:
       - build:
           context: Tilt Slack Context
+      - e2e:
+          context: Tilt Slack Context
+          requires:
+            - build
       - release-dry-run:
           context: Tilt Slack Context
           requires:

Makefile

@@ -11,3 +11,7 @@ generated:
 
 tidy:
 	go mod tidy
+
+e2e:
+	test/e2e.sh
+

pkg/cluster/admin_kind.go

@@ -61,7 +61,10 @@ containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%d"]
     endpoint = ["http://%s:%d"]
-`, registry.Status.HostPort, registry.Name, registry.Status.ContainerPort)
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s:%d"]
+    endpoint = ["http://%s:%d"]
+`, registry.Status.HostPort, registry.Name, registry.Status.ContainerPort,
+			registry.Name, registry.Status.ContainerPort, registry.Name, registry.Status.ContainerPort)
 		in = strings.NewReader(containerdConfig)
 
 		args = append(args, "--config", "-")

pkg/cluster/admin_minikube.go

@@ -130,8 +130,12 @@ func (a *minikubeAdmin) applyContainerdPatch(ctx context.Context, desired *api.C
 			fmt.Sprintf(
 				`s,\\\[plugins.cri.registry.mirrors\\\],[plugins.cri.registry.mirrors]\\\n`+
 					`\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"localhost:%d\\\"]\\\n`+
+					`\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://%s:%d\\\"]\\\n`+
+					`\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"%s:%d\\\"]\\\n`+
 					`\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://%s:%d\\\"],`,
-				registry.Status.HostPort, networkHost, registry.Status.ContainerPort),
+				registry.Status.HostPort, networkHost, registry.Status.ContainerPort,
+				networkHost, registry.Status.ContainerPort,
+				networkHost, registry.Status.ContainerPort),
 			configPath)
 		cmd.Stderr = a.iostreams.ErrOut
 		cmd.Stdout = a.iostreams.Out

test/e2e.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Integration tests that create a full cluster.
+
+set -exo pipefail
+
+cd $(dirname $(dirname $(realpath $0)))
+make install
+test/kind-cluster-network/e2e.sh
+test/minikube-cluster-network/e2e.sh

test/kind-cluster-network/Dockerfile

@@ -0,0 +1,4 @@
+FROM golang:1.15
+RUN go get github.com/google/ko/cmd/ko
+WORKDIR /go/github.com/tilt-dev/ctlptl/test/cluster-network
+ADD . .

test/kind-cluster-network/builder.yaml

@@ -0,0 +1,22 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ko-builder
+spec:
+  template:
+    metadata:
+      labels:
+        app: ko-builder
+    spec:
+      containers:
+      - name: builder
+        image: localhost:5005/ko-builder
+        command: ["bash", "-c",
+          "go get github.com/tilt-dev/ctlptl/test/simple-server && ko publish --insecure-registry github.com/tilt-dev/ctlptl/test/simple-server"]
+        env:
+        - name: KO_DOCKER_REPO
+          value: REGISTRY_HOST_PLACEHOLDER
+        - name: GO111MODULE
+          value: "off"
+      restartPolicy: Never
+  backoffLimit: 4

test/kind-cluster-network/cluster.yaml

@@ -0,0 +1,6 @@
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+name: kind-ctlptl-test-cluster
+product: kind
+registry: ctlptl-test-registry
+

test/kind-cluster-network/e2e.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Tests creating a cluster with a registry,
+# building a container in that cluster,
+# then running that container.
+
+set -exo pipefail
+
+cd $(dirname $(realpath $0))
+ctlptl apply -f registry.yaml
+ctlptl apply -f cluster.yaml
+
+# The ko-builder runs in an image tagged with the host as visible from the local machine.
+docker build -t localhost:5005/ko-builder .
+docker push localhost:5005/ko-builder
+
+# The ko-builder builds an image tagged with the host as visible from the cluster network.
+HOST=$(ctlptl get cluster kind-ctlptl-test-cluster -o template --template '{{.status.localRegistryHosting.hostFromClusterNetwork}}')
+cat builder.yaml | sed "s/REGISTRY_HOST_PLACEHOLDER/$HOST/" | kubectl apply -f -
+kubectl wait --for=condition=complete job/ko-builder --timeout=180s
+cat simple-server.yaml | sed "s/REGISTRY_HOST_PLACEHOLDER/$HOST/" | kubectl apply -f -
+kubectl wait --for=condition=available deployment/simple-server --timeout=60s
+ctlptl delete -f cluster.yaml
+echo "kind-cluster-network test passed!"

test/kind-cluster-network/registry.yaml

@@ -0,0 +1,4 @@
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-test-registry
+port: 5005

test/kind-cluster-network/simple-server.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simple-server
+  labels:
+    app: simple-server
+spec:
+  selector:
+    matchLabels:
+      app: simple-server
+  template:
+    metadata:
+      labels:
+        app: simple-server
+    spec:
+      containers:
+      - name: simple-server
+        image: REGISTRY_HOST_PLACEHOLDER/simple-server-ee41f08d9609e718cc045a4f0190444b
+        ports:
+        - containerPort: 8080
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080

test/minikube-cluster-network/Dockerfile

@@ -0,0 +1,4 @@
+FROM golang:1.15
+RUN go get github.com/google/ko/cmd/ko
+WORKDIR /go/github.com/tilt-dev/ctlptl/test/cluster-network
+ADD . .

test/minikube-cluster-network/builder.yaml

@@ -0,0 +1,22 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ko-builder
+spec:
+  template:
+    metadata:
+      labels:
+        app: ko-builder
+    spec:
+      containers:
+      - name: builder
+        image: localhost:5005/ko-builder
+        command: ["bash", "-c",
+          "go get github.com/tilt-dev/ctlptl/test/simple-server && ko publish --insecure-registry github.com/tilt-dev/ctlptl/test/simple-server"]
+        env:
+        - name: KO_DOCKER_REPO
+          value: REGISTRY_HOST_PLACEHOLDER
+        - name: GO111MODULE
+          value: "off"
+      restartPolicy: Never
+  backoffLimit: 4

test/minikube-cluster-network/cluster.yaml

@@ -0,0 +1,6 @@
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+name: minikube-ctlptl-test-cluster
+product: minikube
+registry: ctlptl-test-registry
+

test/minikube-cluster-network/e2e.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Tests creating a cluster with a registry,
+# building a container in that cluster,
+# then running that container.
+
+set -exo pipefail
+
+cd $(dirname $(realpath $0))
+ctlptl apply -f registry.yaml
+ctlptl apply -f cluster.yaml
+
+# The ko-builder runs in an image tagged with the host as visible from the local machine.
+docker build -t localhost:5005/ko-builder .
+docker push localhost:5005/ko-builder
+
+# The ko-builder builds an image tagged with the host as visible from the cluster network.
+HOST=$(ctlptl get cluster minikube-ctlptl-test-cluster -o template --template '{{.status.localRegistryHosting.hostFromClusterNetwork}}')
+cat builder.yaml | sed "s/REGISTRY_HOST_PLACEHOLDER/$HOST/g" | kubectl apply -f -
+kubectl wait --for=condition=complete job/ko-builder --timeout=180s
+cat simple-server.yaml | sed "s/REGISTRY_HOST_PLACEHOLDER/$HOST/g" | kubectl apply -f -
+kubectl wait --for=condition=available deployment/simple-server --timeout=60s
+ctlptl delete -f cluster.yaml
+
+echo "minikube-cluster-network test passed!"

test/minikube-cluster-network/registry.yaml

@@ -0,0 +1,4 @@
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-test-registry
+port: 5005

test/minikube-cluster-network/simple-server.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simple-server
+  labels:
+    app: simple-server
+spec:
+  selector:
+    matchLabels:
+      app: simple-server
+  template:
+    metadata:
+      labels:
+        app: simple-server
+    spec:
+      containers:
+      - name: simple-server
+        image: REGISTRY_HOST_PLACEHOLDER/simple-server-ee41f08d9609e718cc045a4f0190444b
+        ports:
+        - containerPort: 8080
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080