Add Hosts for Ingress and refractor

Makefile

@@ -8,9 +8,7 @@ GOBIN := $(PROJECTROOT)/bin
 UTILDIR := $(PROJECTROOT)/scripts/utils
 SPINNER := $(UTILDIR)/spinner.sh
 BUILDIR := $(PROJECTROOT)/scripts/build
-CONTROLLER_MANIFEST:= $(PROJECTROOT)/manifests/dev/expose-controller.yml
-HELM_MANIFEST:= $(PROJECTROOT)/manifests/templates/helm-values.yml
-OPENVPN_MANIFEST:= $(PROJECTROOT)/manifests/templates/helm-values.yml
+MANIFEST:= $(PROJECTROOT)/kubernetes/manifests
 
 KEY_NAME := team
 
@@ -83,20 +81,16 @@ gen-certificates:
 	kubectl --namespace $(OPENVPN_NAMESPACE) exec -it $(POD_NAME) cat "/etc/openvpn/certs/pki/$(KEY_NAME)-$$n.ovpn" > $(KEY_NAME)-$$n.ovpn; \
 	done
 
-gen-vpn: set-env
-	helm install openvpn -f $(HELM_MANIFEST) stable/openvpn --namespace openvpn
-	minikube tunnel
-
 set-env: build
 	minikube start --driver=docker && \
 	minikube addons enable ingress  && \
-	kubectl apply -f $(CONTROLLER_MANIFEST) && \
+	kubectl apply -f $(MANIFEST) && \
 	sudo -- sh -c "echo \"$(minikube service nginx-ingress-controller --url -n kube-system | awk '{print substr($0,8)}' | awk '{print substr($0, 1, length($0)-6)}' | head -1)    katana.local\" >> /etc/hosts" &&\
 	cp config.sample.toml config.toml && \
 	./bin/katana run
 
 set-env-prod: build
-	kubectl apply -f $(CONTROLLER_MANIFEST) && \
+	kubectl apply -f $(MANIFEST) && \
 	cp config.sample.toml config.toml && \
 	sudo ./bin/katana run
 

config.sample.toml

@@ -12,9 +12,8 @@ broadcastcount = 2
 broadcastlabel = "broadcast"
 teamcount = 1
 teamlabel = "ctfteam"
-manifest_dir = "manifests/templates"
-manifest_runtime_dir = "manifests/templates/runtime"
-manifests = [
+templated_manifest_dir = "kubernetes/templates"
+templated_manifests = [
     "harbor.yml",
 ]
 

configs/types.go

@@ -6,14 +6,13 @@ type API struct {
 }
 
 type ClusterCfg struct {
-	DeploymentLabel    string   `toml:"deploymentlabel"`
-	BroadcastCount     uint     `toml:"broadcastcount"`
-	BroadcastLabel     string   `toml:"broadcastlabel"`
-	TeamCount          uint     `toml:"teamcount"`
-	TeamLabel          string   `toml:"teamlabel"`
-	ManifestDir        string   `toml:"manifest_dir"`
-	ManifestRuntimeDir string   `toml:"manifest_runtime_dir"`
-	Manifests          []string `toml:"manifests"`
+	DeploymentLabel      string   `toml:"deploymentlabel"`
+	BroadcastCount       uint     `toml:"broadcastcount"`
+	BroadcastLabel       string   `toml:"broadcastlabel"`
+	TeamCount            uint     `toml:"teamcount"`
+	TeamLabel            string   `toml:"teamlabel"`
+	TemplatedManifestDir string   `toml:"templated_manifest_dir"`
+	TemplatedManifests   []string `toml:"templated_manifests"`
 }
 
 type ChallengeDeployerCfg struct {

manifests/templates/ingress.yml → kubernetes/manifests/ingress.yml

@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: ingress-nginx-admission
-  namespace: katana
+  namespace: kube-system
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
@@ -64,14 +64,14 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: ingress-nginx-admission
-    namespace: "katana"
+    namespace: "kube-system"
 ---
 # Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: ingress-nginx-admission
-  namespace: katana
+  namespace: kube-system
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
@@ -97,7 +97,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: ingress-nginx-admission
-  namespace: katana
+  namespace: kube-system
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
@@ -116,14 +116,14 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: ingress-nginx-admission
-    namespace: "katana"
+    namespace: "kube-system"
 ---
 # Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: ingress-nginx-admission-create
-  namespace: katana
+  namespace: kube-system
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
@@ -178,7 +178,7 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: ingress-nginx-admission-patch
-  namespace: katana
+  namespace: kube-system
   annotations:
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
@@ -243,7 +243,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx
-  namespace: katana
+  namespace: kube-system
 automountServiceAccountToken: true
 ---
 # Source: ingress-nginx/templates/controller-configmap.yaml
@@ -259,7 +259,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx-controller
-  namespace: katana
+  namespace: kube-system
 data:
   allow-snippet-annotations: "true"
 ---
@@ -366,7 +366,7 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: ingress-nginx
-    namespace: "katana"
+    namespace: "kube-system"
 ---
 # Source: ingress-nginx/templates/controller-role.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -381,7 +381,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx
-  namespace: katana
+  namespace: kube-system
 rules:
   - apiGroups:
       - ""
@@ -474,15 +474,15 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx
-  namespace: katana
+  namespace: kube-system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: ingress-nginx
 subjects:
   - kind: ServiceAccount
     name: ingress-nginx
-    namespace: "katana"
+    namespace: "kube-system"
 ---
 # Source: ingress-nginx/templates/controller-service-webhook.yaml
 apiVersion: v1
@@ -497,7 +497,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx-controller-admission
-  namespace: katana
+  namespace: kube-system
 spec:
   type: ClusterIP
   ports:
@@ -525,7 +525,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx-controller
-  namespace: katana
+  namespace: kube-system
 spec:
   type: LoadBalancer
   ipFamilyPolicy: SingleStack
@@ -560,7 +560,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/component: controller
   name: ingress-nginx-controller
-  namespace: katana
+  namespace: kube-system
 spec:
   selector:
     matchLabels:
@@ -720,6 +720,6 @@ webhooks:
       - v1
     clientConfig:
       service:
-        namespace: "katana"
+        namespace: "kube-system"
         name: ingress-nginx-controller-admission
         path: /networking/v1/ingresses

kubernetes/templates/namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: katana

lib/deployment/deployment.go

@@ -106,10 +106,10 @@ func DeployCluster(kubeconfig *rest.Config, kubeclientset *kubernetes.Clientset)
 
 	deploymentConfig := utils.DeploymentConfig()
 
-	for _, m := range clusterConfig.Manifests {
+	for _, m := range clusterConfig.TemplatedManifests {
 		manifest := &bytes.Buffer{}
 		log.Printf("Applying: %s\n", m)
-		tmpl, err := template.ParseFiles(filepath.Join(clusterConfig.ManifestDir, m))
+		tmpl, err := template.ParseFiles(filepath.Join(clusterConfig.TemplatedManifestDir, m))
 		if err != nil {
 			return err
 		}

lib/harbor/hosts.go

@@ -33,12 +33,12 @@ func addHarborHostsEntry() error {
 	deploymentName := "katana-release-harbor-core"
 	namespace := "katana"
 
-	err = waitForLoadBalancerExternalIP(client, serviceName)
+	err = utils.WaitForLoadBalancerExternalIP(client, serviceName, namespace)
 	if err != nil {
 		return err
 	}
 
-	err = waitForDeploymentReady(client, deploymentName)
+	err = utils.WaitForDeploymentReady(client, deploymentName, namespace)
 	if err != nil {
 		return err
 	}
@@ -133,7 +133,7 @@ func deployHarborClusterDaemonSet() error {
 
 	manifest := &bytes.Buffer{}
 
-	tmpl, err := template.ParseFiles(filepath.Join(configs.ClusterConfig.ManifestRuntimeDir, "harbor-daemonset.yml"))
+	tmpl, err := template.ParseFiles(filepath.Join(configs.ClusterConfig.TemplatedManifestDir, "runtime", "harbor-daemonset.yml"))
 	if err != nil {
 		return err
 	}

lib/utils/kube.go

@@ -336,3 +336,67 @@ func DeleteConfigMapAndWait(kubeClientset *kubernetes.Clientset, kubeConfig *res
 
 	watcher.Stop()
 }
+
+func WaitForLoadBalancerExternalIP(clientset *kubernetes.Clientset, serviceName string, namespace string) error {
+	service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if service.Status.LoadBalancer.Ingress != nil && len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != "" {
+		return nil
+	}
+
+	watcher, err := clientset.CoreV1().Services(namespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector: "metadata.name=" + serviceName,
+	})
+	if err != nil {
+		return err
+	}
+	defer watcher.Stop()
+
+	for event := range watcher.ResultChan() {
+		service, ok := event.Object.(*v1.Service)
+		if !ok {
+			continue
+		}
+
+		if service.Status.LoadBalancer.Ingress != nil && len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != "" {
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func WaitForDeploymentReady(clientset *kubernetes.Clientset, deploymentName string, namespace string) error {
+	deployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if deployment.Status.ReadyReplicas > 0 {
+		return nil
+	}
+
+	watcher, err := clientset.AppsV1().Deployments(namespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector: "metadata.name=" + deploymentName,
+	})
+	if err != nil {
+		return err
+	}
+	defer watcher.Stop()
+
+	for event := range watcher.ResultChan() {
+		deployment, ok := event.Object.(*appsv1.Deployment)
+		if !ok {
+			continue
+		}
+
+		if deployment.Status.ReadyReplicas > 0 {
+			return nil
+		}
+	}
+
+	return nil
+}