diff --git a/Makefile b/Makefile index d749a64c..329927c4 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,7 @@ GOBIN := $(PROJECTROOT)/bin UTILDIR := $(PROJECTROOT)/scripts/utils SPINNER := $(UTILDIR)/spinner.sh BUILDIR := $(PROJECTROOT)/scripts/build -CONTROLLER_MANIFEST:= $(PROJECTROOT)/manifests/dev/expose-controller.yml -HELM_MANIFEST:= $(PROJECTROOT)/manifests/templates/helm-values.yml -OPENVPN_MANIFEST:= $(PROJECTROOT)/manifests/templates/helm-values.yml +MANIFEST:= $(PROJECTROOT)/kubernetes/manifests KEY_NAME := team @@ -83,20 +81,16 @@ gen-certificates: kubectl --namespace $(OPENVPN_NAMESPACE) exec -it $(POD_NAME) cat "/etc/openvpn/certs/pki/$(KEY_NAME)-$$n.ovpn" > $(KEY_NAME)-$$n.ovpn; \ done -gen-vpn: set-env - helm install openvpn -f $(HELM_MANIFEST) stable/openvpn --namespace openvpn - minikube tunnel - set-env: build minikube start --driver=docker && \ minikube addons enable ingress && \ - kubectl apply -f $(CONTROLLER_MANIFEST) && \ + kubectl apply -f $(MANIFEST) && \ sudo -- sh -c "echo \"$(minikube service nginx-ingress-controller --url -n kube-system | awk '{print substr($0,8)}' | awk '{print substr($0, 1, length($0)-6)}' | head -1) katana.local\" >> /etc/hosts" &&\ cp config.sample.toml config.toml && \ ./bin/katana run set-env-prod: build - kubectl apply -f $(CONTROLLER_MANIFEST) && \ + kubectl apply -f $(MANIFEST) && \ cp config.sample.toml config.toml && \ sudo ./bin/katana run diff --git a/config.sample.toml b/config.sample.toml index 236eb7c9..b596eefa 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -12,9 +12,8 @@ broadcastcount = 2 broadcastlabel = "broadcast" teamcount = 1 teamlabel = "ctfteam" -manifest_dir = "manifests/templates" -manifest_runtime_dir = "manifests/templates/runtime" -manifests = [ +templated_manifest_dir = "kubernetes/templates" +templated_manifests = [ "harbor.yml", ] diff --git a/configs/types.go b/configs/types.go index 48a1ad60..60abd08b 100644 --- a/configs/types.go +++ b/configs/types.go @@ -6,14 +6,13 @@ type API struct { } type ClusterCfg struct { - DeploymentLabel string `toml:"deploymentlabel"` - BroadcastCount uint `toml:"broadcastcount"` - BroadcastLabel string `toml:"broadcastlabel"` - TeamCount uint `toml:"teamcount"` - TeamLabel string `toml:"teamlabel"` - ManifestDir string `toml:"manifest_dir"` - ManifestRuntimeDir string `toml:"manifest_runtime_dir"` - Manifests []string `toml:"manifests"` + DeploymentLabel string `toml:"deploymentlabel"` + BroadcastCount uint `toml:"broadcastcount"` + BroadcastLabel string `toml:"broadcastlabel"` + TeamCount uint `toml:"teamcount"` + TeamLabel string `toml:"teamlabel"` + TemplatedManifestDir string `toml:"templated_manifest_dir"` + TemplatedManifests []string `toml:"templated_manifests"` } type ChallengeDeployerCfg struct { diff --git a/manifests/templates/ingress.yml b/kubernetes/manifests/ingress.yml similarity index 97% rename from manifests/templates/ingress.yml rename to kubernetes/manifests/ingress.yml index d277415d..1f1e4dea 100644 --- a/manifests/templates/ingress.yml +++ b/kubernetes/manifests/ingress.yml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: ingress-nginx-admission - namespace: katana + namespace: kube-system annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded @@ -64,14 +64,14 @@ roleRef: subjects: - kind: ServiceAccount name: ingress-nginx-admission - namespace: "katana" + namespace: "kube-system" --- # Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ingress-nginx-admission - namespace: katana + namespace: kube-system annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded @@ -97,7 +97,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ingress-nginx-admission - namespace: katana + namespace: kube-system annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded @@ -116,14 +116,14 @@ roleRef: subjects: - kind: ServiceAccount name: ingress-nginx-admission - namespace: "katana" + namespace: "kube-system" --- # Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml apiVersion: batch/v1 kind: Job metadata: name: ingress-nginx-admission-create - namespace: katana + namespace: kube-system annotations: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded @@ -178,7 +178,7 @@ apiVersion: batch/v1 kind: Job metadata: name: ingress-nginx-admission-patch - namespace: katana + namespace: kube-system annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded @@ -243,7 +243,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx - namespace: katana + namespace: kube-system automountServiceAccountToken: true --- # Source: ingress-nginx/templates/controller-configmap.yaml @@ -259,7 +259,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller - namespace: katana + namespace: kube-system data: allow-snippet-annotations: "true" --- @@ -366,7 +366,7 @@ roleRef: subjects: - kind: ServiceAccount name: ingress-nginx - namespace: "katana" + namespace: "kube-system" --- # Source: ingress-nginx/templates/controller-role.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -381,7 +381,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx - namespace: katana + namespace: kube-system rules: - apiGroups: - "" @@ -474,7 +474,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx - namespace: katana + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -482,7 +482,7 @@ roleRef: subjects: - kind: ServiceAccount name: ingress-nginx - namespace: "katana" + namespace: "kube-system" --- # Source: ingress-nginx/templates/controller-service-webhook.yaml apiVersion: v1 @@ -497,7 +497,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller-admission - namespace: katana + namespace: kube-system spec: type: ClusterIP ports: @@ -525,7 +525,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller - namespace: katana + namespace: kube-system spec: type: LoadBalancer ipFamilyPolicy: SingleStack @@ -560,7 +560,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller - namespace: katana + namespace: kube-system spec: selector: matchLabels: @@ -720,6 +720,6 @@ webhooks: - v1 clientConfig: service: - namespace: "katana" + namespace: "kube-system" name: ingress-nginx-controller-admission path: /networking/v1/ingresses diff --git a/manifests/templates/gogs.yml b/kubernetes/templates/gogs.yml similarity index 100% rename from manifests/templates/gogs.yml rename to kubernetes/templates/gogs.yml diff --git a/manifests/templates/harbor.yml b/kubernetes/templates/harbor.yml similarity index 100% rename from manifests/templates/harbor.yml rename to kubernetes/templates/harbor.yml diff --git a/manifests/templates/logs.yml b/kubernetes/templates/logs.yml similarity index 100% rename from manifests/templates/logs.yml rename to kubernetes/templates/logs.yml diff --git a/manifests/templates/mongo.yml b/kubernetes/templates/mongo.yml similarity index 100% rename from manifests/templates/mongo.yml rename to kubernetes/templates/mongo.yml diff --git a/manifests/templates/mysql.yml b/kubernetes/templates/mysql.yml similarity index 100% rename from manifests/templates/mysql.yml rename to kubernetes/templates/mysql.yml diff --git a/kubernetes/templates/namespace.yml b/kubernetes/templates/namespace.yml new file mode 100644 index 00000000..044eab04 --- /dev/null +++ b/kubernetes/templates/namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: katana diff --git a/manifests/templates/runtime/harbor-daemonset.yml b/kubernetes/templates/runtime/harbor-daemonset.yml similarity index 100% rename from manifests/templates/runtime/harbor-daemonset.yml rename to kubernetes/templates/runtime/harbor-daemonset.yml diff --git a/manifests/templates/runtime/teams.yml b/kubernetes/templates/runtime/teams.yml similarity index 100% rename from manifests/templates/runtime/teams.yml rename to kubernetes/templates/runtime/teams.yml diff --git a/manifests/templates/web_challenge.yml b/kubernetes/templates/web_challenge.yml similarity index 100% rename from manifests/templates/web_challenge.yml rename to kubernetes/templates/web_challenge.yml diff --git a/lib/deployment/deployment.go b/lib/deployment/deployment.go index 7a396c89..5175d69b 100644 --- a/lib/deployment/deployment.go +++ b/lib/deployment/deployment.go @@ -106,10 +106,10 @@ func DeployCluster(kubeconfig *rest.Config, kubeclientset *kubernetes.Clientset) deploymentConfig := utils.DeploymentConfig() - for _, m := range clusterConfig.Manifests { + for _, m := range clusterConfig.TemplatedManifests { manifest := &bytes.Buffer{} log.Printf("Applying: %s\n", m) - tmpl, err := template.ParseFiles(filepath.Join(clusterConfig.ManifestDir, m)) + tmpl, err := template.ParseFiles(filepath.Join(clusterConfig.TemplatedManifestDir, m)) if err != nil { return err } diff --git a/lib/harbor/hosts.go b/lib/harbor/hosts.go index a6028d06..0043799e 100644 --- a/lib/harbor/hosts.go +++ b/lib/harbor/hosts.go @@ -33,12 +33,12 @@ func addHarborHostsEntry() error { deploymentName := "katana-release-harbor-core" namespace := "katana" - err = waitForLoadBalancerExternalIP(client, serviceName) + err = utils.WaitForLoadBalancerExternalIP(client, serviceName, namespace) if err != nil { return err } - err = waitForDeploymentReady(client, deploymentName) + err = utils.WaitForDeploymentReady(client, deploymentName, namespace) if err != nil { return err } @@ -133,7 +133,7 @@ func deployHarborClusterDaemonSet() error { manifest := &bytes.Buffer{} - tmpl, err := template.ParseFiles(filepath.Join(configs.ClusterConfig.ManifestRuntimeDir, "harbor-daemonset.yml")) + tmpl, err := template.ParseFiles(filepath.Join(configs.ClusterConfig.TemplatedManifestDir, "runtime", "harbor-daemonset.yml")) if err != nil { return err } diff --git a/lib/harbor/kube.go b/lib/harbor/kube.go deleted file mode 100644 index 49beaa50..00000000 --- a/lib/harbor/kube.go +++ /dev/null @@ -1,56 +0,0 @@ -package harbor - -import ( - "context" - - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -func waitForLoadBalancerExternalIP(clientset *kubernetes.Clientset, serviceName string) error { - watcher, err := clientset.CoreV1().Services("katana").Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + serviceName, - }) - if err != nil { - return err - } - defer watcher.Stop() - - for event := range watcher.ResultChan() { - service, ok := event.Object.(*v1.Service) - if !ok { - continue - } - - if service.Status.LoadBalancer.Ingress != nil && len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != "" { - return nil - } - } - - return nil -} - -func waitForDeploymentReady(clientset *kubernetes.Clientset, deploymentName string) error { - watcher, err := clientset.AppsV1().Deployments("katana").Watch(context.TODO(), metav1.ListOptions{ - FieldSelector: "metadata.name=" + deploymentName, - }) - if err != nil { - return err - } - defer watcher.Stop() - - for event := range watcher.ResultChan() { - deployment, ok := event.Object.(*appsv1.Deployment) - if !ok { - continue - } - - if deployment.Status.ReadyReplicas > 0 { - return nil - } - } - - return nil -} diff --git a/lib/utils/kube.go b/lib/utils/kube.go index b702e9cd..99c206ee 100644 --- a/lib/utils/kube.go +++ b/lib/utils/kube.go @@ -336,3 +336,67 @@ func DeleteConfigMapAndWait(kubeClientset *kubernetes.Clientset, kubeConfig *res watcher.Stop() } + +func WaitForLoadBalancerExternalIP(clientset *kubernetes.Clientset, serviceName string, namespace string) error { + service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return err + } + + if service.Status.LoadBalancer.Ingress != nil && len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != "" { + return nil + } + + watcher, err := clientset.CoreV1().Services(namespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + serviceName, + }) + if err != nil { + return err + } + defer watcher.Stop() + + for event := range watcher.ResultChan() { + service, ok := event.Object.(*v1.Service) + if !ok { + continue + } + + if service.Status.LoadBalancer.Ingress != nil && len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != "" { + return nil + } + } + + return nil +} + +func WaitForDeploymentReady(clientset *kubernetes.Clientset, deploymentName string, namespace string) error { + deployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + return err + } + + if deployment.Status.ReadyReplicas > 0 { + return nil + } + + watcher, err := clientset.AppsV1().Deployments(namespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + deploymentName, + }) + if err != nil { + return err + } + defer watcher.Stop() + + for event := range watcher.ResultChan() { + deployment, ok := event.Object.(*appsv1.Deployment) + if !ok { + continue + } + + if deployment.Status.ReadyReplicas > 0 { + return nil + } + } + + return nil +} diff --git a/manifests/dev/expose-controller.yml b/manifests/dev/expose-controller.yml deleted file mode 100644 index b5b99727..00000000 --- a/manifests/dev/expose-controller.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: katana ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx-ingress-controller - namespace: kube-system - labels: - k8s-app: nginx-ingress-controller -spec: - type: NodePort - ports: - - port: 80 - targetPort: 80 - nodePort: 32080 - protocol: TCP - name: http - - port: 443 - targetPort: 443 - nodePort: 32443 - protocol: TCP - name: https - selector: - k8s-app: nginx-ingress-controller diff --git a/manifests/templates/broadcast-service.yml b/manifests/templates/broadcast-service.yml deleted file mode 100644 index f5952b09..00000000 --- a/manifests/templates/broadcast-service.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: broadcast-service - namespace: katana -spec: - type: ClusterIP - selector: - app: {{.BroadcastLabel}} - ports: - - protocol: TCP - port: {{.BroadcastPort}} - targetPort: {{.BroadcastPort}} diff --git a/manifests/templates/broadcast.yml b/manifests/templates/broadcast.yml deleted file mode 100644 index 981889b4..00000000 --- a/manifests/templates/broadcast.yml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: katana-broadcast - namespace: katana -spec: - selector: - matchLabels: - app: {{.BroadcastLabel}} - replicas: {{.BroadcastCount}} - template: - metadata: - labels: - app: {{.BroadcastLabel}} - spec: - containers: - - name: broadcast - image: scar26/sdskatanabroadcast:latest - ports: - - containerPort: {{.BroadcastPort}} - env: - - name: "CHALLENGE_ARTIFACT" - value: {{.ChallengeArtifact}} \ No newline at end of file diff --git a/manifests/templates/helm-values.yml b/manifests/templates/helm-values.yml deleted file mode 100644 index 1b0f4508..00000000 --- a/manifests/templates/helm-values.yml +++ /dev/null @@ -1,30 +0,0 @@ -replicaCount: 1 -image: - repository: jfelten/openvpn-docker - tag: 1.1.0 - pullPolicy: IfNotPresent -service: - name: openvpn - type: LoadBalancer - externalPort: 443 - internalPort: 443 -resources: - limits: - cpu: 300m - memory: 128Mi - requests: - cpu: 300m - memory: 128Mi -persistence: - enabled: true - - existingClaim: openvpn-data-claim - - accessMode: ReadWriteOnce - size: 1Gi -openvpn: - OVPN_NETWORK: 10.240.0.0 - OVPN_SUBNET: 255.255.0.0 - OVPN_PROTO: tcp - OVPN_K8S_POD_NETWORK: "10.244.0.0" - OVPN_K8S_POD_SUBNET: "255.252.0.0" diff --git a/manifests/templates/openvpn.yml b/manifests/templates/openvpn.yml deleted file mode 100644 index c908744a..00000000 --- a/manifests/templates/openvpn.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: "pv-volume" -spec: - capacity: - storage: "1Gi" - accessModes: - - "ReadWriteOnce" - hostPath: - path: /data ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: openvpn-data-claim - namespace: katana -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi \ No newline at end of file diff --git a/services/master/controllers/createTeams.go b/services/master/controllers/createTeams.go index 0055ca4e..6c923400 100644 --- a/services/master/controllers/createTeams.go +++ b/services/master/controllers/createTeams.go @@ -80,7 +80,7 @@ func CreateTeams(c *fiber.Ctx) error { log.Fatal(err) } manifest := &bytes.Buffer{} - tmpl, err := template.ParseFiles(filepath.Join(g.ClusterConfig.ManifestRuntimeDir, "teams.yml")) + tmpl, err := template.ParseFiles(filepath.Join(g.ClusterConfig.TemplatedManifestDir, "runtime", "teams.yml")) if err != nil { return err } diff --git a/services/master/controllers/infraSet.go b/services/master/controllers/infraSet.go index 9b7333cb..4bee31a5 100644 --- a/services/master/controllers/infraSet.go +++ b/services/master/controllers/infraSet.go @@ -28,7 +28,7 @@ func InfraSet(c *fiber.Ctx) error { log.Fatal(err) } - for _, manifests := range configs.ClusterConfig.Manifests { + for _, manifests := range configs.ClusterConfig.TemplatedManifests { if manifests == "harbor.yml" { generateCertsforHarbor() } @@ -38,7 +38,7 @@ func InfraSet(c *fiber.Ctx) error { log.Fatal(err) } - for _, manifests := range configs.ClusterConfig.Manifests { + for _, manifests := range configs.ClusterConfig.TemplatedManifests { if manifests == "harbor.yml" { err = harbor.SetupHarbor() if err != nil { diff --git a/services/master/controllers/setupIngress.go b/services/master/controllers/setupIngress.go new file mode 100644 index 00000000..3dded01f --- /dev/null +++ b/services/master/controllers/setupIngress.go @@ -0,0 +1,96 @@ +package controllers + +import ( + "bufio" + "context" + "fmt" + "os" + "strings" + + "github.com/gofiber/fiber/v2" + "github.com/sdslabs/katana/lib/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + hostsFilePath = "/etc/hosts" +) + +func SetupIngress(c *fiber.Ctx) error { + kubeClient, _ := utils.GetKubeClient() + namespace := "kube-system" + + serviceName := "ingress-nginx-controller" + if err := utils.WaitForLoadBalancerExternalIP(kubeClient, serviceName, namespace); err != nil { + return err + } + + service, err := kubeClient.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return err + } + + externalIP := service.Status.LoadBalancer.Ingress[0].IP + + hosts := []string{ + "mongo.katana.local", + "gogs.katana.local", + "mysql.katana.local", + } + + hostsEntry := "" + for _, host := range hosts { + hostsEntry += " " + host + } + + file, err := os.Open(hostsFilePath) + if err != nil { + return err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + lines := make([]string, 0) + found := false + + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, hostsEntry) { + fields := strings.Fields(line) + if len(fields) > 1 { + fields[0] = externalIP + line = strings.Join(fields, " ") + found = true + } + } + lines = append(lines, line) + } + + if !found { + lines = append(lines, fmt.Sprintf("%s %s", externalIP, hostsEntry)) + } + + if err := scanner.Err(); err != nil { + return err + } + + // Write to file + file, err = os.OpenFile(hostsFilePath, os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, line := range lines { + fmt.Fprintln(w, line) + } + + if w.Flush() != nil { + return err + } else { + return c.JSON(fiber.Map{ + "message": "Successfully setup ingress", + }) + } +} diff --git a/services/master/server.go b/services/master/server.go index dac745ba..85536005 100644 --- a/services/master/server.go +++ b/services/master/server.go @@ -49,6 +49,7 @@ func Server() error { admin.Get("/gitServer", c.GitServer) admin.Get("/cluster/:id", c.ClusterInfo) admin.Get("/deleteChallenge/:chall_name", c.DeleteChallenge) + admin.Get("/setupIngress", c.SetupIngress) log.Printf("Listening on %s:%d\n", cfg.APIConfig.Host, cfg.APIConfig.Port) return app.Listen(fmt.Sprintf("%s:%d", cfg.APIConfig.Host, cfg.APIConfig.Port)) }