diff --git a/.gitignore b/.gitignore index 6d89813bab..3f570ba540 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ junit*.xml .ftl.lock docker-build/ **/.ftl +charts/charts/ \ No newline at end of file diff --git a/Dockerfile.controller b/Dockerfile.controller index 7cf95df75a..22e1b9dbe2 100644 --- a/Dockerfile.controller +++ b/Dockerfile.controller @@ -27,7 +27,7 @@ COPY . /src/ RUN just errtrace # Reset timestamps so that the build state is reset RUN git ls-files -z | xargs -0 touch -r go.mod -RUN just build ftl-controller +RUN just build ftl-controller ftl-initdb # Finally create the runtime image. FROM scratch @@ -37,6 +37,7 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ WORKDIR /service/ COPY --from=builder /src/build/release/ftl-controller . +COPY --from=builder /src/build/release/ftl-initdb . #TODO: do we want to have a separate image for this? EXPOSE 8891 EXPOSE 8892 diff --git a/Dockerfile.initdb b/Dockerfile.initdb new file mode 100644 index 0000000000..05060af727 --- /dev/null +++ b/Dockerfile.initdb @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 AS builder +RUN apt-get update +RUN apt-get install -y curl git zip + +# Copy Hermit bin stubs and install all packages. This is done +# separately so that Docker will cache the tools correctly. +COPY ./bin /src/bin +ENV PATH="/src/bin:$PATH" +WORKDIR /src + +# Seed some of the most common tools - this will be cached +RUN go version +RUN node --version + +# Download Go dependencies separately so Docker will cache them +COPY go.mod go.sum ./ +RUN go mod download -x + +# Download PNPM dependencies separately so Docker will cache them +COPY frontend/console/package.json ./frontend/console/ +COPY frontend/vscode/package.json ./frontend/vscode/ +COPY pnpm-workspace.yaml pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +# Build +COPY . /src/ +RUN just errtrace +# Reset timestamps so that the build state is reset +RUN git ls-files -z | xargs -0 touch -r go.mod +RUN just build ftl-controller + +# Finally create the runtime image. +FROM scratch + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +WORKDIR /service/ + +COPY --from=builder /src/build/release/ftl-controller . +COPY --from=builder /src/build/release/ftl-initdb . #TODO: Remove this line, we need a different migration image, this will get us moving for now though + +EXPOSE 8891 +EXPOSE 8892 + +ENV FTL_CONTROLLER_BIND="http://0.0.0.0:8892" +ENV FTL_CONTROLLER_ADVERTISE="http://127.0.0.1:8892" +ENV FTL_CONTROLLER_CONSOLE_URL="*" +ENV FTL_CONTROLLER_DSN="postgres://host.docker.internal/ftl?sslmode=disable&user=postgres&password=secret" + +CMD ["/service/ftl-controller"] diff --git a/Justfile b/Justfile index 73b9848a63..dd29b053d2 100644 --- a/Justfile +++ b/Justfile @@ -264,3 +264,6 @@ build-docker name: -t ftl0/ftl-{{name}}:"${GITHUB_SHA:-$(git rev-parse HEAD)}" \ -t ftl0/ftl-{{name}}:latest \ -f Dockerfile.{{name}} . + +chart *args: + @cd charts && just {{args}} \ No newline at end of file diff --git a/charts/Chart.lock b/charts/Chart.lock new file mode 100644 index 0000000000..dcbc6a37cc --- /dev/null +++ b/charts/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: postgresql + repository: oci://registry-1.docker.io/bitnamicharts + version: 15.5.29 +- name: harbor + repository: https://helm.goharbor.io + version: 1.15.1 +digest: sha256:54ee3d28306509e5360950f19a6ff0add5552f2a90adef6c50dcecf77d7fd119 +generated: "2024-10-10T14:53:25.059332+11:00" diff --git a/charts/Chart.yaml b/charts/Chart.yaml new file mode 100644 index 0000000000..cb72c95dbe --- /dev/null +++ b/charts/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: ftl +description: | + A Helm chart for Kubernetes that deploys FTL +home: https://github.com/TBD54566975/ftl +version: 0.0.1 +appVersion: v0.354.1 + +dependencies: +- name: postgresql + version: 15.5.29 + repository: oci://registry-1.docker.io/bitnamicharts + condition: postgresql.enabled +- name: harbor + version: v1.15.1 + repository: https://helm.goharbor.io + condition: harbor.enabled diff --git a/charts/Justfile b/charts/Justfile new file mode 100755 index 0000000000..880c6dc242 --- /dev/null +++ b/charts/Justfile @@ -0,0 +1,5 @@ +_help: + @just -l + +dep-update: + helm dep update . \ No newline at end of file diff --git a/charts/templates/_helpers.tpl b/charts/templates/_helpers.tpl new file mode 100644 index 0000000000..b4c60595ab --- /dev/null +++ b/charts/templates/_helpers.tpl @@ -0,0 +1,44 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ftl.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "ftl.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/name: {{ include "ftl.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Values.customLabels -}} +{{ toYaml .Values.customLabels }} +{{- end -}} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "ftl-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ftl.fullname" . }} +app.kubernetes.io/component: controller +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- define "ftl-runner.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ftl.fullname" . }} +app.kubernetes.io/component: runner +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/templates/controller-role.yaml b/charts/templates/controller-role.yaml new file mode 100644 index 0000000000..eb91fe1e9f --- /dev/null +++ b/charts/templates/controller-role.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ftl.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + {{- if .Values.controller.controllersRoleArn }} + annotations: + eks.amazonaws.com/role-arn: {{ .Values.controller.controllersRoleArn }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "ftl.fullname" . }}-controller + labels: + {{- include "ftl.labels" . | nindent 4 }} +rules: + - apiGroups: [ "apps" ] + resources: [ "deployments" ] + verbs: [ "get", "list", "watch", "delete", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "services" , "serviceaccounts"] + verbs: [ "get", "list", "watch", "delete", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + resourceNames: + - ftl-controller-deployment-config + verbs: [ "get"] + - apiGroups: [ "security.istio.io" ] + resources: [ "authorizationpolicies" ] + verbs: [ "get", "list", "watch", "delete", "create", "update", "patch" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "ftl.fullname" . }}-controller + labels: + {{- include "ftl.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ftl.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "ftl.fullname" . }}-controller diff --git a/charts/templates/controller-services.yaml b/charts/templates/controller-services.yaml new file mode 100644 index 0000000000..a77449f904 --- /dev/null +++ b/charts/templates/controller-services.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "ftl.labels" . | nindent 4 }} + name: {{ include "ftl.fullname" . }}-controller + {{- if .Values.controller.service.annotations }} + annotations: + {{- toYaml .Values.controller.service.annotations | nindent 4 }} + {{- end }} +spec: + ports: + {{- range .Values.controller.service.ports }} + - name: {{ .name }} + port: {{ .port }} + protocol: {{ .protocol | default "TCP" }} + {{- if .appProtocol }} + appProtocol: {{ .appProtocol }} + {{- end }} + targetPort: {{ .targetPort }} + {{- end }} + selector: + {{- include "ftl-controller.selectorLabels" . | nindent 4 }} + type: {{ .Values.controller.service.type | default "ClusterIP" }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "ftl.labels" . | nindent 4 }} + name: {{ include "ftl.fullname" . }}-controller-ingress + {{- if .Values.controller.ingressService.annotations }} + annotations: + {{- toYaml .Values.controller.ingressService.annotations | nindent 4 }} + {{- end }} +spec: + ports: + {{- range .Values.controller.ingressService.ports }} + - name: {{ .name }} + port: {{ .port }} + protocol: {{ .protocol | default "TCP" }} + targetPort: {{ .targetPort }} + {{- end }} + selector: + {{- include "ftl-controller.selectorLabels" . | nindent 4 }} + type: {{ .Values.controller.ingressService.type | default "ClusterIP" }} diff --git a/charts/templates/controller.yaml b/charts/templates/controller.yaml new file mode 100644 index 0000000000..4e50c87f45 --- /dev/null +++ b/charts/templates/controller.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ftl.fullname" . }}-controller + labels: + {{- include "ftl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicas }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "ftl-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "ftl-controller.selectorLabels" . | nindent 8 }} + {{- if .Values.controller.podAnnotations }} + annotations: + {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "ftl.fullname" . }}-controller + containers: + - name: app + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + {{- if .Values.controller.envFrom }} + envFrom: + {{- if .Values.controller.envFrom }} + {{- toYaml .Values.controller.envFrom | nindent 12 }} + {{- end }} + {{- else if or .Values.secrets.logEncryptionKey .Values.secrets.asyncEncryptionKey }} + envFrom: + - secretRef: + name: {{ include "ftl.fullname" . }}-secrets + {{- end }} + env: + {{- if .Values.controller.env }} + {{- toYaml .Values.controller.env | nindent 12 }} + {{- end }} + {{- if not .Values.postgresql.enabled }} + - name: FTL_CONTROLLER_DSN + value: "{{ .Values.controller.dbConnectionString }}" + {{- else }} + - name: FTL_CONTROLLER_DSN + value: "postgres://{{ .Release.Name }}-postgresql:5432/{{ .Values.postgresql.global.postgresql.auth.database }}?sslmode=disable&user={{ .Values.postgresql.global.postgresql.auth.username }}&password={{ .Values.postgresql.global.postgresql.auth.password }}" + {{- end }} + {{- if .Values.controller.kmsUri }} + - name: FTL_KMS_URI + value: "{{ .Values.controller.kmsUri }}" + {{- end }} + ports: + {{- range .Values.controller.ports }} + - name: {{ .name }} + containerPort: {{ .containerPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + readinessProbe: + {{- if .Values.controller.readinessProbe }} + {{- toYaml .Values.controller.readinessProbe | nindent 12 }} + {{- else }} + httpGet: + path: /healthz + port: 8892 + initialDelaySeconds: 1 + periodSeconds: 2 + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 15 + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: + {{- toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: + {{- toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} diff --git a/charts/templates/db-migration-job.yaml b/charts/templates/db-migration-job.yaml new file mode 100644 index 0000000000..c927da608a --- /dev/null +++ b/charts/templates/db-migration-job.yaml @@ -0,0 +1,73 @@ +{{- if .Values.dbMigration.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "ftl.fullname" . }}-dbmig-{{ .Values.controller.image.tag }} + labels: + {{- include "ftl.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + template: + metadata: + labels: + {{- include "ftl.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "ftl.fullname" . }}-dbmig + restartPolicy: OnFailure + containers: + - name: dbmig + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + env: + {{- if not .Values.postgresql.enabled }} + - name: FTL_CONTROLLER_DSN + value: "{{ .Values.controller.dbConnectionString }}" + {{- else }} + - name: FTL_CONTROLLER_DSN + value: "postgres://{{ .Release.Name }}-postgresql:5432/{{ .Values.postgresql.global.postgresql.auth.database }}?sslmode=disable&user={{ .Values.postgresql.global.postgresql.auth.username }}&password={{ .Values.postgresql.global.postgresql.auth.password }}" + {{- end }} + command: + - bash + - -c + - | + set -x + set -e + /root/ftl-initdb --dsn="${FTL_CONTROLLER_DSN}" + x=$(echo $?); curl -fsI -X POST http://localhost:15000/quitquitquit && exit $x + backoffLimit: 4 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ftl.fullname" . }}-dbmig + labels: + {{- include "ftl.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "ftl.fullname" . }}-dbmig-role + labels: + {{- include "ftl.labels" . | nindent 4 }} +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "ftl.fullname" . }}-dbmig-rolebinding + labels: + {{- include "ftl.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ftl.fullname" . }}-dbmig-role +subjects: + - kind: ServiceAccount + name: {{ include "ftl.fullname" . }}-dbmig + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/templates/ingress.yaml b/charts/templates/ingress.yaml new file mode 100644 index 0000000000..784185f834 --- /dev/null +++ b/charts/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.ingress.enabled }} +{{ $fullName := include "ftl.fullname" . -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "ftl.fullname" . }}-controller + annotations: + {{- toYaml .Values.ingress.annotations | nindent 4 }} +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ftl-controller + port: + number: 8892 + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host.host }} + http: + paths: + {{- range $host.paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $fullName }}-controller + port: + number: {{ $host.service.port }} + {{- end }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/templates/runner.yaml b/charts/templates/runner.yaml new file mode 100644 index 0000000000..de9b7fc827 --- /dev/null +++ b/charts/templates/runner.yaml @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ftl-controller-deployment-config +data: + serviceTemplate: |- + apiVersion: v1 + kind: Service + metadata: + labels: + app: "(replaced)" + name: "(replaced)" + spec: + ports: + - name: verbs + protocol: TCP + port: 8893 + targetPort: 8893 + type: ClusterIP + deploymentTemplate: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "ftl.fullname" . }}-runner + labels: + {{- include "ftl.labels" . | nindent 8 }} + spec: + replicas: {{ .Values.runner.replicas }} + revisionHistoryLimit: {{ .Values.runner.revisionHistoryLimit }} + minReadySeconds: 30 + selector: + matchLabels: + {{- include "ftl-runner.selectorLabels" . | nindent 10 }} + template: + metadata: + labels: + {{- include "ftl-runner.selectorLabels" . | nindent 12 }} + {{- if .Values.runner.podAnnotations }} + annotations: + {{- toYaml .Values.runner.podAnnotations | nindent 12 }} + {{- end }} + spec: + containers: + - name: app + image: "(replaced)" + {{- if .Values.runner.envFrom }} + envFrom: + {{- toYaml .Values.runner.envFrom | nindent 16 }} + {{- end }} + env: + {{- if .Values.runner.env }} + {{- toYaml .Values.runner.env | nindent 16 }} + {{- end }} + ports: + {{- range .Values.runner.ports }} + - name: {{ .name }} + containerPort: {{ .containerPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + readinessProbe: + {{- if .Values.runner.readinessProbe }} + {{- toYaml .Values.runner.readinessProbe | nindent 16 }} + {{- else }} + httpGet: + path: /healthz + port: 8893 + initialDelaySeconds: 1 + periodSeconds: 2 + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 15 + {{- end }} + {{- if .Values.runner.nodeSelector }} + nodeSelector: + {{- toYaml .Values.runner.nodeSelector | nindent 12 }} + {{- end }} + {{- if .Values.runner.affinity }} + affinity: + {{- toYaml .Values.runner.affinity | nindent 12 }} + {{- end }} + {{- if .Values.runner.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.runner.topologySpreadConstraints | nindent 12 }} + {{- end }} + {{- if .Values.runner.tolerations }} + tolerations: + {{- toYaml .Values.runner.tolerations | nindent 12 }} + {{- end }} diff --git a/charts/templates/secrets.yaml b/charts/templates/secrets.yaml new file mode 100644 index 0000000000..09de4fcf5e --- /dev/null +++ b/charts/templates/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ftl.fullname" . }}-secrets + labels: + {{- include "ftl.labels" . | nindent 4 }} +type: Opaque +stringData: + FTL_LOG_ENCRYPTION_KEY: {{ .Values.secrets.logEncryptionKey }} + FTL_ASYNC_ENCRYPTION_KEY: {{ .Values.secrets.asyncEncryptionKey }} diff --git a/charts/values.yaml b/charts/values.yaml new file mode 100644 index 0000000000..aa9a747137 --- /dev/null +++ b/charts/values.yaml @@ -0,0 +1,162 @@ +fullnameOverride: "" +nameOverride: "" + +ingress: + enabled: false + annotations: + kubernetes.io/ingress.class: nginx + +secrets: + logEncryptionKey: null + asyncEncryptionKey: null + +dbMigration: + enabled: true + +controller: + controllersRoleArn: arn:aws:iam::ftl-controllers-irsa-role + replicas: 2 + revisionHistoryLimit: 0 + + image: + repository: "ftl0/ftl-controller" + tag: "v0.361.2" + + envFrom: null + dbConnectionString: "postgres://$(endpoint):$(port)/tbd?sslmode=disable&user=$(username)&password=$(password)" + kmsUri: "fake-kms://CKbvh_ILElQKSAowdHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuQWVzR2NtS2V5EhIaEE6tD2yE5AWYOirhmkY-r3sYARABGKbvh_ILIAE" + + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FTL_CONTROLLER_BIND + value: "http://$(MY_POD_IP):8892" + - name: FTL_CONTROLLER_INGRESS_BIND + value: "http://$(MY_POD_IP):8891" + - name: FTL_CONTROLLER_ADVERTISE + value: "http://$(MY_POD_IP):8892" + - name: FTL_CONTROLLER_ALLOW_ORIGIN + value: "*" + - name: LOG_LEVEL + value: "debug" + - name: LOG_JSON + value: "true" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +# - name: OTEL_EXPORTER_OTLP_ENDPOINT +# value: "http://$(HOST_IP):4317" +# - name: OTEL_RESOURCE_ATTRIBUTES +# value: "env=ftlDefault" + + ports: + - name: ingress + containerPort: 8891 + protocol: TCP + - name: http + containerPort: 8892 + protocol: TCP + + readinessProbe: null + + service: + type: ClusterIP + annotations: null + ports: + - name: "http-8892" + port: 80 + protocol: TCP + targetPort: 8892 + + ingressService: + type: ClusterIP + annotations: null + ports: + - name: "http-8891" + port: 80 + protocol: TCP + targetPort: 8891 + + podAnnotations: null + nodeSelector: null + affinity: null + topologySpreadConstraints: null + tolerations: null + +runner: + revisionHistoryLimit: 0 + + envFrom: null + + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FTL_ENDPOINT + value: "http://ftl-controller" + - name: FTL_RUNNER_BIND + value: "http://$(MY_POD_IP):8893" + - name: FTL_RUNNER_ADVERTISE + value: "" + - name: FTL_RUNNER_START_DELAY + value: "3s" + - name: LOG_LEVEL + value: "debug" + - name: LOG_JSON + value: "true" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +# - name: OTEL_EXPORTER_OTLP_ENDPOINT +# value: "http://$(HOST_IP):4317" +# - name: OTEL_RESOURCE_ATTRIBUTES +# value: "env=ftlDefault" + + ports: + - name: http + containerPort: 8893 + protocol: TCP + + readinessProbe: null + + headlessService: + ports: + - port: 8893 + targetPort: 8893 + + podAnnotations: null + nodeSelector: null + affinity: null + topologySpreadConstraints: null + tolerations: null + +harbor: + enabled: true + expose: + tls: + enabled: false + type: nodePort + persistence: + enabled: true + resourcePolicy: "" + +postgresql: + enabled: true + architecture: standalone + primary: + resources: + requests: + memory: 512Mi + limits: + memory: 1024Mi + global: + postgresql: + auth: + username: ftl + password: ftl + database: ftl diff --git a/deployment/Justfile b/deployment/Justfile index 9bdf67db8f..2ff3ab1cf6 100755 --- a/deployment/Justfile +++ b/deployment/Justfile @@ -25,8 +25,8 @@ full-deploy: build-controller build-runner setup-cluster wait-for-kube: #!/bin/bash - while [ -z "$(kubectl get pod ftl-pg-cluster-1-0)" ]; do sleep 1; done - kubectl wait --for=condition=ready pod/ftl-pg-cluster-1-0 --timeout=5m + while [ -z "$(kubectl get pod ftl-postgresql-0)" ]; do sleep 1; done + kubectl wait --for=condition=ready pod/ftl-postgresql-0 --timeout=5m kubectl wait --for=condition=available deployment/ftl-controller --timeout=5m sleep 1 ftl status @@ -43,14 +43,10 @@ setup-cluster: setup-registry if [ -z "$(k3d cluster list | grep ftl)" ]; then k3d cluster create ftl --api-port 6550 -p "8892:80@loadbalancer" --agents 2 \ --registry-use {{registry_full}} \ - --registry-config '{{mirrors}}' && \ # Start installing the DB for performance reasons - kubectl apply -k base/db-create || sleep 5 && \ # wait for CRDs to be created, the initial apply will usually fail - kubectl apply -k base/db-create & # background the process, so we can do other things and speed up the test + --registry-config '{{mirrors}}' fi kubectl config set-context --current --namespace=default - - install-istio: #!/bin/bash if [ -z "$(kubectl get namespace | grep istio-system)" ]; then @@ -63,8 +59,6 @@ install-istio: kubectl label namespace default istio-injection=enabled --overwrite kubectl kustomize --load-restrictor=LoadRestrictionsNone istio | kubectl apply -f - - - setup: setup-registry setup-cluster teardown-registry: @@ -76,18 +70,18 @@ teardown-cluster: teardown: teardown-cluster teardown-registry apply: - kubectl kustomize --load-restrictor=LoadRestrictionsNone overlays/dev | kubectl apply -f - + helm upgrade --install ftl ../charts -f values.yaml deploy-version version: setup-cluster # Ideally this would point to https://github.com/TBD54566975/ftl//deployment/overlays/dev/?ref={{version}} but that doesn't work due to symlinks # For now this is using the new config with a released image - kustomize build --load-restrictor=LoadRestrictionsNone overlays/dev | sed 's/ftl:5000/ftl0/g' | sed 's/ftl-controller:latest/ftl-controller:{{version}}/g' | kubectl apply -f - + helm upgrade --install ftl ../charts -f values.yaml --set controller.image.tag={{version}} --set controller.image.repository=ftl0/ftl-controller delete: - kubectl kustomize --load-restrictor=LoadRestrictionsNone overlays/dev | kubectl delete -f - + helm uninstall ftl delete-db: - kubectl delete pvc postgres-db-ftl-pg-cluster-1-0 + kubectl delete pvc data-ftl-postgresql-0 events: kubectl get events -w @@ -105,7 +99,7 @@ enter pod *args="bash": kubectl exec -it {{pod}} -- {{args}} psql: - just enter statefulset.apps/ftl-pg-cluster-1 env PGPASSWORD=secret psql -U postgres ftl + just enter statefulset.apps/ftl-postgresql env PGPASSWORD=ftl psql -U ftl ftl ftl-status: just exec deployment/ftl-controller ./ftl status @@ -150,19 +144,3 @@ ftl *args: export POD=$(kubectl get pods -l app=ftl-controller --no-headers | tail -1 | awk '{print $1}') kubectl exec -it ${POD} -- bash -c "FTL_ENDPOINT=http://ftl-controller:8892 /root/ftl {{args}}" -update-schema: - #!/usr/bin/env bash - cat <base/db-migrate/kustomization.yml - #generated by 'just update-schema' command, DO NOT EDIT - apiVersion: kustomize.config.k8s.io/v1beta1 - kind: Kustomization - resources: - - db-migrate.yml - configMapGenerator: - - name: ftl-db-migrate-config - files: - EOF - for file in $(ls base/db-migrate/schema/*.sql); do - echo " - ./schema/$(basename $file)" >> base/db-migrate/kustomization.yml - done - diff --git a/deployment/base/db-create/kustomization.yml b/deployment/base/db-create/kustomization.yml deleted file mode 100644 index 85b1e81fae..0000000000 --- a/deployment/base/db-create/kustomization.yml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - https://raw.githubusercontent.com/reactive-tech/kubegres/v1.18/kubegres.yaml - - pg-cluster.yaml diff --git a/deployment/base/db-create/pg-cluster.yaml b/deployment/base/db-create/pg-cluster.yaml deleted file mode 100644 index af9ef4d3ec..0000000000 --- a/deployment/base/db-create/pg-cluster.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: pg-secret - namespace: default -type: Opaque -stringData: - superUserPassword: "secret" - replicationUserPassword: "postgresReplicaPsw" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ftl-pg-cluster-conf - namespace: default - -data: - - primary_init_script.sh: | - #!/bin/bash - set -e - - echo "Running init script the 1st time Primary PostgreSql container is created..."; - echo "Running: psql -v ON_ERROR_STOP=1 --username $POSTGRES_USER --dbname $POSTGRES_DB ..."; - - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE DATABASE ftl; - CREATE USER ftl WITH PASSWORD 'secret'; - GRANT ALL PRIVILEGES ON DATABASE "ftl" to ftl; - EOSQL - - echo "Init script is completed"; ---- -apiVersion: kubegres.reactive-tech.io/v1 -kind: Kubegres -metadata: - name: ftl-pg-cluster - namespace: default -spec: - replicas: 1 - image: postgres:14.1 - - database: - size: 200Mi - - customConfig: ftl-pg-cluster-conf - - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: pg-secret - key: superUserPassword - - - name: POSTGRES_REPLICATION_PASSWORD - valueFrom: - secretKeyRef: - name: pg-secret - key: replicationUserPassword diff --git a/deployment/base/db-migrate/db-migrate.yml b/deployment/base/db-migrate/db-migrate.yml deleted file mode 100644 index 1b18490a17..0000000000 --- a/deployment/base/db-migrate/db-migrate.yml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ftl-db-migrate -spec: - template: - spec: - containers: - - name: ftl-db-migrate - image: "ghcr.io/amacneil/dbmate:latest" - command: ["dbmate", "up"] - env: - - name: DATABASE_URL - value: "postgres://postgres:secret@ftl-pg-cluster/ftl?sslmode=disable" - - name: DBMATE_MIGRATIONS_DIR - value: /etc/ftl-migrations - volumeMounts: - - name: ftl-db-migrate-config - mountPath: /etc/ftl-migrations - restartPolicy: OnFailure - volumes: - - name: ftl-db-migrate-config - configMap: - name: ftl-db-migrate-config - parallelism: 1 - backoffLimit: 1000 - ttlSecondsAfterFinished: 1 \ No newline at end of file diff --git a/deployment/base/db-migrate/kustomization.yml b/deployment/base/db-migrate/kustomization.yml deleted file mode 100644 index 0c71c8003e..0000000000 --- a/deployment/base/db-migrate/kustomization.yml +++ /dev/null @@ -1,33 +0,0 @@ -#generated by 'just update-schema' command, DO NOT EDIT -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- db-migrate.yml -configMapGenerator: -- name: ftl-db-migrate-config - files: - - ./schema/20231103205514_init.sql - - ./schema/20240704103403_create_module_secrets.sql - - ./schema/20240725212023_create_fsm_instance_updated_at.sql - - ./schema/20240729002557_catch_async_calls.sql - - ./schema/20240731230343_create_topic_events_caller.sql - - ./schema/20240801160101_edit-db-secrets-and-configuration.sql - - ./schema/20240807174508_pubsub_otel_context.sql - - ./schema/20240812011321_derive_encryption.sql - - ./schema/20240813062546_create_fsm_next_events.sql - - ./schema/20240814060154_rename_events_to_timeline.sql - - ./schema/20240815003340_typesafe_encrypted_columns.sql - - ./schema/20240815053106_update-indexes.sql - - ./schema/20240815164808_async_calls_cron_job_key.sql - - ./schema/20240820011612_encryption_verification.sql - - ./schema/20240902030242_remove_notify.sql - - ./schema/20240903043046_enforce_runner_deployment.sql - - ./schema/20240913035022_encrypted_fsm_next_request.sql - - ./schema/20240913041619_encrypted_topic_events_payload.sql - - ./schema/20240916015906_remove_runner_state.sql - - ./schema/20240916190209_rename_controller_to_controllers.sql - - ./schema/20240917015216_add_ingress_event_type.sql - - ./schema/20240917062716_change_deployments_index.sql - - ./schema/20240919001309_create_identity_keys_table.sql - - ./schema/20240926231955_timeline_async_call_event_types.sql - - ./schema/20241002225509_create_last_activated.sql diff --git a/deployment/base/db-migrate/schema b/deployment/base/db-migrate/schema deleted file mode 120000 index 45985bc500..0000000000 --- a/deployment/base/db-migrate/schema +++ /dev/null @@ -1 +0,0 @@ -../../../backend/controller/sql/schema \ No newline at end of file diff --git a/deployment/base/ftl-controller/deployment-config.yaml b/deployment/base/ftl-controller/deployment-config.yaml deleted file mode 100644 index 0379f7b6f8..0000000000 --- a/deployment/base/ftl-controller/deployment-config.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ftl-controller-deployment-config -data: - serviceTemplate: |- - apiVersion: v1 - kind: Service - metadata: - labels: - app: "(replaced)" - name: "(replaced)" - spec: - ports: - - name: verbs - protocol: TCP - port: 8893 - targetPort: 8893 - type: ClusterIP - deploymentTemplate: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - app: ftl-runner - spec: - selector: - matchLabels: - app: ftl-runner - template: - metadata: - labels: - app: ftl-runner - spec: - containers: - - name: app - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: LOG_LEVEL - value: "debug" - - name: FTL_ENDPOINT - value: "http://ftl-controller:8892" - - name: FTL_RUNNER_BIND - value: "http://$(MY_POD_IP):8893" - - name: FTL_RUNNER_ADVERTISE - value: "http://$(MY_POD_IP):8893" - - name: FTL_LANGUAGE - value: "go,kotlin,java" - - name: LOG_TIMESTAMPS - value: "true" - ports: - - containerPort: 8893 - readinessProbe: - httpGet: - path: /healthz - port: 8893 - initialDelaySeconds: 1 - periodSeconds: 2 - timeoutSeconds: 2 - successThreshold: 1 - failureThreshold: 15 - serviceAccountTemplate: |- - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - app: ftl-runner diff --git a/deployment/base/ftl-controller/ftl-controller.yml b/deployment/base/ftl-controller/ftl-controller.yml deleted file mode 100644 index 7eead27142..0000000000 --- a/deployment/base/ftl-controller/ftl-controller.yml +++ /dev/null @@ -1,109 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ftl-controller - labels: - app: ftl-controller -spec: - replicas: 2 - selector: - matchLabels: - app: ftl-controller - template: - metadata: - labels: - app: ftl-controller - sidecar.istio.io/inject: "true" - spec: - serviceAccountName: ftl-controller - containers: - - name: app - image: ftl0/ftl-controller - imagePullPolicy: Always - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: LOG_LEVEL - value: "debug" - - name: FTL_CONTROLLER_DSN - value: "postgres://ftl-pg-cluster/ftl?sslmode=disable&user=postgres&password=secret" - - name: FTL_CONTROLLER_INGRESS_BIND - value: "http://$(MY_POD_IP):8891" - - name: FTL_CONTROLLER_BIND - value: "http://$(MY_POD_IP):8892" - - name: FTL_CONTROLLER_ADVERTISE - value: "http://$(MY_POD_IP):8892" - - name: AWS_REGION - value: "us-west-2" - - name: AWS_ACCESS_KEY_ID - value: "test" - - name: AWS_SECRET_ACCESS_KEY - value: "test" - - name: AWS_ENDPOINT_URL - value: "http://localstack:4566" - - name: LOG_TIMESTAMPS - value: "true" - ports: - - containerPort: 8891 - - containerPort: 8892 - readinessProbe: - httpGet: - path: /healthz - port: 8892 - initialDelaySeconds: 1 - periodSeconds: 2 - timeoutSeconds: 2 - successThreshold: 1 - failureThreshold: 15 ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: ftl-controller - name: ftl-controller -spec: - ports: - - name: ingress - protocol: TCP - port: 8891 - targetPort: 8891 - - name: controller - protocol: TCP - port: 8892 - targetPort: 8892 - selector: - app: ftl-controller - type: ClusterIP -#--- -#apiVersion: autoscaling/v2 -#kind: HorizontalPodAutoscaler -#metadata: -# name: ftl-controller -#spec: -# scaleTargetRef: -# apiVersion: apps/v1 -# kind: Deployment -# name: ftl-controller -# minReplicas: 1 -# maxReplicas: 10 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ftl-controller - annotations: - ingress.kubernetes.io/ssl-redirect: "false" -spec: - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: ftl-controller - port: - number: 8892 diff --git a/deployment/base/ftl-controller/kustomization.yml b/deployment/base/ftl-controller/kustomization.yml deleted file mode 100644 index 8c1a58899b..0000000000 --- a/deployment/base/ftl-controller/kustomization.yml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ftl-controller.yml -- role.yaml -- role_binding.yaml -- service_account.yaml -- deployment-config.yaml diff --git a/deployment/base/ftl-controller/role.yaml b/deployment/base/ftl-controller/role.yaml deleted file mode 100644 index ba88704d19..0000000000 --- a/deployment/base/ftl-controller/role.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/name: ftl-controller - app.kubernetes.io/managed-by: kustomize - name: ftl-controller -rules: - - apiGroups: [ "apps" ] - resources: [ "deployments" ] - verbs: [ "get", "list", "watch", "create", "update", "patch" ] - - apiGroups: [ "" ] - resources: [ "services" ] - verbs: [ "get", "list", "watch", "delete", "create", "update", "patch" ] - - apiGroups: [ "" ] - resources: [ "serviceaccounts"] - verbs: [ "get", "list", "watch", "create" ] - - apiGroups: [ "" ] - resources: [ "pods" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "configmaps" ] - resourceNames: - - ftl-controller-deployment-config - verbs: [ "get"] - - apiGroups: [ "security.istio.io" ] - resources: [ "authorizationpolicies" ] - verbs: [ "get", "list", "watch", "create" ] diff --git a/deployment/base/ftl-controller/role_binding.yaml b/deployment/base/ftl-controller/role_binding.yaml deleted file mode 100644 index cf1a4ce57d..0000000000 --- a/deployment/base/ftl-controller/role_binding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/name: ftl-controller - app.kubernetes.io/managed-by: kustomize - name: ftl-controller-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ftl-controller -subjects: -- kind: ServiceAccount - name: ftl-controller diff --git a/deployment/base/ftl-controller/service_account.yaml b/deployment/base/ftl-controller/service_account.yaml deleted file mode 100644 index 71a61f3f8d..0000000000 --- a/deployment/base/ftl-controller/service_account.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/name: ftl-controller - app.kubernetes.io/managed-by: kustomize - name: ftl-controller diff --git a/deployment/base/kustomization.yml b/deployment/base/kustomization.yml deleted file mode 100644 index 20e12bb9e7..0000000000 --- a/deployment/base/kustomization.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- db-create -- db-migrate -- ftl-controller -- localstack -images: -- name: ftl0/ftl-controller - newName: ftl0/ftl-controller - newTag: latest -- name: ftl0/ftl-runner - newName: ftl0/ftl-runner - newTag: latest diff --git a/deployment/base/localstack/kustomization.yml b/deployment/base/localstack/kustomization.yml deleted file mode 100644 index 1d1ab04ca5..0000000000 --- a/deployment/base/localstack/kustomization.yml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - localstack.yml diff --git a/deployment/base/localstack/localstack.yml b/deployment/base/localstack/localstack.yml deleted file mode 100644 index b24eb24fa3..0000000000 --- a/deployment/base/localstack/localstack.yml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: localstack -spec: - replicas: 1 - selector: - matchLabels: - app: localstack - template: - metadata: - labels: - app: localstack - spec: - containers: - - name: localstack - image: localstack/localstack - ports: - - containerPort: 4566 - env: - - name: DEBUG - value: "1" - - name: SERVICES - value: "secretsmanager" - volumeMounts: - - name: localstack-data - mountPath: /var/lib/localstack - volumes: - - name: localstack-data - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: localstack -spec: - selector: - app: localstack - type: ClusterIP - ports: - - name: localstack - port: 4566 - targetPort: 4566 diff --git a/deployment/overlays/dev/kustomization.yml b/deployment/overlays/dev/kustomization.yml deleted file mode 100644 index 798373b94e..0000000000 --- a/deployment/overlays/dev/kustomization.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- ../../base - -images: - - name: ftl0/ftl-controller - newName: ftl:5000/ftl-controller - newTag: latest - - name: ftl0/ftl-runner - newName: ftl:5000/ftl-runner - newTag: latest \ No newline at end of file diff --git a/deployment/values.yaml b/deployment/values.yaml new file mode 100644 index 0000000000..9b9385eb49 --- /dev/null +++ b/deployment/values.yaml @@ -0,0 +1,14 @@ +fullnameOverride: "ftl" +ingress: + enabled: true + annotations: + ingress.kubernetes.io/ssl-redirect: "false" + kubernetes.io/ingress.class: traefik +controller: + image: + repository: "ftl:5000/ftl-controller" + tag: "latest" + service: + ports: + - name: "http-8892" + port: 8892 \ No newline at end of file diff --git a/internal/integration/harness.go b/internal/integration/harness.go index d209450694..0b07bab7df 100644 --- a/internal/integration/harness.go +++ b/internal/integration/harness.go @@ -217,6 +217,8 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) { if opts.kube { // This command will build a linux/amd64 version of FTL and deploy it to the kube cluster Infof("Building FTL and deploying to kube") + err = ftlexec.Command(ctx, log.Debug, rootDir, "just", "chart", "dep-update").RunBuffered(ctx) + assert.NoError(t, err) err = ftlexec.Command(ctx, log.Debug, filepath.Join(rootDir, "deployment"), "just", "setup-cluster").RunBuffered(ctx) assert.NoError(t, err) err = ftlexec.Command(ctx, log.Debug, filepath.Join(rootDir, "deployment"), "just", "install-istio").RunBuffered(ctx)