diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 5e29aefd..a05d42a2 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -4,7 +4,7 @@ on: push jobs: pre-commit: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Install asdf @@ -27,7 +27,7 @@ jobs: asdf reshim pre-commit run --color=always --show-diff-on-failure --all-files commitlint: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 with: diff --git a/helm/backup-storage/values-eb75ad-tools-metabase-backup.yaml b/helm/backup-storage/values-eb75ad-tools-metabase-backup.yaml new file mode 100644 index 00000000..d889c8e2 --- /dev/null +++ b/helm/backup-storage/values-eb75ad-tools-metabase-backup.yaml @@ -0,0 +1,20 @@ +# The command to restore the latest backup is: +# ./backup.sh -r metabase-postgresql:5432/metabase +# The command to update the backup in sandbox dev is: +# make install NAME=metabase-backup NAMESPACE=eb75ad-tools +nameOverride: "metabase-backup-storage" +fullnameOverride: "metabase-backup-storage" + +backupConfig: | + postgres=metabase-postgresql:5432/metabase + 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -s -v all + +db: + secretName: metabase-postgresql-credentials + usernameKey: username + passwordKey: password + +persistence: + backup: + size: 1Gi diff --git a/helm/metabase/.helmignore b/helm/metabase/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/metabase/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/metabase/Chart.yaml b/helm/metabase/Chart.yaml new file mode 100644 index 00000000..d702c4df --- /dev/null +++ b/helm/metabase/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: metabase +description: A Helm chart for metabase + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/helm/metabase/Makefile b/helm/metabase/Makefile new file mode 100644 index 00000000..a79f2080 --- /dev/null +++ b/helm/metabase/Makefile @@ -0,0 +1,19 @@ +SHELL := /usr/bin/env bash + +NAMESPACE="" +# This installs metabase in the gold production tools namespace +.PHONY: install +install: + helm install metabase . -n ${NAMESPACE} \ + -f values.yaml \ + -f "values-${NAMESPACE}.yaml" \ + --set postgresql.password="$(shell bash -c 'echo $$RANDOM | md5sum | head -c 32')" + +# Upgrade will change an existing metabase deployment. The secret gets regenerated each time this is run +.PHONY: upgrade +upgrade: + helm upgrade --install metabase . \ + -n ${NAMESPACE} \ + -f values.yaml \ + -f "values-${NAMESPACE}.yaml" \ + --set postgresql.password="$(shell bash -c 'echo $$RANDOM | md5sum | head -c 32')" diff --git a/helm/metabase/README.md b/helm/metabase/README.md new file mode 100644 index 00000000..7b457e70 --- /dev/null +++ b/helm/metabase/README.md @@ -0,0 +1,105 @@ +# Metabase config docs + +## Installing metabase + +### Confirm network policies + +In order for metabase to deploy successfully, pods within the namespace must have access to each other. Ensure the the following two network policies are in place: + +``` +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-same-namespace + namespace: +spec: + podSelector: {} + ingress: + - from: + - podSelector: {} + policyTypes: + - Ingress +``` +and + +``` +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-from-openshift-ingress + namespace: +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: ingress + policyTypes: + - Ingress +``` + +These network policies are fairly generic and may or may not already be configured in the namespace where Metabase is being deployed. + +### Install the helm charts + +The two helm commands are run through the make file in this folder: + + `make install NAMESPACE=eb75ad-tools` + +and + + `make upgrade NAMESPACE=eb75ad-tools` + +This will install and upgrade metabase in the Gold Production Tools namespace. If we need to install it somewhere else the script will need to be generalized. Note this helm chart has not been used to deploy the Silver cluster metabase instance. + +## Adding a databse to metabase: + +The Gold Metabase deployment can query any db in the gold cluster if the credentials are set up properly. In future it may extend to silver as well. The gold configuration requires two steps: + +### Set Network policies + +Metabase needs to access other namespaces. To allow access we must create a network policy in the namespace holding the target database. For the sso-patroni databases, this network policy is: + +``` +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: -metabase-access + namespace: +spec: + podSelector: + matchLabels: + app: + ingress: + - from: + - namespaceSelector: + matchLabels: + environment: + name: + - podSelector: + matchLabels: + app: metabase + policyTypes: + - Ingress +``` + +The config for Gold Metabase are: + +``` + DATABASE CLUSTER TAG: sso-patroni + METABASE NAMESPACE ENVIRONMENT: tools + METABASE NAMESPACE LICENCE PLATE: eb75ad +``` + + +### Connect to the database + +Metabase has a built in workflow for adding new databases. These will require the credentials for the target database. +The one complicated part of adding a db to the metabase account is the host. This follows the patern: + +``` +..svc.cluster.local +``` + +It is important to connect to the read only service if one is available. If the connection attempt times out, it is possible the network policy you created is not set up properly. diff --git a/helm/metabase/templates/NOTES.txt b/helm/metabase/templates/NOTES.txt new file mode 100644 index 00000000..3d4a9365 --- /dev/null +++ b/helm/metabase/templates/NOTES.txt @@ -0,0 +1,3 @@ +1. Get the application URL by running the command: + +oc -n {{.Values.namespace}} get routes diff --git a/helm/metabase/templates/_helpers.tpl b/helm/metabase/templates/_helpers.tpl new file mode 100644 index 00000000..18fcb20b --- /dev/null +++ b/helm/metabase/templates/_helpers.tpl @@ -0,0 +1,65 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "metabase.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "metabase.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "metabase.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "metabase.labels" -}} +app: metabase +service: {{ .Values.metabaseServiceName }} +template: metabase-postgresql-template +helm.sh/chart: {{ include "metabase.chart" . }} +{{ include "metabase.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "metabase.selectorLabels" -}} +app.kubernetes.io/name: {{ include "metabase.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "metabase.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "metabase.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/metabase/templates/deploymentconfig.yaml b/helm/metabase/templates/deploymentconfig.yaml new file mode 100644 index 00000000..48dd5822 --- /dev/null +++ b/helm/metabase/templates/deploymentconfig.yaml @@ -0,0 +1,180 @@ +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }}-postgresql +spec: + replicas: 1 + selector: + name: {{ .Values.metabaseServiceName }}-postgresql + strategy: + type: Recreate + template: + metadata: + labels: + app: metabase + name: {{ .Values.metabaseServiceName }}-postgresql + spec: + containers: + - env: + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + key: database + name: {{ .Values.metabaseServiceName }}-postgresql-credentials + - name: POSTGRESQL_USER + valueFrom: + secretKeyRef: + key: username + name: {{ .Values.metabaseServiceName }}-postgresql-credentials + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: {{ .Values.metabaseServiceName }}-postgresql-credentials + image: ' ' + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -i + - -c + - pg_isready -h 127.0.0.1 -p 5432 + initialDelaySeconds: 30 + timeoutSeconds: 1 + name: {{ .Values.metabaseServiceName }}-postgresql + ports: + - containerPort: 5432 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -i + - -c + - psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c + 'SELECT 1' + initialDelaySeconds: 5 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/pgsql/data + name: {{ .Values.metabaseServiceName }}-postgresql-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: {{ .Values.metabaseServiceName }}-postgresql-data + persistentVolumeClaim: + claimName: {{ .Values.metabaseServiceName }}-postgresql + triggers: + - type: ConfigChange + - imageChangeParams: + automatic: true + containerNames: + - {{ .Values.metabaseServiceName }}-postgresql + from: + kind: ImageStreamTag + name: postgresql:{{ .Values.postgresql.version }} + namespace: {{ .Values.postgresqlImageNamespace }} + lastTriggeredImage: "" + type: ImageChange +--- +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }} +spec: + replicas: 1 + selector: + app: metabase + deploymentconfig: {{ .Values.metabaseServiceName }} + strategy: + type: Recreate + template: + metadata: + labels: + app: metabase + deploymentconfig: {{ .Values.metabaseServiceName }} + service: {{ .Values.metabaseServiceName }} + template: metabase + spec: + containers: + - args: + - -Xmx4096m + - -Xss512k + - -Dfile.encoding=UTF-8 + - -Dlogfile.path=target/log + - -XX:+CMSClassUnloadingEnabled + - -XX:+UseConcMarkSweepGC + - -server + - -jar + - /app/metabase.jar + command: + - java + env: + - name: MB_DB_TYPE + value: postgres + - name: MB_DB_HOST + value: {{ .Values.metabaseServiceName }}-postgresql + - name: MB_DB_PORT + value: "5432" + - name: MB_DB_DBNAME + value: {{ .Values.postgresql.database }} + - name: MB_DB_USER + valueFrom: + secretKeyRef: + key: username + name: {{ .Values.metabaseServiceName }}-postgresql-credentials + - name: MB_DB_PASS + valueFrom: + secretKeyRef: + key: password + name: {{ .Values.metabaseServiceName }}-postgresql-credentials + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + failureThreshold: 30 + httpGet: + path: / + port: 3000 + initialDelaySeconds: 240 + timeoutSeconds: 3 + name: {{ .Values.metabaseServiceName }} + ports: + - containerPort: 3000 + name: {{ .Values.metabaseServiceName }} + protocol: TCP + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 3 + timeoutSeconds: 3 + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 250m + memory: 1Gi + terminationMessagePath: /dev/termination-log + dnsPolicy: ClusterFirst + imagePullSecrets: + - name: artifactory-creds + restartPolicy: Always + serviceAccount: {{ .Values.metabaseServiceName }} + serviceAccountName: {{ .Values.metabaseServiceName }} + terminationGracePeriodSeconds: 30 + triggers: + - type: ConfigChange + - imageChangeParams: + automatic: true + containerNames: + - {{ .Values.metabaseServiceName }} + from: + kind: ImageStreamTag + name: {{ .Values.metabaseServiceName }}:latest + lastTriggeredImage: "" + type: ImageChange diff --git a/helm/metabase/templates/imagestream.yaml b/helm/metabase/templates/imagestream.yaml new file mode 100644 index 00000000..e4f92131 --- /dev/null +++ b/helm/metabase/templates/imagestream.yaml @@ -0,0 +1,16 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + labels: + app: metabase + template: metabase-postgresql-template + name: {{ .Values.metabaseServiceName }} +spec: + tags: + - from: + kind: DockerImage + name: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + generation: 2 + name: latest + referencePolicy: + type: Source diff --git a/helm/metabase/templates/persistentvolumeclaim.yaml b/helm/metabase/templates/persistentvolumeclaim.yaml new file mode 100644 index 00000000..913a9fcd --- /dev/null +++ b/helm/metabase/templates/persistentvolumeclaim.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }}-postgresql +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.postgresql.persistence.size }} diff --git a/helm/metabase/templates/route.yaml b/helm/metabase/templates/route.yaml new file mode 100644 index 00000000..47ab3303 --- /dev/null +++ b/helm/metabase/templates/route.yaml @@ -0,0 +1,12 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }} +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: {{ .Values.metabaseServiceName }} diff --git a/helm/metabase/templates/secret.yaml b/helm/metabase/templates/secret.yaml new file mode 100644 index 00000000..2a25714c --- /dev/null +++ b/helm/metabase/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }}-postgresql-credentials +stringData: + database: {{ .Values.postgresql.database }} + password: {{ .Values.postgresql.password }} + username: {{ .Values.postgresql.user }} +type: Opaque diff --git a/helm/metabase/templates/service.yaml b/helm/metabase/templates/service.yaml new file mode 100644 index 00000000..786ca84c --- /dev/null +++ b/helm/metabase/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + template.openshift.io/expose-uri: http://{.spec.clusterIP}:{.spec.ports[?(.name=="{{ .Values.metabaseServiceName }}")].port} + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }} +spec: + ports: + - name: {{ .Values.metabaseServiceName }} + port: 3000 + protocol: TCP + targetPort: 3000 + selector: + app: metabase + deploymentconfig: {{ .Values.metabaseServiceName }} + sessionAffinity: None + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + template.openshift.io/expose-uri: postgres://{.spec.clusterIP}:{.spec.ports[?(.name=="{{ .Values.metabaseServiceName }}-postgresql")].port} + labels: + app: metabase + template: metabase-postgresql-template + name: {{ .Values.metabaseServiceName }}-postgresql +spec: + ports: + - name: {{ .Values.metabaseServiceName }}-postgresql + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + name: {{ .Values.metabaseServiceName }}-postgresql + sessionAffinity: None + type: ClusterIP diff --git a/helm/metabase/templates/serviceaccount.yaml b/helm/metabase/templates/serviceaccount.yaml new file mode 100644 index 00000000..92211a35 --- /dev/null +++ b/helm/metabase/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: {{ include "metabase.labels" . | nindent 4 }} + name: {{ .Values.metabaseServiceName }} diff --git a/helm/metabase/templates/tests/test-connection.yaml b/helm/metabase/templates/tests/test-connection.yaml new file mode 100644 index 00000000..fef6447b --- /dev/null +++ b/helm/metabase/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "metabase.fullname" . }}-test-connection" + labels: + {{- include "metabase.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "metabase.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/helm/metabase/values-eb75ad-tools.yaml b/helm/metabase/values-eb75ad-tools.yaml new file mode 100644 index 00000000..248500b8 --- /dev/null +++ b/helm/metabase/values-eb75ad-tools.yaml @@ -0,0 +1,4 @@ +namespace: eb75ad-tools + +image: + repository: gcr.io/ggl-cas-storage/metabase diff --git a/helm/metabase/values.yaml b/helm/metabase/values.yaml new file mode 100644 index 00000000..9429898e --- /dev/null +++ b/helm/metabase/values.yaml @@ -0,0 +1,57 @@ +##The first part of the file was autogenerated by `helm create metabase` +## The fields at the bottom were created from the metabase deployment config + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + +## These are the values that were needed by the Metabase helm templates generated by template2helm +## Note The ones above are created by the `healm create metabase`. + +metabaseServiceName: metabase + +image: + repository: docker-remote.artifacts.developer.gov.bc.ca/metabase/metabase + tag: latest + pullPolicy: Always + +postgresql: + database: metabase + password: '' # Generated in the make file + user: metabase + version: "9.6" + persistence: + size: 1Gi + +postgresqlImageNamespace: openshift + +namespace: ""