From 1e572559b0c3439ae8dd477d057fe669aaafee83 Mon Sep 17 00:00:00 2001 From: Dhiraj Bokde Date: Wed, 7 Feb 2024 00:30:26 -0800 Subject: [PATCH] feat: add kustomize manifests for kubeflow, fixes RHOAIENG-1947 --- manifests/kustomize/base/kustomization.yaml | 10 ++ .../base/model-registry-configmap.yaml | 11 ++ .../base/model-registry-deployment.yaml | 103 ++++++++++++++++++ .../kustomize/base/model-registry-sa.yaml | 4 + .../base/model-registry-service.yaml | 17 +++ .../options/istio/destination-rule.yaml | 9 ++ .../istio/istio-authorization-policy.yaml | 11 ++ .../options/istio/kustomization.yaml | 7 ++ .../options/istio/virtual-service.yaml | 21 ++++ .../kustomize/overlays/db/kustomization.yaml | 38 +++++++ .../db/model-registry-db-deployment.yaml | 52 +++++++++ .../overlays/db/model-registry-db-pvc.yaml | 10 ++ .../db/model-registry-db-service.yaml | 14 +++ manifests/kustomize/overlays/db/params.env | 3 + .../db/patches/model-registry-deployment.yaml | 37 +++++++ manifests/kustomize/overlays/db/secrets.env | 2 + .../overlays/postgres/kustomization.yaml | 38 +++++++ .../model-registry-db-deployment.yaml | 43 ++++++++ .../postgres/model-registry-db-pvc.yaml | 10 ++ .../postgres/model-registry-db-service.yaml | 14 +++ .../kustomize/overlays/postgres/params.env | 2 + .../patches/model-registry-deployment.yaml | 28 +++++ .../kustomize/overlays/postgres/secrets.env | 2 + 23 files changed, 486 insertions(+) create mode 100644 manifests/kustomize/base/kustomization.yaml create mode 100644 manifests/kustomize/base/model-registry-configmap.yaml create mode 100644 manifests/kustomize/base/model-registry-deployment.yaml create mode 100644 manifests/kustomize/base/model-registry-sa.yaml create mode 100644 manifests/kustomize/base/model-registry-service.yaml create mode 100644 manifests/kustomize/options/istio/destination-rule.yaml create mode 100644 manifests/kustomize/options/istio/istio-authorization-policy.yaml create mode 100644 manifests/kustomize/options/istio/kustomization.yaml create mode 100644 manifests/kustomize/options/istio/virtual-service.yaml create mode 100644 manifests/kustomize/overlays/db/kustomization.yaml create mode 100644 manifests/kustomize/overlays/db/model-registry-db-deployment.yaml create mode 100644 manifests/kustomize/overlays/db/model-registry-db-pvc.yaml create mode 100644 manifests/kustomize/overlays/db/model-registry-db-service.yaml create mode 100644 manifests/kustomize/overlays/db/params.env create mode 100644 manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml create mode 100644 manifests/kustomize/overlays/db/secrets.env create mode 100644 manifests/kustomize/overlays/postgres/kustomization.yaml create mode 100644 manifests/kustomize/overlays/postgres/model-registry-db-deployment.yaml create mode 100644 manifests/kustomize/overlays/postgres/model-registry-db-pvc.yaml create mode 100644 manifests/kustomize/overlays/postgres/model-registry-db-service.yaml create mode 100644 manifests/kustomize/overlays/postgres/params.env create mode 100644 manifests/kustomize/overlays/postgres/patches/model-registry-deployment.yaml create mode 100644 manifests/kustomize/overlays/postgres/secrets.env diff --git a/manifests/kustomize/base/kustomization.yaml b/manifests/kustomize/base/kustomization.yaml new file mode 100644 index 00000000..5d497c83 --- /dev/null +++ b/manifests/kustomize/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - model-registry-configmap.yaml + - model-registry-deployment.yaml + - model-registry-service.yaml + - model-registry-sa.yaml +images: + - name: gcr.io/ml-pipeline/metadata-envoy + newTag: 2.0.5 diff --git a/manifests/kustomize/base/model-registry-configmap.yaml b/manifests/kustomize/base/model-registry-configmap.yaml new file mode 100644 index 00000000..8d884532 --- /dev/null +++ b/manifests/kustomize/base/model-registry-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: model-registry-configmap + labels: + component: model-registry-server +data: + MODEL_REGISTRY_REST_SERVICE_HOST: "model-registry-service" + MODEL_REGISTRY_REST_SERVICE_PORT: "8080" + MODEL_REGISTRY_GRPC_SERVICE_HOST: "model-registry-service" + MODEL_REGISTRY_GRPC_SERVICE_PORT: "9090" diff --git a/manifests/kustomize/base/model-registry-deployment.yaml b/manifests/kustomize/base/model-registry-deployment.yaml new file mode 100644 index 00000000..7022ee17 --- /dev/null +++ b/manifests/kustomize/base/model-registry-deployment.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment + labels: + component: model-registry-server +spec: + replicas: 1 + selector: + matchLabels: + component: model-registry-server + template: + metadata: + labels: + component: model-registry-server + spec: + containers: + - name: rest-container + args: + - --hostname=0.0.0.0 + - --port=8080 + - --mlmd-hostname=localhost + - --mlmd-port=9090 + command: + - /model-registry + - proxy + image: quay.io/opendatahub/model-registry:latest + # empty placeholder environment for patching + env: [] + ports: + - name: http-api + containerPort: 8080 + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 5 + tcpSocket: + port: http-api + timeoutSeconds: 2 + readinessProbe: + initialDelaySeconds: 3 + periodSeconds: 5 + tcpSocket: + port: http-api + timeoutSeconds: 2 + - name: grpc-container + # ! Sync to the same MLMD version: + # * backend/metadata_writer/requirements.in and requirements.txt + # * @kubeflow/frontend/src/mlmd/generated + # * .cloudbuild.yaml and .release.cloudbuild.yaml + # * manifests/kustomize/base/metadata/base/model-registry-deployment.yaml + # * test/tag_for_hosted.sh + image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 + env: + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: MYSQL_DATABASE + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mlmdDb + - name: MYSQL_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbHost + - name: MYSQL_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbPort + command: ["/bin/metadata_store_server"] + args: ["--grpc_port=9090", + "--mysql_config_database=$(MYSQL_DATABASE)", + "--mysql_config_host=$(MYSQL_HOST)", + "--mysql_config_port=$(MYSQL_PORT)", + "--mysql_config_user=$(DBCONFIG_USER)", + "--mysql_config_password=$(DBCONFIG_PASSWORD)", + "--enable_database_upgrade=true" + ] + ports: + - name: grpc-api + containerPort: 9090 + livenessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + readinessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + serviceAccountName: model-registry-server diff --git a/manifests/kustomize/base/model-registry-sa.yaml b/manifests/kustomize/base/model-registry-sa.yaml new file mode 100644 index 00000000..8cfc77ef --- /dev/null +++ b/manifests/kustomize/base/model-registry-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: model-registry-server diff --git a/manifests/kustomize/base/model-registry-service.yaml b/manifests/kustomize/base/model-registry-service.yaml new file mode 100644 index 00000000..d7d362c3 --- /dev/null +++ b/manifests/kustomize/base/model-registry-service.yaml @@ -0,0 +1,17 @@ +kind: Service +apiVersion: v1 +metadata: + labels: + app: metadata + name: model-registry-service +spec: + selector: + component: model-registry-server + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-api + - port: 9090 + protocol: TCP + name: grpc-api diff --git a/manifests/kustomize/options/istio/destination-rule.yaml b/manifests/kustomize/options/istio/destination-rule.yaml new file mode 100644 index 00000000..179d62d2 --- /dev/null +++ b/manifests/kustomize/options/istio/destination-rule.yaml @@ -0,0 +1,9 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: model-registry-service +spec: + host: model-registry-service.kubeflow.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL diff --git a/manifests/kustomize/options/istio/istio-authorization-policy.yaml b/manifests/kustomize/options/istio/istio-authorization-policy.yaml new file mode 100644 index 00000000..36685d8f --- /dev/null +++ b/manifests/kustomize/options/istio/istio-authorization-policy.yaml @@ -0,0 +1,11 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: model-registry-service +spec: + action: ALLOW + selector: + matchLabels: + component: model-registry-server + rules: + - {} diff --git a/manifests/kustomize/options/istio/kustomization.yaml b/manifests/kustomize/options/istio/kustomization.yaml new file mode 100644 index 00000000..029a6937 --- /dev/null +++ b/manifests/kustomize/options/istio/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- istio-authorization-policy.yaml +- destination-rule.yaml +- virtual-service.yaml diff --git a/manifests/kustomize/options/istio/virtual-service.yaml b/manifests/kustomize/options/istio/virtual-service.yaml new file mode 100644 index 00000000..1cbdaeb2 --- /dev/null +++ b/manifests/kustomize/options/istio/virtual-service.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: model-registry + namespace: kubeflow +spec: + gateways: + - kubeflow-gateway + hosts: + - '*' + http: + - match: + - uri: + prefix: /ml_metadata + rewrite: + uri: /ml_metadata + route: + - destination: + host: metadata-envoy-service.kubeflow.svc.cluster.local + port: + number: 9090 diff --git a/manifests/kustomize/overlays/db/kustomization.yaml b/manifests/kustomize/overlays/db/kustomization.yaml new file mode 100644 index 00000000..42fea9f0 --- /dev/null +++ b/manifests/kustomize/overlays/db/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- model-registry-db-pvc.yaml +- model-registry-db-deployment.yaml +- model-registry-db-service.yaml +- ../../base + +patchesStrategicMerge: +- patches/model-registry-deployment.yaml + +configMapGenerator: +- envs: + - params.env + name: model-registry-db-parameters +secretGenerator: +- envs: + - secrets.env + name: model-registry-db-secrets +generatorOptions: + disableNameSuffixHash: true + + +images: +- name: mysql + newName: mysql + newTag: 8.0.3 + +vars: +- fieldref: + fieldPath: metadata.name + name: MLMD_DB_HOST + objref: + apiVersion: v1 + kind: Service + name: model-registry-db diff --git a/manifests/kustomize/overlays/db/model-registry-db-deployment.yaml b/manifests/kustomize/overlays/db/model-registry-db-deployment.yaml new file mode 100644 index 00000000..7f1477aa --- /dev/null +++ b/manifests/kustomize/overlays/db/model-registry-db-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: db-container + image: mysql:8.0.3 + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + envFrom: + - configMapRef: + name: model-registry-db-parameters + - secretRef: + name: model-registry-db-secrets + ports: + - name: dbapi + containerPort: 3306 + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "mysql -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'" + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + volumeMounts: + - name: metadata-mysql + mountPath: /var/lib/mysql + volumes: + - name: metadata-mysql + persistentVolumeClaim: + claimName: metadata-mysql diff --git a/manifests/kustomize/overlays/db/model-registry-db-pvc.yaml b/manifests/kustomize/overlays/db/model-registry-db-pvc.yaml new file mode 100644 index 00000000..b1c083d9 --- /dev/null +++ b/manifests/kustomize/overlays/db/model-registry-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-mysql +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/manifests/kustomize/overlays/db/model-registry-db-service.yaml b/manifests/kustomize/overlays/db/model-registry-db-service.yaml new file mode 100644 index 00000000..f27c8c76 --- /dev/null +++ b/manifests/kustomize/overlays/db/model-registry-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: model-registry-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + name: dbapi + selector: + component: db diff --git a/manifests/kustomize/overlays/db/params.env b/manifests/kustomize/overlays/db/params.env new file mode 100644 index 00000000..5ab2adb3 --- /dev/null +++ b/manifests/kustomize/overlays/db/params.env @@ -0,0 +1,3 @@ +MYSQL_DATABASE=metadb +MYSQL_PORT=3306 +MYSQL_ALLOW_EMPTY_PASSWORD=true \ No newline at end of file diff --git a/manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml b/manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml new file mode 100644 index 00000000..5d788ed7 --- /dev/null +++ b/manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment +spec: + template: + spec: + containers: + - name: rest-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: model-registry-configmap + args: + - --hostname=0.0.0.0 + - --port=$(MODEL_REGISTRY_REST_SERVICE_PORT) + - --mlmd-hostname=localhost + - --mlmd-port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT) + - name: grpc-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: model-registry-db-parameters + - secretRef: + name: model-registry-db-secrets + - configMapRef: + name: model-registry-configmap + args: ["--grpc_port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)", + "--mysql_config_host=$(MLMD_DB_HOST)", + "--mysql_config_database=$(MYSQL_DATABASE)", + "--mysql_config_port=$(MYSQL_PORT)", + "--mysql_config_user=$(MYSQL_USER_NAME)", + "--mysql_config_password=$(MYSQL_ROOT_PASSWORD)"] diff --git a/manifests/kustomize/overlays/db/secrets.env b/manifests/kustomize/overlays/db/secrets.env new file mode 100644 index 00000000..44ac2ee3 --- /dev/null +++ b/manifests/kustomize/overlays/db/secrets.env @@ -0,0 +1,2 @@ +MYSQL_USER_NAME=root +MYSQL_ROOT_PASSWORD=test \ No newline at end of file diff --git a/manifests/kustomize/overlays/postgres/kustomization.yaml b/manifests/kustomize/overlays/postgres/kustomization.yaml new file mode 100644 index 00000000..facbb163 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +bases: +- ../../base +resources: +- model-registry-db-pvc.yaml +- model-registry-db-deployment.yaml +- model-registry-db-service.yaml + +patchesStrategicMerge: +- patches/model-registry-deployment.yaml + +configMapGenerator: +- name: metadata-postgres-db-parameters + envs: + - params.env +secretGenerator: +- name: metadata-postgres-db-secrets + envs: + - secrets.env +generatorOptions: + disableNameSuffixHash: true + +images: +- name: postgres + newName: postgres + newTag: 14.7-alpine3.17 + +vars: +- name: MLMD_DB_HOST + objref: + kind: Service + name: metadata-postgres-db + apiVersion: v1 + fieldref: + fieldpath: metadata.name diff --git a/manifests/kustomize/overlays/postgres/model-registry-db-deployment.yaml b/manifests/kustomize/overlays/postgres/model-registry-db-deployment.yaml new file mode 100644 index 00000000..061d109e --- /dev/null +++ b/manifests/kustomize/overlays/postgres/model-registry-db-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: db-container + image: postgres + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: metadata-postgres + mountPath: /var/lib/postgresql/data + volumes: + - name: metadata-postgres + persistentVolumeClaim: + claimName: metadata-postgres + diff --git a/manifests/kustomize/overlays/postgres/model-registry-db-pvc.yaml b/manifests/kustomize/overlays/postgres/model-registry-db-pvc.yaml new file mode 100644 index 00000000..13790489 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/model-registry-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/manifests/kustomize/overlays/postgres/model-registry-db-service.yaml b/manifests/kustomize/overlays/postgres/model-registry-db-service.yaml new file mode 100644 index 00000000..63902a66 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/model-registry-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + name: postgres + selector: + component: db diff --git a/manifests/kustomize/overlays/postgres/params.env b/manifests/kustomize/overlays/postgres/params.env new file mode 100644 index 00000000..fce7e267 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/params.env @@ -0,0 +1,2 @@ +POSTGRES_PORT=5432 +POSTGRES_DBNAME=mlmdpostgres \ No newline at end of file diff --git a/manifests/kustomize/overlays/postgres/patches/model-registry-deployment.yaml b/manifests/kustomize/overlays/postgres/patches/model-registry-deployment.yaml new file mode 100644 index 00000000..a97fc2a9 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/patches/model-registry-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment +spec: + template: + spec: + containers: + - name: grpc-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + - configMapRef: + name: model-registry-configmap + args: ["--grpc_port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)", + "--metadata_source_config_type=postgresql", + "--postgres_config_host=$(MLMD_DB_HOST)", + "--postgres_config_port=$(POSTGRES_PORT)", + "--postgres_config_dbname=$(POSTGRES_DBNAME)", + "--postgres_config_user=$(POSTGRES_USER)", + "--postgres_config_password=$(POSTGRES_PASSWORD)", + # "--postgres_config_skip_db_creation=true", + "--enable_database_upgrade=true"] diff --git a/manifests/kustomize/overlays/postgres/secrets.env b/manifests/kustomize/overlays/postgres/secrets.env new file mode 100644 index 00000000..973d1582 --- /dev/null +++ b/manifests/kustomize/overlays/postgres/secrets.env @@ -0,0 +1,2 @@ +POSTGRES_USER=root +POSTGRES_PASSWORD=password \ No newline at end of file