Implement model registry inference service reconciliation (#135)

Implement the model registry InferenceService reconciliation, the implemented workflow is the opposite of the one proposed in #124. Here the direction goes from Cluster to ModelRegistry.

The new reconciler will monitor InferenceService CRs having pre-defined labels, based on those labels will sync the model registry by keeping track of every deployment that occurred in the cluster.
Then will update the InferenceService CR by linking it to the model registry record using a specific label.

* Fix overlays/dev deployment
* Setup InferenceService reconciliation to model registry
* Setup e2e test for new reconciler
* Enable new reconciler
* Create serving env when not found
* Retrieve MR namespace from ISVC label
* Added Kind cluster configuration for local e2e tests
* Check if IS already exists in MR
* Improve e2e testing
* Refactor dev manifests

Signed-off-by: Andrea Lamparelli <[email protected]>
Co-authored-by: Edgar Hernández <[email protected]>

Makefile

@@ -66,7 +66,11 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" POD_NAMESPACE=default go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" POD_NAMESPACE=default go test ./controllers/... -coverprofile cover.out
+
+.PHONY: e2e-test
+e2e-test: manifests generate fmt vet ## Run e2e-tests.
+	POD_NAMESPACE=default go test ./test/e2e/...
 
 ##@ Build
 
@@ -109,6 +113,15 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
+.PHONY: deploy-dev
+deploy-dev: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/overlays/dev | kubectl apply -f -
+
+.PHONY: undeploy-dev
+undeploy-dev: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/overlays/dev | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to

config/crd/external/kustomization.yaml

@@ -1,3 +1,10 @@
 resources:
 - serving.kserve.io_inferenceservices.yaml
+- serving.kserve.io_servingruntimes.yaml
+- route.openshift.io_routes.yaml
+- monitoring.coreos.com_podmonitors.yaml
+- monitoring.coreos.com_servicemonitors.yaml
+- maistra.io_servicemeshmemberrolls.yaml
+- maistra.io_servicemeshmembers.yaml
+- telemetry.istio.io_telemetries.yaml
 # - route.openshift.io_routes.yaml  #minikube

config/overlays/dev/kustomization.yaml

@@ -1,3 +1,30 @@
 resources:
-  - ../../crd/external
-  - ../../manager/namespace.yaml
+- ../../crd/external
+- ../../default
+
+patchesStrategicMerge:
+- odh_model_controller_manager_patch.yaml
+
+namespace: default
+configMapGenerator:
+- envs:
+  - params.env
+  name: odh-model-controller-parameters
+generatorOptions:
+  disableNameSuffixHash: true
+
+vars:
+- fieldref:
+    fieldPath: metadata.namespace
+  name: mesh-namespace
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: odh-model-controller-parameters
+- fieldref:
+    fieldPath: data.monitoring-namespace
+  name: monitoring-namespace
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: odh-model-controller-parameters

config/overlays/dev/odh_model_controller_manager_patch.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: odh-model-controller
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - args:
+        - --leader-elect
+        - --model-registry-inference-reconcile
+        - "--monitoring-namespace"
+        - "$(MONITORING_NS)"
+        env:
+        - name: MONITORING_NS
+          value: $(monitoring-namespace)
+        name: manager
+        imagePullPolicy: IfNotPresent

config/overlays/dev/params.env

@@ -0,0 +1,2 @@
+monitoring-namespace=default
+metadata.namespace=default

config/rbac/role.yaml

@@ -196,6 +196,7 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - serving.kserve.io

controllers/comparators/inferenceservice_comparator.go

@@ -0,0 +1,34 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package comparators
+
+import (
+	"reflect"
+
+	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func GetInferenceServiceComparator() ResourceComparator {
+	return func(deployed client.Object, requested client.Object) bool {
+		deployedInferenceService := deployed.(*kservev1beta1.InferenceService)
+		requestedInferenceService := requested.(*kservev1beta1.InferenceService)
+		return reflect.DeepEqual(deployedInferenceService.Labels[constants.ModelRegistryInferenceServiceIdLabel], requestedInferenceService.Labels[constants.ModelRegistryInferenceServiceIdLabel]) &&
+			reflect.DeepEqual(deployedInferenceService.Labels[constants.ModelRegistryRegisteredModelIdLabel], requestedInferenceService.Labels[constants.ModelRegistryRegisteredModelIdLabel]) &&
+			reflect.DeepEqual(deployedInferenceService.Labels[constants.ModelRegistryModelVersionIdLabel], requestedInferenceService.Labels[constants.ModelRegistryModelVersionIdLabel])
+	}
+}

controllers/constants/constants.go

@@ -21,3 +21,12 @@ const (
 	IstioIngressServiceHTTPPortName  = "http2"
 	IstioIngressServiceHTTPSPortName = "https"
 )
+
+// model registry
+const (
+	MLMDAddressEnv                       = "MLMD_ADDRESS"
+	ModelRegistryNamespaceLabel          = "modelregistry.opendatahub.io/namespace"
+	ModelRegistryInferenceServiceIdLabel = "modelregistry.opendatahub.io/inference-service-id"
+	ModelRegistryModelVersionIdLabel     = "modelregistry.opendatahub.io/model-version-id"
+	ModelRegistryRegisteredModelIdLabel  = "modelregistry.opendatahub.io/registered-model-id"
+)