chore(dash): update minio tenant crd

ulagbulag · Sep 8, 2024 · f84c112 · f84c112
1 parent 054de46
commit f84c112
Show file tree

Hide file tree

Showing 6 changed files with 171 additions and 47 deletions.
diff --git a/crates/dash/api/src/lib.rs b/crates/dash/api/src/lib.rs
@@ -3,6 +3,7 @@ pub mod job;
 pub mod model;
 pub mod model_claim;
 pub mod model_storage_binding;
+pub mod model_user;
 pub mod storage;
 pub mod task;
 

diff --git a/crates/dash/api/src/model_claim.rs b/crates/dash/api/src/model_claim.rs
@@ -1,4 +1,5 @@
 use chrono::{DateTime, Utc};
+use k8s_openapi::api::core::v1::ResourceRequirements;
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -42,12 +43,16 @@ use crate::storage::ModelStorageKind;
 )]
 #[serde(rename_all = "camelCase")]
 pub struct ModelClaimSpec {
+    #[serde(default)]
+    pub affinity: ModelClaimAffinity,
     #[serde(default = "ModelClaimSpec::default_allow_replacement")]
     pub allow_replacement: bool,
     #[serde(default)]
     pub binding_policy: ModelClaimBindingPolicy,
     #[serde(default)]
     pub deletion_policy: ModelClaimDeletionPolicy,
+    #[serde(default)]
+    pub resources: Option<ResourceRequirements>,
     pub storage: Option<ModelStorageKind>,
 }
 
@@ -58,9 +63,11 @@ impl ModelClaimCrd {
 impl Default for ModelClaimSpec {
     fn default() -> Self {
         Self {
+            affinity: ModelClaimAffinity::default(),
             allow_replacement: Self::default_allow_replacement(),
             binding_policy: ModelClaimBindingPolicy::default(),
             deletion_policy: ModelClaimDeletionPolicy::default(),
+            resources: None,
             storage: None,
         }
     }
@@ -72,6 +79,68 @@ impl ModelClaimSpec {
     }
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelClaimAffinity {
+    #[serde(default)]
+    pub placement_affinity: ModelClaimAffinityRequirements,
+    #[serde(default)]
+    pub replacement_affinity: ModelClaimAffinityRequirements,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelClaimAffinityRequirements {
+    #[serde(default)]
+    pub preferred: Vec<ModelClaimPreferredAffinity>,
+    #[serde(default)]
+    pub required: Vec<ModelClaimAffinityPreference>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelClaimPreferredAffinity {
+    #[serde(default, flatten)]
+    pub base: ModelClaimAffinityPreference,
+    pub weight: u8,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelClaimAffinityPreference {
+    #[serde(default)]
+    pub match_expressions: Vec<ModelClaimAffinityExpression>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelClaimAffinityExpression {
+    pub query: String,
+    #[serde(default)]
+    pub source: ModelClaimAffinityExpressionSource,
+}
+
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    Display,
+    Default,
+    EnumString,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+)]
+pub enum ModelClaimAffinityExpressionSource {
+    #[default]
+    Prometheus,
+}
+
 #[derive(
     Copy,
     Clone,

diff --git a/crates/dash/api/src/model_user.rs b/crates/dash/api/src/model_user.rs
@@ -0,0 +1,78 @@
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema, CustomResource)]
+#[kube(
+    group = "dash.ulagbulag.io",
+    version = "v1alpha1",
+    kind = "ModelUser",
+    root = "ModelUserCrd",
+    shortname = "mu",
+    namespaced,
+    printcolumn = r#"{
+        "name": "created-at",
+        "type": "date",
+        "description": "created time",
+        "jsonPath": ".metadata.creationTimestamp"
+    }"#,
+    printcolumn = r#"{
+        "name": "version",
+        "type": "integer",
+        "description": "model user version",
+        "jsonPath": ".metadata.generation"
+    }"#
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelUserSpec {
+    #[serde(default)]
+    pub access_token: Option<ModelUserAccessTokenSpec>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub enum ModelUserAccessTokenSpec {
+    SecretRef(ModelUserAccessTokenSecretRefSpec),
+}
+
+impl Default for ModelUserAccessTokenSpec {
+    fn default() -> Self {
+        Self::SecretRef(ModelUserAccessTokenSecretRefSpec::default())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ModelUserAccessTokenSecretRefSpec {
+    #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_map_access_key")]
+    pub map_access_key: String,
+    #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_map_secret_key")]
+    pub map_secret_key: String,
+
+    #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_name")]
+    pub name: String,
+}
+
+impl Default for ModelUserAccessTokenSecretRefSpec {
+    fn default() -> Self {
+        Self {
+            map_access_key: Self::default_map_access_key(),
+            map_secret_key: Self::default_map_secret_key(),
+            name: Self::default_name(),
+        }
+    }
+}
+
+impl ModelUserAccessTokenSecretRefSpec {
+    fn default_map_access_key() -> String {
+        "CONSOLE_ACCESS_KEY".into()
+    }
+
+    fn default_map_secret_key() -> String {
+        "CONSOLE_SECRET_KEY".into()
+    }
+
+    fn default_name() -> String {
+        "object-storage-user-0".into()
+    }
+}
diff --git a/crates/dash/api/src/storage/object.rs b/crates/dash/api/src/storage/object.rs
@@ -8,6 +8,8 @@ use maplit::btreemap;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
+use crate::model_user::ModelUserAccessTokenSecretRefSpec;
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub enum ModelStorageObjectSpec {
@@ -195,7 +197,7 @@ impl ModelStorageObjectOwnedExternalServiceSpec {
 pub struct ModelStorageObjectRefSpec {
     pub endpoint: Url,
     #[serde(default)]
-    pub secret_ref: ModelStorageObjectRefSecretRefSpec,
+    pub secret_ref: ModelUserAccessTokenSecretRefSpec,
 }
 
 impl ModelStorageObjectRefSpec {
@@ -205,43 +207,17 @@ impl ModelStorageObjectRefSpec {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct ModelStorageObjectRefSecretRefSpec {
-    #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_map_access_key")]
-    pub map_access_key: String,
-    #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_map_secret_key")]
-    pub map_secret_key: String,
-
-    #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_name")]
-    pub name: String,
-}
-
-impl Default for ModelStorageObjectRefSecretRefSpec {
-    fn default() -> Self {
-        Self {
-            map_access_key: Self::default_map_access_key(),
-            map_secret_key: Self::default_map_secret_key(),
-            name: Self::default_name(),
-        }
-    }
-}
-
-impl ModelStorageObjectRefSecretRefSpec {
-    fn default_map_access_key() -> String {
-        "CONSOLE_ACCESS_KEY".into()
-    }
-
-    fn default_map_secret_key() -> String {
-        "CONSOLE_SECRET_KEY".into()
-    }
-
-    fn default_name() -> String {
-        "object-storage-user-0".into()
-    }
+#[inline]
+pub const fn get_default_tenant_name() -> &'static str {
+    "minio"
 }
 
 #[inline]
 pub fn get_kubernetes_minio_endpoint(namespace: &str) -> Option<Url> {
-    format!("http://minio.{namespace}.svc").parse().ok()
+    format!(
+        "http://{service}.{namespace}.svc",
+        service = get_default_tenant_name()
+    )
+    .parse()
+    .ok()
 }
diff --git a/crates/dash/operator/src/ctx/model.rs b/crates/dash/operator/src/ctx/model.rs
@@ -6,6 +6,7 @@ use async_trait::async_trait;
 use chrono::Utc;
 use dash_api::model::{ModelCrd, ModelFieldsNativeSpec, ModelState, ModelStatus};
 use dash_provider::storage::KubernetesStorageClient;
+use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use kube::{
     api::{Patch, PatchParams},
     runtime::controller::Action,
@@ -29,6 +30,10 @@ impl ::ark_core_k8s::manager::Ctx for Ctx {
     const FINALIZER_NAME: &'static str =
         <Self as ::ark_core_k8s::manager::Ctx>::Data::FINALIZER_NAME;
 
+    fn get_subcrds() -> Vec<CustomResourceDefinition> {
+        vec![::dash_api::model_user::ModelUserCrd::crd()]
+    }
+
     #[instrument(level = Level::INFO, skip_all, fields(name = %data.name_any(), namespace = data.namespace()), err(Display))]
     async fn reconcile(
         manager: Arc<Manager<Self>>,

diff --git a/crates/dash/provider/src/storage/object.rs b/crates/dash/provider/src/storage/object.rs
@@ -12,11 +12,11 @@ use dash_api::{
         ModelStorageBindingSyncPolicy, ModelStorageBindingSyncPolicyPull,
         ModelStorageBindingSyncPolicyPush,
     },
+    model_user::ModelUserAccessTokenSecretRefSpec,
     storage::object::{
-        get_kubernetes_minio_endpoint, ModelStorageObjectBorrowedSpec,
+        get_default_tenant_name, get_kubernetes_minio_endpoint, ModelStorageObjectBorrowedSpec,
         ModelStorageObjectClonedSpec, ModelStorageObjectOwnedReplicationSpec,
-        ModelStorageObjectOwnedSpec, ModelStorageObjectRefSecretRefSpec, ModelStorageObjectRefSpec,
-        ModelStorageObjectSpec,
+        ModelStorageObjectOwnedSpec, ModelStorageObjectRefSpec, ModelStorageObjectSpec,
     },
 };
 use dash_provider_api::data::Capacity;
@@ -238,7 +238,7 @@ impl<'model> ObjectStorageSession {
         let ModelStorageObjectRefSpec {
             endpoint,
             secret_ref:
-                ModelStorageObjectRefSecretRefSpec {
+                ModelUserAccessTokenSecretRefSpec {
                     map_access_key,
                     map_secret_key,
                     name: secret_name,
@@ -792,7 +792,7 @@ impl<'model> ObjectStorageSession {
                                 backend: IngressBackend {
                                     resource: None,
                                     service: Some(IngressServiceBackend {
-                                        name: "minio".into(),
+                                        name: get_default_tenant_name().into(),
                                         port: Some(ServiceBackendPort {
                                             name: Some("http-minio".into()),
                                             number: None,
@@ -864,7 +864,7 @@ impl<'model> ObjectStorageSession {
         Ok(ModelStorageObjectRefSpec {
             endpoint: get_kubernetes_minio_endpoint(namespace)
                 .ok_or_else(|| anyhow!("failed to get minio storage endpoint"))?,
-            secret_ref: ModelStorageObjectRefSecretRefSpec {
+            secret_ref: ModelUserAccessTokenSecretRefSpec {
                 map_access_key: "CONSOLE_ACCESS_KEY".into(),
                 map_secret_key: "CONSOLE_SECRET_KEY".into(),
                 name: secret_user_0.name_any(),
@@ -2171,7 +2171,6 @@ export MINIO_ROOT_PASSWORD="{password}"
                             "volumesPerServer": total_volumes_per_node,
                         },
                     ],
-                    "prometheusOperator": false,
                     "requestAutoCert": false,
                     "serviceMetadata": {
                         "consoleServiceAnnotations": {
@@ -2412,10 +2411,6 @@ fn get_default_pod_affinity(tenant_name: &str) -> PodAffinity {
     }
 }
 
-const fn get_default_tenant_name() -> &'static str {
-    "object-storage"
-}
-
 fn get_ingress_host(namespace: &str, tenant_name: &str) -> String {
     format!("{tenant_name}.{namespace}.svc")
 }