From f84c1126c1ec3bc093cf5be64156b3b48d54db4a Mon Sep 17 00:00:00 2001 From: Ho Kim Date: Sun, 8 Sep 2024 21:56:55 +0000 Subject: [PATCH] chore(dash): update minio tenant crd --- crates/dash/api/src/lib.rs | 1 + crates/dash/api/src/model_claim.rs | 69 +++++++++++++++++++ crates/dash/api/src/model_user.rs | 78 ++++++++++++++++++++++ crates/dash/api/src/storage/object.rs | 48 ++++--------- crates/dash/operator/src/ctx/model.rs | 5 ++ crates/dash/provider/src/storage/object.rs | 17 ++--- 6 files changed, 171 insertions(+), 47 deletions(-) create mode 100644 crates/dash/api/src/model_user.rs diff --git a/crates/dash/api/src/lib.rs b/crates/dash/api/src/lib.rs index e9ebeb48..eb2eb879 100644 --- a/crates/dash/api/src/lib.rs +++ b/crates/dash/api/src/lib.rs @@ -3,6 +3,7 @@ pub mod job; pub mod model; pub mod model_claim; pub mod model_storage_binding; +pub mod model_user; pub mod storage; pub mod task; diff --git a/crates/dash/api/src/model_claim.rs b/crates/dash/api/src/model_claim.rs index 571af769..6dab47f5 100644 --- a/crates/dash/api/src/model_claim.rs +++ b/crates/dash/api/src/model_claim.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use k8s_openapi::api::core::v1::ResourceRequirements; use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -42,12 +43,16 @@ use crate::storage::ModelStorageKind; )] #[serde(rename_all = "camelCase")] pub struct ModelClaimSpec { + #[serde(default)] + pub affinity: ModelClaimAffinity, #[serde(default = "ModelClaimSpec::default_allow_replacement")] pub allow_replacement: bool, #[serde(default)] pub binding_policy: ModelClaimBindingPolicy, #[serde(default)] pub deletion_policy: ModelClaimDeletionPolicy, + #[serde(default)] + pub resources: Option, pub storage: Option, } @@ -58,9 +63,11 @@ impl ModelClaimCrd { impl Default for ModelClaimSpec { fn default() -> Self { Self { + affinity: ModelClaimAffinity::default(), allow_replacement: Self::default_allow_replacement(), binding_policy: ModelClaimBindingPolicy::default(), deletion_policy: ModelClaimDeletionPolicy::default(), + resources: None, storage: None, } } @@ -72,6 +79,68 @@ impl ModelClaimSpec { } } +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelClaimAffinity { + #[serde(default)] + pub placement_affinity: ModelClaimAffinityRequirements, + #[serde(default)] + pub replacement_affinity: ModelClaimAffinityRequirements, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelClaimAffinityRequirements { + #[serde(default)] + pub preferred: Vec, + #[serde(default)] + pub required: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelClaimPreferredAffinity { + #[serde(default, flatten)] + pub base: ModelClaimAffinityPreference, + pub weight: u8, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelClaimAffinityPreference { + #[serde(default)] + pub match_expressions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelClaimAffinityExpression { + pub query: String, + #[serde(default)] + pub source: ModelClaimAffinityExpressionSource, +} + +#[derive( + Copy, + Clone, + Debug, + Display, + Default, + EnumString, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +pub enum ModelClaimAffinityExpressionSource { + #[default] + Prometheus, +} + #[derive( Copy, Clone, diff --git a/crates/dash/api/src/model_user.rs b/crates/dash/api/src/model_user.rs new file mode 100644 index 00000000..48ca62a9 --- /dev/null +++ b/crates/dash/api/src/model_user.rs @@ -0,0 +1,78 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema, CustomResource)] +#[kube( + group = "dash.ulagbulag.io", + version = "v1alpha1", + kind = "ModelUser", + root = "ModelUserCrd", + shortname = "mu", + namespaced, + printcolumn = r#"{ + "name": "created-at", + "type": "date", + "description": "created time", + "jsonPath": ".metadata.creationTimestamp" + }"#, + printcolumn = r#"{ + "name": "version", + "type": "integer", + "description": "model user version", + "jsonPath": ".metadata.generation" + }"# +)] +#[serde(rename_all = "camelCase")] +pub struct ModelUserSpec { + #[serde(default)] + pub access_token: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum ModelUserAccessTokenSpec { + SecretRef(ModelUserAccessTokenSecretRefSpec), +} + +impl Default for ModelUserAccessTokenSpec { + fn default() -> Self { + Self::SecretRef(ModelUserAccessTokenSecretRefSpec::default()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ModelUserAccessTokenSecretRefSpec { + #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_map_access_key")] + pub map_access_key: String, + #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_map_secret_key")] + pub map_secret_key: String, + + #[serde(default = "ModelUserAccessTokenSecretRefSpec::default_name")] + pub name: String, +} + +impl Default for ModelUserAccessTokenSecretRefSpec { + fn default() -> Self { + Self { + map_access_key: Self::default_map_access_key(), + map_secret_key: Self::default_map_secret_key(), + name: Self::default_name(), + } + } +} + +impl ModelUserAccessTokenSecretRefSpec { + fn default_map_access_key() -> String { + "CONSOLE_ACCESS_KEY".into() + } + + fn default_map_secret_key() -> String { + "CONSOLE_SECRET_KEY".into() + } + + fn default_name() -> String { + "object-storage-user-0".into() + } +} diff --git a/crates/dash/api/src/storage/object.rs b/crates/dash/api/src/storage/object.rs index cbaa4a56..071d178d 100644 --- a/crates/dash/api/src/storage/object.rs +++ b/crates/dash/api/src/storage/object.rs @@ -8,6 +8,8 @@ use maplit::btreemap; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::model_user::ModelUserAccessTokenSecretRefSpec; + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum ModelStorageObjectSpec { @@ -195,7 +197,7 @@ impl ModelStorageObjectOwnedExternalServiceSpec { pub struct ModelStorageObjectRefSpec { pub endpoint: Url, #[serde(default)] - pub secret_ref: ModelStorageObjectRefSecretRefSpec, + pub secret_ref: ModelUserAccessTokenSecretRefSpec, } impl ModelStorageObjectRefSpec { @@ -205,43 +207,17 @@ impl ModelStorageObjectRefSpec { } } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct ModelStorageObjectRefSecretRefSpec { - #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_map_access_key")] - pub map_access_key: String, - #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_map_secret_key")] - pub map_secret_key: String, - - #[serde(default = "ModelStorageObjectRefSecretRefSpec::default_name")] - pub name: String, -} - -impl Default for ModelStorageObjectRefSecretRefSpec { - fn default() -> Self { - Self { - map_access_key: Self::default_map_access_key(), - map_secret_key: Self::default_map_secret_key(), - name: Self::default_name(), - } - } -} - -impl ModelStorageObjectRefSecretRefSpec { - fn default_map_access_key() -> String { - "CONSOLE_ACCESS_KEY".into() - } - - fn default_map_secret_key() -> String { - "CONSOLE_SECRET_KEY".into() - } - - fn default_name() -> String { - "object-storage-user-0".into() - } +#[inline] +pub const fn get_default_tenant_name() -> &'static str { + "minio" } #[inline] pub fn get_kubernetes_minio_endpoint(namespace: &str) -> Option { - format!("http://minio.{namespace}.svc").parse().ok() + format!( + "http://{service}.{namespace}.svc", + service = get_default_tenant_name() + ) + .parse() + .ok() } diff --git a/crates/dash/operator/src/ctx/model.rs b/crates/dash/operator/src/ctx/model.rs index 9422a7e0..85cddd68 100644 --- a/crates/dash/operator/src/ctx/model.rs +++ b/crates/dash/operator/src/ctx/model.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use chrono::Utc; use dash_api::model::{ModelCrd, ModelFieldsNativeSpec, ModelState, ModelStatus}; use dash_provider::storage::KubernetesStorageClient; +use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::{ api::{Patch, PatchParams}, runtime::controller::Action, @@ -29,6 +30,10 @@ impl ::ark_core_k8s::manager::Ctx for Ctx { const FINALIZER_NAME: &'static str = ::Data::FINALIZER_NAME; + fn get_subcrds() -> Vec { + vec![::dash_api::model_user::ModelUserCrd::crd()] + } + #[instrument(level = Level::INFO, skip_all, fields(name = %data.name_any(), namespace = data.namespace()), err(Display))] async fn reconcile( manager: Arc>, diff --git a/crates/dash/provider/src/storage/object.rs b/crates/dash/provider/src/storage/object.rs index 69a9ebe2..bdfb7922 100644 --- a/crates/dash/provider/src/storage/object.rs +++ b/crates/dash/provider/src/storage/object.rs @@ -12,11 +12,11 @@ use dash_api::{ ModelStorageBindingSyncPolicy, ModelStorageBindingSyncPolicyPull, ModelStorageBindingSyncPolicyPush, }, + model_user::ModelUserAccessTokenSecretRefSpec, storage::object::{ - get_kubernetes_minio_endpoint, ModelStorageObjectBorrowedSpec, + get_default_tenant_name, get_kubernetes_minio_endpoint, ModelStorageObjectBorrowedSpec, ModelStorageObjectClonedSpec, ModelStorageObjectOwnedReplicationSpec, - ModelStorageObjectOwnedSpec, ModelStorageObjectRefSecretRefSpec, ModelStorageObjectRefSpec, - ModelStorageObjectSpec, + ModelStorageObjectOwnedSpec, ModelStorageObjectRefSpec, ModelStorageObjectSpec, }, }; use dash_provider_api::data::Capacity; @@ -238,7 +238,7 @@ impl<'model> ObjectStorageSession { let ModelStorageObjectRefSpec { endpoint, secret_ref: - ModelStorageObjectRefSecretRefSpec { + ModelUserAccessTokenSecretRefSpec { map_access_key, map_secret_key, name: secret_name, @@ -792,7 +792,7 @@ impl<'model> ObjectStorageSession { backend: IngressBackend { resource: None, service: Some(IngressServiceBackend { - name: "minio".into(), + name: get_default_tenant_name().into(), port: Some(ServiceBackendPort { name: Some("http-minio".into()), number: None, @@ -864,7 +864,7 @@ impl<'model> ObjectStorageSession { Ok(ModelStorageObjectRefSpec { endpoint: get_kubernetes_minio_endpoint(namespace) .ok_or_else(|| anyhow!("failed to get minio storage endpoint"))?, - secret_ref: ModelStorageObjectRefSecretRefSpec { + secret_ref: ModelUserAccessTokenSecretRefSpec { map_access_key: "CONSOLE_ACCESS_KEY".into(), map_secret_key: "CONSOLE_SECRET_KEY".into(), name: secret_user_0.name_any(), @@ -2171,7 +2171,6 @@ export MINIO_ROOT_PASSWORD="{password}" "volumesPerServer": total_volumes_per_node, }, ], - "prometheusOperator": false, "requestAutoCert": false, "serviceMetadata": { "consoleServiceAnnotations": { @@ -2412,10 +2411,6 @@ fn get_default_pod_affinity(tenant_name: &str) -> PodAffinity { } } -const fn get_default_tenant_name() -> &'static str { - "object-storage" -} - fn get_ingress_host(namespace: &str, tenant_name: &str) -> String { format!("{tenant_name}.{namespace}.svc") }