diff --git a/Cargo.lock b/Cargo.lock index 92688df7..f182ba0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,6 +189,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "built" version = "0.6.1" @@ -341,6 +350,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -360,6 +378,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.14.4" @@ -441,12 +469,36 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dockerfile-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75799314f5fa405629a365a1f97d80f81edd17f22a0fc9c8ddb3ad191ad8dc08" +dependencies = [ + "enquote", + "lazy_static", + "pest", + "pest_derive", + "regex", + "snafu 0.6.10", +] + [[package]] name = "dyn-clone" version = "1.0.14" @@ -468,6 +520,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -601,6 +662,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1296,6 +1367,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pest_meta" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -1351,8 +1467,8 @@ dependencies = [ [[package]] name = "product-config" -version = "0.5.0" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.5.0#439869d9e6a72fb6d912f6e494649a2f74f41d25" +version = "0.6.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.6.0#ad2c3ea6a291e415d978eb4271fb309e75861ef0" dependencies = [ "fancy-regex", "java-properties", @@ -1361,7 +1477,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "snafu 0.7.5", "xml-rs", ] @@ -1743,6 +1859,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.6" @@ -1776,6 +1903,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +[[package]] +name = "snafu" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7" +dependencies = [ + "doc-comment", + "snafu-derive 0.6.10", +] + [[package]] name = "snafu" version = "0.7.5" @@ -1783,7 +1920,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "snafu-derive", + "snafu-derive 0.7.5", +] + +[[package]] +name = "snafu-derive" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1833,7 +1981,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator", "strum", "tracing", @@ -1844,10 +1992,11 @@ name = "stackable-kafka-operator" version = "0.0.0-dev" dependencies = [ "futures 0.3.28", + "product-config", "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-kafka-crd", "stackable-operator", "strum", @@ -1870,13 +2019,14 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "chrono", "clap", "const_format", "derivative", + "dockerfile-parser", "either", "futures 0.3.28", "json-patch", @@ -1892,7 +2042,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator-derive", "strum", "thiserror", @@ -1904,8 +2054,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -2284,6 +2434,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index 6ba93f8c..b01cd9fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/rust/operator/Cargo.toml b/rust/operator/Cargo.toml index 0ef1d5aa..d555522b 100644 --- a/rust/operator/Cargo.toml +++ b/rust/operator/Cargo.toml @@ -15,6 +15,7 @@ futures.workspace = true serde.workspace = true serde_json.workspace = true stackable-operator.workspace = true +product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 1ec489a6..3b9f5d73 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -1,26 +1,23 @@ //! Ensures that `Pod`s are configured and running for each [`KafkaCluster`] -use crate::operations::pdb::add_pdbs; -use crate::product_logging::{ - extend_role_group_config_map, resolve_vector_aggregator_address, MAX_KAFKA_LOG_FILES_SIZE, - STACKABLE_LOG_DIR, -}; -use crate::{ - discovery::{self, build_discovery_configmaps}, - pod_svc_controller, - product_logging::LOG4J_CONFIG_FILE, - utils::{self, build_recommended_labels}, - ControllerConfig, +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + sync::Arc, }; +use product_config::{ + types::PropertyNameKind, + writer::{to_java_properties_string, PropertiesWriterError}, + ProductConfigManager, +}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_kafka_crd::JVM_SECURITY_PROPERTIES_FILE; use stackable_kafka_crd::{ listener::get_kafka_listener_config, security::KafkaTlsSecurity, Container, KafkaCluster, - KafkaClusterStatus, KafkaConfig, KafkaRole, APP_NAME, DOCKER_IMAGE_BASE_NAME, KAFKA_HEAP_OPTS, - LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, SERVER_PROPERTIES_FILE, - STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_TMP_DIR, + KafkaClusterStatus, KafkaConfig, KafkaRole, APP_NAME, DOCKER_IMAGE_BASE_NAME, + JVM_SECURITY_PROPERTIES_FILE, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, + METRICS_PORT_NAME, OPERATOR_NAME, SERVER_PROPERTIES_FILE, STACKABLE_CONFIG_DIR, + STACKABLE_DATA_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_TMP_DIR, }; - use stackable_operator::{ builder::{ resources::ResourceRequirementsBuilder, ConfigMapBuilder, ContainerBuilder, @@ -51,9 +48,6 @@ use stackable_operator::{ labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, - product_config::{ - types::PropertyNameKind, writer::to_java_properties_string, ProductConfigManager, - }, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, product_logging::{ self, @@ -69,13 +63,20 @@ use stackable_operator::{ }, time::Duration, }; -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - sync::Arc, -}; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::{ + discovery::{self, build_discovery_configmaps}, + operations::pdb::add_pdbs, + pod_svc_controller, + product_logging::{ + extend_role_group_config_map, resolve_vector_aggregator_address, LOG4J_CONFIG_FILE, + MAX_KAFKA_LOG_FILES_SIZE, STACKABLE_LOG_DIR, + }, + utils::{self, build_recommended_labels}, + ControllerConfig, +}; + pub const KAFKA_CONTROLLER_NAME: &str = "kafkacluster"; /// Used as runAsUser in the pod security context. This is specified in the kafka image file pub const KAFKA_UID: i64 = 1000; @@ -141,7 +142,7 @@ pub enum Error { }, #[snafu(display("failed to serialize zoo.cfg for {}", rolegroup))] SerializeZooCfg { - source: stackable_operator::product_config::writer::PropertiesWriterError, + source: PropertiesWriterError, rolegroup: RoleGroupRef, }, #[snafu(display("object is missing metadata to build owner reference"))] @@ -240,7 +241,7 @@ pub enum Error { rolegroup ))] JvmSecurityPoperties { - source: stackable_operator::product_config::writer::PropertiesWriterError, + source: PropertiesWriterError, rolegroup: String, }, #[snafu(display("failed to create PodDisruptionBudget"))] diff --git a/rust/operator/src/lib.rs b/rust/operator/src/lib.rs index 72983c5f..d27ebcbf 100644 --- a/rust/operator/src/lib.rs +++ b/rust/operator/src/lib.rs @@ -1,14 +1,7 @@ -mod discovery; -mod kafka_controller; -mod operations; -mod pod_svc_controller; -mod product_logging; -mod utils; - -use crate::kafka_controller::KAFKA_CONTROLLER_NAME; -use crate::pod_svc_controller::POD_SERVICE_CONTROLLER_NAME; +use std::sync::Arc; use futures::StreamExt; +use product_config::ProductConfigManager; use stackable_kafka_crd::{KafkaCluster, OPERATOR_NAME}; use stackable_operator::{ client::Client, @@ -20,9 +13,18 @@ use stackable_operator::{ kube::runtime::{watcher, Controller}, logging::controller::report_controller_reconciled, namespace::WatchNamespace, - product_config::ProductConfigManager, }; -use std::sync::Arc; + +use crate::{ + kafka_controller::KAFKA_CONTROLLER_NAME, pod_svc_controller::POD_SERVICE_CONTROLLER_NAME, +}; + +mod discovery; +mod kafka_controller; +mod operations; +mod pod_svc_controller; +mod product_logging; +mod utils; mod built_info { pub const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION");