From 2982820030d57b16a9ee6003fcd046d4bb7c1a20 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Tue, 26 Sep 2023 10:02:32 +0800
Subject: [PATCH 01/19] feat: implement is_healthy

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 sidecar/src/backup/mod.rs |  5 +++--
 sidecar/src/operator.rs   |  1 +
 sidecar/src/xline.rs      | 35 ++++++++++++++++++++++++++++++++---
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/sidecar/src/backup/mod.rs b/sidecar/src/backup/mod.rs
index 21b10144..9df2b0f8 100644
--- a/sidecar/src/backup/mod.rs
+++ b/sidecar/src/backup/mod.rs
@@ -3,11 +3,12 @@ pub(crate) mod pv;
 /// The s3 backup
 pub(crate) mod s3;
 
-use anyhow::{anyhow, Result};
-use async_trait::async_trait;
 use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
 use tonic::Streaming;
 use xlineapi::SnapshotResponse;
 
diff --git a/sidecar/src/operator.rs b/sidecar/src/operator.rs
index ffa7f890..e71cd3ac 100644
--- a/sidecar/src/operator.rs
+++ b/sidecar/src/operator.rs
@@ -47,6 +47,7 @@ impl Operator {
             &self.config.container_name,
             backup,
             self.config.xline_port,
+            self.config.xline_members(),
         )?);
         let revision = handle.revision_offline().unwrap_or(1);
         let state = Arc::new(Mutex::new(StatePayload {
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 24742381..be4daf56 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -2,8 +2,7 @@
 #![allow(clippy::unnecessary_wraps)] // TODO remove as it is implemented
 #![allow(clippy::unused_self)] // TODO remove as it is implemented
 
-use crate::backup::Metadata;
-use crate::backup::Provider;
+use std::collections::HashMap;
 
 use anyhow::{anyhow, Result};
 use bytes::Buf;
@@ -17,6 +16,9 @@ use tracing::debug;
 use xline_client::types::kv::RangeRequest;
 use xline_client::Client;
 
+use crate::backup::Metadata;
+use crate::backup::Provider;
+
 /// Meta table name
 pub(crate) const META_TABLE: &str = "meta";
 /// KV table name
@@ -55,15 +57,20 @@ pub(crate) struct XlineHandle {
     health_client: HealthClient<Channel>,
     /// The rocks db engine
     engine: Engine,
+    /// The xline members
+    xline_members: HashMap<String, String>,
+    /// Health retires of xline client
+    is_healthy_retries: usize,
 }
 
 impl XlineHandle {
-    /// Start the xline in pod and return the handle
+    /// Create the xline handle but not start the xline node
     pub(crate) fn open(
         name: &str,
         container_name: &str,
         backup: Option<Box<dyn Provider>>,
         xline_port: u16,
+        xline_members: HashMap<String, String>,
     ) -> Result<Self> {
         debug!("name: {name}, container_name: {container_name}, backup: {backup:?}, xline_port: {xline_port}");
         let endpoint: Endpoint = format!("http://127.0.0.1:{xline_port}").parse()?;
@@ -77,6 +84,8 @@ impl XlineHandle {
             health_client,
             engine,
             client: None, // TODO maybe we could initialize the client here when xline#423 is merged
+            xline_members,
+            is_healthy_retries: 5,
         })
     }
 
@@ -89,6 +98,10 @@ impl XlineHandle {
 
     /// Start the xline server
     pub(crate) fn start(&self) -> Result<()> {
+        // Step 1: Check if there is any node running
+        // Step 2: If there is no node running, start single node cluster
+        // Step 3: If there are some nodes running, start the node as a member to join the cluster
+
         // TODO start xline server
         Ok(())
     }
@@ -99,6 +112,22 @@ impl XlineHandle {
         Ok(())
     }
 
+    /// Return the xline cluster health by sending kv requests
+    pub(crate) async fn is_healthy(&self) -> bool {
+        let client = self.client().kv_client();
+        for _ in 0..self.is_healthy_retries {
+            // send linearized request to check if the xline server is healthy
+            if client
+                .range(RangeRequest::new("health").with_serializable(false))
+                .await
+                .is_ok()
+            {
+                return true;
+            }
+        }
+        false
+    }
+
     /// Return the xline server running state by sending a `gRPC` health request
     pub(crate) async fn is_running(&self) -> bool {
         let mut client = self.health_client.clone();

From 7dc53d0dff1609c44c58dd6bd66ff59d1e42ca7c Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Tue, 26 Sep 2023 10:41:20 +0800
Subject: [PATCH 02/19] feat: implement XlineHandle::start

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 sidecar/src/xline.rs | 43 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index be4daf56..30b1b07a 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -3,10 +3,13 @@
 #![allow(clippy::unused_self)] // TODO remove as it is implemented
 
 use std::collections::HashMap;
+use std::time::Duration;
 
 use anyhow::{anyhow, Result};
 use bytes::Buf;
 use engine::{Engine, EngineType, StorageEngine};
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 use operator_api::consts::DEFAULT_DATA_DIR;
 use tonic::transport::{Channel, Endpoint};
 use tonic_health::pb::health_check_response::ServingStatus;
@@ -14,7 +17,7 @@ use tonic_health::pb::health_client::HealthClient;
 use tonic_health::pb::HealthCheckRequest;
 use tracing::debug;
 use xline_client::types::kv::RangeRequest;
-use xline_client::Client;
+use xline_client::{Client, ClientOptions};
 
 use crate::backup::Metadata;
 use crate::backup::Provider;
@@ -97,18 +100,46 @@ impl XlineHandle {
     }
 
     /// Start the xline server
-    pub(crate) fn start(&self) -> Result<()> {
+    pub(crate) async fn start(&mut self) -> Result<()> {
         // Step 1: Check if there is any node running
         // Step 2: If there is no node running, start single node cluster
         // Step 3: If there are some nodes running, start the node as a member to join the cluster
-
-        // TODO start xline server
+        let endpoints = self
+            .xline_members
+            .values()
+            .map(|addr| {
+                Ok::<_, tonic::transport::Error>(
+                    Endpoint::from_shared(addr.clone())?.connect_timeout(Duration::from_secs(3)),
+                )
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let futs: FuturesUnordered<_> = endpoints.iter().map(Endpoint::connect).collect();
+        // the cluster is started if any of the connection is successful
+        let cluster_started = futs.any(|res| async move { res.is_ok() }).await;
+
+        // start xline server here
+        // TODO define a trait to abstract the start of xline server
+
+        let client = Client::connect(self.xline_members.values(), ClientOptions::default()).await?;
+        if cluster_started {
+            let _cluster_client = client.cluster_client();
+            // send membership change here
+        }
+        let _ig = self.client.replace(client);
         Ok(())
     }
 
     /// Stop the xline server
-    pub(crate) fn stop(&self) -> Result<()> {
-        // TODO stop xline server
+    pub(crate) async fn stop(&self) -> Result<()> {
+        // Step 1: Kill the xline node
+        // Step 2: Remove the xline node from the cluster if the cluster exist
+
+        // kill xline server here
+
+        if self.is_healthy().await {
+            let _cluster_client = self.client().cluster_client();
+            // send membership change here
+        }
         Ok(())
     }
 

From d5f0844e843660a382bc5c67f781616be32e9962 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Tue, 26 Sep 2023 14:42:06 +0800
Subject: [PATCH 03/19] refactor: remove tera config generator

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                | 879 +++++++++++++++-----------------------
 assets/xline_conf.tera    |  17 -
 operator-api/Cargo.toml   |   3 +
 operator-api/src/lib.rs   |   3 +
 operator-api/src/xline.rs |  51 +++
 operator-k8s/Cargo.toml   |   2 +-
 sidecar/Cargo.toml        |   9 +-
 sidecar/src/main.rs       | 226 +++++-----
 sidecar/src/types.rs      |  37 +-
 9 files changed, 544 insertions(+), 683 deletions(-)
 delete mode 100644 assets/xline_conf.tera
 create mode 100644 operator-api/src/xline.rs

diff --git a/Cargo.lock b/Cargo.lock
index d5b47910..f2583b32 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.21.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
 dependencies = [
  "gimli",
 ]
@@ -42,9 +42,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.1.2"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
  "memchr",
 ]
@@ -66,29 +66,30 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.4"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
 dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
  "anstyle-wincon",
  "colorchoice",
+ "is-terminal",
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle"
-version = "1.0.4"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.2"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
 dependencies = [
  "utf8parse",
 ]
@@ -104,9 +105,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.1"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
+checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
 dependencies = [
  "anstyle",
  "windows-sys",
@@ -114,9 +115,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.75"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
 
 [[package]]
 name = "async-channel"
@@ -148,14 +149,14 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "async-task"
-version = "4.4.1"
+version = "4.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921"
+checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"
 
 [[package]]
 name = "async-trait"
@@ -165,7 +166,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -187,9 +188,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "axum"
-version = "0.6.20"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
+checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -247,9 +248,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.69"
+version = "0.3.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
 dependencies = [
  "addr2line",
  "cc",
@@ -274,9 +275,9 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
 
 [[package]]
 name = "base64"
-version = "0.21.4"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
 
 [[package]]
 name = "base64ct"
@@ -305,13 +306,13 @@ dependencies = [
  "lazy_static",
  "lazycell",
  "peeking_take_while",
- "prettyplease 0.2.15",
+ "prettyplease 0.2.12",
  "proc-macro2",
  "quote",
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -322,9 +323,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.4.0"
+version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
 
 [[package]]
 name = "block-buffer"
@@ -335,33 +336,23 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "bstr"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019"
-dependencies = [
- "memchr",
- "serde",
-]
-
 [[package]]
 name = "bumpalo"
-version = "3.14.0"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
 
 [[package]]
 name = "byteorder"
-version = "1.5.0"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.5.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "bzip2-sys"
@@ -376,12 +367,11 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.83"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 dependencies = [
  "jobserver",
- "libc",
 ]
 
 [[package]]
@@ -401,15 +391,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.31"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
  "num-traits",
  "serde",
- "windows-targets",
+ "winapi",
 ]
 
 [[package]]
@@ -442,23 +432,24 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.4.6"
+version = "4.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956"
+checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
 dependencies = [
  "clap_builder",
- "clap_derive 4.4.2",
+ "clap_derive 4.3.2",
+ "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.4.6"
+version = "4.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45"
+checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
 dependencies = [
  "anstream",
  "anstyle",
- "clap_lex 0.5.1",
+ "clap_lex 0.5.0",
  "strsim",
 ]
 
@@ -477,14 +468,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.4.2"
+version = "4.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
+checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -498,9 +489,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.5.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
 
 [[package]]
 name = "clippy-utilities"
@@ -525,9 +516,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "concurrent-queue"
-version = "2.3.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
+checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -557,31 +548,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
-dependencies = [
- "autocfg",
- "cfg-if",
- "crossbeam-utils",
- "memoffset",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-skiplist"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "883a5821d7d079fcf34ac55f27a833ee61678110f6b97637cc74513c0d0b42fc"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
- "scopeguard",
-]
-
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.16"
@@ -604,7 +570,7 @@ dependencies = [
 [[package]]
 name = "curp"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -629,27 +595,25 @@ dependencies = [
  "parking_lot",
  "prost",
  "prost-build",
- "rand",
  "serde",
  "thiserror",
  "tokio-stream 0.1.12",
  "tower",
  "tracing",
  "tracing-opentelemetry",
- "utils",
+ "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
+ "uuid",
 ]
 
 [[package]]
 name = "curp-external-api"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "async-trait",
  "engine",
  "mockall",
- "prost",
  "serde",
- "thiserror",
 ]
 
 [[package]]
@@ -697,7 +661,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -719,22 +683,28 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "dashmap"
-version = "5.5.3"
+version = "5.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28"
 dependencies = [
  "cfg-if",
- "hashbrown 0.14.1",
+ "hashbrown 0.14.0",
  "lock_api",
  "once_cell",
  "parking_lot_core",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
+
 [[package]]
 name = "deranged"
 version = "0.3.8"
@@ -838,20 +808,20 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.14"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd"
+checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
 
 [[package]]
 name = "either"
-version = "1.9.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
 [[package]]
 name = "engine"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "async-trait",
  "bincode",
@@ -867,20 +837,31 @@ dependencies = [
 
 [[package]]
 name = "equivalent"
-version = "1.0.1"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
 
 [[package]]
 name = "errno"
-version = "0.3.5"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
 dependencies = [
+ "errno-dragonfly",
  "libc",
  "windows-sys",
 ]
 
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "etcd-client"
 version = "0.11.1"
@@ -905,9 +886,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
 [[package]]
 name = "fastrand"
-version = "2.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
 
 [[package]]
 name = "fixedbitset"
@@ -1029,7 +1010,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -1082,7 +1063,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -1122,9 +1103,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.28.0"
+version = "0.27.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
 
 [[package]]
 name = "glob"
@@ -1132,30 +1113,6 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
-[[package]]
-name = "globset"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d"
-dependencies = [
- "aho-corasick",
- "bstr",
- "fnv",
- "log",
- "regex",
-]
-
-[[package]]
-name = "globwalk"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
-dependencies = [
- "bitflags 1.3.2",
- "ignore",
- "walkdir",
-]
-
 [[package]]
 name = "h2"
 version = "0.3.21"
@@ -1183,9 +1140,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.14.1"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 
 [[package]]
 name = "heck"
@@ -1204,9 +1161,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.3"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 
 [[package]]
 name = "hex"
@@ -1223,15 +1180,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "home"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
-dependencies = [
- "windows-sys",
-]
-
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -1256,9 +1204,9 @@ dependencies = [
 
 [[package]]
 name = "http-range-header"
-version = "0.3.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
+checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
 
 [[package]]
 name = "httparse"
@@ -1268,9 +1216,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
 [[package]]
 name = "httpdate"
-version = "1.0.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
 [[package]]
 name = "hyper"
@@ -1289,7 +1237,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.4.9",
+ "socket2",
  "tokio",
  "tower-service",
  "tracing",
@@ -1365,23 +1313,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "ignore"
-version = "0.4.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492"
-dependencies = [
- "globset",
- "lazy_static",
- "log",
- "memchr",
- "regex",
- "same-file",
- "thread_local",
- "walkdir",
- "winapi-util",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -1395,12 +1326,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.0.2"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.1",
+ "hashbrown 0.14.0",
 ]
 
 [[package]]
@@ -1418,6 +1349,17 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1429,15 +1371,15 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.9"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a"
 
 [[package]]
 name = "jobserver"
-version = "0.1.27"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
+checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
 dependencies = [
  "libc",
 ]
@@ -1453,9 +1395,9 @@ dependencies = [
 
 [[package]]
 name = "json-patch"
-version = "1.2.0"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6"
+checksum = "1f54898088ccb91df1b492cc80029a6fdf1c48ca0db7c6822a8babad69c94658"
 dependencies = [
  "serde",
  "serde_json",
@@ -1480,7 +1422,7 @@ version = "8.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378"
 dependencies = [
- "base64 0.21.4",
+ "base64 0.21.2",
  "pem",
  "ring",
  "serde",
@@ -1494,7 +1436,7 @@ version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788"
 dependencies = [
- "base64 0.21.4",
+ "base64 0.21.2",
  "bytes",
  "chrono",
  "http",
@@ -1542,12 +1484,14 @@ dependencies = [
  "openssl",
  "pem",
  "pin-project",
+ "rand",
  "secrecy",
  "serde",
  "serde_json",
  "serde_yaml",
  "thiserror",
  "tokio",
+ "tokio-tungstenite",
  "tokio-util",
  "tower",
  "tower-http",
@@ -1624,9 +1568,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.149"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "libloading"
@@ -1682,9 +1626,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.10"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
+checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
 
 [[package]]
 name = "lock_api"
@@ -1698,9 +1642,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "lz4-sys"
@@ -1738,7 +1682,7 @@ dependencies = [
  "spin 0.9.8",
  "tokio",
  "tokio-util",
- "toml 0.7.8",
+ "toml 0.7.6",
  "tracing",
  "tracing-subscriber",
 ]
@@ -1758,7 +1702,7 @@ dependencies = [
  "spin 0.9.8",
  "thiserror",
  "tokio",
- "toml 0.7.8",
+ "toml 0.7.6",
  "tonic",
  "tracing",
 ]
@@ -1820,29 +1764,20 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
 dependencies = [
- "regex-automata 0.1.10",
+ "regex-automata",
 ]
 
 [[package]]
 name = "matchit"
-version = "0.7.3"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b"
 
 [[package]]
 name = "memchr"
-version = "2.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
-
-[[package]]
-name = "memoffset"
-version = "0.9.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
-dependencies = [
- "autocfg",
-]
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "merged_range"
@@ -1982,9 +1917,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.17"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
  "autocfg",
 ]
@@ -1995,7 +1930,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.3",
+ "hermit-abi 0.3.1",
  "libc",
 ]
 
@@ -2007,9 +1942,9 @@ checksum = "89c16f12ae83bcd4510aff7778f043464991f8b5a10a27d7083f75361de32ac3"
 
 [[package]]
 name = "object"
-version = "0.32.1"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
 dependencies = [
  "memchr",
 ]
@@ -2022,11 +1957,11 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
-version = "0.10.57"
+version = "0.10.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
+checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 1.3.2",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -2043,14 +1978,14 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.93"
+version = "0.9.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
+checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
 dependencies = [
  "cc",
  "libc",
@@ -2149,6 +2084,9 @@ dependencies = [
 name = "operator-api"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
+ "k8s-openapi",
+ "kube",
  "serde",
 ]
 
@@ -2163,9 +2101,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "2.10.1"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
+checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"
 dependencies = [
  "num-traits",
 ]
@@ -2255,51 +2193,6 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
-[[package]]
-name = "pest"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4"
-dependencies = [
- "memchr",
- "thiserror",
- "ucd-trie",
-]
-
-[[package]]
-name = "pest_derive"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8"
-dependencies = [
- "pest",
- "pest_generator",
-]
-
-[[package]]
-name = "pest_generator"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a"
-dependencies = [
- "pest",
- "pest_meta",
- "proc-macro2",
- "quote",
- "syn 2.0.38",
-]
-
-[[package]]
-name = "pest_meta"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d"
-dependencies = [
- "once_cell",
- "pest",
- "sha2",
-]
-
 [[package]]
 name = "petgraph"
 version = "0.6.4"
@@ -2307,34 +2200,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap 2.0.2",
+ "indexmap 2.0.0",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.1.3"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.3"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.13"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
+checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
 
 [[package]]
 name = "pin-utils"
@@ -2396,12 +2289,12 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.15"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
+checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
 dependencies = [
  "proc-macro2",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -2440,28 +2333,13 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.69"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
+checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "prometheus"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
-dependencies = [
- "cfg-if",
- "fnv",
- "lazy_static",
- "memchr",
- "parking_lot",
- "protobuf",
- "thiserror",
-]
-
 [[package]]
 name = "prost"
 version = "0.11.9"
@@ -2516,17 +2394,11 @@ dependencies = [
  "prost",
 ]
 
-[[package]]
-name = "protobuf"
-version = "2.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
-
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
  "proc-macro2",
 ]
@@ -2601,14 +2473,13 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.0"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.1",
- "regex-syntax 0.8.1",
+ "regex-syntax 0.7.2",
 ]
 
 [[package]]
@@ -2620,17 +2491,6 @@ dependencies = [
  "regex-syntax 0.6.29",
 ]
 
-[[package]]
-name = "regex-automata"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax 0.8.1",
-]
-
 [[package]]
 name = "regex-syntax"
 version = "0.6.29"
@@ -2639,9 +2499,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33"
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
 [[package]]
 name = "ring"
@@ -2682,11 +2542,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
 name = "rustix"
-version = "0.38.18"
+version = "0.38.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c"
+checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.3.3",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -2701,24 +2561,15 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "ryu"
-version = "1.0.15"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
-
-[[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
+checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9"
 
 [[package]]
 name = "schemars"
-version = "0.8.15"
+version = "0.8.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c"
+checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f"
 dependencies = [
  "dyn-clone",
  "schemars_derive",
@@ -2728,9 +2579,9 @@ dependencies = [
 
 [[package]]
 name = "schemars_derive"
-version = "0.8.15"
+version = "0.8.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c"
+checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2740,9 +2591,9 @@ dependencies = [
 
 [[package]]
 name = "scopeguard"
-version = "1.2.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "secrecy"
@@ -2756,9 +2607,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.188"
+version = "1.0.166"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
+checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8"
 dependencies = [
  "serde_derive",
 ]
@@ -2769,19 +2620,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"
 dependencies = [
- "ordered-float 2.10.1",
+ "ordered-float 2.10.0",
  "serde",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.188"
+version = "1.0.166"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
+checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -2797,11 +2648,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.107"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
+checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
 dependencies = [
- "indexmap 2.0.2",
+ "indexmap 2.0.0",
  "itoa",
  "ryu",
  "serde",
@@ -2863,27 +2714,38 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "serde_yaml"
-version = "0.9.25"
+version = "0.9.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85"
 dependencies = [
- "indexmap 2.0.2",
+ "indexmap 2.0.0",
  "itoa",
  "ryu",
  "serde",
  "unsafe-libyaml",
 ]
 
+[[package]]
+name = "sha1"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "sha2"
-version = "0.10.8"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -2892,18 +2754,18 @@ dependencies = [
 
 [[package]]
 name = "sharded-slab"
-version = "0.1.7"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
 dependencies = [
  "lazy_static",
 ]
 
 [[package]]
 name = "shlex"
-version = "1.2.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
+checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
 
 [[package]]
 name = "signal-hook-registry"
@@ -2928,18 +2790,18 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "smallvec"
-version = "1.11.1"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
@@ -2951,16 +2813,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "socket2"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e"
-dependencies = [
- "libc",
- "windows-sys",
-]
-
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -3001,9 +2853,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.38"
+version = "2.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3029,27 +2881,11 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "tera"
-version = "1.19.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "970dff17c11e884a4a09bc76e3a17ef71e01bb13447a11e85226e254fe6d10b8"
-dependencies = [
- "globwalk",
- "lazy_static",
- "pest",
- "pest_derive",
- "regex",
- "serde",
- "serde_json",
- "unic-segment",
-]
-
 [[package]]
 name = "termcolor"
-version = "1.3.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
@@ -3068,22 +2904,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
 [[package]]
 name = "thiserror"
-version = "1.0.49"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4"
+checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.49"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
+checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -3120,9 +2956,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.29"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe"
+checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07"
 dependencies = [
  "deranged",
  "itoa",
@@ -3133,15 +2969,15 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.2"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.15"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
+checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451"
 dependencies = [
  "time-core",
 ]
@@ -3163,10 +2999,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.33.0"
+version = "1.29.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
+checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
 dependencies = [
+ "autocfg",
  "backtrace",
  "bytes",
  "libc",
@@ -3174,7 +3011,7 @@ dependencies = [
  "num_cpus",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2 0.5.4",
+ "socket2",
  "tokio-macros",
  "windows-sys",
 ]
@@ -3197,7 +3034,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -3233,11 +3070,23 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-tungstenite"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c"
+dependencies = [
+ "futures-util",
+ "log",
+ "tokio",
+ "tungstenite",
+]
+
 [[package]]
 name = "tokio-util"
-version = "0.7.9"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d"
+checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
 dependencies = [
  "bytes",
  "futures-core",
@@ -3259,9 +3108,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.8"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257"
+checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -3280,11 +3129,11 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.19.15"
+version = "0.19.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
+checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
 dependencies = [
- "indexmap 2.0.2",
+ "indexmap 2.0.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -3299,7 +3148,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
 dependencies = [
  "async-trait",
  "axum",
- "base64 0.21.4",
+ "base64 0.21.2",
  "bytes",
  "futures-core",
  "futures-util",
@@ -3367,12 +3216,12 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.4.4"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
+checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
 dependencies = [
- "base64 0.21.4",
- "bitflags 2.4.0",
+ "base64 0.20.0",
+ "bitflags 2.3.3",
  "bytes",
  "futures-core",
  "futures-util",
@@ -3430,7 +3279,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -3502,66 +3351,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
-name = "typenum"
-version = "1.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
-
-[[package]]
-name = "ucd-trie"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
-
-[[package]]
-name = "unic-char-property"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
-dependencies = [
- "unic-char-range",
-]
-
-[[package]]
-name = "unic-char-range"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
-
-[[package]]
-name = "unic-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
-
-[[package]]
-name = "unic-segment"
-version = "0.9.0"
+name = "tungstenite"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23"
+checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67"
 dependencies = [
- "unic-ucd-segment",
-]
-
-[[package]]
-name = "unic-ucd-segment"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
+ "byteorder",
+ "bytes",
+ "data-encoding",
+ "http",
+ "httparse",
+ "log",
+ "rand",
+ "sha1",
+ "thiserror",
+ "url",
+ "utf-8",
 ]
 
 [[package]]
-name = "unic-ucd-version"
-version = "0.9.0"
+name = "typenum"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
-dependencies = [
- "unic-common",
-]
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicode-bidi"
@@ -3571,9 +3383,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73"
 
 [[package]]
 name = "unicode-normalization"
@@ -3586,9 +3398,9 @@ dependencies = [
 
 [[package]]
 name = "unsafe-libyaml"
-version = "0.2.9"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
+checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6"
 
 [[package]]
 name = "untrusted"
@@ -3598,15 +3410,21 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.4.1"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
+checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
 dependencies = [
  "form_urlencoded",
  "idna",
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
@@ -3616,12 +3434,19 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+dependencies = [
+ "anyhow",
+ "uuid",
+]
+
+[[package]]
+name = "utils"
+version = "0.1.0"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "clippy-utilities 0.2.0",
  "derive_builder",
  "getset",
- "madsim-tokio",
  "madsim-tonic",
  "opentelemetry",
  "parking_lot",
@@ -3660,16 +3485,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "walkdir"
-version = "2.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
-dependencies = [
- "same-file",
- "winapi-util",
-]
-
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -3706,7 +3521,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
  "wasm-bindgen-shared",
 ]
 
@@ -3728,7 +3543,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.23",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3751,14 +3566,13 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "4.4.2"
+version = "4.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
 dependencies = [
  "either",
- "home",
+ "libc",
  "once_cell",
- "rustix",
 ]
 
 [[package]]
@@ -3779,9 +3593,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.6"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
 dependencies = [
  "winapi",
 ]
@@ -3812,9 +3626,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.48.5"
+version = "0.48.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -3827,51 +3641,51 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.5"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
 [[package]]
 name = "winnow"
-version = "0.5.16"
+version = "0.5.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907"
+checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97"
 dependencies = [
  "memchr",
 ]
@@ -3879,7 +3693,7 @@ dependencies = [
 [[package]]
 name = "xline"
 version = "0.4.1"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -3887,14 +3701,12 @@ dependencies = [
  "bytes",
  "clap 3.2.25",
  "clippy-utilities 0.1.0",
- "crossbeam-skiplist",
  "curp",
  "engine",
  "event-listener",
  "flume",
  "futures",
  "getset",
- "hyper",
  "itertools",
  "jsonwebtoken",
  "log",
@@ -3921,7 +3733,7 @@ dependencies = [
  "tracing-appender",
  "tracing-opentelemetry",
  "tracing-subscriber",
- "utils",
+ "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
  "uuid",
  "xlineapi",
 ]
@@ -3929,7 +3741,7 @@ dependencies = [
 [[package]]
 name = "xline-client"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
  "async-stream",
  "clippy-utilities 0.1.0",
@@ -3943,7 +3755,8 @@ dependencies = [
  "pbkdf2",
  "thiserror",
  "tower",
- "utils",
+ "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
+ "uuid",
  "xline",
  "xlineapi",
 ]
@@ -3955,24 +3768,22 @@ dependencies = [
  "anyhow",
  "async-trait",
  "axum",
- "clap 4.4.6",
+ "clap 4.3.10",
  "clippy-utilities 0.2.0",
  "flume",
  "futures",
  "garde",
  "k8s-openapi",
  "kube",
- "lazy_static",
  "operator-api",
- "prometheus",
  "schemars",
  "serde",
  "serde_json",
- "serde_yaml",
  "thiserror",
  "tokio",
  "tracing",
  "tracing-subscriber",
+ "utils 0.1.0",
 ]
 
 [[package]]
@@ -3983,22 +3794,20 @@ dependencies = [
  "async-trait",
  "axum",
  "bytes",
- "clap 4.4.6",
+ "clap 4.3.10",
  "engine",
  "event-listener",
  "futures",
- "operator-api",
  "serde",
  "serde_json",
- "tera",
  "thiserror",
  "tokio",
- "toml 0.7.8",
+ "toml 0.7.6",
  "tonic",
  "tonic-health",
  "tracing",
  "tracing-subscriber",
- "uuid",
+ "utils 0.1.0",
  "xline-client",
  "xlineapi",
 ]
@@ -4006,9 +3815,8 @@ dependencies = [
 [[package]]
 name = "xlineapi"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#b9e4660debf4feb06458bd604473a509e299cfd7"
+source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
 dependencies = [
- "curp-external-api",
  "madsim-etcd-client",
  "madsim-tonic",
  "madsim-tonic-build",
@@ -4024,10 +3832,11 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.9+zstd.1.5.5"
+version = "2.0.8+zstd.1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
+checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
 dependencies = [
  "cc",
+ "libc",
  "pkg-config",
 ]
diff --git a/assets/xline_conf.tera b/assets/xline_conf.tera
deleted file mode 100644
index 19ee24a7..00000000
--- a/assets/xline_conf.tera
+++ /dev/null
@@ -1,17 +0,0 @@
-# Required
-
-[cluster]
-name = "{{ name }}"
-is_leader = {{ is_leader }}
-
-[cluster.members]{% for name, addr in members %}
-{{ name }} = "{{ addr }}"
-{%- endfor %}
-
-[storage]
-engine = "{{ storage_engine }}"
-data_dir = "{{ data_dir }}"
-
-# Additional
-
-{{ additional }}
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index 0972c217..a94f14e7 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -11,4 +11,7 @@ categories = ["API"]
 keywords = ["operator", "API", "operator"]
 
 [dependencies]
+async-trait = "0.1.72"
+k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
+kube = { version = "0.83.0", features = ["runtime", "derive", "ws"] }
 serde = { version = "1.0.130", features = ["derive"] }
diff --git a/operator-api/src/lib.rs b/operator-api/src/lib.rs
index 9f6fa36c..16701eed 100644
--- a/operator-api/src/lib.rs
+++ b/operator-api/src/lib.rs
@@ -1,3 +1,6 @@
+/// Xline handle
+mod xline;
+
 pub mod consts;
 
 use serde::{Deserialize, Serialize};
diff --git a/operator-api/src/xline.rs b/operator-api/src/xline.rs
new file mode 100644
index 00000000..2f0bf515
--- /dev/null
+++ b/operator-api/src/xline.rs
@@ -0,0 +1,51 @@
+use async_trait::async_trait;
+use k8s_openapi::api::core::v1::Pod;
+use kube::api::{AttachParams, AttachedProcess};
+use kube::Api;
+
+/// xline handle abstraction
+#[async_trait]
+pub trait XlineHandle {
+    /// the err during start and kill
+    type Err;
+
+    /// start a xline node
+    async fn start(&mut self) -> Result<(), Self::Err>;
+
+    /// kill a xline node
+    async fn kill(&mut self) -> Result<(), Self::Err>;
+}
+
+/// K8s xline handle
+pub struct K8sXlineHandle {
+    /// the pod name
+    pod_name: String,
+    /// the container name of xline
+    container_name: String,
+    /// k8s pods api
+    pods_api: Api<Pod>,
+    /// the attached process of xline
+    process: Option<AttachedProcess>,
+}
+
+#[async_trait]
+impl XlineHandle for K8sXlineHandle {
+    type Err = kube::Error;
+
+    async fn start(&mut self) -> Result<(), Self::Err> {
+        let process = self
+            .pods_api
+            .exec(
+                &self.pod_name,
+                vec!["sh"],
+                &AttachParams::default().container(&self.container_name),
+            )
+            .await?;
+        self.process = Some(process);
+        todo!()
+    }
+
+    async fn kill(&mut self) -> Result<(), Self::Err> {
+        todo!()
+    }
+}
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index c7c05611..e1b2a814 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -21,8 +21,8 @@ flume = "0.10.14"
 futures = "0.3.28"
 k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
 kube = { version = "0.83.0", features = ["runtime", "derive"] }
-lazy_static = "1.4.0"
 operator-api = { path = "../operator-api" }
+lazy_static = "1.4.0"
 prometheus = "0.13.3"
 schemars = "0.8.6"
 serde = { version = "1.0.130", features = ["derive"] }
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 19b4be18..c14ae129 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -23,13 +23,12 @@ futures = "0.3.28"
 operator-api = { path = "../operator-api" }
 serde = { version = "1.0.130", features = ["derive"] }
 serde_json = "1.0.97"
-tera = { version = "1.17.1", default-features = false }
 thiserror = "1.0.40"
 tokio = { version = "1.0", features = [
-    "rt-multi-thread",
-    "time",
-    "macros",
-    "net",
+  "rt-multi-thread",
+  "time",
+  "macros",
+  "net",
 ] }
 toml = "0.7.4"
 tonic = "0.9.2"
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index 2a8e8c87..94ea3c0e 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -136,13 +136,10 @@
 )]
 
 use std::collections::HashMap;
-use std::fs::write;
 use std::path::PathBuf;
-use std::time::Duration;
 
 use anyhow::{anyhow, Result};
-use clap::{Parser, Subcommand};
-use tera::{Context, Tera};
+use clap::Parser;
 use tracing::debug;
 
 use xline_sidecar::operator::Operator;
@@ -155,57 +152,78 @@ struct Cli {
     /// The name of this node
     #[arg(long)]
     name: String, // used in xline and deployment operator to identify this node
-
     /// The host ip of each member, [node_name] -> [node_host]
     #[arg(long, value_parser = parse_members)]
     members: HashMap<String, String>,
-
     /// The xline server port
     #[arg(long)]
     xline_port: u16,
-
-    /// Sub commands
-    #[command(subcommand)]
-    command: Commands,
+    /// Operator web server port
+    #[arg(long)]
+    operator_port: u16,
+    /// The xline container name
+    #[arg(long)]
+    container_name: String,
+    /// Check health interval, default 20 [unit: seconds]
+    #[arg(long, default_value = "20")]
+    check_interval: u64,
+    /// Enable backup, choose a storage type, e.g. s3:bucket_name or pv:/path/to/dir
+    #[arg(long, value_parser=parse_backup_type)]
+    backup: Option<Backup>,
+    /// The xline executable path, default "xline"
+    #[arg(long, default_value = "xline")]
+    xline_executable: String,
+    /// Storage engine used in xline
+    #[arg(long)]
+    storage_engine: String,
+    /// The directory path contains xline server data if the storage_engine is rocksdb
+    #[arg(long)]
+    data_dir: PathBuf,
+    /// Whether this node is leader or not
+    #[arg(long, default_value = "false")]
+    is_leader: bool,
+    /// Additional arguments, it will be appended behind the required parameters,
+    /// e.g "--jaeger_offline true"
+    #[arg(long)]
+    additional: Option<String>,
 }
 
-/// Sub commands
-#[allow(variant_size_differences)] // required by clap
-#[derive(Subcommand, Debug)]
-enum Commands {
-    /// Run sidecar operator
-    Run {
-        /// The xline container name
-        #[arg(long)]
-        container_name: String,
-        /// Operator web server port
-        #[arg(long)]
-        operator_port: u16,
-        /// Check health interval, default 20 [unit: seconds]
-        #[arg(long, default_value = "20")]
-        check_interval: u64,
-        /// Enable backup, choose a storage type, e.g. s3:bucket_name:secret_key or pv:/path/to/dir
-        #[arg(long, value_parser=parse_backup_type)]
-        backup: Option<Backup>,
-    },
-    /// Generate xline configuration file
-    Gen {
-        /// The file path where the xline kvserver reads configs
-        #[arg(long)]
-        path: PathBuf,
-        /// Storage engine used in xline
-        #[arg(long)]
-        storage_engine: String,
-        /// The directory path contains xline kvserver data is the storage_engine is rocksdb
-        #[arg(long)]
-        data_dir: PathBuf,
-        /// Whether this node is leader or not
-        #[arg(long)]
-        is_leader: bool,
-        /// Additional arguments [format: JSON]
-        #[arg(long)]
-        additional: Option<String>,
-    },
+impl From<Cli> for Config {
+    fn from(value: Cli) -> Self {
+        let mut config = Self {
+            start_cmd: String::new(),
+            name: value.name.clone(),
+            container_name: value.container_name,
+            xline_port: value.xline_port,
+            operator_port: value.operator_port,
+            check_interval: std::time::Duration::from_secs(value.check_interval),
+            backup: value.backup,
+            members: value.members,
+        };
+        config.start_cmd = format!(
+            "{} --name {} --members {} --storage-engine {} --data-dir {}",
+            value.xline_executable,
+            value.name,
+            config
+                .xline_members()
+                .into_iter()
+                .map(|(name, addr)| format!("{name}={addr}"))
+                .collect::<Vec<_>>()
+                .join(","),
+            value.storage_engine,
+            value.data_dir.to_string_lossy(),
+        );
+        if value.is_leader {
+            config.start_cmd.push(' ');
+            config.start_cmd.push_str("--is-leader");
+        }
+        if let Some(additional) = value.additional {
+            config.start_cmd.push(' ');
+            let pat: &[_] = &['\'', '"'];
+            config.start_cmd.push_str(additional.trim_matches(pat));
+        }
+        config
+    }
 }
 
 /// parse backup type
@@ -262,9 +280,6 @@ pub fn parse_members(s: &str) -> Result<HashMap<String, String>> {
     Ok(map)
 }
 
-/// Xline config template
-pub static XLINE_CONF: &str = include_str!("../../assets/xline_conf.tera");
-
 #[tokio::main]
 async fn main() -> Result<()> {
     tracing_subscriber::fmt::init();
@@ -272,53 +287,68 @@ async fn main() -> Result<()> {
     let cli = Cli::parse();
     debug!("{:?}", cli);
 
-    match cli.command {
-        Commands::Run {
-            container_name,
-            operator_port,
-            check_interval,
-            backup,
-        } => {
-            let config = Config::new(
-                cli.name,
-                container_name,
-                cli.members,
-                cli.xline_port,
-                operator_port,
-                Duration::from_secs(check_interval),
-                backup,
-            );
-            Operator::new(config).run().await
-        }
-        Commands::Gen {
-            path,
-            storage_engine,
-            data_dir,
-            is_leader,
-            additional,
-        } => {
-            let mut ctx = Context::new();
-            let members: HashMap<_, _> = cli
-                .members
-                .into_iter()
-                .map(|(node_name, host)| (node_name, format!("{host}:{}", cli.xline_port)))
-                .collect();
-            ctx.insert("name", &cli.name);
-            ctx.insert("is_leader", &is_leader);
-            ctx.insert("members", &members);
-            ctx.insert("storage_engine", &storage_engine);
-            ctx.insert("data_dir", &data_dir);
-            let additional = if let Some(json) = additional.as_deref() {
-                let value: serde_json::Value = serde_json::from_str(json)?;
-                toml::to_string_pretty(&value)?
-            } else {
-                String::new()
-            };
-            ctx.insert("additional", &additional);
-            let conf = Tera::one_off(XLINE_CONF, &ctx, false)?;
-            debug!("generate config: \n{}", conf);
-            write(path, conf)?;
-            Ok(())
-        }
+    Operator::new(cli.into()).run().await
+}
+
+#[cfg(test)]
+mod test {
+    use crate::Cli;
+    use clap::Parser;
+    use std::collections::HashMap;
+    use xline_sidecar::types::{Backup, Config};
+
+    fn full_parameter() -> Vec<&'static str> {
+        vec![
+            "sidecar_exe",
+            "--name=node1",
+            "--members=node1=127.0.0.1",
+            "--xline-port=2379",
+            "--operator-port=2380",
+            "--container-name=xline",
+            "--check-interval=60",
+            "--backup=s3:bucket_name",
+            "--xline-executable=/usr/local/bin/xline",
+            "--storage-engine=rocksdb",
+            "--data-dir=/usr/local/xline/data-dir",
+            "--is-leader",
+            "--additional='--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'",
+        ]
+    }
+
+    #[test]
+    fn test_parse_cli_should_success() {
+        let cli = Cli::parse_from(full_parameter());
+        assert_eq!(cli.name, "node1");
+        assert_eq!(
+            cli.members,
+            HashMap::from([("node1".to_owned(), "127.0.0.1".to_owned())])
+        );
+        assert_eq!(cli.xline_port, 2379);
+        assert_eq!(cli.operator_port, 2380);
+        assert_eq!(cli.container_name, "xline");
+        assert_eq!(cli.check_interval, 60);
+        assert_eq!(
+            cli.backup,
+            Some(Backup::S3 {
+                bucket: "bucket_name".to_owned(),
+            })
+        );
+        assert_eq!(cli.xline_executable, "/usr/local/bin/xline");
+        assert_eq!(cli.storage_engine, "rocksdb");
+        assert_eq!(cli.data_dir.to_string_lossy(), "/usr/local/xline/data-dir");
+        assert!(cli.is_leader);
+        assert_eq!(
+            cli.additional,
+            Some(
+                "'--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'"
+                    .to_owned()
+            )
+        );
+    }
+
+    #[test]
+    fn test_gen_start_cmd() {
+        let config: Config = Cli::parse_from(full_parameter()).into();
+        assert_eq!(config.start_cmd, "/usr/local/bin/xline --name node1 --members node1=127.0.0.1:2379 --storage-engine rocksdb --data-dir /usr/local/xline/data-dir --is-leader --auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem");
     }
 }
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 77e35669..87887f85 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -7,7 +7,7 @@ use std::time::Duration;
 
 /// Sidecar operator config
 #[derive(Debug, Clone)]
-#[non_exhaustive]
+#[allow(clippy::exhaustive_structs)] // it is exhaustive
 pub struct Config {
     /// Name of this node
     pub name: String,
@@ -22,11 +22,13 @@ pub struct Config {
     /// Backup storage config
     pub backup: Option<Backup>,
     /// Operators hosts, [pod_name]->[pod_host]
-    members: HashMap<String, String>,
+    pub members: HashMap<String, String>,
+    /// The xline start cmd
+    pub start_cmd: String,
 }
 
 /// Backup storage config
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum Backup {
     /// S3 storage
@@ -42,31 +44,10 @@ pub enum Backup {
 }
 
 impl Config {
-    /// Constructor
+    /// Get the operator members
     #[must_use]
     #[inline]
-    pub fn new(
-        name: String,
-        container_name: String,
-        members: HashMap<String, String>,
-        xline_port: u16,
-        operator_port: u16,
-        check_interval: Duration,
-        backup: Option<Backup>,
-    ) -> Self {
-        Self {
-            name,
-            container_name,
-            xline_port,
-            operator_port,
-            check_interval,
-            backup,
-            members,
-        }
-    }
-
-    /// Get the operator members
-    pub(crate) fn operator_members(&self) -> HashMap<String, String> {
+    pub fn operator_members(&self) -> HashMap<String, String> {
         self.members
             .clone()
             .into_iter()
@@ -75,7 +56,9 @@ impl Config {
     }
 
     /// Get the xline members
-    pub(crate) fn xline_members(&self) -> HashMap<String, String> {
+    #[must_use]
+    #[inline]
+    pub fn xline_members(&self) -> HashMap<String, String> {
         self.members
             .clone()
             .into_iter()

From 4d1c4be8c7d2eed6ecea6440f5133a29eea6f755 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Tue, 26 Sep 2023 17:21:06 +0800
Subject: [PATCH 04/19] feat: implement xline handle

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                |   6 +-
 operator-api/Cargo.toml   |   1 +
 operator-api/src/lib.rs   |   2 +
 operator-api/src/xline.rs | 118 ++++++++++++++++++++---
 sidecar/Cargo.toml        |   1 +
 sidecar/src/main.rs       | 197 ++++++++++++++++++++++++++++++++++----
 sidecar/src/operator.rs   |  48 +++++++---
 sidecar/src/types.rs      |  21 +++-
 sidecar/src/xline.rs      |  19 ++--
 9 files changed, 353 insertions(+), 60 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f2583b32..dbec4a35 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -115,9 +115,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.71"
+version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
 [[package]]
 name = "async-channel"
@@ -2084,6 +2084,7 @@ dependencies = [
 name = "operator-api"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "async-trait",
  "k8s-openapi",
  "kube",
@@ -3798,6 +3799,7 @@ dependencies = [
  "engine",
  "event-listener",
  "futures",
+ "operator-api",
  "serde",
  "serde_json",
  "thiserror",
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index a94f14e7..af23fc4e 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -15,3 +15,4 @@ async-trait = "0.1.72"
 k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
 kube = { version = "0.83.0", features = ["runtime", "derive", "ws"] }
 serde = { version = "1.0.130", features = ["derive"] }
+anyhow = "1.0.72"
diff --git a/operator-api/src/lib.rs b/operator-api/src/lib.rs
index 16701eed..fceef081 100644
--- a/operator-api/src/lib.rs
+++ b/operator-api/src/lib.rs
@@ -3,6 +3,8 @@ mod xline;
 
 pub mod consts;
 
+pub use xline::{K8sXlineHandle, LocalXlineHandle, XlineHandle};
+
 use serde::{Deserialize, Serialize};
 
 /// Heartbeat status
diff --git a/operator-api/src/xline.rs b/operator-api/src/xline.rs
index 2f0bf515..11a161ef 100644
--- a/operator-api/src/xline.rs
+++ b/operator-api/src/xline.rs
@@ -2,21 +2,58 @@ use async_trait::async_trait;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::{AttachParams, AttachedProcess};
 use kube::Api;
+use std::fmt::{Debug, Formatter};
+
+use std::process::{Child, Command};
 
 /// xline handle abstraction
 #[async_trait]
-pub trait XlineHandle {
-    /// the err during start and kill
-    type Err;
-
+pub trait XlineHandle: Debug + Send + Sync + 'static {
     /// start a xline node
-    async fn start(&mut self) -> Result<(), Self::Err>;
+    async fn start(&mut self) -> anyhow::Result<()>; // we truly dont care about what failure happened when start, it just failed
 
     /// kill a xline node
-    async fn kill(&mut self) -> Result<(), Self::Err>;
+    async fn kill(&mut self) -> anyhow::Result<()>;
+}
+
+/// Local xline handle, it will execute the xline in the local
+/// machine with the start_cmd
+#[derive(Debug)]
+pub struct LocalXlineHandle {
+    start_cmd: String,
+    child_proc: Option<Child>,
+}
+
+impl LocalXlineHandle {
+    /// New a local xline handle
+    pub fn new(start_cmd: String) -> Self {
+        Self {
+            start_cmd,
+            child_proc: None,
+        }
+    }
+}
+
+#[async_trait]
+impl XlineHandle for LocalXlineHandle {
+    async fn start(&mut self) -> anyhow::Result<()> {
+        if let Some((exe, args)) = self.start_cmd.split_once(' ') {
+            let proc = Command::new(exe).args(args.split(' ')).spawn()?;
+            self.child_proc = Some(proc);
+        }
+        unreachable!("the start_cmd must be valid");
+    }
+
+    async fn kill(&mut self) -> anyhow::Result<()> {
+        if let Some(mut proc) = self.child_proc.take() {
+            return Ok(proc.kill()?);
+        }
+        Ok(())
+    }
 }
 
-/// K8s xline handle
+/// K8s xline handle, it will execute the xline start_cmd
+/// in pod
 pub struct K8sXlineHandle {
     /// the pod name
     pod_name: String,
@@ -26,26 +63,79 @@ pub struct K8sXlineHandle {
     pods_api: Api<Pod>,
     /// the attached process of xline
     process: Option<AttachedProcess>,
+    /// the xline start cmd, parameters are split by ' '
+    start_cmd: String,
+}
+
+impl Debug for K8sXlineHandle {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("K8sXlineHandle")
+            .field("pod_name", &self.pod_name)
+            .field("container_name", &self.container_name)
+            .field("pods_api", &self.pods_api)
+            .field("start_cmd", &self.start_cmd)
+            .finish()
+    }
+}
+
+impl K8sXlineHandle {
+    /// New with default k8s client
+    pub async fn new_with_default(
+        pod_name: String,
+        container_name: String,
+        namespace: &str,
+        start_cmd: String,
+    ) -> Self {
+        let client = kube::Client::try_default()
+            .await
+            .unwrap_or_else(|_ig| unreachable!("it must be setup in k8s environment"));
+        Self {
+            pod_name,
+            container_name,
+            pods_api: Api::namespaced(client, namespace),
+            process: None,
+            start_cmd,
+        }
+    }
+
+    /// New with the provided k8s client
+    pub fn new_with_client(
+        pod_name: String,
+        container_name: String,
+        client: kube::Client,
+        namespace: &str,
+        start_cmd: String,
+    ) -> Self {
+        Self {
+            pod_name,
+            container_name,
+            pods_api: Api::namespaced(client, namespace),
+            process: None,
+            start_cmd,
+        }
+    }
 }
 
 #[async_trait]
 impl XlineHandle for K8sXlineHandle {
-    type Err = kube::Error;
-
-    async fn start(&mut self) -> Result<(), Self::Err> {
+    async fn start(&mut self) -> anyhow::Result<()> {
+        let start_cmd: Vec<&str> = self.start_cmd.split(' ').collect();
         let process = self
             .pods_api
             .exec(
                 &self.pod_name,
-                vec!["sh"],
+                start_cmd,
                 &AttachParams::default().container(&self.container_name),
             )
             .await?;
         self.process = Some(process);
-        todo!()
+        Ok(())
     }
 
-    async fn kill(&mut self) -> Result<(), Self::Err> {
-        todo!()
+    async fn kill(&mut self) -> anyhow::Result<()> {
+        if let Some(process) = self.process.take() {
+            process.abort();
+        }
+        Ok(())
     }
 }
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index c14ae129..40d40a6e 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -38,3 +38,4 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 uuid = { version = "1.4.1", features = ["v4"] }
 xline-client = { git = "https://github.com/xline-kv/Xline.git", package = "xline-client" }
 xlineapi = { git = "https://github.com/xline-kv/Xline.git", package = "xlineapi" }
+operator-api = { path = "../operator-api" }
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index 94ea3c0e..256c9bb0 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -138,20 +138,20 @@
 use std::collections::HashMap;
 use std::path::PathBuf;
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use clap::Parser;
 use tracing::debug;
 
 use xline_sidecar::operator::Operator;
-use xline_sidecar::types::{Backup, Config};
+use xline_sidecar::types::{Backend, Backup, Config};
 
 /// Command line interface
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Cli {
-    /// The name of this node
+    /// The name of this sidecar, and is shared with xline node name
     #[arg(long)]
-    name: String, // used in xline and deployment operator to identify this node
+    name: String,
     /// The host ip of each member, [node_name] -> [node_host]
     #[arg(long, value_parser = parse_members)]
     members: HashMap<String, String>,
@@ -161,9 +161,6 @@ struct Cli {
     /// Operator web server port
     #[arg(long)]
     operator_port: u16,
-    /// The xline container name
-    #[arg(long)]
-    container_name: String,
     /// Check health interval, default 20 [unit: seconds]
     #[arg(long, default_value = "20")]
     check_interval: u64,
@@ -186,6 +183,12 @@ struct Cli {
     /// e.g "--jaeger_offline true"
     #[arg(long)]
     additional: Option<String>,
+    /// The sidecar backend, when you use different operators, the backend may different.
+    /// e.g:
+    ///   "k8s,pod=xline-pod-1,container=xline,namespace=default" for k8s backend
+    ///   "local" for local backend
+    #[arg(long, value_parser=parse_backend)]
+    backend: Backend,
 }
 
 impl From<Cli> for Config {
@@ -193,12 +196,12 @@ impl From<Cli> for Config {
         let mut config = Self {
             start_cmd: String::new(),
             name: value.name.clone(),
-            container_name: value.container_name,
             xline_port: value.xline_port,
             operator_port: value.operator_port,
             check_interval: std::time::Duration::from_secs(value.check_interval),
             backup: value.backup,
             members: value.members,
+            backend: value.backend,
         };
         config.start_cmd = format!(
             "{} --name {} --members {} --storage-engine {} --data-dir {}",
@@ -228,11 +231,10 @@ impl From<Cli> for Config {
 
 /// parse backup type
 fn parse_backup_type(value: &str) -> Result<Backup, String> {
-    debug!("parse backup type: {}", value);
-    let mut items: Vec<_> = value.split([':', ' ', ',', '-']).collect();
-    if items.is_empty() {
+    if value.is_empty() {
         return Err("backup type is empty".to_owned());
     }
+    let mut items: Vec<_> = value.split([':', ' ', ',', '-']).collect();
     let backup_type = items.remove(0);
     match backup_type {
         "s3" => {
@@ -261,19 +263,55 @@ fn parse_backup_type(value: &str) -> Result<Backup, String> {
     }
 }
 
+/// Parse backend
+fn parse_backend(value: &str) -> Result<Backend, String> {
+    if value.is_empty() {
+        return Err("backend is empty".to_owned());
+    }
+    let mut items: Vec<_> = value.split(',').collect();
+    let backend = items.remove(0);
+    match backend {
+        "k8s" => {
+            let mut pod_name = String::new();
+            let mut container_name = String::new();
+            let mut namespace = String::new();
+            while let Some(item) = items.pop() {
+                let Some((k, v)) = item.split_once('=') else {
+                    return Err(format!("k8s backend got unexpected argument {item}, expect <key>=<value>"));
+                };
+                match k {
+                    "pod" => pod_name = v.to_owned(),
+                    "container" => container_name = v.to_owned(),
+                    "namespace" => namespace = v.to_owned(),
+                    _ => return Err(format!("k8s backend got unexpect argument {item}, expect one of 'pod', 'container', 'namespace'")),
+                }
+            }
+            if pod_name.is_empty() || container_name.is_empty() || namespace.is_empty() {
+                return Err("k8s backend must set 'pod', 'container', 'namespace'".to_owned());
+            }
+            Ok(Backend::K8s {
+                pod_name,
+                container_name,
+                namespace,
+            })
+        }
+        "local" => Ok(Backend::Local),
+        _ => Err(format!("unknown backend: {backend}")),
+    }
+}
+
 /// parse members from string
 /// # Errors
 /// Return error when pass wrong args
 #[inline]
-pub fn parse_members(s: &str) -> Result<HashMap<String, String>> {
+pub fn parse_members(s: &str) -> Result<HashMap<String, String>, String> {
     let mut map = HashMap::new();
     for pair in s.split(',') {
         if let Some((id, addr)) = pair.split_once('=') {
             let _ignore = map.insert(id.to_owned(), addr.to_owned());
         } else {
-            return Err(anyhow!(
-                "parse the pair '{}' error, expect '<id>=<addr>'",
-                pair
+            return Err(format!(
+                "parse the pair '{pair}' error, expect '<id>=<addr>'",
             ));
         }
     }
@@ -292,10 +330,11 @@ async fn main() -> Result<()> {
 
 #[cfg(test)]
 mod test {
-    use crate::Cli;
+    use crate::{parse_backend, parse_backup_type, parse_members, Cli};
     use clap::Parser;
     use std::collections::HashMap;
-    use xline_sidecar::types::{Backup, Config};
+    use std::path::PathBuf;
+    use xline_sidecar::types::{Backend, Backup, Config};
 
     fn full_parameter() -> Vec<&'static str> {
         vec![
@@ -304,7 +343,6 @@ mod test {
             "--members=node1=127.0.0.1",
             "--xline-port=2379",
             "--operator-port=2380",
-            "--container-name=xline",
             "--check-interval=60",
             "--backup=s3:bucket_name",
             "--xline-executable=/usr/local/bin/xline",
@@ -312,9 +350,123 @@ mod test {
             "--data-dir=/usr/local/xline/data-dir",
             "--is-leader",
             "--additional='--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'",
+            "--backend=k8s,pod=xline-pod-1,container=xline,namespace=default",
         ]
     }
 
+    #[test]
+    fn test_parse_backup_type() {
+        let test_cases = [
+            ("", Err("backup type is empty".to_owned())),
+            (
+                "s3:bucket_name",
+                Ok(Backup::S3 {
+                    bucket: "bucket_name".to_owned(),
+                }),
+            ),
+            (
+                "s3:bucket:name",
+                Err("s3 backup type requires 1 arguments, got 2".to_owned()),
+            ),
+            (
+                "s3",
+                Err("s3 backup type requires 1 arguments, got 0".to_owned()),
+            ),
+            (
+                "pv:/home",
+                Ok(Backup::PV {
+                    path: PathBuf::from("/home"),
+                }),
+            ),
+            (
+                "pv:/home:/paopao",
+                Err("pv backup type requires 1 argument, got 2".to_owned()),
+            ),
+            (
+                "pv",
+                Err("pv backup type requires 1 argument, got 0".to_owned()),
+            ),
+            (
+                "_invalid_",
+                Err("unknown backup type: _invalid_".to_owned()),
+            ),
+        ];
+        for (test_case, res) in test_cases {
+            assert_eq!(parse_backup_type(test_case), res);
+        }
+    }
+
+    #[test]
+    fn test_parse_backend() {
+        let test_cases = [
+            (
+                "k8s,pod=my-pod,container=my-container,namespace=my-namespace",
+                Ok(Backend::K8s {
+                    pod_name: "my-pod".to_owned(),
+                    container_name: "my-container".to_owned(),
+                    namespace: "my-namespace".to_owned(),
+                }),
+            ),
+            ("local", Ok(Backend::Local)),
+            ("", Err("backend is empty".to_owned())),
+            (
+                "k8s,pod=my-pod,invalid-arg,namespace=my-namespace",
+                Err(
+                    "k8s backend got unexpected argument invalid-arg, expect <key>=<value>"
+                        .to_owned(),
+                ),
+            ),
+            (
+                "k8s,pod=my-pod,container=my-container",
+                Err("k8s backend must set 'pod', 'container', 'namespace'".to_owned()),
+            ),
+            (
+                "unknown-backend",
+                Err("unknown backend: unknown-backend".to_owned()),
+            ),
+        ];
+        for (input, expected) in test_cases {
+            let result = parse_backend(input);
+            assert_eq!(result, expected);
+        }
+    }
+
+    #[test]
+    fn test_parse_members() {
+        let test_cases = vec![
+            (
+                "id1=addr1,id2=addr2,id3=addr3",
+                Ok([("id1", "addr1"), ("id2", "addr2"), ("id3", "addr3")]
+                    .iter()
+                    .map(|&(id, addr)| (id.to_owned(), addr.to_owned()))
+                    .collect()),
+            ),
+            (
+                "id1=addr1",
+                Ok(std::iter::once(&("id1", "addr1"))
+                    .map(|&(id, addr)| (id.to_owned(), addr.to_owned()))
+                    .collect()),
+            ),
+            (
+                "",
+                Err("parse the pair '' error, expect '<id>=<addr>'".to_owned()),
+            ),
+            (
+                "id1=addr1,id2",
+                Err("parse the pair 'id2' error, expect '<id>=<addr>'".to_owned()),
+            ),
+            (
+                "id1=addr1,id2=addr2,",
+                Err("parse the pair '' error, expect '<id>=<addr>'".to_owned()),
+            ),
+        ];
+
+        for (input, expected) in test_cases {
+            let result = parse_members(input);
+            assert_eq!(result, expected);
+        }
+    }
+
     #[test]
     fn test_parse_cli_should_success() {
         let cli = Cli::parse_from(full_parameter());
@@ -325,7 +477,6 @@ mod test {
         );
         assert_eq!(cli.xline_port, 2379);
         assert_eq!(cli.operator_port, 2380);
-        assert_eq!(cli.container_name, "xline");
         assert_eq!(cli.check_interval, 60);
         assert_eq!(
             cli.backup,
@@ -344,6 +495,14 @@ mod test {
                     .to_owned()
             )
         );
+        assert_eq!(
+            cli.backend,
+            Backend::K8s {
+                pod_name: "xline-pod-1".to_owned(),
+                container_name: "xline".to_owned(),
+                namespace: "default".to_owned(),
+            }
+        );
     }
 
     #[test]
diff --git a/sidecar/src/operator.rs b/sidecar/src/operator.rs
index e71cd3ac..4d157e39 100644
--- a/sidecar/src/operator.rs
+++ b/sidecar/src/operator.rs
@@ -5,6 +5,7 @@ use anyhow::{anyhow, Result};
 use axum::routing::{get, post};
 use axum::{Extension, Router};
 use futures::{FutureExt, TryFutureExt};
+use operator_api::{K8sXlineHandle, LocalXlineHandle};
 use tokio::sync::watch::{Receiver, Sender};
 use tokio::sync::Mutex;
 use tokio::time::interval;
@@ -15,7 +16,7 @@ use crate::backup::Provider;
 use crate::controller::Controller;
 use crate::controller::Error;
 use crate::routers;
-use crate::types::{Backup, Config, State, StatePayload};
+use crate::types::{Backend, Backup, Config, State, StatePayload};
 use crate::xline::XlineHandle;
 
 /// Sidecar operator
@@ -41,14 +42,7 @@ impl Operator {
     pub async fn run(&self) -> Result<()> {
         let (graceful_shutdown_event, _) = tokio::sync::watch::channel(());
         let forceful_shutdown = self.forceful_shutdown(&graceful_shutdown_event);
-        let backup = self.init_backup();
-        let handle = Arc::new(XlineHandle::open(
-            &self.config.name,
-            &self.config.container_name,
-            backup,
-            self.config.xline_port,
-            self.config.xline_members(),
-        )?);
+        let handle = Arc::new(self.init_xline_handle().await?);
         let revision = handle.revision_offline().unwrap_or(1);
         let state = Arc::new(Mutex::new(StatePayload {
             state: State::Start,
@@ -82,9 +76,10 @@ impl Operator {
         Ok(())
     }
 
-    /// Initialize backup
-    fn init_backup(&self) -> Option<Box<dyn Provider>> {
-        self.config
+    /// Initialize xline handle
+    async fn init_xline_handle(&self) -> Result<XlineHandle> {
+        let backup = self
+            .config
             .backup
             .as_ref()
             .and_then(|backup| match *backup {
@@ -95,7 +90,34 @@ impl Operator {
                     });
                     Some(pv)
                 }
-            })
+            });
+        let inner: Box<dyn operator_api::XlineHandle> = match self.config.backend.clone() {
+            Backend::K8s {
+                pod_name,
+                container_name,
+                namespace,
+            } => {
+                let handle = K8sXlineHandle::new_with_default(
+                    pod_name,
+                    container_name,
+                    &namespace,
+                    self.config.start_cmd.clone(),
+                )
+                .await;
+                Box::new(handle)
+            }
+            Backend::Local => {
+                let handle = LocalXlineHandle::new(self.config.start_cmd.clone());
+                Box::new(handle)
+            }
+        };
+        XlineHandle::open(
+            &self.config.name,
+            backup,
+            inner,
+            self.config.xline_port,
+            self.config.xline_members(),
+        )
     }
 
     /// Forceful shutdown
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 87887f85..44f2e162 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -11,8 +11,6 @@ use std::time::Duration;
 pub struct Config {
     /// Name of this node
     pub name: String,
-    /// Xline container name
-    pub container_name: String,
     /// The xline server port
     pub xline_port: u16,
     /// The operator web server port
@@ -25,6 +23,25 @@ pub struct Config {
     pub members: HashMap<String, String>,
     /// The xline start cmd
     pub start_cmd: String,
+    /// The backend
+    pub backend: Backend,
+}
+
+/// Sidecar backend, it determinate how xline could be setup
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum Backend {
+    /// K8s backend
+    K8s {
+        /// The pod name of this node
+        pod_name: String,
+        /// The xline container name, used to attach on it
+        container_name: String,
+        /// The namespace of this node
+        namespace: String,
+    },
+    /// Local backend
+    Local,
 }
 
 /// Backup storage config
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 30b1b07a..6e38a7eb 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -3,6 +3,7 @@
 #![allow(clippy::unused_self)] // TODO remove as it is implemented
 
 use std::collections::HashMap;
+use std::fmt::Debug;
 use std::time::Duration;
 
 use anyhow::{anyhow, Result};
@@ -50,8 +51,6 @@ pub(crate) const XLINE_TABLES: [&str; 6] = [
 pub(crate) struct XlineHandle {
     /// The name of the operator
     name: String,
-    /// The xline container name in the pod
-    container_name: String,
     /// The xline backup provider
     backup: Option<Box<dyn Provider>>,
     /// The xline client
@@ -64,31 +63,33 @@ pub(crate) struct XlineHandle {
     xline_members: HashMap<String, String>,
     /// Health retires of xline client
     is_healthy_retries: usize,
+    /// The detailed xline process handle
+    inner: Box<dyn operator_api::XlineHandle>,
 }
 
 impl XlineHandle {
     /// Create the xline handle but not start the xline node
     pub(crate) fn open(
         name: &str,
-        container_name: &str,
         backup: Option<Box<dyn Provider>>,
+        inner: Box<dyn operator_api::XlineHandle>,
         xline_port: u16,
         xline_members: HashMap<String, String>,
     ) -> Result<Self> {
-        debug!("name: {name}, container_name: {container_name}, backup: {backup:?}, xline_port: {xline_port}");
+        debug!("name: {name}, backup: {backup:?}, xline_port: {xline_port}");
         let endpoint: Endpoint = format!("http://127.0.0.1:{xline_port}").parse()?;
         let channel = Channel::balance_list(std::iter::once(endpoint));
         let health_client = HealthClient::new(channel);
         let engine = Engine::new(EngineType::Rocks(DEFAULT_DATA_DIR.parse()?), &XLINE_TABLES)?;
         Ok(Self {
             name: name.to_owned(),
-            container_name: container_name.to_owned(),
             backup,
             health_client,
             engine,
             client: None, // TODO maybe we could initialize the client here when xline#423 is merged
             xline_members,
             is_healthy_retries: 5,
+            inner,
         })
     }
 
@@ -117,8 +118,7 @@ impl XlineHandle {
         // the cluster is started if any of the connection is successful
         let cluster_started = futs.any(|res| async move { res.is_ok() }).await;
 
-        // start xline server here
-        // TODO define a trait to abstract the start of xline server
+        self.inner.start().await?;
 
         let client = Client::connect(self.xline_members.values(), ClientOptions::default()).await?;
         if cluster_started {
@@ -130,11 +130,10 @@ impl XlineHandle {
     }
 
     /// Stop the xline server
-    pub(crate) async fn stop(&self) -> Result<()> {
+    pub(crate) async fn stop(&mut self) -> Result<()> {
         // Step 1: Kill the xline node
         // Step 2: Remove the xline node from the cluster if the cluster exist
-
-        // kill xline server here
+        self.inner.kill().await?;
 
         if self.is_healthy().await {
             let _cluster_client = self.client().cluster_client();

From 4d35f3d5e7258371783a081b0e67888909991e8a Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Wed, 11 Oct 2023 19:05:09 +0800
Subject: [PATCH 05/19] chore: fix xline handle start_cmd splitting

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 operator-api/src/xline.rs | 18 +++++++++++-------
 sidecar/src/xline.rs      |  2 +-
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/operator-api/src/xline.rs b/operator-api/src/xline.rs
index 11a161ef..351bc138 100644
--- a/operator-api/src/xline.rs
+++ b/operator-api/src/xline.rs
@@ -10,7 +10,7 @@ use std::process::{Child, Command};
 #[async_trait]
 pub trait XlineHandle: Debug + Send + Sync + 'static {
     /// start a xline node
-    async fn start(&mut self) -> anyhow::Result<()>; // we truly dont care about what failure happened when start, it just failed
+    async fn start(&mut self) -> anyhow::Result<()>; // we dont care about what failure happened when start, it just failed
 
     /// kill a xline node
     async fn kill(&mut self) -> anyhow::Result<()>;
@@ -37,11 +37,15 @@ impl LocalXlineHandle {
 #[async_trait]
 impl XlineHandle for LocalXlineHandle {
     async fn start(&mut self) -> anyhow::Result<()> {
-        if let Some((exe, args)) = self.start_cmd.split_once(' ') {
-            let proc = Command::new(exe).args(args.split(' ')).spawn()?;
-            self.child_proc = Some(proc);
-        }
-        unreachable!("the start_cmd must be valid");
+        let mut cmds = self.start_cmd.split_whitespace();
+        let Some((exe, args)) = cmds
+            .next()
+            .map(|exe| (exe, cmds.collect::<Vec<_>>())) else {
+            unreachable!("the start_cmd must be valid");
+        };
+        let proc = Command::new(exe).args(args).spawn()?;
+        self.child_proc = Some(proc);
+        Ok(())
     }
 
     async fn kill(&mut self) -> anyhow::Result<()> {
@@ -119,7 +123,7 @@ impl K8sXlineHandle {
 #[async_trait]
 impl XlineHandle for K8sXlineHandle {
     async fn start(&mut self) -> anyhow::Result<()> {
-        let start_cmd: Vec<&str> = self.start_cmd.split(' ').collect();
+        let start_cmd: Vec<&str> = self.start_cmd.split_whitespace().collect();
         let process = self
             .pods_api
             .exec(
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 6e38a7eb..a2b19995 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -86,7 +86,7 @@ impl XlineHandle {
             backup,
             health_client,
             engine,
-            client: None, // TODO maybe we could initialize the client here when xline#423 is merged
+            client: None,
             xline_members,
             is_healthy_retries: 5,
             inner,

From 9970d4bc26ebf665eca27daf2f967b6cd8c2edd7 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Tue, 17 Oct 2023 13:56:58 +0800
Subject: [PATCH 06/19] feat: implement membership change

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                | 954 +++++++++++++++++++++-----------------
 operator-api/Cargo.toml   |   2 +-
 operator-api/src/xline.rs |   2 +
 operator-k8s/Cargo.toml   |   1 -
 sidecar/Cargo.toml        |   2 -
 sidecar/src/backup/mod.rs |   2 +-
 sidecar/src/backup/pv.rs  |   2 +-
 sidecar/src/xline.rs      |  61 ++-
 8 files changed, 571 insertions(+), 455 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dbec4a35..029027a2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.20.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
 dependencies = [
  "gimli",
 ]
@@ -42,13 +42,19 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@@ -66,30 +72,29 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.3.2"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
 dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
  "anstyle-wincon",
  "colorchoice",
- "is-terminal",
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle"
-version = "1.0.1"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
+checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
 dependencies = [
  "utf8parse",
 ]
@@ -105,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "1.0.1"
+version = "3.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
 dependencies = [
  "anstyle",
  "windows-sys",
@@ -149,24 +154,24 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "async-task"
-version = "4.4.0"
+version = "4.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"
+checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921"
 
 [[package]]
 name = "async-trait"
-version = "0.1.73"
+version = "0.1.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
+checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -188,9 +193,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "axum"
-version = "0.6.19"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c"
+checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -248,9 +253,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.68"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
 dependencies = [
  "addr2line",
  "cc",
@@ -275,9 +280,9 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
 
 [[package]]
 name = "base64"
-version = "0.21.2"
+version = "0.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
 
 [[package]]
 name = "base64ct"
@@ -306,13 +311,13 @@ dependencies = [
  "lazy_static",
  "lazycell",
  "peeking_take_while",
- "prettyplease 0.2.12",
+ "prettyplease 0.2.15",
  "proc-macro2",
  "quote",
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -323,9 +328,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.3.3"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
 
 [[package]]
 name = "block-buffer"
@@ -338,21 +343,21 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.13.0"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
 
 [[package]]
 name = "byteorder"
-version = "1.4.3"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
 [[package]]
 name = "bzip2-sys"
@@ -367,11 +372,12 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
 dependencies = [
  "jobserver",
+ "libc",
 ]
 
 [[package]]
@@ -391,15 +397,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.26"
+version = "0.4.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
  "num-traits",
  "serde",
- "winapi",
+ "windows-targets",
 ]
 
 [[package]]
@@ -432,24 +438,23 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.3.10"
+version = "4.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
+checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956"
 dependencies = [
  "clap_builder",
- "clap_derive 4.3.2",
- "once_cell",
+ "clap_derive 4.4.2",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.3.10"
+version = "4.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
+checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45"
 dependencies = [
  "anstream",
  "anstyle",
- "clap_lex 0.5.0",
+ "clap_lex 0.5.1",
  "strsim",
 ]
 
@@ -468,14 +473,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.3.2"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
+checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -489,9 +494,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
 
 [[package]]
 name = "clippy-utilities"
@@ -516,13 +521,23 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "concurrent-queue"
-version = "2.2.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
+checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
 dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.4"
@@ -548,6 +563,31 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-skiplist"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "883a5821d7d079fcf34ac55f27a833ee61678110f6b97637cc74513c0d0b42fc"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.16"
@@ -570,7 +610,7 @@ dependencies = [
 [[package]]
 name = "curp"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -595,6 +635,7 @@ dependencies = [
  "parking_lot",
  "prost",
  "prost-build",
+ "rand",
  "serde",
  "thiserror",
  "tokio-stream 0.1.12",
@@ -602,18 +643,19 @@ dependencies = [
  "tracing",
  "tracing-opentelemetry",
  "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
- "uuid",
 ]
 
 [[package]]
 name = "curp-external-api"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "async-trait",
  "engine",
  "mockall",
+ "prost",
  "serde",
+ "thiserror",
 ]
 
 [[package]]
@@ -661,7 +703,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -683,17 +725,17 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "dashmap"
-version = "5.5.1"
+version = "5.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
- "hashbrown 0.14.0",
+ "hashbrown 0.14.1",
  "lock_api",
  "once_cell",
  "parking_lot_core",
@@ -707,10 +749,11 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
 
 [[package]]
 name = "deranged"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
+checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
 dependencies = [
+ "powerfmt",
  "serde",
 ]
 
@@ -773,27 +816,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
 [[package]]
 name = "downcast"
 version = "0.11.0"
@@ -808,20 +830,20 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
+checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd"
 
 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
 [[package]]
 name = "engine"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "async-trait",
  "bincode",
@@ -837,31 +859,20 @@ dependencies = [
 
 [[package]]
 name = "equivalent"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
-version = "0.3.1"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
 dependencies = [
- "errno-dragonfly",
  "libc",
  "windows-sys",
 ]
 
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
 [[package]]
 name = "etcd-client"
 version = "0.11.1"
@@ -886,9 +897,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
 [[package]]
 name = "fastrand"
-version = "2.0.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
+checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
 [[package]]
 name = "fixedbitset"
@@ -924,21 +935,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.2.0"
@@ -1010,7 +1006,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -1045,9 +1041,9 @@ dependencies = [
 
 [[package]]
 name = "garde"
-version = "0.11.2"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d959ef7bda0bda7cc0f6fbebfbac6202f810394f50e07059eeea8ec31e69e4b0"
+checksum = "efbc4e06c5b4a8fef3ad41939c6116ea06a0d93f647c6430154d924bf021c374"
 dependencies = [
  "garde_derive",
  "once_cell",
@@ -1056,14 +1052,14 @@ dependencies = [
 
 [[package]]
 name = "garde_derive"
-version = "0.11.2"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e89f7fce035bb3a3718e23efff13709a0b21b694c4eae20a32e1a3e4e27c6a2"
+checksum = "6699dea5b266ce9ca75ab4cbc5a7e98d5abfdf3197cecef5374a623023939acd"
 dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -1103,9 +1099,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.27.3"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
 
 [[package]]
 name = "glob"
@@ -1140,9 +1136,13 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.14.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
+dependencies = [
+ "ahash 0.8.3",
+ "allocator-api2",
+]
 
 [[package]]
 name = "heck"
@@ -1161,9 +1161,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.1"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 
 [[package]]
 name = "hex"
@@ -1180,6 +1180,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -1204,9 +1213,9 @@ dependencies = [
 
 [[package]]
 name = "http-range-header"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
+checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 
 [[package]]
 name = "httparse"
@@ -1216,9 +1225,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
 [[package]]
 name = "httpdate"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "hyper"
@@ -1237,7 +1246,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2",
+ "socket2 0.4.9",
  "tokio",
  "tower-service",
  "tracing",
@@ -1245,21 +1254,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "hyper-openssl"
-version = "0.9.2"
+name = "hyper-rustls"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b"
+checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
 dependencies = [
+ "futures-util",
  "http",
  "hyper",
- "linked_hash_set",
- "once_cell",
- "openssl",
- "openssl-sys",
- "parking_lot",
+ "log",
+ "rustls",
+ "rustls-native-certs",
  "tokio",
- "tokio-openssl",
- "tower-layer",
+ "tokio-rustls",
 ]
 
 [[package]]
@@ -1326,12 +1333,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.0.0"
+version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.0",
+ "hashbrown 0.14.1",
 ]
 
 [[package]]
@@ -1349,17 +1356,6 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
-[[package]]
-name = "is-terminal"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
-dependencies = [
- "hermit-abi 0.3.1",
- "rustix",
- "windows-sys",
-]
-
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1371,15 +1367,15 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jobserver"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
 dependencies = [
  "libc",
 ]
@@ -1395,9 +1391,9 @@ dependencies = [
 
 [[package]]
 name = "json-patch"
-version = "1.0.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f54898088ccb91df1b492cc80029a6fdf1c48ca0db7c6822a8babad69c94658"
+checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6"
 dependencies = [
  "serde",
  "serde_json",
@@ -1422,8 +1418,8 @@ version = "8.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378"
 dependencies = [
- "base64 0.21.2",
- "pem",
+ "base64 0.21.4",
+ "pem 1.1.1",
  "ring",
  "serde",
  "serde_json",
@@ -1432,27 +1428,24 @@ dependencies = [
 
 [[package]]
 name = "k8s-openapi"
-version = "0.18.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788"
+checksum = "edc3606fd16aca7989db2f84bb25684d0270c6d6fa1dbcd0025af7b4130523a6"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.4",
  "bytes",
  "chrono",
- "http",
- "percent-encoding",
  "schemars",
  "serde",
  "serde-value",
  "serde_json",
- "url",
 ]
 
 [[package]]
 name = "kube"
-version = "0.83.0"
+version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f468b2fa6c5ef92117813238758f79e394c2d7688bd6faa3e77243f90260b0"
+checksum = "f8647c2211a9b480d910b155d573602c52cd5f646acecb06a03d594865dc4784"
 dependencies = [
  "k8s-openapi",
  "kube-client",
@@ -1463,28 +1456,29 @@ dependencies = [
 
 [[package]]
 name = "kube-client"
-version = "0.83.0"
+version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "337eb332d253036adc3247936248d0742c6c743f51eb38a684fd9b3b2878b27c"
+checksum = "af8952521f3e8ce11920229e5f2965fef70525aecd9efc7b65e39bf9e2c6f66d"
 dependencies = [
  "base64 0.20.0",
  "bytes",
  "chrono",
- "dirs-next",
  "either",
  "futures",
+ "home",
  "http",
  "http-body",
  "hyper",
- "hyper-openssl",
+ "hyper-rustls",
  "hyper-timeout",
  "jsonpath_lib",
  "k8s-openapi",
  "kube-core",
- "openssl",
- "pem",
+ "pem 3.0.2",
  "pin-project",
  "rand",
+ "rustls",
+ "rustls-pemfile",
  "secrecy",
  "serde",
  "serde_json",
@@ -1500,9 +1494,9 @@ dependencies = [
 
 [[package]]
 name = "kube-core"
-version = "0.83.0"
+version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f924177ad71936cfe612641b45bb9879890696d3c026f0846423529f4fa449af"
+checksum = "7608a0cd05dfa36167d2da982bb70f17feb5450f73ec601f6d428bbcf991c5b9"
 dependencies = [
  "chrono",
  "form_urlencoded",
@@ -1518,9 +1512,9 @@ dependencies = [
 
 [[package]]
 name = "kube-derive"
-version = "0.83.0"
+version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ce7c7a14cf3fe567ca856de41db0d61394867675cfb0d65094c55f0fa2df2e0"
+checksum = "a8dd623cf49cd632da4727a70e05d9cb948d5ea1098a1af49b1fd3bc9ec60b3c"
 dependencies = [
  "darling 0.14.4",
  "proc-macro2",
@@ -1531,15 +1525,16 @@ dependencies = [
 
 [[package]]
 name = "kube-runtime"
-version = "0.83.0"
+version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d5e4d09df25250ffcb09df3f31105a5f49eb8f8a08da9b776ea5b6431ec476f"
+checksum = "fde2bd0b2d248be72f30c658b728f87e84c68495bec2c689dff7a3479eb29afd"
 dependencies = [
  "ahash 0.8.3",
  "async-trait",
  "backoff",
  "derivative",
  "futures",
+ "hashbrown 0.14.1",
  "json-patch",
  "k8s-openapi",
  "kube-client",
@@ -1568,9 +1563,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.147"
+version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
 
 [[package]]
 name = "libloading"
@@ -1609,26 +1604,11 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "linked-hash-map"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
-
-[[package]]
-name = "linked_hash_set"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588"
-dependencies = [
- "linked-hash-map",
-]
-
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.3"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
 
 [[package]]
 name = "lock_api"
@@ -1642,9 +1622,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "lz4-sys"
@@ -1682,7 +1662,7 @@ dependencies = [
  "spin 0.9.8",
  "tokio",
  "tokio-util",
- "toml 0.7.6",
+ "toml 0.7.8",
  "tracing",
  "tracing-subscriber",
 ]
@@ -1702,7 +1682,7 @@ dependencies = [
  "spin 0.9.8",
  "thiserror",
  "tokio",
- "toml 0.7.6",
+ "toml 0.7.8",
  "tonic",
  "tracing",
 ]
@@ -1764,20 +1744,29 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
 dependencies = [
- "regex-automata",
+ "regex-automata 0.1.10",
 ]
 
 [[package]]
 name = "matchit"
-version = "0.7.1"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b"
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
+
+[[package]]
+name = "memoffset"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
 
 [[package]]
 name = "merged_range"
@@ -1917,9 +1906,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
+checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
 dependencies = [
  "autocfg",
 ]
@@ -1930,7 +1919,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.1",
+ "hermit-abi 0.3.3",
  "libc",
 ]
 
@@ -1942,9 +1931,9 @@ checksum = "89c16f12ae83bcd4510aff7778f043464991f8b5a10a27d7083f75361de32ac3"
 
 [[package]]
 name = "object"
-version = "0.31.1"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "memchr",
 ]
@@ -1956,42 +1945,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
-name = "openssl"
-version = "0.10.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
-dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-macros"
-version = "0.1.1"
+name = "openssl-probe"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.90"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "opentelemetry"
@@ -2102,18 +2059,18 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "2.10.0"
+version = "2.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"
+checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
 dependencies = [
  "num-traits",
 ]
 
 [[package]]
 name = "os_str_bytes"
-version = "6.5.1"
+version = "6.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac"
+checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
 
 [[package]]
 name = "overload"
@@ -2145,7 +2102,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.3.5",
+ "redox_syscall",
  "smallvec",
  "windows-targets",
 ]
@@ -2188,6 +2145,16 @@ dependencies = [
  "base64 0.13.1",
 ]
 
+[[package]]
+name = "pem"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923"
+dependencies = [
+ "base64 0.21.4",
+ "serde",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.0"
@@ -2201,34 +2168,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap 2.0.0",
+ "indexmap 2.0.2",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
+checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
+checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.10"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
+checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
 
 [[package]]
 name = "pin-utils"
@@ -2242,6 +2209,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -2290,12 +2263,12 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.12"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
+checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
 dependencies = [
  "proc-macro2",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -2334,13 +2307,28 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.63"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
+checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
 dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prometheus"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
+dependencies = [
+ "cfg-if",
+ "fnv",
+ "lazy_static",
+ "memchr",
+ "parking_lot",
+ "protobuf",
+ "thiserror",
+]
+
 [[package]]
 name = "prost"
 version = "0.11.9"
@@ -2395,11 +2383,17 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "protobuf"
+version = "2.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
+
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
@@ -2443,15 +2437,6 @@ dependencies = [
  "rand_core",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.3.5"
@@ -2461,26 +2446,16 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom",
- "redox_syscall 0.2.16",
- "thiserror",
-]
-
 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.2",
+ "regex-automata 0.4.3",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -2492,6 +2467,17 @@ dependencies = [
  "regex-syntax 0.6.29",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.8.2",
+]
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.29"
@@ -2500,9 +2486,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.2"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "ring"
@@ -2543,17 +2529,60 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
 name = "rustix"
-version = "0.38.2"
+version = "0.38.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
+checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
  "errno",
  "libc",
  "linux-raw-sys",
  "windows-sys",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-webpki",
+ "sct",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
+dependencies = [
+ "base64 0.21.4",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.101.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.14"
@@ -2562,15 +2591,24 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "ryu"
-version = "1.0.14"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
+[[package]]
+name = "schannel"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+dependencies = [
+ "windows-sys",
+]
 
 [[package]]
 name = "schemars"
-version = "0.8.12"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f"
+checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c"
 dependencies = [
  "dyn-clone",
  "schemars_derive",
@@ -2580,9 +2618,9 @@ dependencies = [
 
 [[package]]
 name = "schemars_derive"
-version = "0.8.12"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c"
+checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2592,9 +2630,19 @@ dependencies = [
 
 [[package]]
 name = "scopeguard"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "sct"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+dependencies = [
+ "ring",
+ "untrusted",
+]
 
 [[package]]
 name = "secrecy"
@@ -2606,11 +2654,34 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "security-framework"
+version = "2.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "serde"
-version = "1.0.166"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8"
+checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
 dependencies = [
  "serde_derive",
 ]
@@ -2621,19 +2692,19 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"
 dependencies = [
- "ordered-float 2.10.0",
+ "ordered-float 2.10.1",
  "serde",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.166"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6"
+checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -2649,11 +2720,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.99"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
+checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap 2.0.2",
  "itoa",
  "ryu",
  "serde",
@@ -2715,16 +2786,16 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "serde_yaml"
-version = "0.9.22"
+version = "0.9.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85"
+checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap 2.0.2",
  "itoa",
  "ryu",
  "serde",
@@ -2733,9 +2804,9 @@ dependencies = [
 
 [[package]]
 name = "sha1"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -2744,9 +2815,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.7"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -2755,18 +2826,18 @@ dependencies = [
 
 [[package]]
 name = "sharded-slab"
-version = "0.1.4"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
 dependencies = [
  "lazy_static",
 ]
 
 [[package]]
 name = "shlex"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
 
 [[package]]
 name = "signal-hook-registry"
@@ -2791,18 +2862,18 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "smallvec"
-version = "1.10.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
 
 [[package]]
 name = "socket2"
@@ -2814,6 +2885,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -2854,9 +2935,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.23"
+version = "2.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
+checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2877,16 +2958,16 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall 0.3.5",
+ "redox_syscall",
  "rustix",
  "windows-sys",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.2.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64"
 dependencies = [
  "winapi-util",
 ]
@@ -2905,22 +2986,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
 [[package]]
 name = "thiserror"
-version = "1.0.40"
+version = "1.0.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.40"
+version = "1.0.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -2957,12 +3038,13 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.26"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
 dependencies = [
  "deranged",
  "itoa",
+ "powerfmt",
  "serde",
  "time-core",
  "time-macros",
@@ -2970,15 +3052,15 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
-version = "0.2.12"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451"
+checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
 dependencies = [
  "time-core",
 ]
@@ -3000,11 +3082,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.29.1"
+version = "1.33.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
+checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
 dependencies = [
- "autocfg",
  "backtrace",
  "bytes",
  "libc",
@@ -3012,7 +3093,7 @@ dependencies = [
  "num_cpus",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2",
+ "socket2 0.5.4",
  "tokio-macros",
  "windows-sys",
 ]
@@ -3035,18 +3116,16 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
-name = "tokio-openssl"
-version = "0.6.3"
+name = "tokio-rustls"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a"
+checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "futures-util",
- "openssl",
- "openssl-sys",
+ "rustls",
  "tokio",
 ]
 
@@ -3073,9 +3152,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-tungstenite"
-version = "0.19.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c"
+checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c"
 dependencies = [
  "futures-util",
  "log",
@@ -3085,9 +3164,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.8"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
+checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d"
 dependencies = [
  "bytes",
  "futures-core",
@@ -3109,9 +3188,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.6"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
+checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -3130,11 +3209,11 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.19.14"
+version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
+checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap 2.0.2",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -3149,7 +3228,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
 dependencies = [
  "async-trait",
  "axum",
- "base64 0.21.2",
+ "base64 0.21.4",
  "bytes",
  "futures-core",
  "futures-util",
@@ -3217,12 +3296,12 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.4.1"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
+checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
 dependencies = [
- "base64 0.20.0",
- "bitflags 2.3.3",
+ "base64 0.21.4",
+ "bitflags 2.4.1",
  "bytes",
  "futures-core",
  "futures-util",
@@ -3250,11 +3329,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.37"
+version = "0.1.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9"
 dependencies = [
- "cfg-if",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -3274,20 +3352,20 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "tracing-core"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
 dependencies = [
  "once_cell",
  "valuable",
@@ -3353,9 +3431,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
 name = "tungstenite"
-version = "0.19.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67"
+checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9"
 dependencies = [
  "byteorder",
  "bytes",
@@ -3372,9 +3450,9 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "unicode-bidi"
@@ -3384,9 +3462,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.10"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
 [[package]]
 name = "unicode-normalization"
@@ -3399,9 +3477,9 @@ dependencies = [
 
 [[package]]
 name = "unsafe-libyaml"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6"
+checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
 
 [[package]]
 name = "untrusted"
@@ -3411,9 +3489,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
+checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -3443,11 +3521,12 @@ dependencies = [
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "clippy-utilities 0.2.0",
  "derive_builder",
  "getset",
+ "madsim-tokio",
  "madsim-tonic",
  "opentelemetry",
  "parking_lot",
@@ -3522,7 +3601,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
  "wasm-bindgen-shared",
 ]
 
@@ -3544,7 +3623,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.38",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3567,13 +3646,14 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "4.4.0"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
 dependencies = [
  "either",
- "libc",
+ "home",
  "once_cell",
+ "rustix",
 ]
 
 [[package]]
@@ -3594,9 +3674,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
 dependencies = [
  "winapi",
 ]
@@ -3627,9 +3707,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.48.1"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -3642,51 +3722,51 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "winnow"
-version = "0.5.14"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97"
+checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c"
 dependencies = [
  "memchr",
 ]
@@ -3694,7 +3774,7 @@ dependencies = [
 [[package]]
 name = "xline"
 version = "0.4.1"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -3702,16 +3782,17 @@ dependencies = [
  "bytes",
  "clap 3.2.25",
  "clippy-utilities 0.1.0",
+ "crossbeam-skiplist",
  "curp",
  "engine",
  "event-listener",
  "flume",
  "futures",
  "getset",
+ "hyper",
  "itertools",
  "jsonwebtoken",
  "log",
- "madsim-etcd-client",
  "madsim-tokio",
  "madsim-tonic",
  "madsim-tonic-build",
@@ -3742,7 +3823,7 @@ dependencies = [
 [[package]]
 name = "xline-client"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
  "async-stream",
  "clippy-utilities 0.1.0",
@@ -3757,7 +3838,6 @@ dependencies = [
  "thiserror",
  "tower",
  "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
- "uuid",
  "xline",
  "xlineapi",
 ]
@@ -3769,7 +3849,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "axum",
- "clap 4.3.10",
+ "clap 4.4.6",
  "clippy-utilities 0.2.0",
  "flume",
  "futures",
@@ -3777,6 +3857,7 @@ dependencies = [
  "k8s-openapi",
  "kube",
  "operator-api",
+ "prometheus",
  "schemars",
  "serde",
  "serde_json",
@@ -3795,7 +3876,7 @@ dependencies = [
  "async-trait",
  "axum",
  "bytes",
- "clap 4.3.10",
+ "clap 4.4.6",
  "engine",
  "event-listener",
  "futures",
@@ -3804,21 +3885,21 @@ dependencies = [
  "serde_json",
  "thiserror",
  "tokio",
- "toml 0.7.6",
+ "toml 0.7.8",
  "tonic",
  "tonic-health",
  "tracing",
  "tracing-subscriber",
  "utils 0.1.0",
  "xline-client",
- "xlineapi",
 ]
 
 [[package]]
 name = "xlineapi"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#fadc65670fb6cf5ca2540fe4a25c8f1b4df1b766"
+source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
 dependencies = [
+ "curp-external-api",
  "madsim-etcd-client",
  "madsim-tonic",
  "madsim-tonic-build",
@@ -3834,11 +3915,10 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.8+zstd.1.5.5"
+version = "2.0.9+zstd.1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
+checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
 dependencies = [
  "cc",
- "libc",
  "pkg-config",
 ]
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index af23fc4e..82df28e1 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -11,8 +11,8 @@ categories = ["API"]
 keywords = ["operator", "API", "operator"]
 
 [dependencies]
+anyhow = "1.0.72"
 async-trait = "0.1.72"
 k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
 kube = { version = "0.83.0", features = ["runtime", "derive", "ws"] }
 serde = { version = "1.0.130", features = ["derive"] }
-anyhow = "1.0.72"
diff --git a/operator-api/src/xline.rs b/operator-api/src/xline.rs
index 351bc138..b740f760 100644
--- a/operator-api/src/xline.rs
+++ b/operator-api/src/xline.rs
@@ -37,6 +37,7 @@ impl LocalXlineHandle {
 #[async_trait]
 impl XlineHandle for LocalXlineHandle {
     async fn start(&mut self) -> anyhow::Result<()> {
+        self.kill().await?;
         let mut cmds = self.start_cmd.split_whitespace();
         let Some((exe, args)) = cmds
             .next()
@@ -123,6 +124,7 @@ impl K8sXlineHandle {
 #[async_trait]
 impl XlineHandle for K8sXlineHandle {
     async fn start(&mut self) -> anyhow::Result<()> {
+        self.kill().await?;
         let start_cmd: Vec<&str> = self.start_cmd.split_whitespace().collect();
         let process = self
             .pods_api
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index e1b2a814..1e8b9e0a 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -22,7 +22,6 @@ futures = "0.3.28"
 k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
 kube = { version = "0.83.0", features = ["runtime", "derive"] }
 operator-api = { path = "../operator-api" }
-lazy_static = "1.4.0"
 prometheus = "0.13.3"
 schemars = "0.8.6"
 serde = { version = "1.0.130", features = ["derive"] }
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 40d40a6e..c1f17009 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -37,5 +37,3 @@ tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 uuid = { version = "1.4.1", features = ["v4"] }
 xline-client = { git = "https://github.com/xline-kv/Xline.git", package = "xline-client" }
-xlineapi = { git = "https://github.com/xline-kv/Xline.git", package = "xlineapi" }
-operator-api = { path = "../operator-api" }
diff --git a/sidecar/src/backup/mod.rs b/sidecar/src/backup/mod.rs
index 9df2b0f8..45e167ed 100644
--- a/sidecar/src/backup/mod.rs
+++ b/sidecar/src/backup/mod.rs
@@ -10,7 +10,7 @@ use std::time::Duration;
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
 use tonic::Streaming;
-use xlineapi::SnapshotResponse;
+use xline_client::types::maintenance::SnapshotResponse;
 
 /// Snapshot file suffix
 const SNAPSHOT_SUFFIX: &str = "xline.backup";
diff --git a/sidecar/src/backup/pv.rs b/sidecar/src/backup/pv.rs
index edba1cd1..d0cb27e8 100644
--- a/sidecar/src/backup/pv.rs
+++ b/sidecar/src/backup/pv.rs
@@ -11,7 +11,7 @@ use tokio::io;
 use tokio::io::AsyncWriteExt;
 use tonic::Streaming;
 use tracing::debug;
-use xlineapi::SnapshotResponse;
+use xline_client::types::maintenance::SnapshotResponse;
 
 use crate::backup::{Metadata, Provider, SNAPSHOT_SUFFIX};
 
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index a2b19995..57ce23b6 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -17,6 +17,7 @@ use tonic_health::pb::health_check_response::ServingStatus;
 use tonic_health::pb::health_client::HealthClient;
 use tonic_health::pb::HealthCheckRequest;
 use tracing::debug;
+use xline_client::types::cluster::{MemberAddRequest, MemberListRequest, MemberRemoveRequest};
 use xline_client::types::kv::RangeRequest;
 use xline_client::{Client, ClientOptions};
 
@@ -53,10 +54,12 @@ pub(crate) struct XlineHandle {
     name: String,
     /// The xline backup provider
     backup: Option<Box<dyn Provider>>,
-    /// The xline client
+    /// The xline client, used to connect to the cluster
     client: Option<Client>,
-    /// The xline health client
+    /// The xline health client, used to check self health
     health_client: HealthClient<Channel>,
+    /// The self xline server id
+    server_id: Option<u64>,
     /// The rocks db engine
     engine: Engine,
     /// The xline members
@@ -87,6 +90,7 @@ impl XlineHandle {
             health_client,
             engine,
             client: None,
+            server_id: None,
             xline_members,
             is_healthy_retries: 5,
             inner,
@@ -102,6 +106,8 @@ impl XlineHandle {
 
     /// Start the xline server
     pub(crate) async fn start(&mut self) -> Result<()> {
+        // TODO: hold a distributed lock during start
+
         // Step 1: Check if there is any node running
         // Step 2: If there is no node running, start single node cluster
         // Step 3: If there are some nodes running, start the node as a member to join the cluster
@@ -121,24 +127,55 @@ impl XlineHandle {
         self.inner.start().await?;
 
         let client = Client::connect(self.xline_members.values(), ClientOptions::default()).await?;
-        if cluster_started {
-            let _cluster_client = client.cluster_client();
-            // send membership change here
-        }
-        let _ig = self.client.replace(client);
+        let mut cluster_client = client.cluster_client();
+        let member = if cluster_started {
+            let peer_addr = self
+                .xline_members
+                .get(&self.name)
+                .unwrap_or_else(|| unreachable!("member should contain self"))
+                .clone();
+            let resp = cluster_client
+                .member_add(MemberAddRequest::new(vec![peer_addr], false))
+                .await?;
+            let Some(member) = resp.member else {
+                unreachable!("self member should be set when member add request success")
+            };
+            member
+        } else {
+            let mut members = cluster_client
+                .member_list(MemberListRequest::new(false))
+                .await?
+                .members;
+            if members.len() != 1 {
+                return Err(anyhow!(
+                    "there should be only one member(self) if the cluster if not start"
+                ));
+            }
+            members.remove(0)
+        };
+        debug!("xline server started, member: {:?}", member);
+        _ = self.server_id.replace(member.id);
+        _ = self.client.replace(client);
         Ok(())
     }
 
     /// Stop the xline server
     pub(crate) async fn stop(&mut self) -> Result<()> {
-        // Step 1: Kill the xline node
-        // Step 2: Remove the xline node from the cluster if the cluster exist
-        self.inner.kill().await?;
+        // Step 1: Remove the xline node from the cluster if the cluster exist
+        // Step 2: Kill the xline node
+        let server_id = self
+            .server_id
+            .take()
+            .ok_or_else(|| anyhow!("xline server should not be stopped before started"))?;
 
         if self.is_healthy().await {
-            let _cluster_client = self.client().cluster_client();
-            // send membership change here
+            let mut cluster_client = self.client().cluster_client();
+            _ = cluster_client
+                .member_remove(MemberRemoveRequest::new(server_id))
+                .await?;
         }
+
+        self.inner.kill().await?;
         Ok(())
     }
 

From f45408193ce7b39fb750f3d7e8367a1cbcee0c48 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Sun, 22 Oct 2023 22:39:01 +0800
Subject: [PATCH 07/19] chore: bump crate version

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                                    | 202 ++++++++++++------
 operator-api/Cargo.toml                       |   4 +-
 operator-k8s/Cargo.toml                       |   6 +-
 .../src/controller/cluster/v1alpha1.rs        |   2 +-
 sidecar/Cargo.toml                            |   3 +-
 sidecar/src/main.rs                           |   2 +-
 6 files changed, 139 insertions(+), 80 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 029027a2..3437372a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -30,14 +30,15 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+checksum = "72832d73be48bac96a5d7944568f305d829ed55b0ce3b483647089dfaf6cf704"
 dependencies = [
  "cfg-if",
  "getrandom",
  "once_cell",
  "version_check",
+ "zerocopy",
 ]
 
 [[package]]
@@ -159,9 +160,9 @@ dependencies = [
 
 [[package]]
 name = "async-task"
-version = "4.4.1"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921"
+checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1"
 
 [[package]]
 name = "async-trait"
@@ -546,9 +547,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
+checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4"
 dependencies = [
  "libc",
 ]
@@ -610,7 +611,7 @@ dependencies = [
 [[package]]
 name = "curp"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -623,7 +624,7 @@ dependencies = [
  "derive_builder",
  "engine",
  "event-listener",
- "flume",
+ "flume 0.10.14",
  "futures",
  "indexmap 1.9.3",
  "itertools",
@@ -637,18 +638,19 @@ dependencies = [
  "prost-build",
  "rand",
  "serde",
+ "serde_json",
  "thiserror",
  "tokio-stream 0.1.12",
  "tower",
  "tracing",
  "tracing-opentelemetry",
- "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
+ "utils",
 ]
 
 [[package]]
 name = "curp-external-api"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "async-trait",
  "engine",
@@ -735,7 +737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
- "hashbrown 0.14.1",
+ "hashbrown 0.14.2",
  "lock_api",
  "once_cell",
  "parking_lot_core",
@@ -843,7 +845,7 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 [[package]]
 name = "engine"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "async-trait",
  "bincode",
@@ -929,6 +931,18 @@ dependencies = [
  "spin 0.9.8",
 ]
 
+[[package]]
+name = "flume"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "nanorand",
+ "spin 0.9.8",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1041,9 +1055,9 @@ dependencies = [
 
 [[package]]
 name = "garde"
-version = "0.14.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efbc4e06c5b4a8fef3ad41939c6116ea06a0d93f647c6430154d924bf021c374"
+checksum = "d959ef7bda0bda7cc0f6fbebfbac6202f810394f50e07059eeea8ec31e69e4b0"
 dependencies = [
  "garde_derive",
  "once_cell",
@@ -1052,9 +1066,9 @@ dependencies = [
 
 [[package]]
 name = "garde_derive"
-version = "0.14.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6699dea5b266ce9ca75ab4cbc5a7e98d5abfdf3197cecef5374a623023939acd"
+checksum = "7e89f7fce035bb3a3718e23efff13709a0b21b694c4eae20a32e1a3e4e27c6a2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1136,11 +1150,11 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.14.1"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
+checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
 dependencies = [
- "ahash 0.8.3",
+ "ahash 0.8.4",
  "allocator-api2",
 ]
 
@@ -1246,7 +1260,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.4.9",
+ "socket2 0.4.10",
  "tokio",
  "tower-service",
  "tracing",
@@ -1283,16 +1297,16 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.57"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
+checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows",
+ "windows-core",
 ]
 
 [[package]]
@@ -1338,7 +1352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.1",
+ "hashbrown 0.14.2",
 ]
 
 [[package]]
@@ -1529,12 +1543,12 @@ version = "0.86.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fde2bd0b2d248be72f30c658b728f87e84c68495bec2c689dff7a3479eb29afd"
 dependencies = [
- "ahash 0.8.3",
+ "ahash 0.8.4",
  "async-trait",
  "backoff",
  "derivative",
  "futures",
- "hashbrown 0.14.1",
+ "hashbrown 0.14.2",
  "json-patch",
  "k8s-openapi",
  "kube-client",
@@ -1612,9 +1626,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
 
 [[package]]
 name = "lock_api"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -2096,13 +2110,13 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.8"
+version = "0.9.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
+checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
  "smallvec",
  "windows-targets",
 ]
@@ -2446,6 +2460,15 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "regex"
 version = "1.10.2"
@@ -2529,9 +2552,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
 name = "rustix"
-version = "0.38.19"
+version = "0.38.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
+checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0"
 dependencies = [
  "bitflags 2.4.1",
  "errno",
@@ -2877,9 +2900,9 @@ checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
 
 [[package]]
 name = "socket2"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"
 dependencies = [
  "libc",
  "winapi",
@@ -2887,9 +2910,9 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.5.4"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e"
+checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
 dependencies = [
  "libc",
  "windows-sys",
@@ -2958,7 +2981,7 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall",
+ "redox_syscall 0.3.5",
  "rustix",
  "windows-sys",
 ]
@@ -2986,18 +3009,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
 [[package]]
 name = "thiserror"
-version = "1.0.49"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4"
+checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.49"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
+checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3093,7 +3116,7 @@ dependencies = [
  "num_cpus",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2 0.5.4",
+ "socket2 0.5.5",
  "tokio-macros",
  "windows-sys",
 ]
@@ -3195,7 +3218,19 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit",
+ "toml_edit 0.19.15",
+]
+
+[[package]]
+name = "toml"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit 0.20.2",
 ]
 
 [[package]]
@@ -3220,6 +3255,19 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "toml_edit"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
+dependencies = [
+ "indexmap 2.0.2",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "tonic"
 version = "0.9.2"
@@ -3329,9 +3377,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.39"
+version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9"
+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
 dependencies = [
  "log",
  "pin-project-lite",
@@ -3513,15 +3561,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 [[package]]
 name = "utils"
 version = "0.1.0"
-dependencies = [
- "anyhow",
- "uuid",
-]
-
-[[package]]
-name = "utils"
-version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "clippy-utilities 0.2.0",
  "derive_builder",
@@ -3540,9 +3580,9 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.4.1"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
+checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
 dependencies = [
  "getrandom",
 ]
@@ -3688,10 +3728,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
-name = "windows"
-version = "0.48.0"
+name = "windows-core"
+version = "0.51.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
 dependencies = [
  "windows-targets",
 ]
@@ -3774,7 +3814,7 @@ dependencies = [
 [[package]]
 name = "xline"
 version = "0.4.1"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -3786,7 +3826,7 @@ dependencies = [
  "curp",
  "engine",
  "event-listener",
- "flume",
+ "flume 0.10.14",
  "futures",
  "getset",
  "hyper",
@@ -3815,7 +3855,7 @@ dependencies = [
  "tracing-appender",
  "tracing-opentelemetry",
  "tracing-subscriber",
- "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
+ "utils",
  "uuid",
  "xlineapi",
 ]
@@ -3823,8 +3863,9 @@ dependencies = [
 [[package]]
 name = "xline-client"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
+ "anyhow",
  "async-stream",
  "clippy-utilities 0.1.0",
  "curp",
@@ -3837,7 +3878,7 @@ dependencies = [
  "pbkdf2",
  "thiserror",
  "tower",
- "utils 0.1.0 (git+https://github.com/xline-kv/Xline.git)",
+ "utils",
  "xline",
  "xlineapi",
 ]
@@ -3851,7 +3892,7 @@ dependencies = [
  "axum",
  "clap 4.4.6",
  "clippy-utilities 0.2.0",
- "flume",
+ "flume 0.11.0",
  "futures",
  "garde",
  "k8s-openapi",
@@ -3861,11 +3902,11 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "serde_yaml",
  "thiserror",
  "tokio",
  "tracing",
  "tracing-subscriber",
- "utils 0.1.0",
 ]
 
 [[package]]
@@ -3878,26 +3919,25 @@ dependencies = [
  "bytes",
  "clap 4.4.6",
  "engine",
- "event-listener",
  "futures",
  "operator-api",
  "serde",
  "serde_json",
  "thiserror",
  "tokio",
- "toml 0.7.8",
+ "toml 0.8.2",
  "tonic",
  "tonic-health",
  "tracing",
  "tracing-subscriber",
- "utils 0.1.0",
+ "uuid",
  "xline-client",
 ]
 
 [[package]]
 name = "xlineapi"
 version = "0.1.0"
-source = "git+https://github.com/xline-kv/Xline.git#6e9588157852816dd7e44935d8b0ae4edc5075d7"
+source = "git+https://github.com/xline-kv/Xline.git#5cee0118d363e69e8040fdd6961f605821dbd177"
 dependencies = [
  "curp-external-api",
  "madsim-etcd-client",
@@ -3907,6 +3947,26 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c19fae0c8a9efc6a8281f2e623db8af1db9e57852e04cde3e754dd2dc29340f"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc56589e9ddd1f1c28d4b4b5c773ce232910a6bb67a70133d61c9e347585efe9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.6.0"
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index 82df28e1..cb3cd76c 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -13,6 +13,6 @@ keywords = ["operator", "API", "operator"]
 [dependencies]
 anyhow = "1.0.72"
 async-trait = "0.1.72"
-k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
-kube = { version = "0.83.0", features = ["runtime", "derive", "ws"] }
+k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
+kube = { version = "0.86.0", features = ["runtime", "derive", "ws"] }
 serde = { version = "1.0.130", features = ["derive"] }
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index 1e8b9e0a..ffd9f3fc 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -17,10 +17,10 @@ async-trait = "0.1.68"
 axum = "0.6.18"
 clap = { version = "4.3.4", features = ["derive"] }
 clippy-utilities = "0.2.0"
-flume = "0.10.14"
+flume = "0.11.0"
 futures = "0.3.28"
-k8s-openapi = { version = "0.18.0", features = ["v1_26", "schemars"] }
-kube = { version = "0.83.0", features = ["runtime", "derive"] }
+k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
+kube = { version = "0.86.0", features = ["runtime", "derive"] }
 operator-api = { path = "../operator-api" }
 prometheus = "0.13.3"
 schemars = "0.8.6"
diff --git a/operator-k8s/src/controller/cluster/v1alpha1.rs b/operator-k8s/src/controller/cluster/v1alpha1.rs
index 4765f8cc..292f6cb0 100644
--- a/operator-k8s/src/controller/cluster/v1alpha1.rs
+++ b/operator-k8s/src/controller/cluster/v1alpha1.rs
@@ -33,7 +33,7 @@ impl MetricsLabeled for kube::Error {
             Self::Service(_) => vec!["service error"],
             Self::FromUtf8(_) | Self::SerdeError(_) => vec!["encode/decode error"],
             Self::Auth(_) => vec!["authorization error"],
-            Self::OpensslTls(_) => vec!["tls error"],
+            Self::RustlsTls(_) => vec!["tls error"],
             Self::HyperError(_) | Self::HttpError(_) => vec!["http error"],
             _ => vec!["unknown"],
         }
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index c1f17009..2a8c8415 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -18,7 +18,6 @@ axum = "0.6.18"
 bytes = "1.4.0"
 clap = { version = "4.3.4", features = ["derive"] }
 engine = { git = "https://github.com/xline-kv/Xline.git", package = "engine" }
-event-listener = "2.5.3"
 futures = "0.3.28"
 operator-api = { path = "../operator-api" }
 serde = { version = "1.0.130", features = ["derive"] }
@@ -30,7 +29,7 @@ tokio = { version = "1.0", features = [
   "macros",
   "net",
 ] }
-toml = "0.7.4"
+toml = "0.8.2"
 tonic = "0.9.2"
 tonic-health = "0.9.2"
 tracing = "0.1.37"
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index 256c9bb0..85d1b082 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -283,7 +283,7 @@ fn parse_backend(value: &str) -> Result<Backend, String> {
                     "pod" => pod_name = v.to_owned(),
                     "container" => container_name = v.to_owned(),
                     "namespace" => namespace = v.to_owned(),
-                    _ => return Err(format!("k8s backend got unexpect argument {item}, expect one of 'pod', 'container', 'namespace'")),
+                    _ => return Err(format!("k8s backend got unexpected argument {item}, expect one of 'pod', 'container', 'namespace'")),
                 }
             }
             if pod_name.is_empty() || container_name.is_empty() || namespace.is_empty() {

From 57905e48b20989207cd192a220eddeafa52f6186 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Mon, 23 Oct 2023 20:13:42 +0800
Subject: [PATCH 08/19] chore: fix start

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 sidecar/src/xline.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 57ce23b6..3df3406f 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -111,16 +111,17 @@ impl XlineHandle {
         // Step 1: Check if there is any node running
         // Step 2: If there is no node running, start single node cluster
         // Step 3: If there are some nodes running, start the node as a member to join the cluster
-        let endpoints = self
+        let others = self
             .xline_members
-            .values()
-            .map(|addr| {
+            .iter()
+            .filter(|&(name, _)| name != &self.name)
+            .map(|(_, addr)| {
                 Ok::<_, tonic::transport::Error>(
                     Endpoint::from_shared(addr.clone())?.connect_timeout(Duration::from_secs(3)),
                 )
             })
             .collect::<Result<Vec<_>, _>>()?;
-        let futs: FuturesUnordered<_> = endpoints.iter().map(Endpoint::connect).collect();
+        let futs: FuturesUnordered<_> = others.iter().map(Endpoint::connect).collect();
         // the cluster is started if any of the connection is successful
         let cluster_started = futs.any(|res| async move { res.is_ok() }).await;
 

From c4b9f4def35fe452cbc0fc5e9c40c99533db71ab Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Wed, 25 Oct 2023 14:17:45 +0800
Subject: [PATCH 09/19] refactor: refactor operator monitor

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                           | 191 ++++++++++++++++
 operator-api/src/consts.rs           |   4 +
 operator-api/src/lib.rs              |  43 +++-
 operator-k8s/src/config.rs           |   8 +-
 operator-k8s/src/crd/mod.rs          |  31 +++
 operator-k8s/src/crd/v1alpha1/mod.rs | 156 +++++++++++++
 operator-k8s/src/monitor.rs          | 323 ++++++++++++++++++---------
 operator-k8s/src/operator.rs         | 178 +--------------
 operator-k8s/src/router.rs           |   4 +-
 9 files changed, 651 insertions(+), 287 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3437372a..4b490f4a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -842,6 +842,15 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "engine"
 version = "0.1.0"
@@ -949,6 +958,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.0"
@@ -1295,6 +1319,19 @@ dependencies = [
  "tokio-io-timeout",
 ]
 
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.58"
@@ -1370,6 +1407,12 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "ipnet"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1871,6 +1914,24 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -1958,12 +2019,50 @@ version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
+[[package]]
+name = "openssl"
+version = "0.10.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
+dependencies = [
+ "bitflags 2.4.1",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-sys"
+version = "0.9.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.18.0"
@@ -2513,6 +2612,44 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
+[[package]]
+name = "reqwest"
+version = "0.11.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
+dependencies = [
+ "base64 0.21.4",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -2973,6 +3110,27 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.8.0"
@@ -3142,6 +3300,16 @@ dependencies = [
  "syn 2.0.38",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
@@ -3645,6 +3813,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.87"
@@ -3811,6 +3991,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys",
+]
+
 [[package]]
 name = "xline"
 version = "0.4.1"
@@ -3921,6 +4111,7 @@ dependencies = [
  "engine",
  "futures",
  "operator-api",
+ "reqwest",
  "serde",
  "serde_json",
  "thiserror",
diff --git a/operator-api/src/consts.rs b/operator-api/src/consts.rs
index c7b0727c..f56985ad 100644
--- a/operator-api/src/consts.rs
+++ b/operator-api/src/consts.rs
@@ -2,3 +2,7 @@
 pub const DEFAULT_BACKUP_DIR: &str = "/xline-backup";
 /// Default xline data dir, this path cannot be mounted by user
 pub const DEFAULT_DATA_DIR: &str = "/usr/local/xline/data-dir";
+/// the URL ROUTE that sidecar sends heartbeat status to
+pub const OPERATOR_MONITOR_ROUTE: &str = "/monitor";
+/// the URL ROUTE of each sidecar for backup
+pub const SIDECAR_BACKUP_ROUTE: &str = "/backup";
diff --git a/operator-api/src/lib.rs b/operator-api/src/lib.rs
index fceef081..fd1728c7 100644
--- a/operator-api/src/lib.rs
+++ b/operator-api/src/lib.rs
@@ -1,20 +1,33 @@
+/// constants shared by the operator and the sidecar
+pub mod consts;
+
 /// Xline handle
 mod xline;
 
-pub mod consts;
-
+use std::time::{SystemTime, UNIX_EPOCH};
 pub use xline::{K8sXlineHandle, LocalXlineHandle, XlineHandle};
 
 use serde::{Deserialize, Serialize};
 
-/// Heartbeat status
+/// Heartbeat status, sort by timestamp.
+/// The clock of each machine may be different, which may cause heartbeat to be unable to assist
+/// the operator in detecting the dropped sidecar.
+///
+/// FIXME: May cause misjudgment under extreme conditions:
+/// Assume a 3-node cluster. One of the sidecars has a slow clock. When network fluctuations occur,
+/// the two sidecars with faster clocks fail to send heartbeats. At this time, the sidecar with
+/// slower clocks successfully sends heartbeats and communicates with some outdated data stored on
+/// the operator. The heartbeats satisfaction interval is not greater than the heartbeat period, and
+/// the results obtained by the operator at this time may be out of date.
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct HeartbeatStatus {
-    /// the name of the sidecar operator
+    /// the cluster name of this sidecar
+    pub cluster_name: String,
+    /// the name of the sidecar
     pub name: String,
-    /// the timestamp of this status
+    /// the timestamp of this status in seconds
     pub timestamp: u64,
-    /// reachable sidecar operator ids
+    /// reachable sidecar names
     pub reachable: Vec<String>,
 }
 
@@ -31,9 +44,23 @@ impl Ord for HeartbeatStatus {
 }
 
 impl HeartbeatStatus {
-    /// Creates a new `HeartbeatStatus`
-    pub fn new(name: String, timestamp: u64, reachable: Vec<String>) -> Self {
+    /// Create a new `HeartbeatStatus` with current timestamp
+    pub fn current(cluster_name: String, name: String, reachable: Vec<String>) -> Self {
+        let dur = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_else(|_| unreachable!("time turns back!"));
+        Self {
+            cluster_name,
+            name,
+            timestamp: dur.as_secs(),
+            reachable,
+        }
+    }
+
+    /// Create a new `HeartbeatStatus`
+    pub fn new(cluster_name: String, name: String, timestamp: u64, reachable: Vec<String>) -> Self {
         Self {
+            cluster_name,
             name,
             timestamp,
             reachable,
diff --git a/operator-k8s/src/config.rs b/operator-k8s/src/config.rs
index 46ff29bf..7741361c 100644
--- a/operator-k8s/src/config.rs
+++ b/operator-k8s/src/config.rs
@@ -11,10 +11,12 @@ pub struct Config {
     /// The address on which the heartbeat HTTP server will listen to
     #[arg(long, default_value = "0.0.0.0:8080")]
     pub listen_addr: String,
-    /// Whether to create CRD regardless of current version on k8s
+    /// Enable manage-crd to create CRD automatically
+    /// This requires a ClusterRole of CRD manipulate
     #[arg(long, default_value = "false")]
-    pub create_crd: bool,
-    /// Whether to enable auto migration if CRD version is less than current version
+    pub manage_crd: bool,
+    /// Enable auto migration if CRD version is less than current version
+    /// (--manage-crd should be set to true)
     #[arg(long, default_value = "false")]
     pub auto_migration: bool,
     /// The kubernetes cluster DNS suffix
diff --git a/operator-k8s/src/crd/mod.rs b/operator-k8s/src/crd/mod.rs
index 565aa1b5..60e971d6 100644
--- a/operator-k8s/src/crd/mod.rs
+++ b/operator-k8s/src/crd/mod.rs
@@ -9,3 +9,34 @@ pub(crate) mod version;
 
 /// Current CRD `XineCluster`
 pub(crate) use v1alpha1::Cluster;
+
+use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
+use kube::runtime::conditions;
+use kube::runtime::wait::await_condition;
+use kube::{Api, Client};
+use tracing::debug;
+
+use std::time::Duration;
+
+/// wait crd to establish timeout
+const CRD_ESTABLISH_TIMEOUT: Duration = Duration::from_secs(20);
+
+/// Setup CRD
+pub(crate) async fn setup(
+    kube_client: &Client,
+    manage_crd: bool,
+    auto_migration: bool,
+) -> anyhow::Result<()> {
+    v1alpha1::set_up(kube_client, manage_crd, auto_migration).await
+}
+
+/// Wait for CRD to be established
+async fn wait_crd_established(
+    crd_api: Api<CustomResourceDefinition>,
+    crd_name: &str,
+) -> anyhow::Result<()> {
+    let establish = await_condition(crd_api, crd_name, conditions::is_crd_established());
+    debug!("wait for crd established");
+    _ = tokio::time::timeout(CRD_ESTABLISH_TIMEOUT, establish).await??;
+    Ok(())
+}
diff --git a/operator-k8s/src/crd/v1alpha1/mod.rs b/operator-k8s/src/crd/v1alpha1/mod.rs
index b0a1a7d0..8c028015 100644
--- a/operator-k8s/src/crd/v1alpha1/mod.rs
+++ b/operator-k8s/src/crd/v1alpha1/mod.rs
@@ -1,3 +1,159 @@
 pub(crate) use cluster::Cluster;
 
 mod cluster;
+
+use crate::consts::FIELD_MANAGER;
+use crate::crd::version::ApiVersion;
+use crate::crd::wait_crd_established;
+use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
+use kube::api::{DynamicObject, ListParams, Patch, PatchParams};
+use kube::core::crd::merge_crds;
+use kube::{Api, Client, CustomResourceExt, Resource};
+use tracing::{debug, info};
+
+/// Setup CRD
+pub(super) async fn set_up(
+    kube_client: &Client,
+    manage_crd: bool,
+    auto_migration: bool,
+) -> anyhow::Result<()> {
+    if !manage_crd {
+        info!("--manage-crd set to false, skip checking CRD");
+        return Ok(());
+    }
+
+    let crd_api: Api<CustomResourceDefinition> = Api::all(kube_client.clone());
+    let definition = Cluster::crd();
+    let current_version: ApiVersion<Cluster> = Cluster::version(&()).as_ref().parse()?;
+
+    let ret = crd_api.get(Cluster::crd_name()).await;
+    if let Err(kube::Error::Api(kube::error::ErrorResponse { code: 404, .. })) = ret {
+        // the following code needs `customresourcedefinitions` write permission
+        debug!("cannot found XlineCluster CRD, try to init it");
+        _ = crd_api
+            .patch(
+                Cluster::crd_name(),
+                &PatchParams::apply(FIELD_MANAGER),
+                &Patch::Apply(definition.clone()),
+            )
+            .await?;
+        wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
+        return Ok(());
+    }
+
+    debug!("found XlineCluster CRD, current version: {current_version}");
+
+    let mut add = true;
+    let mut storage = String::new();
+
+    let mut crds = ret?
+        .spec
+        .versions
+        .iter()
+        .cloned()
+        .map(|ver| {
+            let mut crd = definition.clone();
+            if ver.name == current_version.to_string() {
+                add = false;
+            }
+            if ver.storage {
+                storage = ver.name.clone();
+            }
+            crd.spec.versions = vec![ver];
+            crd
+        })
+        .collect::<Vec<_>>();
+
+    if add {
+        crds.push(definition.clone());
+    } else {
+        debug!("current version already exists, try to migrate");
+        try_migration(
+            kube_client,
+            crds,
+            &current_version,
+            &storage,
+            auto_migration,
+        )
+        .await?;
+        return Ok(());
+    }
+
+    let merged_crd = merge_crds(crds.clone(), &storage)?;
+    debug!("try to update crd");
+    _ = crd_api
+        .patch(
+            Cluster::crd_name(),
+            &PatchParams::apply(FIELD_MANAGER),
+            &Patch::Apply(merged_crd),
+        )
+        .await?;
+    wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
+
+    debug!("crd updated, try to migrate");
+    try_migration(
+        kube_client,
+        crds,
+        &current_version,
+        &storage,
+        auto_migration,
+    )
+    .await?;
+
+    Ok(())
+}
+
+/// Try to migrate CRD
+#[allow(clippy::indexing_slicing)] // there is at least one element in `versions`
+#[allow(clippy::expect_used)]
+async fn try_migration(
+    kube_client: &Client,
+    crds: Vec<CustomResourceDefinition>,
+    current_version: &ApiVersion<Cluster>,
+    storage: &str,
+    auto_migration: bool,
+) -> anyhow::Result<()> {
+    if !auto_migration {
+        debug!("auto migration is disabled, skip migration");
+        return Ok(());
+    }
+    if current_version.to_string() == storage {
+        // stop migration if current version is already in storage
+        debug!("current version is already in storage, skip migration");
+        return Ok(());
+    }
+    let versions: Vec<ApiVersion<Cluster>> = crds
+        .iter()
+        .map(|crd| crd.spec.versions[0].name.parse())
+        .collect::<anyhow::Result<_>>()?;
+    if versions.iter().any(|ver| current_version < ver) {
+        // stop migration if current version is less than any version in `versions`
+        debug!("current version is less than some version in crd, skip migration");
+        return Ok(());
+    }
+    let group = kube::discovery::group(kube_client, Cluster::group(&()).as_ref()).await?;
+    let Some((ar, _)) = group
+        .versioned_resources(storage)
+        .into_iter()
+        .find(|res| res.0.kind == Cluster::kind(&())) else { return Ok(()) };
+    let api: Api<DynamicObject> = Api::all_with(kube_client.clone(), &ar);
+    let clusters = api.list(&ListParams::default()).await?.items;
+    if !clusters.is_empty() && !current_version.compat_with(&storage.parse()?) {
+        // there is some clusters with storage version and is not compat with current version, stop migration
+        // TODO add a flag to these clusters to indicate that they need to be migrated
+        return Ok(());
+    }
+    // start migration as there is no cluster with storage version
+    let merged_crd = merge_crds(crds, &current_version.to_string())?;
+    let crd_api: Api<CustomResourceDefinition> = Api::all(kube_client.clone());
+    debug!("try to migrate crd from {storage} to {current_version}");
+    _ = crd_api
+        .patch(
+            Cluster::crd_name(),
+            &PatchParams::apply(FIELD_MANAGER),
+            &Patch::Apply(merged_crd),
+        )
+        .await?;
+    wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
+    Ok(())
+}
diff --git a/operator-k8s/src/monitor.rs b/operator-k8s/src/monitor.rs
index 34fc627f..0c05013f 100644
--- a/operator-k8s/src/monitor.rs
+++ b/operator-k8s/src/monitor.rs
@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::time::SystemTime;
 
 use anyhow::Result;
 use clippy_utilities::OverflowArithmetic;
@@ -6,28 +7,38 @@ use flume::Receiver;
 use futures::Future;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::DeleteParams;
-use kube::{Api, CustomResourceExt};
+use kube::Api;
 use operator_api::HeartbeatStatus;
 use tracing::{debug, error};
 
 use crate::crd::Cluster;
 
-/// Sidecar monitor
-pub(crate) struct SidecarMonitor {
-    /// Map for each sidecar operator and its status
-    statuses: HashMap<String, HeartbeatStatus>,
+/// Sidecar monitor context
+struct Context {
     /// Receiver for heartbeat status
     status_rx: Receiver<HeartbeatStatus>,
     /// Maximum interval between accepted `HeartbeatStatus`
     heartbeat_period: u64,
+    /// Unreachable counter threshold
+    unreachable_thresh: usize,
     /// Api for Cluster
     cluster_api: Api<Cluster>,
     /// Api for Pods
     pod_api: Api<Pod>,
+}
+
+/// A sidecar cluster states map
+type SidecarClusterOwned<T> = HashMap<String, T>;
+
+/// Sidecar monitor.
+/// It monitors the communication of all sidecars, finds and tries to recover the dropped sidecar.
+pub(crate) struct SidecarMonitor {
+    /// Map for each sidecar clusters and their status
+    statuses: HashMap<String, SidecarClusterOwned<HeartbeatStatus>>,
     /// Unreachable cache
-    unreachable: HashMap<String, usize>,
-    /// Unreachable counter threshold
-    unreachable_thresh: usize,
+    unreachable: HashMap<String, SidecarClusterOwned<usize>>,
+    /// Context for sidecar monitor
+    ctx: Context,
 }
 
 impl SidecarMonitor {
@@ -35,22 +46,25 @@ impl SidecarMonitor {
     pub(crate) fn new(
         status_rx: Receiver<HeartbeatStatus>,
         heartbeat_period: u64,
+        unreachable_thresh: usize,
         cluster_api: Api<Cluster>,
         pod_api: Api<Pod>,
-        unreachable_thresh: usize,
     ) -> Self {
         Self {
             statuses: HashMap::new(),
-            status_rx,
-            heartbeat_period,
-            cluster_api,
-            pod_api,
             unreachable: HashMap::new(),
-            unreachable_thresh,
+            ctx: Context {
+                status_rx,
+                heartbeat_period,
+                unreachable_thresh,
+                cluster_api,
+                pod_api,
+            },
         }
     }
 
     /// Run the state update task with graceful shutdown.
+    /// Return fatal error if run failed.
     /// The task that update the state received from sidecar operators
     #[allow(clippy::integer_arithmetic)] // required by tokio::select
     pub(crate) async fn run_with_graceful_shutdown(
@@ -61,106 +75,97 @@ impl SidecarMonitor {
             _ = graceful_shutdown => {
                 Ok(())
             }
-            res = self.state_update_inner() => {
+            res = self.state_update() => {
                 res
             }
         }
     }
 
     /// Inner task for state update, return the unrecoverable error
-    async fn state_update_inner(mut self) -> Result<()> {
+    async fn state_update(mut self) -> Result<()> {
         loop {
-            let status = self.status_rx.recv_async().await?;
+            let status = self.ctx.status_rx.recv_async().await?;
             debug!("received status: {status:?}");
-            let _prev = self.statuses.insert(status.name.clone(), status);
+            self.state_update_inner(status).await;
+        }
+    }
 
-            let spec_size = match self.get_spec_size().await {
-                Ok(spec_size) => spec_size,
-                Err(err) => {
-                    error!("get cluster size failed, error: {err}");
-                    continue;
-                }
+    /// State update inner
+    async fn state_update_inner(&mut self, status: HeartbeatStatus) {
+        let spec_size = match self.get_spec_size(&status.cluster_name).await {
+            Ok(spec_size) => spec_size,
+            Err(err) => {
+                error!("get cluster size failed, error: {err}");
+                return;
+            }
+        };
+        let majority = (spec_size / 2).overflow_add(1);
+
+        debug!(
+            "cluster: {}, spec.size: {spec_size}, majority: {majority}",
+            status.cluster_name
+        );
+
+        let cluster_unreachable = self
+            .unreachable
+            .entry(status.cluster_name.clone())
+            .or_default();
+        let cluster_status = self
+            .statuses
+            .entry(status.cluster_name.clone())
+            .or_default();
+        let _prev = cluster_status.insert(status.name.clone(), status);
+
+        let reachable: HashMap<_, _> =
+            match get_reachable_counts(cluster_status, self.ctx.heartbeat_period, majority) {
+                None => return,
+                Some(counts) => cluster_status
+                    .keys()
+                    .map(|name| (name.clone(), counts.get(name).copied().unwrap_or(0)))
+                    .collect(),
             };
-            let majority = (spec_size / 2).overflow_add(1);
-            debug!("spec.size: {spec_size}, majority: {majority}");
 
-            let Some(reachable_counts) =
-                    Self::get_reachable_counts(&self.statuses, self.heartbeat_period, majority)
-                else {
-                    continue;
-                };
-            debug!("reachable_counts: {reachable_counts:?}");
-
-            for name in self.statuses.keys() {
-                let count = reachable_counts.get(name).copied().unwrap_or(0);
-                // The sidecar operator is considered offline
-                if count < majority {
-                    // If already in unreachable cache, increment the counter.
-                    // We would consider the recovery is failed if the counter reach
-                    // the threshold.
-                    if let Some(cnt) = self.unreachable.get_mut(name) {
-                        *cnt = cnt.overflow_add(1);
-                        if *cnt == self.unreachable_thresh {
-                            error!("failed to recover the operator: {name}");
-                            let _ignore = self.unreachable.remove(name);
-                            // TODO: notify the administrator
-                        }
+        for (name, count) in reachable {
+            // The sidecar operator is considered offline
+            if count < majority {
+                // If already in unreachable cache, increment the counter.
+                // We would consider the recovery is failed if the counter reach
+                // the threshold.
+                if let Some(cnt) = cluster_unreachable.get_mut(&name) {
+                    *cnt = cnt.overflow_add(1);
+                    if *cnt == self.ctx.unreachable_thresh {
+                        error!("failed to recover the operator: {name}");
+                        let _ignore = cluster_unreachable.remove(&name);
+                        let _ig = cluster_status.remove(&name);
+                        // TODO: notify the administrator
                     }
-                    // Otherwise delete the pod, which will trigger k8s to recreate it
-                    else {
-                        debug!("{name} is unreachable, count: {count}, deleting the pod");
-
-                        if let Err(e) = self.pod_api.delete(name, &DeleteParams::default()).await {
-                            error!("failed to delete pod {name}, {e}");
-                        }
-                        let _ignore = self.unreachable.insert(name.clone(), 0);
-                    }
-                // If recovered, remove it from the cache
-                } else if self.unreachable.remove(name).is_some() {
-                    debug!("operator {name} recovered");
-                } else {
-                    debug!("operator {name} online");
+                    continue;
                 }
+                // Otherwise delete the pod, which will trigger k8s to recreate it
+                debug!("{name} is unreachable, count: {count}, deleting the pod");
+                if let Err(e) = self
+                    .ctx
+                    .pod_api
+                    .delete(&name, &DeleteParams::default())
+                    .await
+                {
+                    error!("failed to delete pod {name}, {e}");
+                }
+                let _ignore = cluster_unreachable.insert(name, 0);
+                continue;
+            }
+            // If recovered, remove it from the cache
+            if cluster_unreachable.remove(&name).is_some() {
+                debug!("sidecar {name} recovered");
+            } else {
+                debug!("sidecar {name} online");
             }
         }
     }
 
-    /// Get the count for each reachable sidecar name
-    fn get_reachable_counts(
-        statuses: &HashMap<String, HeartbeatStatus>,
-        heartbeat_period: u64,
-        majority: usize,
-    ) -> Option<HashMap<String, usize>> {
-        let latest_ts = statuses
-            .values()
-            .max()
-            .unwrap_or_else(|| unreachable!("there should be at least one status"))
-            .timestamp;
-
-        // Take timestamps that within the period from the latest
-        let accepted = statuses
-            .values()
-            .filter(|s| s.timestamp.overflow_add(heartbeat_period) >= latest_ts);
-
-        // The current accepted status is less than half
-        if accepted.clone().count() < majority {
-            return None;
-        }
-
-        Some(
-            accepted
-                .flat_map(|s| s.reachable.iter())
-                .fold(HashMap::new(), |mut map, name| {
-                    let v = map.entry(name.clone()).or_insert(0);
-                    *v = v.overflow_add(1);
-                    map
-                }),
-        )
-    }
-
-    /// Get the cluster size in the specification
-    async fn get_spec_size(&self) -> Result<usize> {
-        let cluster = self.cluster_api.get(Cluster::crd_name()).await?;
+    /// Get the certain cluster size in the specification
+    async fn get_spec_size(&self, cluster_name: &str) -> Result<usize> {
+        let cluster = self.ctx.cluster_api.get(cluster_name).await?;
         Ok(cluster
             .spec
             .size
@@ -169,6 +174,45 @@ impl SidecarMonitor {
     }
 }
 
+/// Get reachable counts for a sidecar cluster
+fn get_reachable_counts(
+    statuses: &SidecarClusterOwned<HeartbeatStatus>,
+    heartbeat_period: u64,
+    majority: usize,
+) -> Option<HashMap<String, usize>> {
+    let latest_ts = statuses.values().max()?.timestamp;
+
+    let my_ts = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap_or_else(|_| unreachable!("time turns back!"))
+        .as_secs();
+
+    if my_ts.abs_diff(latest_ts) > heartbeat_period {
+        debug!("serious clock skew, sidecar latest timestamp: {latest_ts}, operator latest timestamp: {my_ts}");
+        return None;
+    }
+
+    // Take timestamps that within the period from the latest
+    let accepted = statuses
+        .values()
+        .filter(|s| s.timestamp.overflow_add(heartbeat_period) >= latest_ts);
+
+    // The current accepted status is less than half
+    if accepted.clone().count() < majority {
+        return None;
+    }
+
+    let counts = accepted
+        .flat_map(|s| s.reachable.iter())
+        .fold(HashMap::new(), |mut map, name| {
+            let v = map.entry(name.clone()).or_insert(0);
+            *v = v.overflow_add(1);
+            map
+        });
+
+    Some(counts)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -180,40 +224,103 @@ mod test {
         let id0 = "o0".to_owned();
         let id1 = "o1".to_owned();
         let id2 = "o2".to_owned();
+        let cur_ts = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap_or_else(|_| unreachable!("time turns back!"))
+            .as_secs();
 
         let statuses1 = [
-            (id0.clone(), HeartbeatStatus::new(id0.clone(), 0, vec![])),
-            (id1.clone(), HeartbeatStatus::new(id1.clone(), 1, vec![])),
-            (id2.clone(), HeartbeatStatus::new(id1.clone(), 10, vec![])),
+            (
+                id0.clone(),
+                HeartbeatStatus::new("c1".to_owned(), id0.clone(), cur_ts, vec![]),
+            ),
+            (
+                id1.clone(),
+                HeartbeatStatus::new("c1".to_owned(), id1.clone(), cur_ts + 1, vec![]),
+            ),
+            (
+                id2.clone(),
+                HeartbeatStatus::new("c1".to_owned(), id1.clone(), cur_ts + 10, vec![]),
+            ),
         ];
 
         assert!(
-            SidecarMonitor::get_reachable_counts(&statuses1.into(), heartbeat_period, majority)
-                .is_none(),
+            get_reachable_counts(&statuses1.into(), heartbeat_period, majority).is_none(),
             "the reachable status should not be accepted"
         );
 
         let statuses0 = [
             (
                 id0.clone(),
-                HeartbeatStatus::new(id0.clone(), 0, vec![id0.clone(), id1.clone(), id2.clone()]),
+                HeartbeatStatus::new(
+                    "c1".to_owned(),
+                    id0.clone(),
+                    cur_ts - 10,
+                    vec![id0.clone(), id1.clone(), id2.clone()],
+                ),
             ),
             (
                 id1.clone(),
-                HeartbeatStatus::new(id1.clone(), 10, vec![id1.clone(), id2.clone()]),
+                HeartbeatStatus::new(
+                    "c1".to_owned(),
+                    id1.clone(),
+                    cur_ts,
+                    vec![id1.clone(), id2.clone()],
+                ),
             ),
             (
                 id2.clone(),
-                HeartbeatStatus::new(id1.clone(), 11, vec![id2.clone(), id0.clone(), id1.clone()]),
+                HeartbeatStatus::new(
+                    "c1".to_owned(),
+                    id1.clone(),
+                    cur_ts + 1,
+                    vec![id2.clone(), id0.clone(), id1.clone()],
+                ),
             ),
         ];
 
-        let counts =
-            SidecarMonitor::get_reachable_counts(&statuses0.into(), heartbeat_period, majority)
-                .expect("the status not accepted");
+        let counts = get_reachable_counts(&statuses0.into(), heartbeat_period, majority)
+            .expect("the status not accepted");
 
         assert_eq!(counts[&id0], 1);
         assert_eq!(counts[&id1], 2);
         assert_eq!(counts[&id2], 2);
     }
+
+    #[test]
+    fn serious_clock_skew_should_be_detected() {
+        let id0 = "o0".to_owned();
+        let id1 = "o1".to_owned();
+        let id2 = "o2".to_owned();
+
+        let statuses = [
+            (
+                id0.clone(),
+                HeartbeatStatus::new(
+                    "c1".to_owned(),
+                    id0.clone(),
+                    10,
+                    vec![id0.clone(), id1.clone()],
+                ),
+            ),
+            (
+                id1.clone(),
+                HeartbeatStatus::new(
+                    "c1".to_owned(),
+                    id1.clone(),
+                    10,
+                    vec![id0.clone(), id1.clone()],
+                ),
+            ),
+            (
+                id2.clone(),
+                HeartbeatStatus::new("c1".to_owned(), id1.clone(), 10, vec![id0, id2, id1]),
+            ),
+        ];
+
+        assert!(
+            get_reachable_counts(&statuses.into(), 20, 2).is_none(),
+            "Did you test it on the original computer?"
+        );
+    }
 }
diff --git a/operator-k8s/src/operator.rs b/operator-k8s/src/operator.rs
index 9525c030..f084e881 100644
--- a/operator-k8s/src/operator.rs
+++ b/operator-k8s/src/operator.rs
@@ -1,5 +1,4 @@
 use std::sync::Arc;
-use std::time::Duration;
 
 use anyhow::Result;
 use axum::routing::any;
@@ -7,27 +6,19 @@ use axum::routing::post;
 use axum::{Extension, Router};
 use futures::FutureExt;
 use k8s_openapi::api::core::v1::Pod;
-use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
-use kube::api::{DynamicObject, ListParams, Patch, PatchParams};
-use kube::core::crd::merge_crds;
-use kube::runtime::wait::{await_condition, conditions};
-use kube::{Api, Client, CustomResourceExt, Resource};
+use kube::{Api, Client};
+use operator_api::consts::OPERATOR_MONITOR_ROUTE;
 use operator_api::HeartbeatStatus;
 use prometheus::Registry;
 use tokio::sync::watch::{Receiver, Sender};
-use tracing::{debug, error, info, warn};
+use tracing::{error, info, warn};
 
 use crate::config::{Config, Namespace};
-use crate::consts::FIELD_MANAGER;
 use crate::controller::cluster::{ClusterController, ClusterMetrics};
 use crate::controller::{Controller, Metrics};
-use crate::crd::version::ApiVersion;
 use crate::crd::Cluster;
 use crate::monitor::SidecarMonitor;
-use crate::router::{healthz, metrics, sidecar_state};
-
-/// wait crd to establish timeout
-const CRD_ESTABLISH_TIMEOUT: Duration = Duration::from_secs(20);
+use crate::router::{healthz, metrics, sidecar_monitor};
 
 /// Xline Operator for k8s
 #[derive(Debug)]
@@ -52,7 +43,12 @@ impl Operator {
     #[inline]
     pub async fn run(&self) -> Result<()> {
         let kube_client: Client = Client::try_default().await?;
-        self.prepare_crd(&kube_client).await?;
+        crate::crd::setup(
+            &kube_client,
+            self.config.manage_crd,
+            self.config.auto_migration,
+        )
+        .await?;
         let (cluster_api, pod_api): (Api<Cluster>, Api<Pod>) = match self.config.namespace {
             Namespace::Single(ref namespace) => (
                 Api::namespaced(kube_client.clone(), namespace.as_str()),
@@ -160,9 +156,9 @@ impl Operator {
         let monitor = SidecarMonitor::new(
             status_rx,
             self.config.heartbeat_period,
+            self.config.unreachable_thresh,
             cluster_api,
             pod_api,
-            self.config.unreachable_thresh,
         );
 
         let _ig = tokio::spawn(async move {
@@ -185,7 +181,7 @@ impl Operator {
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
         let status = Router::new()
-            .route("/status", post(sidecar_state))
+            .route(OPERATOR_MONITOR_ROUTE, post(sidecar_monitor))
             .route("/metrics", any(metrics))
             .route("/healthz", any(healthz))
             .layer(Extension(status_tx))
@@ -206,154 +202,4 @@ impl Operator {
 
         Ok(())
     }
-
-    /// Wait for CRD to be established
-    async fn wait_crd_established(
-        crd_api: Api<CustomResourceDefinition>,
-        crd_name: &str,
-    ) -> Result<()> {
-        let establish = await_condition(crd_api, crd_name, conditions::is_crd_established());
-        debug!("wait for crd established");
-        _ = tokio::time::timeout(CRD_ESTABLISH_TIMEOUT, establish).await??;
-        Ok(())
-    }
-
-    /// Prepare CRD
-    /// This method attempts to initialize the CRD if it does not already exist.
-    /// Additionally, it could migrate CRD with the version of `CURRENT_VERSION`.
-    async fn prepare_crd(&self, kube_client: &Client) -> Result<()> {
-        let crd_api: Api<CustomResourceDefinition> = Api::all(kube_client.clone());
-        let definition = Cluster::crd();
-        let current_version: ApiVersion<Cluster> = Cluster::version(&()).as_ref().parse()?;
-
-        let ret = crd_api.get(Cluster::crd_name()).await;
-        if let Err(kube::Error::Api(kube::error::ErrorResponse { code: 404, .. })) = ret {
-            if !self.config.create_crd {
-                return Err(anyhow::anyhow!(
-                    "cannot found XlineCluster CRD, please set --create-crd to true or apply the CRD manually"
-                ));
-            }
-            // the following code needs `customresourcedefinitions` write permission
-            debug!("cannot found XlineCluster CRD, try to init it");
-            _ = crd_api
-                .patch(
-                    Cluster::crd_name(),
-                    &PatchParams::apply(FIELD_MANAGER),
-                    &Patch::Apply(definition.clone()),
-                )
-                .await?;
-            Self::wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
-            return Ok(());
-        }
-
-        debug!("found XlineCluster CRD, current version: {current_version}");
-
-        let mut add = true;
-        let mut storage = String::new();
-
-        let mut crds = ret?
-            .spec
-            .versions
-            .iter()
-            .cloned()
-            .map(|ver| {
-                let mut crd = definition.clone();
-                if ver.name == current_version.to_string() {
-                    add = false;
-                }
-                if ver.storage {
-                    storage = ver.name.clone();
-                }
-                crd.spec.versions = vec![ver];
-                crd
-            })
-            .collect::<Vec<_>>();
-
-        if add {
-            crds.push(definition.clone());
-        } else {
-            debug!("current version already exists, try to migrate");
-            self.try_migration(kube_client, crds, &current_version, &storage)
-                .await?;
-            return Ok(());
-        }
-
-        if !self.config.create_crd {
-            return Err(anyhow::anyhow!(
-                "cannot found XlineCluster CRD with version {current_version}, please set --create-crd to true or apply the CRD manually"
-            ));
-        }
-
-        let merged_crd = merge_crds(crds.clone(), &storage)?;
-        debug!("try to update crd");
-        _ = crd_api
-            .patch(
-                Cluster::crd_name(),
-                &PatchParams::apply(FIELD_MANAGER),
-                &Patch::Apply(merged_crd),
-            )
-            .await?;
-        Self::wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
-
-        debug!("crd updated, try to migrate");
-        self.try_migration(kube_client, crds, &current_version, &storage)
-            .await?;
-
-        Ok(())
-    }
-
-    /// Try to migrate CRD
-    #[allow(clippy::indexing_slicing)] // there is at least one element in `versions`
-    #[allow(clippy::expect_used)]
-    async fn try_migration(
-        &self,
-        kube_client: &Client,
-        crds: Vec<CustomResourceDefinition>,
-        current_version: &ApiVersion<Cluster>,
-        storage: &str,
-    ) -> Result<()> {
-        if !self.config.auto_migration {
-            debug!("auto migration is disabled, skip migration");
-            return Ok(());
-        }
-        if current_version.to_string() == storage {
-            // stop migration if current version is already in storage
-            debug!("current version is already in storage, skip migration");
-            return Ok(());
-        }
-        let versions: Vec<ApiVersion<Cluster>> = crds
-            .iter()
-            .map(|crd| crd.spec.versions[0].name.parse())
-            .collect::<Result<_>>()?;
-        if versions.iter().any(|ver| current_version < ver) {
-            // stop migration if current version is less than any version in `versions`
-            debug!("current version is less than some version in crd, skip migration");
-            return Ok(());
-        }
-        let group = kube::discovery::group(kube_client, Cluster::group(&()).as_ref()).await?;
-        let Some((ar, _)) = group
-            .versioned_resources(storage)
-            .into_iter()
-            .find(|res| res.0.kind == Cluster::kind(&())) else { return Ok(()) };
-        let api: Api<DynamicObject> = Api::all_with(kube_client.clone(), &ar);
-        let clusters = api.list(&ListParams::default()).await?.items;
-        if !clusters.is_empty() && !current_version.compat_with(&storage.parse()?) {
-            // there is some clusters with storage version and is not compat with current version, stop migration
-            // TODO add a flag to these clusters to indicate that they need to be migrated
-            return Ok(());
-        }
-        // start migration as there is no cluster with storage version
-        let merged_crd = merge_crds(crds, &current_version.to_string())?;
-        let crd_api: Api<CustomResourceDefinition> = Api::all(kube_client.clone());
-        debug!("try to migrate crd from {storage} to {current_version}");
-        _ = crd_api
-            .patch(
-                Cluster::crd_name(),
-                &PatchParams::apply(FIELD_MANAGER),
-                &Patch::Apply(merged_crd),
-            )
-            .await?;
-        Self::wait_crd_established(crd_api.clone(), Cluster::crd_name()).await?;
-        Ok(())
-    }
 }
diff --git a/operator-k8s/src/router.rs b/operator-k8s/src/router.rs
index 41436a56..3887e186 100644
--- a/operator-k8s/src/router.rs
+++ b/operator-k8s/src/router.rs
@@ -30,9 +30,9 @@ pub(crate) async fn healthz() -> &'static str {
     "healthy"
 }
 
-/// sidecar state handler
+/// sidecar monitor handler
 #[allow(clippy::unused_async)] // require by axum
-pub(crate) async fn sidecar_state(
+pub(crate) async fn sidecar_monitor(
     Extension(status_tx): Extension<Sender<HeartbeatStatus>>,
     Json(status): Json<HeartbeatStatus>,
 ) {

From 21eca5639e35200f784e2f6d66fdefd8039f896b Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Wed, 25 Oct 2023 17:45:13 +0800
Subject: [PATCH 10/19] feat: implement heartbeat in sidecar

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                              |   3 +-
 operator-api/Cargo.toml                 |   2 +
 operator-api/src/consts.rs              |   4 +
 operator-api/src/lib.rs                 |  88 ++++++++--
 operator-api/src/xline.rs               |  72 ++++++--
 sidecar/src/lib.rs                      |   6 +-
 sidecar/src/main.rs                     | 215 +++++++++++++-----------
 sidecar/src/{operator.rs => sidecar.rs} |  77 +++++++--
 sidecar/src/types.rs                    |  58 ++++---
 sidecar/src/xline.rs                    |   2 +-
 10 files changed, 359 insertions(+), 168 deletions(-)
 rename sidecar/src/{operator.rs => sidecar.rs} (69%)

diff --git a/Cargo.lock b/Cargo.lock
index 4b490f4a..02609ce9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2156,8 +2156,10 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "futures",
  "k8s-openapi",
  "kube",
+ "reqwest",
  "serde",
 ]
 
@@ -4111,7 +4113,6 @@ dependencies = [
  "engine",
  "futures",
  "operator-api",
- "reqwest",
  "serde",
  "serde_json",
  "thiserror",
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index cb3cd76c..116a171c 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -16,3 +16,5 @@ async-trait = "0.1.72"
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive", "ws"] }
 serde = { version = "1.0.130", features = ["derive"] }
+reqwest = { version = "0.11", features = ["json"] }
+futures = "0.3.28"
diff --git a/operator-api/src/consts.rs b/operator-api/src/consts.rs
index f56985ad..161cb8f8 100644
--- a/operator-api/src/consts.rs
+++ b/operator-api/src/consts.rs
@@ -6,3 +6,7 @@ pub const DEFAULT_DATA_DIR: &str = "/usr/local/xline/data-dir";
 pub const OPERATOR_MONITOR_ROUTE: &str = "/monitor";
 /// the URL ROUTE of each sidecar for backup
 pub const SIDECAR_BACKUP_ROUTE: &str = "/backup";
+/// the URL ROUTE of each sidecar member for health checking
+pub const SIDECAR_HEALTH_ROUTE: &str = "/health";
+/// the URL ROUTE of each sidecar member for getting states
+pub const SIDECAR_STATE_ROUTE: &str = "/state";
diff --git a/operator-api/src/lib.rs b/operator-api/src/lib.rs
index fd1728c7..c8f53856 100644
--- a/operator-api/src/lib.rs
+++ b/operator-api/src/lib.rs
@@ -4,11 +4,20 @@ pub mod consts;
 /// Xline handle
 mod xline;
 
-use std::time::{SystemTime, UNIX_EPOCH};
-pub use xline::{K8sXlineHandle, LocalXlineHandle, XlineHandle};
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
+use reqwest::StatusCode;
+use std::collections::HashMap;
+use std::sync::OnceLock;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+pub use xline::*;
 
 use serde::{Deserialize, Serialize};
 
+/// Heartbeat http client
+static HEARTBEAT_CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+
 /// Heartbeat status, sort by timestamp.
 /// The clock of each machine may be different, which may cause heartbeat to be unable to assist
 /// the operator in detecting the dropped sidecar.
@@ -44,26 +53,83 @@ impl Ord for HeartbeatStatus {
 }
 
 impl HeartbeatStatus {
-    /// Create a new `HeartbeatStatus` with current timestamp
-    pub fn current(cluster_name: String, name: String, reachable: Vec<String>) -> Self {
-        let dur = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .unwrap_or_else(|_| unreachable!("time turns back!"));
+    const DEFAULT_HEALTH_CHECK_TIMEOUT: Duration = Duration::from_secs(10);
+
+    /// Create a new `HeartbeatStatus`
+    pub fn new(cluster_name: String, name: String, timestamp: u64, reachable: Vec<String>) -> Self {
         Self {
             cluster_name,
             name,
-            timestamp: dur.as_secs(),
+            timestamp,
             reachable,
         }
     }
 
-    /// Create a new `HeartbeatStatus`
-    pub fn new(cluster_name: String, name: String, timestamp: u64, reachable: Vec<String>) -> Self {
+    /// Create a new `HeartbeatStatus` from gathered information
+    pub async fn gather(
+        cluster_name: String,
+        name: String,
+        sidecars: &HashMap<String, String>,
+    ) -> Self {
+        use consts::SIDECAR_HEALTH_ROUTE;
+
+        let client = HEARTBEAT_CLIENT.get_or_init(|| {
+            reqwest::Client::builder()
+                .timeout(Self::DEFAULT_HEALTH_CHECK_TIMEOUT)
+                .build()
+                .unwrap_or_else(|err| unreachable!("http client build error {err}"))
+        });
+
+        let mut reachable: Vec<_> = sidecars
+            .iter()
+            .map(|(name, addr)| async move {
+                (
+                    name,
+                    client
+                        .get(format!("http://{addr}{SIDECAR_HEALTH_ROUTE}"))
+                        .send()
+                        .await,
+                )
+            })
+            .collect::<FuturesUnordered<_>>()
+            .filter_map(|(name, resp)| async {
+                resp.is_ok_and(|r| r.status() == StatusCode::OK)
+                    .then(|| name.clone())
+            })
+            .collect()
+            .await;
+
+        // make sure self name should be inside
+        if !reachable.contains(&name) {
+            reachable.push(name.clone());
+        }
+
+        let ts = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_else(|_| unreachable!("time turns back!"));
         Self {
             cluster_name,
             name,
-            timestamp,
+            timestamp: ts.as_secs(),
             reachable,
         }
     }
+
+    /// Report status to monitor
+    pub async fn report(&self, monitor_addr: &str) -> anyhow::Result<()> {
+        use consts::OPERATOR_MONITOR_ROUTE;
+
+        let url = format!("http://{monitor_addr}{OPERATOR_MONITOR_ROUTE}");
+
+        let client = HEARTBEAT_CLIENT.get_or_init(|| {
+            reqwest::Client::builder()
+                .timeout(Self::DEFAULT_HEALTH_CHECK_TIMEOUT)
+                .build()
+                .unwrap_or_else(|err| unreachable!("http client build error {err}"))
+        });
+
+        let _ig = client.post(url).json(self).send().await?;
+
+        Ok(())
+    }
 }
diff --git a/operator-api/src/xline.rs b/operator-api/src/xline.rs
index b740f760..4bc633d7 100644
--- a/operator-api/src/xline.rs
+++ b/operator-api/src/xline.rs
@@ -2,15 +2,53 @@ use async_trait::async_trait;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::{AttachParams, AttachedProcess};
 use kube::Api;
+use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 
 use std::process::{Child, Command};
 
+#[derive(Debug, Clone)]
+pub struct XlineConfig {
+    pub name: String,
+    pub executable: String,
+    pub storage_engine: String,
+    pub data_dir: String,
+    pub is_leader: bool,
+    pub additional: Option<String>,
+}
+
+impl XlineConfig {
+    fn gen_start_cmd(&self, members: &HashMap<String, String>) -> String {
+        let mut start_cmd = format!(
+            "{} --name {} --members {} --storage-engine {} --data-dir {}",
+            self.executable,
+            self.name,
+            members
+                .iter()
+                .map(|(name, addr)| format!("{name}={addr}"))
+                .collect::<Vec<_>>()
+                .join(","),
+            self.storage_engine,
+            self.data_dir,
+        );
+        if self.is_leader {
+            start_cmd.push(' ');
+            start_cmd.push_str("--is-leader");
+        }
+        if let Some(additional) = &self.additional {
+            start_cmd.push(' ');
+            let pat: &[_] = &['\'', '"'];
+            start_cmd.push_str(additional.trim_matches(pat));
+        }
+        start_cmd
+    }
+}
+
 /// xline handle abstraction
 #[async_trait]
 pub trait XlineHandle: Debug + Send + Sync + 'static {
     /// start a xline node
-    async fn start(&mut self) -> anyhow::Result<()>; // we dont care about what failure happened when start, it just failed
+    async fn start(&mut self, members: &HashMap<String, String>) -> anyhow::Result<()>; // we dont care about what failure happened when start, it just failed
 
     /// kill a xline node
     async fn kill(&mut self) -> anyhow::Result<()>;
@@ -20,15 +58,15 @@ pub trait XlineHandle: Debug + Send + Sync + 'static {
 /// machine with the start_cmd
 #[derive(Debug)]
 pub struct LocalXlineHandle {
-    start_cmd: String,
+    config: XlineConfig,
     child_proc: Option<Child>,
 }
 
 impl LocalXlineHandle {
     /// New a local xline handle
-    pub fn new(start_cmd: String) -> Self {
+    pub fn new(config: XlineConfig) -> Self {
         Self {
-            start_cmd,
+            config,
             child_proc: None,
         }
     }
@@ -36,9 +74,10 @@ impl LocalXlineHandle {
 
 #[async_trait]
 impl XlineHandle for LocalXlineHandle {
-    async fn start(&mut self) -> anyhow::Result<()> {
+    async fn start(&mut self, members: &HashMap<String, String>) -> anyhow::Result<()> {
         self.kill().await?;
-        let mut cmds = self.start_cmd.split_whitespace();
+        let cmd = self.config.gen_start_cmd(members);
+        let mut cmds = cmd.split_whitespace();
         let Some((exe, args)) = cmds
             .next()
             .map(|exe| (exe, cmds.collect::<Vec<_>>())) else {
@@ -68,8 +107,8 @@ pub struct K8sXlineHandle {
     pods_api: Api<Pod>,
     /// the attached process of xline
     process: Option<AttachedProcess>,
-    /// the xline start cmd, parameters are split by ' '
-    start_cmd: String,
+    /// the xline config
+    config: XlineConfig,
 }
 
 impl Debug for K8sXlineHandle {
@@ -78,7 +117,7 @@ impl Debug for K8sXlineHandle {
             .field("pod_name", &self.pod_name)
             .field("container_name", &self.container_name)
             .field("pods_api", &self.pods_api)
-            .field("start_cmd", &self.start_cmd)
+            .field("config", &self.config)
             .finish()
     }
 }
@@ -89,7 +128,7 @@ impl K8sXlineHandle {
         pod_name: String,
         container_name: String,
         namespace: &str,
-        start_cmd: String,
+        config: XlineConfig,
     ) -> Self {
         let client = kube::Client::try_default()
             .await
@@ -99,7 +138,7 @@ impl K8sXlineHandle {
             container_name,
             pods_api: Api::namespaced(client, namespace),
             process: None,
-            start_cmd,
+            config,
         }
     }
 
@@ -109,28 +148,29 @@ impl K8sXlineHandle {
         container_name: String,
         client: kube::Client,
         namespace: &str,
-        start_cmd: String,
+        config: XlineConfig,
     ) -> Self {
         Self {
             pod_name,
             container_name,
             pods_api: Api::namespaced(client, namespace),
             process: None,
-            start_cmd,
+            config,
         }
     }
 }
 
 #[async_trait]
 impl XlineHandle for K8sXlineHandle {
-    async fn start(&mut self) -> anyhow::Result<()> {
+    async fn start(&mut self, members: &HashMap<String, String>) -> anyhow::Result<()> {
         self.kill().await?;
-        let start_cmd: Vec<&str> = self.start_cmd.split_whitespace().collect();
+        let cmd = self.config.gen_start_cmd(members);
+        let cmds: Vec<&str> = cmd.split_whitespace().collect();
         let process = self
             .pods_api
             .exec(
                 &self.pod_name,
-                start_cmd,
+                cmds,
                 &AttachParams::default().container(&self.container_name),
             )
             .await?;
diff --git a/sidecar/src/lib.rs b/sidecar/src/lib.rs
index 775a2e55..6bca3d96 100644
--- a/sidecar/src/lib.rs
+++ b/sidecar/src/lib.rs
@@ -87,7 +87,7 @@
     clippy::rc_mutex,
     clippy::rest_pat_in_fully_bound_structs,
     clippy::same_name_method,
-    clippy::self_named_module_files,
+    // clippy::self_named_module_files, false positive
     // clippy::shadow_reuse, it’s a common pattern in Rust code
     // clippy::shadow_same, it’s a common pattern in Rust code
     clippy::shadow_unrelated,
@@ -152,10 +152,10 @@
 mod backup;
 /// Sidecar operator controller
 mod controller;
-/// Sidecar operator
-pub mod operator;
 /// The web server routers
 mod routers;
+/// Sidecar operator
+pub mod sidecar;
 /// Sidecar operator types
 pub mod types;
 /// Some utils
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index 85d1b082..86175ed3 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -86,7 +86,7 @@
     clippy::rc_mutex,
     clippy::rest_pat_in_fully_bound_structs,
     clippy::same_name_method,
-    clippy::self_named_module_files,
+    // clippy::self_named_module_files, false positive
     // clippy::shadow_reuse, it’s a common pattern in Rust code
     // clippy::shadow_same, it’s a common pattern in Rust code
     clippy::shadow_unrelated,
@@ -135,15 +135,23 @@
     clippy::multiple_crate_versions, // caused by the dependency, can't be fixed
 )]
 
+use std::borrow::ToOwned;
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::time::Duration;
 
 use anyhow::Result;
 use clap::Parser;
+use operator_api::consts::DEFAULT_DATA_DIR;
+use operator_api::XlineConfig;
 use tracing::debug;
+use xline_sidecar::sidecar::Sidecar;
+use xline_sidecar::types::{BackendConfig, BackupConfig, Config, MonitorConfig};
 
-use xline_sidecar::operator::Operator;
-use xline_sidecar::types::{Backend, Backup, Config};
+/// `DEFAULT_DATA_DIR` to String
+fn default_data_dir() -> String {
+    DEFAULT_DATA_DIR.to_owned()
+}
 
 /// Command line interface
 #[derive(Parser, Debug)]
@@ -152,85 +160,86 @@ struct Cli {
     /// The name of this sidecar, and is shared with xline node name
     #[arg(long)]
     name: String,
-    /// The host ip of each member, [node_name] -> [node_host]
+    /// The cluster name of this sidecar
+    #[arg(long)]
+    cluster_name: String,
+    /// The host of each member at initial, [node_name] -> [node_host]
+    /// Need to include at least the pair of this node
     #[arg(long, value_parser = parse_members)]
-    members: HashMap<String, String>,
+    init_members: HashMap<String, String>,
     /// The xline server port
     #[arg(long)]
     xline_port: u16,
-    /// Operator web server port
+    /// Sidecar web server port
     #[arg(long)]
-    operator_port: u16,
-    /// Check health interval, default 20 [unit: seconds]
+    sidecar_port: u16,
+    /// Reconcile cluster interval, default 20 [unit: seconds]
     #[arg(long, default_value = "20")]
-    check_interval: u64,
-    /// Enable backup, choose a storage type, e.g. s3:bucket_name or pv:/path/to/dir
-    #[arg(long, value_parser=parse_backup_type)]
-    backup: Option<Backup>,
-    /// The xline executable path, default "xline"
+    reconcile_interval: u64,
+    /// The sidecar backend, when you use different operators, the backend may different.
+    /// e.g:
+    ///   "k8s,pod=xline-pod-1,container=xline,namespace=default" for k8s backend
+    ///   "local" for local backend
+    #[arg(long, value_parser = parse_backend)]
+    backend: BackendConfig,
+    /// The xline executable path, default to "xline"
     #[arg(long, default_value = "xline")]
     xline_executable: String,
-    /// Storage engine used in xline
-    #[arg(long)]
-    storage_engine: String,
-    /// The directory path contains xline server data if the storage_engine is rocksdb
-    #[arg(long)]
-    data_dir: PathBuf,
-    /// Whether this node is leader or not
+    /// The xline storage engine, default to "rocksdb"
+    #[arg(long, default_value = "rocksdb")]
+    xline_storage_engine: String,
+    /// The xline data directory, default to "/usr/local/xline/data-dir"
+    #[arg(long, default_value_t = default_data_dir())]
+    xline_data_dir: String,
+    /// Set if this xline node is a leader node, default to false
     #[arg(long, default_value = "false")]
-    is_leader: bool,
-    /// Additional arguments, it will be appended behind the required parameters,
-    /// e.g "--jaeger_offline true"
+    xline_is_leader: bool,
+    /// The xline additional parameter
     #[arg(long)]
-    additional: Option<String>,
-    /// The sidecar backend, when you use different operators, the backend may different.
+    xline_additional: Option<String>,
+    /// Enable backup, choose a storage type.
     /// e.g:
-    ///   "k8s,pod=xline-pod-1,container=xline,namespace=default" for k8s backend
-    ///   "local" for local backend
-    #[arg(long, value_parser=parse_backend)]
-    backend: Backend,
+    ///    s3:bucket_name    for s3 (not available)
+    ///    pv:/path/to/dir   for pv
+    #[arg(long, value_parser = parse_backup_type)]
+    backup: Option<BackupConfig>,
+    /// Monitor(Operator) address, set to enable heartbeat and configuration discovery
+    #[arg(long, alias = "operator-addr")]
+    monitor_addr: Option<String>,
+    /// Heartbeat interval, it is enabled if --monitor_addr is set.
+    #[arg(long, alias = "operator_heartbeat_interval", default_value = "10")]
+    heartbeat_interval: u64,
 }
 
 impl From<Cli> for Config {
     fn from(value: Cli) -> Self {
-        let mut config = Self {
-            start_cmd: String::new(),
+        Self {
             name: value.name.clone(),
+            cluster_name: value.cluster_name,
+            init_members: value.init_members,
             xline_port: value.xline_port,
-            operator_port: value.operator_port,
-            check_interval: std::time::Duration::from_secs(value.check_interval),
-            backup: value.backup,
-            members: value.members,
+            sidecar_port: value.sidecar_port,
+            reconcile_interval: Duration::from_secs(value.reconcile_interval),
             backend: value.backend,
-        };
-        config.start_cmd = format!(
-            "{} --name {} --members {} --storage-engine {} --data-dir {}",
-            value.xline_executable,
-            value.name,
-            config
-                .xline_members()
-                .into_iter()
-                .map(|(name, addr)| format!("{name}={addr}"))
-                .collect::<Vec<_>>()
-                .join(","),
-            value.storage_engine,
-            value.data_dir.to_string_lossy(),
-        );
-        if value.is_leader {
-            config.start_cmd.push(' ');
-            config.start_cmd.push_str("--is-leader");
-        }
-        if let Some(additional) = value.additional {
-            config.start_cmd.push(' ');
-            let pat: &[_] = &['\'', '"'];
-            config.start_cmd.push_str(additional.trim_matches(pat));
+            xline: XlineConfig {
+                name: value.name, // xline server has a same name with sidecar
+                executable: value.xline_executable,
+                storage_engine: value.xline_storage_engine,
+                data_dir: value.xline_data_dir,
+                is_leader: value.xline_is_leader,
+                additional: value.xline_additional,
+            },
+            backup: value.backup,
+            monitor: value.monitor_addr.map(|addr| MonitorConfig {
+                monitor_addr: addr,
+                heartbeat_interval: Duration::from_secs(value.heartbeat_interval),
+            }),
         }
-        config
     }
 }
 
 /// parse backup type
-fn parse_backup_type(value: &str) -> Result<Backup, String> {
+fn parse_backup_type(value: &str) -> Result<BackupConfig, String> {
     if value.is_empty() {
         return Err("backup type is empty".to_owned());
     }
@@ -245,7 +254,7 @@ fn parse_backup_type(value: &str) -> Result<Backup, String> {
                 ));
             }
             let bucket = items.remove(0).to_owned();
-            Ok(Backup::S3 { bucket })
+            Ok(BackupConfig::S3 { bucket })
         }
         "pv" => {
             if items.len() != 1 {
@@ -255,7 +264,7 @@ fn parse_backup_type(value: &str) -> Result<Backup, String> {
                 ));
             }
             let path = items.remove(0).to_owned();
-            Ok(Backup::PV {
+            Ok(BackupConfig::PV {
                 path: PathBuf::from(path),
             })
         }
@@ -264,7 +273,7 @@ fn parse_backup_type(value: &str) -> Result<Backup, String> {
 }
 
 /// Parse backend
-fn parse_backend(value: &str) -> Result<Backend, String> {
+fn parse_backend(value: &str) -> Result<BackendConfig, String> {
     if value.is_empty() {
         return Err("backend is empty".to_owned());
     }
@@ -289,13 +298,13 @@ fn parse_backend(value: &str) -> Result<Backend, String> {
             if pod_name.is_empty() || container_name.is_empty() || namespace.is_empty() {
                 return Err("k8s backend must set 'pod', 'container', 'namespace'".to_owned());
             }
-            Ok(Backend::K8s {
+            Ok(BackendConfig::K8s {
                 pod_name,
                 container_name,
                 namespace,
             })
         }
-        "local" => Ok(Backend::Local),
+        "local" => Ok(BackendConfig::Local),
         _ => Err(format!("unknown backend: {backend}")),
     }
 }
@@ -303,8 +312,7 @@ fn parse_backend(value: &str) -> Result<Backend, String> {
 /// parse members from string
 /// # Errors
 /// Return error when pass wrong args
-#[inline]
-pub fn parse_members(s: &str) -> Result<HashMap<String, String>, String> {
+fn parse_members(s: &str) -> Result<HashMap<String, String>, String> {
     let mut map = HashMap::new();
     for pair in s.split(',') {
         if let Some((id, addr)) = pair.split_once('=') {
@@ -325,7 +333,7 @@ async fn main() -> Result<()> {
     let cli = Cli::parse();
     debug!("{:?}", cli);
 
-    Operator::new(cli.into()).run().await
+    Sidecar::new(cli.into()).run().await
 }
 
 #[cfg(test)]
@@ -334,23 +342,33 @@ mod test {
     use clap::Parser;
     use std::collections::HashMap;
     use std::path::PathBuf;
-    use xline_sidecar::types::{Backend, Backup, Config};
+    use xline_sidecar::types::{BackendConfig, BackupConfig};
 
     fn full_parameter() -> Vec<&'static str> {
         vec![
             "sidecar_exe",
             "--name=node1",
-            "--members=node1=127.0.0.1",
+            "--cluster-name=my-xline-cluster",
+
+            "--init-members=node1=127.0.0.1,node2=127.0.0.2,node3=127.0.0.3",
+
             "--xline-port=2379",
-            "--operator-port=2380",
-            "--check-interval=60",
-            "--backup=s3:bucket_name",
-            "--xline-executable=/usr/local/bin/xline",
-            "--storage-engine=rocksdb",
-            "--data-dir=/usr/local/xline/data-dir",
-            "--is-leader",
-            "--additional='--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'",
+            "--sidecar-port=2380",
+
+            "--reconcile-interval=20",
             "--backend=k8s,pod=xline-pod-1,container=xline,namespace=default",
+
+            "--xline-executable=/usr/local/bin/xline",
+            "--xline-storage-engine=rocksdb",
+            "--xline-data-dir=/usr/local/xline/data-dir",
+            "--xline-is-leader",
+            "--xline-additional='--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'",
+
+            "--backup=s3:bucket_name",
+
+            "--operator-addr=xline-operator.svc.default.cluster.local:8080",
+
+            "--heartbeat-interval=10",
         ]
     }
 
@@ -360,7 +378,7 @@ mod test {
             ("", Err("backup type is empty".to_owned())),
             (
                 "s3:bucket_name",
-                Ok(Backup::S3 {
+                Ok(BackupConfig::S3 {
                     bucket: "bucket_name".to_owned(),
                 }),
             ),
@@ -374,7 +392,7 @@ mod test {
             ),
             (
                 "pv:/home",
-                Ok(Backup::PV {
+                Ok(BackupConfig::PV {
                     path: PathBuf::from("/home"),
                 }),
             ),
@@ -401,13 +419,13 @@ mod test {
         let test_cases = [
             (
                 "k8s,pod=my-pod,container=my-container,namespace=my-namespace",
-                Ok(Backend::K8s {
+                Ok(BackendConfig::K8s {
                     pod_name: "my-pod".to_owned(),
                     container_name: "my-container".to_owned(),
                     namespace: "my-namespace".to_owned(),
                 }),
             ),
-            ("local", Ok(Backend::Local)),
+            ("local", Ok(BackendConfig::Local)),
             ("", Err("backend is empty".to_owned())),
             (
                 "k8s,pod=my-pod,invalid-arg,namespace=my-namespace",
@@ -472,24 +490,33 @@ mod test {
         let cli = Cli::parse_from(full_parameter());
         assert_eq!(cli.name, "node1");
         assert_eq!(
-            cli.members,
-            HashMap::from([("node1".to_owned(), "127.0.0.1".to_owned())])
+            cli.init_members,
+            HashMap::from([
+                ("node1".to_owned(), "127.0.0.1".to_owned()),
+                ("node2".to_owned(), "127.0.0.2".to_owned()),
+                ("node3".to_owned(), "127.0.0.3".to_owned()),
+            ])
         );
         assert_eq!(cli.xline_port, 2379);
-        assert_eq!(cli.operator_port, 2380);
-        assert_eq!(cli.check_interval, 60);
+        assert_eq!(cli.sidecar_port, 2380);
+        assert_eq!(
+            cli.monitor_addr.unwrap_or_default(),
+            "xline-operator.svc.default.cluster.local:8080"
+        );
+        assert_eq!(cli.reconcile_interval, 20);
+        assert_eq!(cli.heartbeat_interval, 10);
         assert_eq!(
             cli.backup,
-            Some(Backup::S3 {
+            Some(BackupConfig::S3 {
                 bucket: "bucket_name".to_owned(),
             })
         );
         assert_eq!(cli.xline_executable, "/usr/local/bin/xline");
-        assert_eq!(cli.storage_engine, "rocksdb");
-        assert_eq!(cli.data_dir.to_string_lossy(), "/usr/local/xline/data-dir");
-        assert!(cli.is_leader);
+        assert_eq!(cli.xline_storage_engine, "rocksdb");
+        assert_eq!(cli.xline_data_dir, "/usr/local/xline/data-dir");
+        assert!(cli.xline_is_leader);
         assert_eq!(
-            cli.additional,
+            cli.xline_additional,
             Some(
                 "'--auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem'"
                     .to_owned()
@@ -497,17 +524,11 @@ mod test {
         );
         assert_eq!(
             cli.backend,
-            Backend::K8s {
+            BackendConfig::K8s {
                 pod_name: "xline-pod-1".to_owned(),
                 container_name: "xline".to_owned(),
                 namespace: "default".to_owned(),
             }
         );
     }
-
-    #[test]
-    fn test_gen_start_cmd() {
-        let config: Config = Cli::parse_from(full_parameter()).into();
-        assert_eq!(config.start_cmd, "/usr/local/bin/xline --name node1 --members node1=127.0.0.1:2379 --storage-engine rocksdb --data-dir /usr/local/xline/data-dir --is-leader --auth-public-key /mnt/public.pem --auth-private-key /mnt/private.pem");
-    }
 }
diff --git a/sidecar/src/operator.rs b/sidecar/src/sidecar.rs
similarity index 69%
rename from sidecar/src/operator.rs
rename to sidecar/src/sidecar.rs
index 4d157e39..2cc08ba4 100644
--- a/sidecar/src/operator.rs
+++ b/sidecar/src/sidecar.rs
@@ -5,10 +5,11 @@ use anyhow::{anyhow, Result};
 use axum::routing::{get, post};
 use axum::{Extension, Router};
 use futures::{FutureExt, TryFutureExt};
-use operator_api::{K8sXlineHandle, LocalXlineHandle};
+use operator_api::consts::{SIDECAR_BACKUP_ROUTE, SIDECAR_HEALTH_ROUTE, SIDECAR_STATE_ROUTE};
+use operator_api::{HeartbeatStatus, K8sXlineHandle, LocalXlineHandle};
 use tokio::sync::watch::{Receiver, Sender};
 use tokio::sync::Mutex;
-use tokio::time::interval;
+use tokio::time::{interval, MissedTickBehavior};
 use tracing::{debug, error, info, warn};
 
 use crate::backup::pv::Pv;
@@ -16,17 +17,17 @@ use crate::backup::Provider;
 use crate::controller::Controller;
 use crate::controller::Error;
 use crate::routers;
-use crate::types::{Backend, Backup, Config, State, StatePayload};
+use crate::types::{BackendConfig, BackupConfig, Config, State, StatePayload};
 use crate::xline::XlineHandle;
 
-/// Sidecar operator
+/// Sidecar
 #[derive(Debug)]
-pub struct Operator {
+pub struct Sidecar {
     /// Operator config
     config: Config,
 }
 
-impl Operator {
+impl Sidecar {
     /// Constructor
     #[must_use]
     #[inline]
@@ -59,6 +60,7 @@ impl Operator {
             Arc::clone(&state),
             graceful_shutdown_event.subscribe(),
         )?;
+        self.start_heartbeat(graceful_shutdown_event.subscribe());
 
         tokio::pin!(forceful_shutdown);
 
@@ -83,8 +85,8 @@ impl Operator {
             .backup
             .as_ref()
             .and_then(|backup| match *backup {
-                Backup::S3 { .. } => None, // TODO S3 backup
-                Backup::PV { ref path } => {
+                BackupConfig::S3 { .. } => None, // TODO S3 backup
+                BackupConfig::PV { ref path } => {
                     let pv: Box<dyn Provider> = Box::new(Pv {
                         backup_path: path.clone(),
                     });
@@ -92,7 +94,7 @@ impl Operator {
                 }
             });
         let inner: Box<dyn operator_api::XlineHandle> = match self.config.backend.clone() {
-            Backend::K8s {
+            BackendConfig::K8s {
                 pod_name,
                 container_name,
                 namespace,
@@ -101,13 +103,13 @@ impl Operator {
                     pod_name,
                     container_name,
                     &namespace,
-                    self.config.start_cmd.clone(),
+                    self.config.xline.clone(),
                 )
                 .await;
                 Box::new(handle)
             }
-            Backend::Local => {
-                let handle = LocalXlineHandle::new(self.config.start_cmd.clone());
+            BackendConfig::Local => {
+                let handle = LocalXlineHandle::new(self.config.xline.clone());
                 Box::new(handle)
             }
         };
@@ -116,7 +118,7 @@ impl Operator {
             backup,
             inner,
             self.config.xline_port,
-            self.config.xline_members(),
+            self.config.init_xline_members(),
         )
     }
 
@@ -129,6 +131,45 @@ impl Operator {
         let _ctrl_c_c = tokio::signal::ctrl_c().await;
     }
 
+    /// Start heartbeat
+    fn start_heartbeat(&self, graceful_shutdown: Receiver<()>) {
+        let Some(monitor) = self.config.monitor.clone() else {
+            return;
+        };
+        let cluster_name = self.config.cluster_name.clone();
+        let name = self.config.name.clone();
+        let init_members = self.config.init_sidecar_members();
+
+        #[allow(clippy::integer_arithmetic)] // this error originates in the macro `tokio::select`
+        let _ig = tokio::spawn(async move {
+            let mut shutdown = graceful_shutdown;
+
+            let heartbeat_task = async move {
+                let mut tick = interval(monitor.heartbeat_interval);
+                // ensure a fixed heartbeat interval
+                tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
+                loop {
+                    let instant = tick.tick().await;
+                    debug!("send heartbeat at {instant:?}");
+                    let status =
+                        HeartbeatStatus::gather(cluster_name.clone(), name.clone(), &init_members)
+                            .await;
+                    debug!("sidecar gathered status: {status:?}");
+                    if let Err(e) = status.report(&monitor.monitor_addr).await {
+                        error!("heartbeat report failed, error {e}");
+                    }
+                }
+            };
+
+            tokio::select! {
+                _ = shutdown.changed() => {
+                    info!("heartbeat task graceful shutdown");
+                },
+                _ = heartbeat_task => {}
+            }
+        });
+    }
+
     /// Start controller
     fn start_controller(
         &self,
@@ -138,7 +179,7 @@ impl Operator {
     ) {
         let mut controller = Controller::new(
             handle,
-            interval(self.config.check_interval),
+            interval(self.config.reconcile_interval),
             state,
             graceful_shutdown,
         );
@@ -170,7 +211,7 @@ impl Operator {
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
-        let members = self.config.operator_members();
+        let members = self.config.init_sidecar_members();
         let advertise_url = members.get(&self.config.name).ok_or(anyhow!(
             "node name {} not found in members",
             self.config.name
@@ -180,9 +221,9 @@ impl Operator {
         ))?;
 
         let app = Router::new()
-            .route("/health", get(routers::health))
-            .route("/backup", get(routers::backup))
-            .route("/state", get(routers::state))
+            .route(SIDECAR_HEALTH_ROUTE, get(routers::health))
+            .route(SIDECAR_BACKUP_ROUTE, get(routers::backup))
+            .route(SIDECAR_STATE_ROUTE, get(routers::state))
             .route("/membership", post(routers::membership))
             .layer(Extension(handle))
             .layer(Extension(state));
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 44f2e162..cdf0940a 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
+use operator_api::XlineConfig;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -7,30 +8,45 @@ use std::time::Duration;
 
 /// Sidecar operator config
 #[derive(Debug, Clone)]
-#[allow(clippy::exhaustive_structs)] // it is exhaustive
+#[allow(clippy::exhaustive_structs)] // It is only constructed once
 pub struct Config {
     /// Name of this node
     pub name: String,
+    /// The cluster name
+    pub cluster_name: String,
+    /// Sidecar initial hosts, [pod_name]->[pod_host]
+    pub init_members: HashMap<String, String>,
     /// The xline server port
     pub xline_port: u16,
-    /// The operator web server port
-    pub operator_port: u16,
-    /// Check cluster health interval
-    pub check_interval: Duration,
+    /// The sidecar web server port
+    pub sidecar_port: u16,
+    /// Reconcile cluster interval
+    pub reconcile_interval: Duration,
+    /// The xline config
+    pub xline: XlineConfig,
+    /// The backend to run xline
+    pub backend: BackendConfig,
+    /// The sidecar monitor (operator) config, set to enable
+    /// heartbeat and configuration discovery
+    pub monitor: Option<MonitorConfig>,
     /// Backup storage config
-    pub backup: Option<Backup>,
-    /// Operators hosts, [pod_name]->[pod_host]
-    pub members: HashMap<String, String>,
-    /// The xline start cmd
-    pub start_cmd: String,
-    /// The backend
-    pub backend: Backend,
+    pub backup: Option<BackupConfig>,
+}
+
+/// Monitor(Operator) config
+#[derive(Debug, Clone)]
+#[allow(clippy::exhaustive_structs)] // It is only constructed once
+pub struct MonitorConfig {
+    /// Monitor address
+    pub monitor_addr: String,
+    /// heartbeat interval
+    pub heartbeat_interval: Duration,
 }
 
 /// Sidecar backend, it determinate how xline could be setup
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
-pub enum Backend {
+pub enum BackendConfig {
     /// K8s backend
     K8s {
         /// The pod name of this node
@@ -47,7 +63,7 @@ pub enum Backend {
 /// Backup storage config
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
-pub enum Backup {
+pub enum BackupConfig {
     /// S3 storage
     S3 {
         /// S3 bucket name
@@ -61,22 +77,22 @@ pub enum Backup {
 }
 
 impl Config {
-    /// Get the operator members
+    /// Get the initial sidecar members
     #[must_use]
     #[inline]
-    pub fn operator_members(&self) -> HashMap<String, String> {
-        self.members
+    pub fn init_sidecar_members(&self) -> HashMap<String, String> {
+        self.init_members
             .clone()
             .into_iter()
-            .map(|(name, host)| (name, format!("{host}:{}", self.operator_port)))
+            .map(|(name, host)| (name, format!("{host}:{}", self.sidecar_port)))
             .collect()
     }
 
-    /// Get the xline members
+    /// Get the initial xline members
     #[must_use]
     #[inline]
-    pub fn xline_members(&self) -> HashMap<String, String> {
-        self.members
+    pub fn init_xline_members(&self) -> HashMap<String, String> {
+        self.init_members
             .clone()
             .into_iter()
             .map(|(name, host)| (name, format!("{host}:{}", self.xline_port)))
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 3df3406f..bddc185a 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -125,7 +125,7 @@ impl XlineHandle {
         // the cluster is started if any of the connection is successful
         let cluster_started = futs.any(|res| async move { res.is_ok() }).await;
 
-        self.inner.start().await?;
+        self.inner.start(&self.xline_members).await?;
 
         let client = Client::connect(self.xline_members.values(), ClientOptions::default()).await?;
         let mut cluster_client = client.cluster_client();

From 865404770d4db31bd7774a27415425d14a144ceb Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Thu, 26 Oct 2023 10:22:26 +0800
Subject: [PATCH 11/19] feat: implement state gather

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock              |  1 +
 operator-api/Cargo.toml |  4 ++--
 sidecar/Cargo.toml      |  1 +
 sidecar/src/types.rs    | 42 ++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 02609ce9..dfac1ad6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4113,6 +4113,7 @@ dependencies = [
  "engine",
  "futures",
  "operator-api",
+ "reqwest",
  "serde",
  "serde_json",
  "thiserror",
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index 116a171c..f552553c 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -13,8 +13,8 @@ keywords = ["operator", "API", "operator"]
 [dependencies]
 anyhow = "1.0.72"
 async-trait = "0.1.72"
+futures = "0.3.28"
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive", "ws"] }
-serde = { version = "1.0.130", features = ["derive"] }
 reqwest = { version = "0.11", features = ["json"] }
-futures = "0.3.28"
+serde = { version = "1.0.130", features = ["derive"] }
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 2a8c8415..51ec75de 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -20,6 +20,7 @@ clap = { version = "4.3.4", features = ["derive"] }
 engine = { git = "https://github.com/xline-kv/Xline.git", package = "engine" }
 futures = "0.3.28"
 operator-api = { path = "../operator-api" }
+reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0.130", features = ["derive"] }
 serde_json = "1.0.97"
 thiserror = "1.0.40"
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index cdf0940a..5d7a3326 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -2,7 +2,9 @@
 
 use operator_api::XlineConfig;
 use serde::{Deserialize, Serialize};
+
 use std::collections::HashMap;
+use std::ops::AddAssign;
 use std::path::PathBuf;
 use std::time::Duration;
 
@@ -101,7 +103,7 @@ impl Config {
 }
 
 /// Sidecar operator state
-#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)]
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Hash)]
 pub(crate) enum State {
     /// When this operator is trying to start it's kvserver
     Start,
@@ -120,6 +122,44 @@ pub(crate) struct StatePayload {
     pub(crate) revision: i64,
 }
 
+/// The gathered states from sidecars
+#[derive(Debug, Clone)]
+pub(crate) struct StateStatus {
+    /// A sidecar with highest revision is considered as "seeder".
+    /// There could be more than one "seeder".
+    pub(crate) seeder: String,
+    /// State count, used to determine cluster status
+    pub(crate) states: HashMap<State, usize>,
+}
+
+impl StateStatus {
+    /// Gather status from sidecars
+    pub(crate) async fn gather(sidecars: &HashMap<String, String>) -> anyhow::Result<Self> {
+        use operator_api::consts::SIDECAR_STATE_ROUTE;
+
+        let mut seeder = "";
+        let mut max_rev = i64::MIN;
+        let mut states = HashMap::<State, usize>::new();
+
+        for (name, addr) in sidecars {
+            let url = format!("http://{addr}{SIDECAR_STATE_ROUTE}");
+            let state: StatePayload = reqwest::get(url).await?.json().await?;
+            if state.revision > max_rev {
+                max_rev = state.revision;
+                seeder = name;
+            }
+            let _ig = states
+                .entry(state.state)
+                .and_modify(|cnt| cnt.add_assign(1))
+                .or_default();
+        }
+        Ok(Self {
+            seeder: seeder.to_owned(),
+            states,
+        })
+    }
+}
+
 /// The membership change request sent by other sidecar operators when they are shutting down
 #[derive(Debug, Deserialize, Serialize)]
 pub(crate) struct MembershipChange {

From c025e6d52766a287dd55ab2bd005f5d87702c48c Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Thu, 26 Oct 2023 21:22:50 +0800
Subject: [PATCH 12/19] chore: refactor controller run reconcile

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                   |  1 -
 operator-k8s/src/operator.rs |  6 ++--
 sidecar/Cargo.toml           |  1 -
 sidecar/src/controller.rs    | 67 ++++++++++++++++++------------------
 sidecar/src/sidecar.rs       | 33 ++++++------------
 5 files changed, 47 insertions(+), 61 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dfac1ad6..f631711e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4116,7 +4116,6 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
- "thiserror",
  "tokio",
  "toml 0.8.2",
  "tonic",
diff --git a/operator-k8s/src/operator.rs b/operator-k8s/src/operator.rs
index f084e881..dedd0868 100644
--- a/operator-k8s/src/operator.rs
+++ b/operator-k8s/src/operator.rs
@@ -168,8 +168,9 @@ impl Operator {
                 .await;
             if let Err(err) = res {
                 error!("monitor run failed, error: {err}");
+            } else {
+                info!("sidecar monitor shutdown");
             }
-            info!("sidecar monitor shutdown");
         });
     }
 
@@ -196,8 +197,9 @@ impl Operator {
                 .await;
             if let Err(err) = res {
                 error!("web server starts failed, error: {err}");
+            } else {
+                info!("web server shut down");
             }
-            info!("web server shut down");
         });
 
         Ok(())
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 51ec75de..08a99161 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -23,7 +23,6 @@ operator-api = { path = "../operator-api" }
 reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0.130", features = ["derive"] }
 serde_json = "1.0.97"
-thiserror = "1.0.40"
 tokio = { version = "1.0", features = [
   "rt-multi-thread",
   "time",
diff --git a/sidecar/src/controller.rs b/sidecar/src/controller.rs
index 141f0c4f..abe583ce 100644
--- a/sidecar/src/controller.rs
+++ b/sidecar/src/controller.rs
@@ -1,11 +1,14 @@
 #![allow(dead_code)] // TODO remove when it is implemented
 
+use std::future::Future;
 use std::sync::Arc;
-use thiserror::Error;
+use std::time::Duration;
+
+use anyhow::Result;
 use tokio::select;
-use tokio::sync::watch::Receiver;
 use tokio::sync::Mutex;
-use tokio::time::{Instant, Interval};
+use tokio::time::{interval, MissedTickBehavior};
+use tracing::debug;
 
 use crate::types::StatePayload;
 use crate::xline::XlineHandle;
@@ -18,61 +21,57 @@ pub(crate) struct Controller {
     /// Xline handle
     handle: Arc<XlineHandle>,
     /// Check interval
-    check_interval: Interval,
-    /// graceful shutdown signal
-    graceful_shutdown: Receiver<()>,
-}
-
-/// All possible errors
-#[derive(Error, Debug, PartialEq)]
-pub(crate) enum Error {
-    /// Graceful shutdown error
-    #[error("operator has been shutdown")]
-    Shutdown,
+    reconcile_interval: Duration,
 }
 
-/// Controller result
-type Result<T> = std::result::Result<T, Error>;
-
 impl Controller {
     /// Constructor
     pub(crate) fn new(
-        handle: Arc<XlineHandle>,
-        check_interval: Interval,
         state: Arc<Mutex<StatePayload>>,
-        graceful_shutdown: Receiver<()>,
+        handle: Arc<XlineHandle>,
+        reconcile_interval: Duration,
     ) -> Self {
         Self {
             state,
             handle,
-            check_interval,
-            graceful_shutdown,
+            reconcile_interval,
         }
     }
 
-    /// Perform a reconciliation
+    /// Run reconcile loop with shutdown
     #[allow(clippy::integer_arithmetic)] // this error originates in the macro `tokio::select`
-    pub(crate) async fn reconcile_once(&mut self) -> Result<Instant> {
+    pub(crate) async fn run_reconcile_with_shutdown(
+        self,
+        graceful_shutdown: impl Future<Output = ()>,
+    ) -> Result<()> {
         select! {
-            _ = self.graceful_shutdown.changed() => {
-                // TODO notify the cluster of this node's shutdown
-                Err(Error::Shutdown)
+            _ = graceful_shutdown => {
+                Ok(())
             }
-            instant = self.check_interval.tick() => {
-                self.reconcile_inner().await.map(|_| instant)
+            res = self.run_reconcile() => {
+                res
             }
         }
     }
 
-    /// Reconciliation inner
-    async fn reconcile_inner(&mut self) -> Result<()> {
-        self.evaluate().await?;
-        self.execute().await
+    /// Run reconcile loop
+    pub(crate) async fn run_reconcile(self) -> Result<()> {
+        let mut tick = interval(self.reconcile_interval);
+        tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
+        loop {
+            let instant = tick.tick().await;
+            let _result = self.evaluate().await;
+            let _result1 = self.execute().await;
+            debug!(
+                "successfully reconcile the cluster states within {:?}",
+                instant.elapsed()
+            );
+        }
     }
 
     /// Evaluate cluster states
     #[allow(clippy::unused_async)] // TODO remove when it is implemented
-    async fn evaluate(&mut self) -> Result<()> {
+    async fn evaluate(&self) -> Result<()> {
         // TODO evaluate states
         Ok(())
     }
diff --git a/sidecar/src/sidecar.rs b/sidecar/src/sidecar.rs
index 2cc08ba4..575c8df5 100644
--- a/sidecar/src/sidecar.rs
+++ b/sidecar/src/sidecar.rs
@@ -15,7 +15,6 @@ use tracing::{debug, error, info, warn};
 use crate::backup::pv::Pv;
 use crate::backup::Provider;
 use crate::controller::Controller;
-use crate::controller::Error;
 use crate::routers;
 use crate::types::{BackendConfig, BackupConfig, Config, State, StatePayload};
 use crate::xline::XlineHandle;
@@ -134,6 +133,7 @@ impl Sidecar {
     /// Start heartbeat
     fn start_heartbeat(&self, graceful_shutdown: Receiver<()>) {
         let Some(monitor) = self.config.monitor.clone() else {
+            info!("monitor did not set, disable heartbeat");
             return;
         };
         let cluster_name = self.config.cluster_name.clone();
@@ -177,29 +177,16 @@ impl Sidecar {
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) {
-        let mut controller = Controller::new(
-            handle,
-            interval(self.config.reconcile_interval),
-            state,
-            graceful_shutdown,
-        );
+        let controller = Controller::new(state, handle, self.config.reconcile_interval);
         let _ig = tokio::spawn(async move {
-            loop {
-                match controller.reconcile_once().await {
-                    Ok(instant) => {
-                        debug!(
-                            "successfully reconcile the cluster states within {:?}",
-                            instant.elapsed()
-                        );
-                    }
-                    Err(err) => {
-                        if err == Error::Shutdown {
-                            info!("controller graceful shutdown");
-                            break;
-                        }
-                        error!("reconcile failed, error: {}", err);
-                    }
-                }
+            let mut shutdown = graceful_shutdown;
+            let res = controller
+                .run_reconcile_with_shutdown(shutdown.changed().map(|_| ()))
+                .await;
+            if let Err(err) = res {
+                error!("controller run failed, error: {err}");
+            } else {
+                info!("controller shutdown");
             }
         });
     }

From 23a7b2f46743da0b2ba6bb3238df884337f81eaf Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Fri, 27 Oct 2023 09:20:05 +0800
Subject: [PATCH 13/19] feat: implement reconcile and evaluate

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 sidecar/src/controller.rs | 112 +++++++++++++++++++++-----
 sidecar/src/routers.rs    |   6 +-
 sidecar/src/sidecar.rs    |  22 +++--
 sidecar/src/types.rs      |  60 +++++++++++---
 sidecar/src/xline.rs      | 165 ++++++++++++++++++++++++++++----------
 5 files changed, 284 insertions(+), 81 deletions(-)

diff --git a/sidecar/src/controller.rs b/sidecar/src/controller.rs
index abe583ce..2b5b358f 100644
--- a/sidecar/src/controller.rs
+++ b/sidecar/src/controller.rs
@@ -6,20 +6,22 @@ use std::time::Duration;
 
 use anyhow::Result;
 use tokio::select;
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, RwLock};
 use tokio::time::{interval, MissedTickBehavior};
-use tracing::debug;
+use tracing::{debug, error, info};
 
-use crate::types::StatePayload;
+use crate::types::{MemberConfig, State, StatePayload, StateStatus};
 use crate::xline::XlineHandle;
 
 /// Sidecar operator controller
 #[derive(Debug)]
 pub(crate) struct Controller {
+    /// The name of this sidecar
+    name: String,
     /// The state of this operator
     state: Arc<Mutex<StatePayload>>,
     /// Xline handle
-    handle: Arc<XlineHandle>,
+    handle: Arc<RwLock<XlineHandle>>,
     /// Check interval
     reconcile_interval: Duration,
 }
@@ -27,11 +29,13 @@ pub(crate) struct Controller {
 impl Controller {
     /// Constructor
     pub(crate) fn new(
+        name: String,
         state: Arc<Mutex<StatePayload>>,
-        handle: Arc<XlineHandle>,
+        handle: Arc<RwLock<XlineHandle>>,
         reconcile_interval: Duration,
     ) -> Self {
         Self {
+            name,
             state,
             handle,
             reconcile_interval,
@@ -42,26 +46,35 @@ impl Controller {
     #[allow(clippy::integer_arithmetic)] // this error originates in the macro `tokio::select`
     pub(crate) async fn run_reconcile_with_shutdown(
         self,
+        init_member_config: MemberConfig,
         graceful_shutdown: impl Future<Output = ()>,
     ) -> Result<()> {
         select! {
             _ = graceful_shutdown => {
                 Ok(())
             }
-            res = self.run_reconcile() => {
+            res = self.run_reconcile(init_member_config) => {
                 res
             }
         }
     }
 
     /// Run reconcile loop
-    pub(crate) async fn run_reconcile(self) -> Result<()> {
+    pub(crate) async fn run_reconcile(self, init_member_config: MemberConfig) -> Result<()> {
         let mut tick = interval(self.reconcile_interval);
         tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
+        let member_config = init_member_config;
+        self.handle
+            .write()
+            .await
+            .start(&member_config.xline_members())
+            .await?;
         loop {
             let instant = tick.tick().await;
-            let _result = self.evaluate().await;
-            let _result1 = self.execute().await;
+            if let Err(err) = self.reconcile_once(&member_config).await {
+                error!("reconcile failed, error: {err}");
+                continue;
+            }
             debug!(
                 "successfully reconcile the cluster states within {:?}",
                 instant.elapsed()
@@ -69,17 +82,80 @@ impl Controller {
         }
     }
 
-    /// Evaluate cluster states
-    #[allow(clippy::unused_async)] // TODO remove when it is implemented
-    async fn evaluate(&self) -> Result<()> {
-        // TODO evaluate states
+    /// Reconcile inner
+    async fn reconcile_once(&self, member_config: &MemberConfig) -> Result<()> {
+        let mut handle = self.handle.write().await;
+
+        let sidecar_members = member_config.sidecar_members();
+        let xline_members = member_config.xline_members();
+        let cluster_size = member_config.members.len();
+        let majority = member_config.majority_cnt();
+
+        let cluster_health = handle.is_healthy().await;
+        let xline_running = handle.is_running().await;
+        let states = StateStatus::gather(&sidecar_members).await?;
+
+        match (cluster_health, xline_running) {
+            (true, true) => {
+                self.set_state(State::OK).await;
+
+                info!("status: cluster healthy + xline running");
+            }
+            (true, false) => {
+                self.set_state(State::Pending).await;
+
+                info!("status: cluster healthy + xline not running, joining the cluster");
+                handle.start(&xline_members).await?;
+            }
+            (false, true) => {
+                self.set_state(State::Pending).await;
+
+                if states
+                    .states
+                    .get(&State::OK)
+                    .is_some_and(|c| *c >= majority)
+                {
+                    info!("status: cluster unhealthy + xline running + quorum ok, waiting...");
+                } else {
+                    info!(
+                        "status: cluster unhealthy + xline running + quorum loss, backup and start failure recovery"
+                    );
+                    handle.backup().await?;
+                    handle.stop().await?;
+                }
+            }
+            (false, false) => {
+                let is_seeder = states.seeder == self.name;
+                if !is_seeder {
+                    info!("status: cluster unhealthy + xline not running + not seeder, try to install backup");
+                    handle.install_backup().await?;
+                }
+
+                self.set_state(State::Start).await;
+
+                if states
+                    .states
+                    .get(&State::Start)
+                    .is_some_and(|c| *c != cluster_size)
+                {
+                    info!("status: cluster unhealthy + xline not running + wait all start");
+                    return Ok(());
+                }
+
+                if is_seeder {
+                    info!(
+                        "status: cluster unhealthy + xline not running + all start + seeder, seed cluster"
+                    );
+                    handle.start(&xline_members).await?;
+                }
+            }
+        }
+
         Ok(())
     }
 
-    /// Execute reconciliation based on evaluation
-    #[allow(clippy::unused_async)] // TODO remove when it is implemented
-    async fn execute(&self) -> Result<()> {
-        // TODO execute reconciliation
-        Ok(())
+    /// Set current state
+    async fn set_state(&self, state: State) {
+        self.state.lock().await.state = state;
     }
 }
diff --git a/sidecar/src/routers.rs b/sidecar/src/routers.rs
index a846f0e9..fdcb3261 100644
--- a/sidecar/src/routers.rs
+++ b/sidecar/src/routers.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use axum::http::StatusCode;
 use axum::{Extension, Json};
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, RwLock};
 
 use crate::types::{MembershipChange, StatePayload};
 use crate::utils::{check_backup_volume, check_data_volume};
@@ -22,8 +22,8 @@ pub(crate) async fn health() -> StatusCode {
 }
 
 /// Backup hook
-pub(crate) async fn backup(Extension(handle): Extension<Arc<XlineHandle>>) -> StatusCode {
-    if handle.backup().await.is_err() {
+pub(crate) async fn backup(Extension(handle): Extension<Arc<RwLock<XlineHandle>>>) -> StatusCode {
+    if handle.read().await.backup().await.is_err() {
         return StatusCode::INTERNAL_SERVER_ERROR;
     }
     StatusCode::OK
diff --git a/sidecar/src/sidecar.rs b/sidecar/src/sidecar.rs
index 575c8df5..0ed51955 100644
--- a/sidecar/src/sidecar.rs
+++ b/sidecar/src/sidecar.rs
@@ -8,7 +8,7 @@ use futures::{FutureExt, TryFutureExt};
 use operator_api::consts::{SIDECAR_BACKUP_ROUTE, SIDECAR_HEALTH_ROUTE, SIDECAR_STATE_ROUTE};
 use operator_api::{HeartbeatStatus, K8sXlineHandle, LocalXlineHandle};
 use tokio::sync::watch::{Receiver, Sender};
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, RwLock};
 use tokio::time::{interval, MissedTickBehavior};
 use tracing::{debug, error, info, warn};
 
@@ -42,8 +42,8 @@ impl Sidecar {
     pub async fn run(&self) -> Result<()> {
         let (graceful_shutdown_event, _) = tokio::sync::watch::channel(());
         let forceful_shutdown = self.forceful_shutdown(&graceful_shutdown_event);
-        let handle = Arc::new(self.init_xline_handle().await?);
-        let revision = handle.revision_offline().unwrap_or(1);
+        let handle = Arc::new(RwLock::new(self.init_xline_handle().await?));
+        let revision = handle.read().await.revision_offline().unwrap_or(1);
         let state = Arc::new(Mutex::new(StatePayload {
             state: State::Start,
             revision,
@@ -114,10 +114,10 @@ impl Sidecar {
         };
         XlineHandle::open(
             &self.config.name,
+            &self.config.xline.data_dir,
             backup,
             inner,
             self.config.xline_port,
-            self.config.init_xline_members(),
         )
     }
 
@@ -173,15 +173,21 @@ impl Sidecar {
     /// Start controller
     fn start_controller(
         &self,
-        handle: Arc<XlineHandle>,
+        handle: Arc<RwLock<XlineHandle>>,
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) {
-        let controller = Controller::new(state, handle, self.config.reconcile_interval);
+        let controller = Controller::new(
+            self.config.name.clone(),
+            state,
+            handle,
+            self.config.reconcile_interval,
+        );
+        let init_member_config = self.config.init_member_config();
         let _ig = tokio::spawn(async move {
             let mut shutdown = graceful_shutdown;
             let res = controller
-                .run_reconcile_with_shutdown(shutdown.changed().map(|_| ()))
+                .run_reconcile_with_shutdown(init_member_config, shutdown.changed().map(|_| ()))
                 .await;
             if let Err(err) = res {
                 error!("controller run failed, error: {err}");
@@ -194,7 +200,7 @@ impl Sidecar {
     /// Run a web server to expose current state to other sidecar operators and k8s
     fn start_web_server(
         &self,
-        handle: Arc<XlineHandle>,
+        handle: Arc<RwLock<XlineHandle>>,
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 5d7a3326..61c33ef7 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -4,7 +4,7 @@ use operator_api::XlineConfig;
 use serde::{Deserialize, Serialize};
 
 use std::collections::HashMap;
-use std::ops::AddAssign;
+use std::ops::{Add, AddAssign, Div};
 use std::path::PathBuf;
 use std::time::Duration;
 
@@ -16,7 +16,7 @@ pub struct Config {
     pub name: String,
     /// The cluster name
     pub cluster_name: String,
-    /// Sidecar initial hosts, [pod_name]->[pod_host]
+    /// Nodes initial hosts, [pod_name]->[pod_host]
     pub init_members: HashMap<String, String>,
     /// The xline server port
     pub xline_port: u16,
@@ -45,6 +45,17 @@ pub struct MonitorConfig {
     pub heartbeat_interval: Duration,
 }
 
+/// Member config
+#[derive(Debug, Clone)]
+pub(crate) struct MemberConfig {
+    /// Nodes hosts, [pod_name]->[pod_host]
+    pub(crate) members: HashMap<String, String>,
+    /// The xline server port
+    pub(crate) xline_port: u16,
+    /// The sidecar web server port
+    pub(crate) sidecar_port: u16,
+}
+
 /// Sidecar backend, it determinate how xline could be setup
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
@@ -80,9 +91,7 @@ pub enum BackupConfig {
 
 impl Config {
     /// Get the initial sidecar members
-    #[must_use]
-    #[inline]
-    pub fn init_sidecar_members(&self) -> HashMap<String, String> {
+    pub(crate) fn init_sidecar_members(&self) -> HashMap<String, String> {
         self.init_members
             .clone()
             .into_iter()
@@ -91,15 +100,48 @@ impl Config {
     }
 
     /// Get the initial xline members
-    #[must_use]
-    #[inline]
-    pub fn init_xline_members(&self) -> HashMap<String, String> {
+    pub(crate) fn init_xline_members(&self) -> HashMap<String, String> {
         self.init_members
             .clone()
             .into_iter()
             .map(|(name, host)| (name, format!("{host}:{}", self.xline_port)))
             .collect()
     }
+
+    /// Get the initial member config
+    pub(crate) fn init_member_config(&self) -> MemberConfig {
+        MemberConfig {
+            members: self.init_members.clone(),
+            xline_port: self.xline_port,
+            sidecar_port: self.sidecar_port,
+        }
+    }
+}
+
+impl MemberConfig {
+    /// Get the xline members
+    pub(crate) fn xline_members(&self) -> HashMap<String, String> {
+        self.members
+            .clone()
+            .into_iter()
+            .map(|(name, host)| (name, format!("{host}:{}", self.xline_port)))
+            .collect()
+    }
+
+    /// Get the sidecar members
+    pub(crate) fn sidecar_members(&self) -> HashMap<String, String> {
+        self.members
+            .clone()
+            .into_iter()
+            .map(|(name, host)| (name, format!("{host}:{}", self.sidecar_port)))
+            .collect()
+    }
+
+    /// Get the majority count
+
+    pub(crate) fn majority_cnt(&self) -> usize {
+        self.members.len().div(2).add(1)
+    }
 }
 
 /// Sidecar operator state
@@ -144,7 +186,7 @@ impl StateStatus {
         for (name, addr) in sidecars {
             let url = format!("http://{addr}{SIDECAR_STATE_ROUTE}");
             let state: StatePayload = reqwest::get(url).await?.json().await?;
-            if state.revision > max_rev {
+            if (state.revision, name.as_str()) > (max_rev, seeder) {
                 max_rev = state.revision;
                 seeder = name;
             }
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index bddc185a..b8f403d3 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -4,20 +4,21 @@
 
 use std::collections::HashMap;
 use std::fmt::Debug;
+use std::io;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use anyhow::{anyhow, Result};
 use bytes::Buf;
 use engine::{Engine, EngineType, StorageEngine};
-use futures::stream::FuturesUnordered;
-use futures::StreamExt;
-use operator_api::consts::DEFAULT_DATA_DIR;
 use tonic::transport::{Channel, Endpoint};
 use tonic_health::pb::health_check_response::ServingStatus;
 use tonic_health::pb::health_client::HealthClient;
 use tonic_health::pb::HealthCheckRequest;
-use tracing::debug;
-use xline_client::types::cluster::{MemberAddRequest, MemberListRequest, MemberRemoveRequest};
+use tracing::{debug, info};
+use xline_client::types::cluster::{
+    MemberAddRequest, MemberListRequest, MemberRemoveRequest, MemberUpdateRequest,
+};
 use xline_client::types::kv::RangeRequest;
 use xline_client::{Client, ClientOptions};
 
@@ -52,6 +53,8 @@ pub(crate) const XLINE_TABLES: [&str; 6] = [
 pub(crate) struct XlineHandle {
     /// The name of the operator
     name: String,
+    /// The xline data dir
+    data_dir: PathBuf,
     /// The xline backup provider
     backup: Option<Box<dyn Provider>>,
     /// The xline client, used to connect to the cluster
@@ -62,8 +65,6 @@ pub(crate) struct XlineHandle {
     server_id: Option<u64>,
     /// The rocks db engine
     engine: Engine,
-    /// The xline members
-    xline_members: HashMap<String, String>,
     /// Health retires of xline client
     is_healthy_retries: usize,
     /// The detailed xline process handle
@@ -74,24 +75,25 @@ impl XlineHandle {
     /// Create the xline handle but not start the xline node
     pub(crate) fn open(
         name: &str,
+        data_dir: &str,
         backup: Option<Box<dyn Provider>>,
         inner: Box<dyn operator_api::XlineHandle>,
         xline_port: u16,
-        xline_members: HashMap<String, String>,
     ) -> Result<Self> {
         debug!("name: {name}, backup: {backup:?}, xline_port: {xline_port}");
         let endpoint: Endpoint = format!("http://127.0.0.1:{xline_port}").parse()?;
         let channel = Channel::balance_list(std::iter::once(endpoint));
         let health_client = HealthClient::new(channel);
-        let engine = Engine::new(EngineType::Rocks(DEFAULT_DATA_DIR.parse()?), &XLINE_TABLES)?;
+        let data_path: PathBuf = data_dir.parse()?;
+        let engine = Engine::new(EngineType::Rocks(data_path.clone()), &XLINE_TABLES)?;
         Ok(Self {
             name: name.to_owned(),
+            data_dir: data_path,
             backup,
             health_client,
             engine,
             client: None,
             server_id: None,
-            xline_members,
             is_healthy_retries: 5,
             inner,
         })
@@ -104,49 +106,76 @@ impl XlineHandle {
             .unwrap_or_else(|| panic!("xline client not initialized"))
     }
 
+    /// Cleanup data directory
+    pub(crate) async fn cleanup(&self) -> Result<()> {
+        tokio::fs::remove_dir_all(&self.data_dir).await?;
+        Ok(())
+    }
+
     /// Start the xline server
-    pub(crate) async fn start(&mut self) -> Result<()> {
+    pub(crate) async fn start(&mut self, xlines: &HashMap<String, String>) -> Result<()> {
+        /// Timeout for test start
+        const TEST_START_TIMEOUT: Duration = Duration::from_secs(3);
+
         // TODO: hold a distributed lock during start
 
         // Step 1: Check if there is any node running
         // Step 2: If there is no node running, start single node cluster
-        // Step 3: If there are some nodes running, start the node as a member to join the cluster
-        let others = self
-            .xline_members
-            .iter()
-            .filter(|&(name, _)| name != &self.name)
-            .map(|(_, addr)| {
-                Ok::<_, tonic::transport::Error>(
-                    Endpoint::from_shared(addr.clone())?.connect_timeout(Duration::from_secs(3)),
-                )
-            })
-            .collect::<Result<Vec<_>, _>>()?;
-        let futs: FuturesUnordered<_> = others.iter().map(Endpoint::connect).collect();
+        // Step 3: If there are some nodes running, start the node as a member to join/update the cluster
+
+        let self_addr = xlines
+            .get(&self.name)
+            .ok_or_else(|| anyhow!("self name should be in xline members"))?
+            .clone();
+        let mut start_members = HashMap::from([(self.name.clone(), self_addr.clone())]);
         // the cluster is started if any of the connection is successful
-        let cluster_started = futs.any(|res| async move { res.is_ok() }).await;
+        let mut cluster_started = false;
+
+        for (name, addr) in xlines.iter().filter(|&(name, _)| name != &self.name) {
+            let online = Endpoint::from_shared(addr.clone())?
+                .connect_timeout(TEST_START_TIMEOUT)
+                .connect()
+                .await
+                .is_ok();
+            if online {
+                cluster_started = true;
+                let _ig = start_members.insert(name.clone(), addr.clone());
+            }
+        }
 
-        self.inner.start(&self.xline_members).await?;
+        self.inner.start(&start_members).await?;
 
-        let client = Client::connect(self.xline_members.values(), ClientOptions::default()).await?;
+        let client = Client::connect(xlines.values(), ClientOptions::default()).await?;
         let mut cluster_client = client.cluster_client();
+
+        let mut members = cluster_client
+            .member_list(MemberListRequest::new(false))
+            .await?
+            .members;
+
         let member = if cluster_started {
-            let peer_addr = self
-                .xline_members
-                .get(&self.name)
-                .unwrap_or_else(|| unreachable!("member should contain self"))
-                .clone();
-            let resp = cluster_client
-                .member_add(MemberAddRequest::new(vec![peer_addr], false))
-                .await?;
-            let Some(member) = resp.member else {
-                unreachable!("self member should be set when member add request success")
-            };
-            member
+            let joined = members.iter().find(|mem| mem.name == self.name);
+            if let Some(old_member) = joined {
+                let _ig = cluster_client
+                    .member_update(MemberUpdateRequest::new(
+                        old_member.id,
+                        vec![self_addr.clone()],
+                    ))
+                    .await?;
+                let mut new_member = old_member.clone();
+                new_member.peer_ur_ls = vec![self_addr.clone()];
+                new_member.client_ur_ls = vec![self_addr.clone()];
+                new_member
+            } else {
+                let resp = cluster_client
+                    .member_add(MemberAddRequest::new(vec![self_addr.clone()], false))
+                    .await?;
+                let Some(member) = resp.member else {
+                    unreachable!("self member should be set when member add request success")
+                };
+                member
+            }
         } else {
-            let mut members = cluster_client
-                .member_list(MemberListRequest::new(false))
-                .await?
-                .members;
             if members.len() != 1 {
                 return Err(anyhow!(
                     "there should be only one member(self) if the cluster if not start"
@@ -169,6 +198,7 @@ impl XlineHandle {
             .take()
             .ok_or_else(|| anyhow!("xline server should not be stopped before started"))?;
 
+        // double check for cluster health
         if self.is_healthy().await {
             let mut cluster_client = self.client().cluster_client();
             _ = cluster_client
@@ -253,7 +283,10 @@ impl XlineHandle {
         //         If the local revision is less than remote, abort backup
         // Step 3. Start backup
         let backup = match self.backup.as_ref() {
-            None => return Err(anyhow!("no backup specified")),
+            None => {
+                info!("no backup config found, skip backup");
+                return Ok(());
+            }
             Some(backup) => backup,
         };
         let remote = backup.latest().await?.map(|metadata| metadata.revision);
@@ -280,4 +313,50 @@ impl XlineHandle {
             .await?;
         Ok(())
     }
+
+    /// Install backup, make sure that xline is shutdown
+    pub(crate) async fn install_backup(&self) -> Result<()> {
+        let backup = match self.backup.as_ref() {
+            None => {
+                info!("no backup config found, skip install backup");
+                return Ok(());
+            }
+            Some(backup) => backup,
+        };
+        let Some(latest) = backup.latest().await? else {
+            info!("no backup found, skip install backup");
+            return Ok(())
+        };
+        if !self.data_dir.exists() {
+            debug!("data directory not found, install backup");
+            let local = backup.load(&latest).await?;
+            copy_recursively(&local, &self.data_dir)?;
+            tokio::fs::remove_dir_all(local).await?;
+            return Ok(());
+        }
+        if latest.revision <= self.revision_offline()? {
+            info!("remote revision is less than local, skip install backup");
+            return Ok(());
+        }
+        tokio::fs::remove_dir_all(&self.data_dir).await?;
+        let local = backup.load(&latest).await?;
+        copy_recursively(&local, &self.data_dir)?;
+        tokio::fs::remove_dir_all(local).await?;
+        Ok(())
+    }
+}
+
+/// Copy directory
+fn copy_recursively(source: impl AsRef<Path>, destination: impl AsRef<Path>) -> io::Result<()> {
+    std::fs::create_dir_all(&destination)?;
+    for entry in std::fs::read_dir(source)? {
+        let entry = entry?;
+        let filetype = entry.file_type()?;
+        if filetype.is_dir() {
+            copy_recursively(entry.path(), destination.as_ref().join(entry.file_name()))?;
+        } else {
+            let _ig = std::fs::copy(entry.path(), destination.as_ref().join(entry.file_name()))?;
+        }
+    }
+    Ok(())
 }

From fbbdf62ca0344398453e777a22a23ef5bff0a1f2 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Fri, 27 Oct 2023 09:53:57 +0800
Subject: [PATCH 14/19] chore: add member registry in cluster status

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 operator-k8s/src/crd/v1alpha1/cluster.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/operator-k8s/src/crd/v1alpha1/cluster.rs b/operator-k8s/src/crd/v1alpha1/cluster.rs
index ed840f2f..daeef442 100644
--- a/operator-k8s/src/crd/v1alpha1/cluster.rs
+++ b/operator-k8s/src/crd/v1alpha1/cluster.rs
@@ -8,6 +8,7 @@ use k8s_openapi::api::core::v1::{Affinity, Container, PersistentVolumeClaim};
 use k8s_openapi::serde::{Deserialize, Serialize};
 use kube::CustomResource;
 use schemars::JsonSchema;
+use std::collections::HashMap;
 
 /// Xline cluster specification
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
@@ -113,6 +114,8 @@ pub(crate) struct S3Spec {
 pub(crate) struct ClusterStatus {
     /// The available nodes' number in the cluster
     pub(crate) available: i32,
+    /// The members registry
+    pub(crate) members: HashMap<String, String>,
 }
 
 #[cfg(test)]

From 136f352e6f5a14447ac984827cbdb380079c9a89 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Fri, 27 Oct 2023 10:24:21 +0800
Subject: [PATCH 15/19] refactor: move crd into crd-api crate

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                                    | 19 +++++++++--
 Cargo.toml                                    |  1 +
 crd-api/Cargo.toml                            | 26 ++++++++++++++
 .../src/crd/mod.rs => crd-api/src/lib.rs      |  8 ++---
 .../crd => crd-api/src}/v1alpha1/cluster.rs   | 34 +++++++++----------
 .../src/crd => crd-api/src}/v1alpha1/mod.rs   | 10 +++---
 .../src/crd => crd-api/src}/version.rs        | 12 +++----
 operator-api/Cargo.toml                       |  4 +--
 operator-k8s/Cargo.toml                       |  4 +--
 operator-k8s/src/consts.rs                    |  2 +-
 operator-k8s/src/lib.rs                       |  7 ++--
 11 files changed, 85 insertions(+), 42 deletions(-)
 create mode 100644 crd-api/Cargo.toml
 rename operator-k8s/src/crd/mod.rs => crd-api/src/lib.rs (90%)
 rename {operator-k8s/src/crd => crd-api/src}/v1alpha1/cluster.rs (91%)
 rename {operator-k8s/src/crd => crd-api/src}/v1alpha1/mod.rs (97%)
 rename {operator-k8s/src/crd => crd-api/src}/version.rs (96%)

diff --git a/Cargo.lock b/Cargo.lock
index f631711e..89c08b06 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -554,6 +554,22 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crd-api"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "garde",
+ "k8s-openapi",
+ "kube",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.8"
@@ -4084,9 +4100,9 @@ dependencies = [
  "axum",
  "clap 4.4.6",
  "clippy-utilities 0.2.0",
+ "crd-api",
  "flume 0.11.0",
  "futures",
- "garde",
  "k8s-openapi",
  "kube",
  "operator-api",
@@ -4094,7 +4110,6 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "serde_yaml",
  "thiserror",
  "tokio",
  "tracing",
diff --git a/Cargo.toml b/Cargo.toml
index e711873b..5bb67c36 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "crd-api",
     "operator-api",
     "operator-k8s",
     "sidecar",
diff --git a/crd-api/Cargo.toml b/crd-api/Cargo.toml
new file mode 100644
index 00000000..d877cfca
--- /dev/null
+++ b/crd-api/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "crd-api"
+version = "0.1.0"
+edition = "2021"
+authors = ["DatenLord <dev@datenlord.io>"]
+description = "Custom Resource Definition"
+license = "Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/xline-kv/xline-operator/tree/main/crd-api"
+categories = ["API"]
+keywords = ["operator", "API"]
+
+[dependencies]
+k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
+kube = { version = "0.86.0", features = ["runtime", "derive"] }
+schemars = "0.8.6"
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.97"
+tracing = "0.1.37"
+anyhow = "1.0.75"
+tokio = { version = "1.0", features = ["time"] }
+
+
+[dev-dependencies]
+garde = { version = "0.11.2", default-features = false, features = ["derive", "pattern"] }
+serde_yaml = "0.9.25"
\ No newline at end of file
diff --git a/operator-k8s/src/crd/mod.rs b/crd-api/src/lib.rs
similarity index 90%
rename from operator-k8s/src/crd/mod.rs
rename to crd-api/src/lib.rs
index 60e971d6..80cc9fd7 100644
--- a/operator-k8s/src/crd/mod.rs
+++ b/crd-api/src/lib.rs
@@ -2,13 +2,13 @@
 /// Features:
 ///     1. Xline sidecar
 ///     2. PV backup
-pub(crate) mod v1alpha1;
+pub mod v1alpha1;
 
 /// CRD version
-pub(crate) mod version;
+pub mod version;
 
 /// Current CRD `XineCluster`
-pub(crate) use v1alpha1::Cluster;
+pub use v1alpha1::Cluster;
 
 use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use kube::runtime::conditions;
@@ -22,7 +22,7 @@ use std::time::Duration;
 const CRD_ESTABLISH_TIMEOUT: Duration = Duration::from_secs(20);
 
 /// Setup CRD
-pub(crate) async fn setup(
+pub async fn setup(
     kube_client: &Client,
     manage_crd: bool,
     auto_migration: bool,
diff --git a/operator-k8s/src/crd/v1alpha1/cluster.rs b/crd-api/src/v1alpha1/cluster.rs
similarity index 91%
rename from operator-k8s/src/crd/v1alpha1/cluster.rs
rename to crd-api/src/v1alpha1/cluster.rs
index daeef442..f864f6df 100644
--- a/operator-k8s/src/crd/v1alpha1/cluster.rs
+++ b/crd-api/src/v1alpha1/cluster.rs
@@ -29,53 +29,53 @@ use std::collections::HashMap;
     printcolumn = r#"{"name":"Backup Cron", "type":"string", "description":"The cron spec defining the interval a backup CronJob is run", "jsonPath":".spec.backup.cron"}"#,
     printcolumn = r#"{"name":"Age", "type":"date", "description":"The cluster age", "jsonPath":".metadata.creationTimestamp"}"#
 )]
-pub(crate) struct ClusterSpec {
+pub struct ClusterSpec {
     /// Size of the xline cluster, less than 3 is not allowed
     #[cfg_attr(test, garde(range(min = 3)))]
     #[schemars(range(min = 3))]
-    pub(crate) size: i32,
+    pub size: i32,
     /// Xline container specification
     #[cfg_attr(test, garde(skip))]
-    pub(crate) container: Container,
+    pub container: Container,
     /// The affinity of the xline node
     #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) affinity: Option<Affinity>,
+    pub affinity: Option<Affinity>,
     /// Backup specification
     #[cfg_attr(test, garde(custom(option_backup_dive)))]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) backup: Option<BackupSpec>,
+    pub backup: Option<BackupSpec>,
     /// The data PVC, if it is not specified, then use emptyDir instead
     #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) data: Option<PersistentVolumeClaim>,
+    pub data: Option<PersistentVolumeClaim>,
     /// Some user defined persistent volume claim templates
     #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) pvcs: Option<Vec<PersistentVolumeClaim>>,
+    pub pvcs: Option<Vec<PersistentVolumeClaim>>,
 }
 
 /// Xline cluster backup specification
 #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
 #[cfg_attr(test, derive(Validate))]
-pub(crate) struct BackupSpec {
+pub struct BackupSpec {
     /// Cron Spec
     #[cfg_attr(test, garde(pattern(r"^(?:\*|[0-5]?\d)(?:[-/,]?(?:\*|[0-5]?\d))*(?: +(?:\*|1?[0-9]|2[0-3])(?:[-/,]?(?:\*|1?[0-9]|2[0-3]))*){4}$")))]
     #[schemars(regex(
         pattern = r"^(?:\*|[0-5]?\d)(?:[-/,]?(?:\*|[0-5]?\d))*(?: +(?:\*|1?[0-9]|2[0-3])(?:[-/,]?(?:\*|1?[0-9]|2[0-3]))*){4}$"
     ))]
-    pub(crate) cron: String,
+    pub cron: String,
     /// Backup storage type
     #[cfg_attr(test, garde(dive))]
     #[serde(flatten)]
-    pub(crate) storage: StorageSpec,
+    pub storage: StorageSpec,
 }
 
 /// Xline cluster backup storage specification
 #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
 #[cfg_attr(test, derive(Validate))]
 #[serde(untagged)]
-pub(crate) enum StorageSpec {
+pub enum StorageSpec {
     /// S3 backup type
     S3 {
         /// S3 backup specification
@@ -91,7 +91,7 @@ pub(crate) enum StorageSpec {
 }
 
 impl StorageSpec {
-    pub(crate) fn as_pvc(&self) -> Option<&PersistentVolumeClaim> {
+    pub fn as_pvc(&self) -> Option<&PersistentVolumeClaim> {
         match *self {
             Self::Pvc { ref pvc } => Some(pvc),
             Self::S3 { .. } => None,
@@ -102,20 +102,20 @@ impl StorageSpec {
 /// Xline cluster backup S3 specification
 #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
 #[cfg_attr(test, derive(Validate))]
-pub(crate) struct S3Spec {
+pub struct S3Spec {
     /// S3 bucket name to use for backup
     #[cfg_attr(test, garde(pattern(r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$")))]
     #[schemars(regex(pattern = r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$"))]
-    pub(crate) bucket: String,
+    pub bucket: String,
 }
 
 /// Xline cluster status
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
-pub(crate) struct ClusterStatus {
+pub struct ClusterStatus {
     /// The available nodes' number in the cluster
-    pub(crate) available: i32,
+    pub available: i32,
     /// The members registry
-    pub(crate) members: HashMap<String, String>,
+    pub members: HashMap<String, String>,
 }
 
 #[cfg(test)]
diff --git a/operator-k8s/src/crd/v1alpha1/mod.rs b/crd-api/src/v1alpha1/mod.rs
similarity index 97%
rename from operator-k8s/src/crd/v1alpha1/mod.rs
rename to crd-api/src/v1alpha1/mod.rs
index 8c028015..f27a5c30 100644
--- a/operator-k8s/src/crd/v1alpha1/mod.rs
+++ b/crd-api/src/v1alpha1/mod.rs
@@ -1,16 +1,18 @@
-pub(crate) use cluster::Cluster;
+pub use cluster::Cluster;
 
 mod cluster;
 
-use crate::consts::FIELD_MANAGER;
-use crate::crd::version::ApiVersion;
-use crate::crd::wait_crd_established;
 use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use kube::api::{DynamicObject, ListParams, Patch, PatchParams};
 use kube::core::crd::merge_crds;
 use kube::{Api, Client, CustomResourceExt, Resource};
 use tracing::{debug, info};
 
+use crate::version::ApiVersion;
+use crate::wait_crd_established;
+
+const FIELD_MANAGER: &str = "xlineoperator.datenlord.io/crd";
+
 /// Setup CRD
 pub(super) async fn set_up(
     kube_client: &Client,
diff --git a/operator-k8s/src/crd/version.rs b/crd-api/src/version.rs
similarity index 96%
rename from operator-k8s/src/crd/version.rs
rename to crd-api/src/version.rs
index 7bccb2ad..41bd0522 100644
--- a/operator-k8s/src/crd/version.rs
+++ b/crd-api/src/version.rs
@@ -1,5 +1,3 @@
-#![allow(unused)] // TODO Remove
-
 use std::cmp::Ordering;
 use std::fmt::{Display, Formatter};
 use std::marker::PhantomData;
@@ -10,7 +8,7 @@ use anyhow::anyhow;
 /// The api version, the generic CR here to prevent from comparing `ApiVersion<CR1>` to `ApiVersion<CR2>`
 #[non_exhaustive]
 #[derive(Clone, Debug)]
-pub(crate) enum ApiVersion<CR> {
+pub enum ApiVersion<CR> {
     /// alpha version e.g. v1alpha1 => Alpha(1, 1)
     Alpha(u32, u32, PhantomData<CR>),
     /// beta version e.g. v2beta3 => Beta(2, 3)
@@ -23,21 +21,21 @@ impl<CR> ApiVersion<CR> {
     /// Create alpha version
     #[must_use]
     #[inline]
-    pub(crate) fn alpha(main: u32, sub: u32) -> Self {
+    pub fn alpha(main: u32, sub: u32) -> Self {
         Self::Alpha(main, sub, PhantomData)
     }
 
     /// Create beta version
     #[must_use]
     #[inline]
-    pub(crate) fn beta(main: u32, sub: u32) -> Self {
+    pub fn beta(main: u32, sub: u32) -> Self {
         Self::Beta(main, sub, PhantomData)
     }
 
     /// Create beta version
     #[must_use]
     #[inline]
-    pub(crate) fn stable(main: u32) -> Self {
+    pub fn stable(main: u32) -> Self {
         Self::Stable(main, PhantomData)
     }
 
@@ -45,7 +43,7 @@ impl<CR> ApiVersion<CR> {
     /// We promise that we keep compatible with the same main version
     #[must_use]
     #[inline]
-    pub(crate) fn compat_with(&self, other: &Self) -> bool {
+    pub fn compat_with(&self, other: &Self) -> bool {
         self.main_version() == other.main_version()
     }
 
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index f552553c..f58449f0 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -3,12 +3,12 @@ name = "operator-api"
 version = "0.1.0"
 edition = "2021"
 authors = ["DatenLord <dev@datenlord.io>"]
-description = "API types that used in operators"
+description = "API between operators and sidecars"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/xline-kv/xline-operator/tree/main/operator-api"
 categories = ["API"]
-keywords = ["operator", "API", "operator"]
+keywords = ["operator", "API"]
 
 [dependencies]
 anyhow = "1.0.72"
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index ffd9f3fc..8f0216e5 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -35,7 +35,5 @@ tokio = { version = "1.0", features = [
 ] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
+crd-api = { path = "../crd-api" }
 
-[dev-dependencies]
-garde = { version = "0.11.2", default-features = false, features = ["derive", "pattern"] }
-serde_yaml = "0.9.25"
diff --git a/operator-k8s/src/consts.rs b/operator-k8s/src/consts.rs
index 607e86e8..36a1116a 100644
--- a/operator-k8s/src/consts.rs
+++ b/operator-k8s/src/consts.rs
@@ -5,7 +5,7 @@ use std::time::Duration;
 /// The default requeue duration to achieve eventual consistency
 pub(crate) const DEFAULT_REQUEUE_DURATION: Duration = Duration::from_secs(600);
 /// The field manager identifier of xline operator
-pub(crate) const FIELD_MANAGER: &str = "xlineoperator.datenlord.io";
+pub(crate) const FIELD_MANAGER: &str = "xlineoperator.datenlord.io/operator";
 /// The image used for cronjob to trigger backup
 /// The following command line tool should be available in this image
 /// 1. curl
diff --git a/operator-k8s/src/lib.rs b/operator-k8s/src/lib.rs
index a3ad1015..6395a0ed 100644
--- a/operator-k8s/src/lib.rs
+++ b/operator-k8s/src/lib.rs
@@ -155,8 +155,6 @@ pub mod config;
 mod consts;
 /// Custom resource controller
 mod controller;
-/// Custom resource definition
-mod crd;
 /// Custom resource manager
 mod manager;
 /// Sidecar monitor
@@ -165,3 +163,8 @@ mod monitor;
 pub mod operator;
 /// Xline operator web server router
 mod router;
+
+/// Custom resource definition
+mod crd {
+    pub(crate) use crd_api::*;
+}

From 059ce227953635e99c51d6d74f2d0223c71a244e Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Fri, 27 Oct 2023 11:25:10 +0800
Subject: [PATCH 16/19] feat: implement custom resource registry

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                   |  1 +
 crd-api/Cargo.toml           |  7 ++--
 operator-api/Cargo.toml      |  1 +
 operator-api/src/lib.rs      |  3 ++
 operator-api/src/registry.rs | 78 ++++++++++++++++++++++++++++++++++++
 operator-k8s/Cargo.toml      |  2 +-
 6 files changed, 87 insertions(+), 5 deletions(-)
 create mode 100644 operator-api/src/registry.rs

diff --git a/Cargo.lock b/Cargo.lock
index 89c08b06..7dd855c2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2172,6 +2172,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "crd-api",
  "futures",
  "k8s-openapi",
  "kube",
diff --git a/crd-api/Cargo.toml b/crd-api/Cargo.toml
index d877cfca..9ccafbc0 100644
--- a/crd-api/Cargo.toml
+++ b/crd-api/Cargo.toml
@@ -11,16 +11,15 @@ categories = ["API"]
 keywords = ["operator", "API"]
 
 [dependencies]
+anyhow = "1.0.75"
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive"] }
 schemars = "0.8.6"
 serde = { version = "1.0.130", features = ["derive"] }
 serde_json = "1.0.97"
-tracing = "0.1.37"
-anyhow = "1.0.75"
 tokio = { version = "1.0", features = ["time"] }
-
+tracing = "0.1.37"
 
 [dev-dependencies]
 garde = { version = "0.11.2", default-features = false, features = ["derive", "pattern"] }
-serde_yaml = "0.9.25"
\ No newline at end of file
+serde_yaml = "0.9.25"
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index f58449f0..03d5882d 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -13,6 +13,7 @@ keywords = ["operator", "API"]
 [dependencies]
 anyhow = "1.0.72"
 async-trait = "0.1.72"
+crd-api = { path = "../crd-api" }
 futures = "0.3.28"
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive", "ws"] }
diff --git a/operator-api/src/lib.rs b/operator-api/src/lib.rs
index c8f53856..7470dd2c 100644
--- a/operator-api/src/lib.rs
+++ b/operator-api/src/lib.rs
@@ -1,6 +1,9 @@
 /// constants shared by the operator and the sidecar
 pub mod consts;
 
+/// Config registry
+pub mod registry;
+
 /// Xline handle
 mod xline;
 
diff --git a/operator-api/src/registry.rs b/operator-api/src/registry.rs
new file mode 100644
index 00000000..0a90a22b
--- /dev/null
+++ b/operator-api/src/registry.rs
@@ -0,0 +1,78 @@
+#![allow(unused)] // TODO remove
+
+use anyhow::anyhow;
+use async_trait::async_trait;
+use crd_api::Cluster;
+use k8s_openapi::serde_json::json;
+use kube::api::{Patch, PatchParams};
+use kube::core::object::HasStatus;
+use kube::{Api, Client};
+use std::collections::HashMap;
+
+pub struct Config {
+    /// Members [node_name] => [node_ip]
+    pub members: HashMap<String, String>,
+    /// Cluster size
+    pub cluster_size: usize,
+}
+
+#[async_trait]
+pub trait Registry {
+    const FIELD_MANAGER: &'static str = "xlineoperator.datenlord.io/registry";
+
+    async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config>;
+}
+
+pub struct CustomResourceRegistry {
+    cluster_name: String,
+    cluster_api: Api<Cluster>,
+}
+
+impl CustomResourceRegistry {
+    /// New a registry with default kube client
+    pub async fn new_with_default(cluster_name: String, namespace: &str) -> Self {
+        let kube_client = Client::try_default()
+            .await
+            .unwrap_or_else(|_ig| unreachable!("it must be setup in k8s environment"));
+        Self {
+            cluster_name,
+            cluster_api: Api::namespaced(kube_client, namespace),
+        }
+    }
+
+    pub async fn new(cluster_name: String, namespace: &str, kube_client: Client) -> Self {
+        Self {
+            cluster_name,
+            cluster_api: Api::namespaced(kube_client, namespace),
+        }
+    }
+}
+
+#[async_trait]
+impl Registry for CustomResourceRegistry {
+    async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config> {
+        /// TODO: hold a distributed lock here
+        let cluster = self.cluster_api.get_status(&self.cluster_name).await?;
+        let mut status = cluster
+            .status
+            .ok_or_else(|| anyhow!("no status found in cluster {}", self.cluster_name))?;
+        if status.members.get(&self_name) != Some(&self_ip) {
+            status.members.insert(self_name, self_ip);
+            let patch = json!({
+                "status": status,
+            });
+            let _ig = self
+                .cluster_api
+                .patch_status(
+                    &self.cluster_name,
+                    &PatchParams::apply(Self::FIELD_MANAGER),
+                    &Patch::Apply(patch),
+                )
+                .await?;
+        }
+        Ok(Config {
+            members: status.members,
+            cluster_size: cluster.spec.size as usize,
+        })
+    }
+}
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index 8f0216e5..be24aa06 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -17,6 +17,7 @@ async-trait = "0.1.68"
 axum = "0.6.18"
 clap = { version = "4.3.4", features = ["derive"] }
 clippy-utilities = "0.2.0"
+crd-api = { path = "../crd-api" }
 flume = "0.11.0"
 futures = "0.3.28"
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
@@ -35,5 +36,4 @@ tokio = { version = "1.0", features = [
 ] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
-crd-api = { path = "../crd-api" }
 

From d7d1fbc248905fab1c2e56b0a9dd5360955a7d25 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Fri, 27 Oct 2023 17:44:29 +0800
Subject: [PATCH 17/19] chore: bump crd dependencies version

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                          | 73 +++++++++++++-------------
 crd-api/Cargo.toml                  | 10 ++--
 crd-api/src/v1alpha1/cluster.rs     | 79 ++++++++++++-----------------
 operator-api/src/registry.rs        | 30 ++++++++---
 operator-k8s/Cargo.toml             |  4 +-
 operator-k8s/src/manager/cluster.rs |  5 +-
 operator-k8s/src/monitor.rs         |  6 +--
 sidecar/Cargo.toml                  |  2 -
 sidecar/src/main.rs                 | 10 ++--
 sidecar/src/sidecar.rs              |  8 +--
 sidecar/src/types.rs                | 46 +++--------------
 sidecar/src/xline.rs                |  8 +++
 12 files changed, 132 insertions(+), 149 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7dd855c2..79ec6c14 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -371,6 +371,15 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "castaway"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.83"
@@ -520,6 +529,19 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
+[[package]]
+name = "compact_str"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f86b9c4c00838774a6d902ef931eff7470720c51d90c2e32cfe15dc304737b3f"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "ryu",
+ "static_assertions",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.3.0"
@@ -562,10 +584,10 @@ dependencies = [
  "garde",
  "k8s-openapi",
  "kube",
+ "regex",
  "schemars",
  "serde",
  "serde_json",
- "serde_yaml",
  "tokio",
  "tracing",
 ]
@@ -1095,20 +1117,23 @@ dependencies = [
 
 [[package]]
 name = "garde"
-version = "0.11.2"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d959ef7bda0bda7cc0f6fbebfbac6202f810394f50e07059eeea8ec31e69e4b0"
+checksum = "d16596022bab79d38f74999f49b2fa08db8e4e568b43a95a2bd78afbee13f55c"
 dependencies = [
+ "compact_str",
  "garde_derive",
  "once_cell",
  "regex",
+ "smallvec",
+ "url",
 ]
 
 [[package]]
 name = "garde_derive"
-version = "0.11.2"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e89f7fce035bb3a3718e23efff13709a0b21b694c4eae20a32e1a3e4e27c6a2"
+checksum = "38ae3270e3d5914fcf8ac7b3f77d2e7d424018913e62f5f3ade51995da109412"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3089,6 +3114,12 @@ dependencies = [
  "lock_api",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -3405,19 +3436,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.19.15",
-]
-
-[[package]]
-name = "toml"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit 0.20.2",
+ "toml_edit",
 ]
 
 [[package]]
@@ -3442,19 +3461,6 @@ dependencies = [
  "winnow",
 ]
 
-[[package]]
-name = "toml_edit"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
-dependencies = [
- "indexmap 2.0.2",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "winnow",
-]
-
 [[package]]
 name = "tonic"
 version = "0.9.2"
@@ -4108,10 +4114,9 @@ dependencies = [
  "kube",
  "operator-api",
  "prometheus",
- "schemars",
  "serde",
  "serde_json",
- "thiserror",
+ "serde_yaml",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -4131,9 +4136,7 @@ dependencies = [
  "operator-api",
  "reqwest",
  "serde",
- "serde_json",
  "tokio",
- "toml 0.8.2",
  "tonic",
  "tonic-health",
  "tracing",
diff --git a/crd-api/Cargo.toml b/crd-api/Cargo.toml
index 9ccafbc0..10cf1c64 100644
--- a/crd-api/Cargo.toml
+++ b/crd-api/Cargo.toml
@@ -12,14 +12,16 @@ keywords = ["operator", "API"]
 
 [dependencies]
 anyhow = "1.0.75"
+garde = { version = "0.16.1", default-features = false, features = ["derive", "pattern", "url"] }
 k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive"] }
+regex = { version = "1", default-features = false, features = ["unicode-perl"] } # garde did not enable regex unicode-perl feature by default, but we need it
 schemars = "0.8.6"
 serde = { version = "1.0.130", features = ["derive"] }
-serde_json = "1.0.97"
+serde_json = "1"
 tokio = { version = "1.0", features = ["time"] }
 tracing = "0.1.37"
 
-[dev-dependencies]
-garde = { version = "0.11.2", default-features = false, features = ["derive", "pattern"] }
-serde_yaml = "0.9.25"
+# Some false positive deps...
+[package.metadata.cargo-machete]
+ignored = ["regex", "serde_json"]
diff --git a/crd-api/src/v1alpha1/cluster.rs b/crd-api/src/v1alpha1/cluster.rs
index f864f6df..db80454b 100644
--- a/crd-api/src/v1alpha1/cluster.rs
+++ b/crd-api/src/v1alpha1/cluster.rs
@@ -2,17 +2,16 @@
 #![allow(clippy::str_to_string)]
 #![allow(clippy::missing_docs_in_private_items)]
 
-#[cfg(test)]
 use garde::Validate;
 use k8s_openapi::api::core::v1::{Affinity, Container, PersistentVolumeClaim};
 use k8s_openapi::serde::{Deserialize, Serialize};
 use kube::CustomResource;
 use schemars::JsonSchema;
 use std::collections::HashMap;
+use std::net::IpAddr;
 
 /// Xline cluster specification
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[cfg_attr(test, derive(Validate))]
+#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Validate)]
 #[kube(
     group = "xlineoperator.xline.cloud",
     version = "v1alpha1",
@@ -29,63 +28,59 @@ use std::collections::HashMap;
     printcolumn = r#"{"name":"Backup Cron", "type":"string", "description":"The cron spec defining the interval a backup CronJob is run", "jsonPath":".spec.backup.cron"}"#,
     printcolumn = r#"{"name":"Age", "type":"date", "description":"The cluster age", "jsonPath":".metadata.creationTimestamp"}"#
 )]
+#[schemars(rename_all = "camelCase")]
+#[garde(allow_unvalidated)]
 pub struct ClusterSpec {
     /// Size of the xline cluster, less than 3 is not allowed
-    #[cfg_attr(test, garde(range(min = 3)))]
+    #[garde(range(min = 3))]
     #[schemars(range(min = 3))]
-    pub size: i32,
+    pub size: usize,
     /// Xline container specification
-    #[cfg_attr(test, garde(skip))]
     pub container: Container,
     /// The affinity of the xline node
-    #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub affinity: Option<Affinity>,
     /// Backup specification
-    #[cfg_attr(test, garde(custom(option_backup_dive)))]
+    #[garde(dive)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub backup: Option<BackupSpec>,
     /// The data PVC, if it is not specified, then use emptyDir instead
-    #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub data: Option<PersistentVolumeClaim>,
     /// Some user defined persistent volume claim templates
-    #[cfg_attr(test, garde(skip))]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub pvcs: Option<Vec<PersistentVolumeClaim>>,
 }
 
 /// Xline cluster backup specification
-#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
-#[cfg_attr(test, derive(Validate))]
+#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, Validate)]
 pub struct BackupSpec {
     /// Cron Spec
-    #[cfg_attr(test, garde(pattern(r"^(?:\*|[0-5]?\d)(?:[-/,]?(?:\*|[0-5]?\d))*(?: +(?:\*|1?[0-9]|2[0-3])(?:[-/,]?(?:\*|1?[0-9]|2[0-3]))*){4}$")))]
+    #[garde(pattern(r"^(?:\*|[0-5]?\d)(?:[-/,]?(?:\*|[0-5]?\d))*(?: +(?:\*|1?[0-9]|2[0-3])(?:[-/,]?(?:\*|1?[0-9]|2[0-3]))*){4}$"))]
     #[schemars(regex(
         pattern = r"^(?:\*|[0-5]?\d)(?:[-/,]?(?:\*|[0-5]?\d))*(?: +(?:\*|1?[0-9]|2[0-3])(?:[-/,]?(?:\*|1?[0-9]|2[0-3]))*){4}$"
     ))]
     pub cron: String,
     /// Backup storage type
-    #[cfg_attr(test, garde(dive))]
+    #[garde(dive)]
     #[serde(flatten)]
     pub storage: StorageSpec,
 }
 
 /// Xline cluster backup storage specification
-#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
-#[cfg_attr(test, derive(Validate))]
+#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, Validate)]
 #[serde(untagged)]
 pub enum StorageSpec {
     /// S3 backup type
     S3 {
         /// S3 backup specification
-        #[cfg_attr(test, garde(dive))]
+        #[garde(dive)]
         s3: S3Spec,
     },
     /// Persistent volume backup type
     Pvc {
         /// Persistent volume claim
-        #[cfg_attr(test, garde(skip))]
+        #[garde(skip)]
         pvc: PersistentVolumeClaim,
     },
 }
@@ -100,32 +95,24 @@ impl StorageSpec {
 }
 
 /// Xline cluster backup S3 specification
-#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
-#[cfg_attr(test, derive(Validate))]
+#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, Validate)]
 pub struct S3Spec {
     /// S3 bucket name to use for backup
-    #[cfg_attr(test, garde(pattern(r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$")))]
+    #[garde(pattern(r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$"))]
     #[schemars(regex(pattern = r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$"))]
     pub bucket: String,
 }
 
 /// Xline cluster status
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default, Validate)]
+#[garde(context(ClusterSpec as ctx))]
 pub struct ClusterStatus {
     /// The available nodes' number in the cluster
-    pub available: i32,
+    #[garde(range(max = ctx.size))]
+    pub available: usize,
     /// The members registry
-    pub members: HashMap<String, String>,
-}
-
-#[cfg(test)]
-#[allow(clippy::trivially_copy_pass_by_ref)] // required bt garde
-fn option_backup_dive(value: &Option<BackupSpec>, _cx: &()) -> garde::Result {
-    if let Some(spec) = value.as_ref() {
-        spec.validate(&())
-            .map_err(|e| garde::Error::new(e.to_string()))?;
-    }
-    Ok(())
+    #[garde(skip)]
+    pub members: HashMap<String, IpAddr>,
 }
 
 #[cfg(test)]
@@ -168,10 +155,10 @@ mod test {
             pvcs: None,
             data: None,
         };
-        assert_eq!(
-            Validate::validate(&bad_size, &()).unwrap_err().flatten()[0].0,
-            "value.size"
-        );
+        assert!(Validate::validate(&bad_size, &())
+            .unwrap_err()
+            .to_string()
+            .contains("size"));
     }
 
     #[test]
@@ -189,10 +176,10 @@ mod test {
             pvcs: None,
             data: None,
         };
-        assert_eq!(
-            Validate::validate(&bad_cron, &()).unwrap_err().flatten()[0].0,
-            "value.backup"
-        );
+        assert!(Validate::validate(&bad_cron, &())
+            .unwrap_err()
+            .to_string()
+            .contains("backup.cron"));
     }
 
     #[test]
@@ -212,9 +199,9 @@ mod test {
             pvcs: None,
             data: None,
         };
-        assert_eq!(
-            Validate::validate(&bad_bucket, &()).unwrap_err().flatten()[0].0,
-            "value.backup"
-        );
+        assert!(Validate::validate(&bad_bucket, &())
+            .unwrap_err()
+            .to_string()
+            .contains("backup.storage.s3.bucket"))
     }
 }
diff --git a/operator-api/src/registry.rs b/operator-api/src/registry.rs
index 0a90a22b..1b9a6db3 100644
--- a/operator-api/src/registry.rs
+++ b/operator-api/src/registry.rs
@@ -7,7 +7,9 @@ use k8s_openapi::serde_json::json;
 use kube::api::{Patch, PatchParams};
 use kube::core::object::HasStatus;
 use kube::{Api, Client};
+
 use std::collections::HashMap;
+use std::net::ToSocketAddrs;
 
 pub struct Config {
     /// Members [node_name] => [node_ip]
@@ -23,12 +25,13 @@ pub trait Registry {
     async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config>;
 }
 
-pub struct CustomResourceRegistry {
+/// K8s custom resource `Cluster` status registry
+pub struct K8sClusterStatusRegistry {
     cluster_name: String,
     cluster_api: Api<Cluster>,
 }
 
-impl CustomResourceRegistry {
+impl K8sClusterStatusRegistry {
     /// New a registry with default kube client
     pub async fn new_with_default(cluster_name: String, namespace: &str) -> Self {
         let kube_client = Client::try_default()
@@ -49,15 +52,23 @@ impl CustomResourceRegistry {
 }
 
 #[async_trait]
-impl Registry for CustomResourceRegistry {
+impl Registry for K8sClusterStatusRegistry {
     async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config> {
         /// TODO: hold a distributed lock here
         let cluster = self.cluster_api.get_status(&self.cluster_name).await?;
         let mut status = cluster
             .status
             .ok_or_else(|| anyhow!("no status found in cluster {}", self.cluster_name))?;
-        if status.members.get(&self_name) != Some(&self_ip) {
-            status.members.insert(self_name, self_ip);
+
+        // dns may not change, but ip can
+        let ip = format!("{self_ip}:0")
+            .to_socket_addrs()?
+            .next()
+            .ok_or_else(|| anyhow!("cannot resolve dns {self_ip}"))?
+            .ip();
+
+        if status.members.get(&self_name) != Some(&ip) {
+            status.members.insert(self_name, ip);
             let patch = json!({
                 "status": status,
             });
@@ -70,9 +81,14 @@ impl Registry for CustomResourceRegistry {
                 )
                 .await?;
         }
+
         Ok(Config {
-            members: status.members,
-            cluster_size: cluster.spec.size as usize,
+            members: status
+                .members
+                .into_iter()
+                .map(|(k, v)| (k, v.to_string()))
+                .collect(),
+            cluster_size: cluster.spec.size,
         })
     }
 }
diff --git a/operator-k8s/Cargo.toml b/operator-k8s/Cargo.toml
index be24aa06..2984035b 100644
--- a/operator-k8s/Cargo.toml
+++ b/operator-k8s/Cargo.toml
@@ -24,10 +24,8 @@ k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive"] }
 operator-api = { path = "../operator-api" }
 prometheus = "0.13.3"
-schemars = "0.8.6"
 serde = { version = "1.0.130", features = ["derive"] }
 serde_json = "1.0.97"
-thiserror = "1.0.40"
 tokio = { version = "1.0", features = [
     "rt-multi-thread",
     "time",
@@ -37,3 +35,5 @@ tokio = { version = "1.0", features = [
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 
+[dev-dependencies]
+serde_yaml = "0.9.25"
diff --git a/operator-k8s/src/manager/cluster.rs b/operator-k8s/src/manager/cluster.rs
index b3bfce45..ea92d634 100644
--- a/operator-k8s/src/manager/cluster.rs
+++ b/operator-k8s/src/manager/cluster.rs
@@ -374,7 +374,10 @@ impl Factory {
         StatefulSet {
             metadata: self.general_metadata(Component::Nodes),
             spec: Some(StatefulSetSpec {
-                replicas: Some(size),
+                replicas: Some(
+                    i32::try_from(size)
+                        .unwrap_or_else(|_| unreachable!("size should not overflow i32::MAX")),
+                ),
                 selector: LabelSelector {
                     match_expressions: None,
                     match_labels: Some(labels),
diff --git a/operator-k8s/src/monitor.rs b/operator-k8s/src/monitor.rs
index 0c05013f..0fd0a3b6 100644
--- a/operator-k8s/src/monitor.rs
+++ b/operator-k8s/src/monitor.rs
@@ -166,11 +166,7 @@ impl SidecarMonitor {
     /// Get the certain cluster size in the specification
     async fn get_spec_size(&self, cluster_name: &str) -> Result<usize> {
         let cluster = self.ctx.cluster_api.get(cluster_name).await?;
-        Ok(cluster
-            .spec
-            .size
-            .try_into()
-            .unwrap_or_else(|_| unreachable!("the spec size should not be negative")))
+        Ok(cluster.spec.size)
     }
 }
 
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 08a99161..94bba3d3 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -22,14 +22,12 @@ futures = "0.3.28"
 operator-api = { path = "../operator-api" }
 reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0.130", features = ["derive"] }
-serde_json = "1.0.97"
 tokio = { version = "1.0", features = [
   "rt-multi-thread",
   "time",
   "macros",
   "net",
 ] }
-toml = "0.8.2"
 tonic = "0.9.2"
 tonic-health = "0.9.2"
 tracing = "0.1.37"
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index 86175ed3..df0dcc79 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -146,7 +146,7 @@ use operator_api::consts::DEFAULT_DATA_DIR;
 use operator_api::XlineConfig;
 use tracing::debug;
 use xline_sidecar::sidecar::Sidecar;
-use xline_sidecar::types::{BackendConfig, BackupConfig, Config, MonitorConfig};
+use xline_sidecar::types::{BackendConfig, BackupConfig, Config, MemberConfig, MonitorConfig};
 
 /// `DEFAULT_DATA_DIR` to String
 fn default_data_dir() -> String {
@@ -216,9 +216,11 @@ impl From<Cli> for Config {
         Self {
             name: value.name.clone(),
             cluster_name: value.cluster_name,
-            init_members: value.init_members,
-            xline_port: value.xline_port,
-            sidecar_port: value.sidecar_port,
+            init_member: MemberConfig {
+                members: value.init_members,
+                xline_port: value.xline_port,
+                sidecar_port: value.sidecar_port,
+            },
             reconcile_interval: Duration::from_secs(value.reconcile_interval),
             backend: value.backend,
             xline: XlineConfig {
diff --git a/sidecar/src/sidecar.rs b/sidecar/src/sidecar.rs
index 0ed51955..14694243 100644
--- a/sidecar/src/sidecar.rs
+++ b/sidecar/src/sidecar.rs
@@ -117,7 +117,7 @@ impl Sidecar {
             &self.config.xline.data_dir,
             backup,
             inner,
-            self.config.xline_port,
+            self.config.init_member.xline_port,
         )
     }
 
@@ -138,7 +138,7 @@ impl Sidecar {
         };
         let cluster_name = self.config.cluster_name.clone();
         let name = self.config.name.clone();
-        let init_members = self.config.init_sidecar_members();
+        let init_members = self.config.init_member.sidecar_members();
 
         #[allow(clippy::integer_arithmetic)] // this error originates in the macro `tokio::select`
         let _ig = tokio::spawn(async move {
@@ -183,7 +183,7 @@ impl Sidecar {
             handle,
             self.config.reconcile_interval,
         );
-        let init_member_config = self.config.init_member_config();
+        let init_member_config = self.config.init_member.clone();
         let _ig = tokio::spawn(async move {
             let mut shutdown = graceful_shutdown;
             let res = controller
@@ -204,7 +204,7 @@ impl Sidecar {
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
-        let members = self.config.init_sidecar_members();
+        let members = self.config.init_member.sidecar_members();
         let advertise_url = members.get(&self.config.name).ok_or(anyhow!(
             "node name {} not found in members",
             self.config.name
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 61c33ef7..2a5bcc5c 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -16,12 +16,8 @@ pub struct Config {
     pub name: String,
     /// The cluster name
     pub cluster_name: String,
-    /// Nodes initial hosts, [pod_name]->[pod_host]
-    pub init_members: HashMap<String, String>,
-    /// The xline server port
-    pub xline_port: u16,
-    /// The sidecar web server port
-    pub sidecar_port: u16,
+    /// Initial member config
+    pub init_member: MemberConfig,
     /// Reconcile cluster interval
     pub reconcile_interval: Duration,
     /// The xline config
@@ -47,13 +43,14 @@ pub struct MonitorConfig {
 
 /// Member config
 #[derive(Debug, Clone)]
-pub(crate) struct MemberConfig {
+#[allow(clippy::exhaustive_structs)] // It is only constructed once
+pub struct MemberConfig {
     /// Nodes hosts, [pod_name]->[pod_host]
-    pub(crate) members: HashMap<String, String>,
+    pub members: HashMap<String, String>,
     /// The xline server port
-    pub(crate) xline_port: u16,
+    pub xline_port: u16,
     /// The sidecar web server port
-    pub(crate) sidecar_port: u16,
+    pub sidecar_port: u16,
 }
 
 /// Sidecar backend, it determinate how xline could be setup
@@ -89,35 +86,6 @@ pub enum BackupConfig {
     },
 }
 
-impl Config {
-    /// Get the initial sidecar members
-    pub(crate) fn init_sidecar_members(&self) -> HashMap<String, String> {
-        self.init_members
-            .clone()
-            .into_iter()
-            .map(|(name, host)| (name, format!("{host}:{}", self.sidecar_port)))
-            .collect()
-    }
-
-    /// Get the initial xline members
-    pub(crate) fn init_xline_members(&self) -> HashMap<String, String> {
-        self.init_members
-            .clone()
-            .into_iter()
-            .map(|(name, host)| (name, format!("{host}:{}", self.xline_port)))
-            .collect()
-    }
-
-    /// Get the initial member config
-    pub(crate) fn init_member_config(&self) -> MemberConfig {
-        MemberConfig {
-            members: self.init_members.clone(),
-            xline_port: self.xline_port,
-            sidecar_port: self.sidecar_port,
-        }
-    }
-}
-
 impl MemberConfig {
     /// Get the xline members
     pub(crate) fn xline_members(&self) -> HashMap<String, String> {
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index b8f403d3..1d97e415 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -112,6 +112,14 @@ impl XlineHandle {
         Ok(())
     }
 
+    /// Update member update the client
+    /// TODO: should xline client automatically discovery xlines?
+    pub(crate) async fn update_member(&mut self, xlines: &HashMap<String, String>) -> Result<()> {
+        let new_client = Client::connect(xlines.values(), ClientOptions::default()).await?;
+        _ = self.client.replace(new_client);
+        Ok(())
+    }
+
     /// Start the xline server
     pub(crate) async fn start(&mut self, xlines: &HashMap<String, String>) -> Result<()> {
         /// Timeout for test start

From 65e15281b9f0237d14acfd80b851bd4cf65114c9 Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Mon, 30 Oct 2023 22:15:07 +0800
Subject: [PATCH 18/19] feat: implement registry

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                      |   1 +
 crd-api/src/v1alpha1/cluster.rs |   5 -
 operator-api/Cargo.toml         |   1 +
 operator-api/src/consts.rs      |   2 +
 operator-api/src/registry.rs    | 237 ++++++++++++++++++++++++--------
 operator-k8s/src/config.rs      |   5 +-
 operator-k8s/src/lib.rs         |   7 +
 operator-k8s/src/monitor.rs     |   8 +-
 operator-k8s/src/operator.rs    |  33 +++--
 operator-k8s/src/registry.rs    |  76 ++++++++++
 operator-k8s/src/router.rs      |  33 ++++-
 sidecar/src/controller.rs       |  51 ++++++-
 sidecar/src/main.rs             |  95 ++++++++++++-
 sidecar/src/sidecar.rs          |  59 ++++++--
 sidecar/src/types.rs            |  29 +++-
 sidecar/src/xline.rs            |  43 +++++-
 16 files changed, 585 insertions(+), 100 deletions(-)
 create mode 100644 operator-k8s/src/registry.rs

diff --git a/Cargo.lock b/Cargo.lock
index 79ec6c14..7fa61d0e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2203,6 +2203,7 @@ dependencies = [
  "kube",
  "reqwest",
  "serde",
+ "tokio",
 ]
 
 [[package]]
diff --git a/crd-api/src/v1alpha1/cluster.rs b/crd-api/src/v1alpha1/cluster.rs
index db80454b..6c375802 100644
--- a/crd-api/src/v1alpha1/cluster.rs
+++ b/crd-api/src/v1alpha1/cluster.rs
@@ -7,8 +7,6 @@ use k8s_openapi::api::core::v1::{Affinity, Container, PersistentVolumeClaim};
 use k8s_openapi::serde::{Deserialize, Serialize};
 use kube::CustomResource;
 use schemars::JsonSchema;
-use std::collections::HashMap;
-use std::net::IpAddr;
 
 /// Xline cluster specification
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Validate)]
@@ -110,9 +108,6 @@ pub struct ClusterStatus {
     /// The available nodes' number in the cluster
     #[garde(range(max = ctx.size))]
     pub available: usize,
-    /// The members registry
-    #[garde(skip)]
-    pub members: HashMap<String, IpAddr>,
 }
 
 #[cfg(test)]
diff --git a/operator-api/Cargo.toml b/operator-api/Cargo.toml
index 03d5882d..b8672786 100644
--- a/operator-api/Cargo.toml
+++ b/operator-api/Cargo.toml
@@ -19,3 +19,4 @@ k8s-openapi = { version = "0.20.0", features = ["v1_28", "schemars"] }
 kube = { version = "0.86.0", features = ["runtime", "derive", "ws"] }
 reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0.130", features = ["derive"] }
+tokio = { version = "1.0", features = ["time"] }
diff --git a/operator-api/src/consts.rs b/operator-api/src/consts.rs
index 161cb8f8..c9bd5e2d 100644
--- a/operator-api/src/consts.rs
+++ b/operator-api/src/consts.rs
@@ -4,6 +4,8 @@ pub const DEFAULT_BACKUP_DIR: &str = "/xline-backup";
 pub const DEFAULT_DATA_DIR: &str = "/usr/local/xline/data-dir";
 /// the URL ROUTE that sidecar sends heartbeat status to
 pub const OPERATOR_MONITOR_ROUTE: &str = "/monitor";
+/// the URL ROUTE that sidecar sends fetch config to
+pub const OPERATOR_REGISTRY_ROUTE: &str = "/registry";
 /// the URL ROUTE of each sidecar for backup
 pub const SIDECAR_BACKUP_ROUTE: &str = "/backup";
 /// the URL ROUTE of each sidecar member for health checking
diff --git a/operator-api/src/registry.rs b/operator-api/src/registry.rs
index 1b9a6db3..1f531d72 100644
--- a/operator-api/src/registry.rs
+++ b/operator-api/src/registry.rs
@@ -3,92 +3,217 @@
 use anyhow::anyhow;
 use async_trait::async_trait;
 use crd_api::Cluster;
+use k8s_openapi::api::apps::v1::StatefulSet;
 use k8s_openapi::serde_json::json;
 use kube::api::{Patch, PatchParams};
 use kube::core::object::HasStatus;
 use kube::{Api, Client};
 
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::net::ToSocketAddrs;
+use std::sync::OnceLock;
+use std::time::Duration;
 
+use crate::consts::OPERATOR_REGISTRY_ROUTE;
+
+pub use dummy::*;
+pub use http::*;
+pub use k8s::*;
+
+#[derive(Deserialize, Debug, Default, Clone, Serialize)]
 pub struct Config {
-    /// Members [node_name] => [node_ip]
+    /// Members [node_name] => [node_host]
     pub members: HashMap<String, String>,
     /// Cluster size
     pub cluster_size: usize,
 }
 
+const WAIT_DELAY: Duration = Duration::from_secs(5);
+const WAIT_THRESHOLD: usize = 60;
+
 #[async_trait]
 pub trait Registry {
-    const FIELD_MANAGER: &'static str = "xlineoperator.datenlord.io/registry";
+    async fn send_fetch(&self, self_name: String, self_host: String) -> anyhow::Result<Config>;
 
-    async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config>;
+    async fn wait_full_fetch(
+        &self,
+        self_name: String,
+        self_host: String,
+    ) -> anyhow::Result<Config> {
+        let mut retry = 0;
+        loop {
+            let config = self
+                .send_fetch(self_name.clone(), self_host.clone())
+                .await?;
+            if config.members.len() == config.cluster_size {
+                break Ok(config);
+            }
+            retry += 1;
+            if retry > WAIT_THRESHOLD {
+                break Err(anyhow!("wait for full config timeout"));
+            }
+            tokio::time::sleep(WAIT_DELAY).await;
+        }
+    }
 }
 
-/// K8s custom resource `Cluster` status registry
-pub struct K8sClusterStatusRegistry {
-    cluster_name: String,
-    cluster_api: Api<Cluster>,
-}
+mod k8s {
+    use super::*;
 
-impl K8sClusterStatusRegistry {
-    /// New a registry with default kube client
-    pub async fn new_with_default(cluster_name: String, namespace: &str) -> Self {
-        let kube_client = Client::try_default()
-            .await
-            .unwrap_or_else(|_ig| unreachable!("it must be setup in k8s environment"));
-        Self {
-            cluster_name,
-            cluster_api: Api::namespaced(kube_client, namespace),
+    const FIELD_MANAGER: &str = "xlineoperator.datenlord.io/registry";
+
+    /// K8s statefulset registry
+    pub struct K8sStsRegistry {
+        sts_name: String,
+        sts_api: Api<StatefulSet>,
+        namespace: String,
+        dns_suffix: String,
+    }
+
+    impl K8sStsRegistry {
+        /// New a k8s statefulset registry with default kube client
+        pub async fn new_with_default(
+            sts_name: String,
+            namespace: String,
+            dns_suffix: String,
+        ) -> Self {
+            let kube_client = Client::try_default()
+                .await
+                .unwrap_or_else(|_ig| unreachable!("it must be setup in k8s environment"));
+            Self {
+                sts_name,
+                sts_api: Api::namespaced(kube_client, &namespace),
+                namespace,
+                dns_suffix,
+            }
+        }
+
+        pub fn new(
+            sts_name: String,
+            namespace: String,
+            kube_client: Client,
+            dns_suffix: String,
+        ) -> Self {
+            Self {
+                sts_name,
+                sts_api: Api::namespaced(kube_client, &namespace),
+                namespace,
+                dns_suffix,
+            }
         }
     }
 
-    pub async fn new(cluster_name: String, namespace: &str, kube_client: Client) -> Self {
-        Self {
-            cluster_name,
-            cluster_api: Api::namespaced(kube_client, namespace),
+    #[async_trait]
+    impl Registry for K8sStsRegistry {
+        async fn send_fetch(&self, _: String, _: String) -> anyhow::Result<Config> {
+            let sts = self.sts_api.get(&self.sts_name).await?;
+            let spec = sts
+                .spec
+                .unwrap_or_else(|| unreachable!(".spec should be set in statefulset"));
+            let replicas = spec
+                .replicas
+                .unwrap_or_else(|| unreachable!(".spec.replicas should be set in statefulset"));
+            let start_at = spec
+                .ordinals
+                .into_iter()
+                .flat_map(|ordinal| ordinal.start)
+                .next()
+                .unwrap_or(0);
+            let members = (start_at..)
+                .take(replicas as usize)
+                .map(|idx| {
+                    let name = format!("{}-{idx}", self.sts_name);
+                    let host = format!(
+                        "{name}.{}.{}.svc.{}",
+                        spec.service_name, self.namespace, self.dns_suffix
+                    );
+                    (name, host)
+                })
+                .collect();
+            Ok(Config {
+                members,
+                cluster_size: replicas as usize,
+            })
         }
     }
 }
 
-#[async_trait]
-impl Registry for K8sClusterStatusRegistry {
-    async fn send_fetch(&self, self_name: String, self_ip: String) -> anyhow::Result<Config> {
-        /// TODO: hold a distributed lock here
-        let cluster = self.cluster_api.get_status(&self.cluster_name).await?;
-        let mut status = cluster
-            .status
-            .ok_or_else(|| anyhow!("no status found in cluster {}", self.cluster_name))?;
-
-        // dns may not change, but ip can
-        let ip = format!("{self_ip}:0")
-            .to_socket_addrs()?
-            .next()
-            .ok_or_else(|| anyhow!("cannot resolve dns {self_ip}"))?
-            .ip();
-
-        if status.members.get(&self_name) != Some(&ip) {
-            status.members.insert(self_name, ip);
-            let patch = json!({
-                "status": status,
+mod http {
+    use super::*;
+
+    static DEFAULT_HTTP_CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+
+    #[derive(Debug, Deserialize, Serialize, Clone)]
+    pub struct RegisterQuery {
+        pub cluster: String,
+        pub name: String,
+        pub host: String,
+    }
+
+    /// HTTP server registry
+    pub struct HttpRegistry {
+        server_addr: String,
+        cluster_name: String,
+    }
+
+    impl HttpRegistry {
+        pub fn new(server_addr: String, cluster_name: String) -> Self {
+            Self {
+                server_addr,
+                cluster_name,
+            }
+        }
+    }
+
+    #[async_trait]
+    impl Registry for HttpRegistry {
+        async fn send_fetch(&self, self_name: String, self_host: String) -> anyhow::Result<Config> {
+            let client = DEFAULT_HTTP_CLIENT.get_or_init(|| {
+                reqwest::Client::builder().build().unwrap_or_else(|err| {
+                    unreachable!("cannot build http client to register config, err: {err}")
+                })
             });
-            let _ig = self
-                .cluster_api
-                .patch_status(
-                    &self.cluster_name,
-                    &PatchParams::apply(Self::FIELD_MANAGER),
-                    &Patch::Apply(patch),
-                )
+            let config: Config = client
+                .get(format!(
+                    "http://{}{}",
+                    self.server_addr, OPERATOR_REGISTRY_ROUTE
+                ))
+                .query(&RegisterQuery {
+                    cluster: self.cluster_name.clone(),
+                    name: self_name,
+                    host: self_host,
+                })
+                .send()
+                .await?
+                .json()
                 .await?;
+            Ok(config)
+        }
+    }
+}
+
+mod dummy {
+    use super::*;
+
+    /// Dummy registry does not register anything, it keeps the original config
+    pub struct DummyRegistry {
+        init_members: HashMap<String, String>,
+    }
+
+    impl DummyRegistry {
+        pub fn new(init_members: HashMap<String, String>) -> Self {
+            Self { init_members }
         }
+    }
 
-        Ok(Config {
-            members: status
-                .members
-                .into_iter()
-                .map(|(k, v)| (k, v.to_string()))
-                .collect(),
-            cluster_size: cluster.spec.size,
-        })
+    #[async_trait]
+    impl Registry for DummyRegistry {
+        async fn send_fetch(&self, _: String, _: String) -> anyhow::Result<Config> {
+            Ok(Config {
+                members: self.init_members.clone(),
+                cluster_size: self.init_members.len(),
+            })
+        }
     }
 }
diff --git a/operator-k8s/src/config.rs b/operator-k8s/src/config.rs
index 7741361c..3346552b 100644
--- a/operator-k8s/src/config.rs
+++ b/operator-k8s/src/config.rs
@@ -23,11 +23,14 @@ pub struct Config {
     #[arg(long, default_value = "cluster.local")]
     pub cluster_suffix: String,
     /// Maximum interval between accepted `HeartbeatStatus`
-    #[arg(long, default_value = "2")]
+    #[arg(long, default_value = "5")]
     pub heartbeat_period: u64,
     /// Sidecar unreachable counter threshold
     #[arg(long, default_value = "4")]
     pub unreachable_thresh: usize,
+    /// Sidecar config time to live, in seconds. No longer than 300.
+    #[arg(long, default_value = "30")]
+    pub registry_ttl: u64,
 }
 
 /// The namespace to work, `ClusterWide` means work with all namespaces
diff --git a/operator-k8s/src/lib.rs b/operator-k8s/src/lib.rs
index 6395a0ed..ba1fb12a 100644
--- a/operator-k8s/src/lib.rs
+++ b/operator-k8s/src/lib.rs
@@ -149,6 +149,11 @@
     )
 )]
 
+use std::collections::HashMap;
+
+/// A sidecar cluster states map
+type SidecarClusterOwned<T> = HashMap<String, T>;
+
 /// Xline operator config
 pub mod config;
 /// Some constants
@@ -161,6 +166,8 @@ mod manager;
 mod monitor;
 /// Xline operator
 pub mod operator;
+/// HTTP server registry for sidecars to find each other
+mod registry;
 /// Xline operator web server router
 mod router;
 
diff --git a/operator-k8s/src/monitor.rs b/operator-k8s/src/monitor.rs
index 0fd0a3b6..9af24cc3 100644
--- a/operator-k8s/src/monitor.rs
+++ b/operator-k8s/src/monitor.rs
@@ -12,6 +12,7 @@ use operator_api::HeartbeatStatus;
 use tracing::{debug, error};
 
 use crate::crd::Cluster;
+use crate::SidecarClusterOwned;
 
 /// Sidecar monitor context
 struct Context {
@@ -27,9 +28,6 @@ struct Context {
     pod_api: Api<Pod>,
 }
 
-/// A sidecar cluster states map
-type SidecarClusterOwned<T> = HashMap<String, T>;
-
 /// Sidecar monitor.
 /// It monitors the communication of all sidecars, finds and tries to recover the dropped sidecar.
 pub(crate) struct SidecarMonitor {
@@ -65,7 +63,7 @@ impl SidecarMonitor {
 
     /// Run the state update task with graceful shutdown.
     /// Return fatal error if run failed.
-    /// The task that update the state received from sidecar operators
+    /// The task that update the state received from sidecars
     #[allow(clippy::integer_arithmetic)] // required by tokio::select
     pub(crate) async fn run_with_graceful_shutdown(
         self,
@@ -81,7 +79,7 @@ impl SidecarMonitor {
         }
     }
 
-    /// Inner task for state update, return the unrecoverable error
+    /// Task for state update, return the unrecoverable error
     async fn state_update(mut self) -> Result<()> {
         loop {
             let status = self.ctx.status_rx.recv_async().await?;
diff --git a/operator-k8s/src/operator.rs b/operator-k8s/src/operator.rs
index dedd0868..f68d1f23 100644
--- a/operator-k8s/src/operator.rs
+++ b/operator-k8s/src/operator.rs
@@ -1,16 +1,17 @@
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::Result;
-use axum::routing::any;
 use axum::routing::post;
+use axum::routing::{any, get};
 use axum::{Extension, Router};
 use futures::FutureExt;
 use k8s_openapi::api::core::v1::Pod;
 use kube::{Api, Client};
-use operator_api::consts::OPERATOR_MONITOR_ROUTE;
+use operator_api::consts::{OPERATOR_MONITOR_ROUTE, OPERATOR_REGISTRY_ROUTE};
 use operator_api::HeartbeatStatus;
-use prometheus::Registry;
 use tokio::sync::watch::{Receiver, Sender};
+use tokio::sync::Mutex;
 use tracing::{error, info, warn};
 
 use crate::config::{Config, Namespace};
@@ -18,7 +19,8 @@ use crate::controller::cluster::{ClusterController, ClusterMetrics};
 use crate::controller::{Controller, Metrics};
 use crate::crd::Cluster;
 use crate::monitor::SidecarMonitor;
-use crate::router::{healthz, metrics, sidecar_monitor};
+use crate::registry::Registry;
+use crate::router::{healthz, metrics, sidecar_monitor, sidecar_registry};
 
 /// Xline Operator for k8s
 #[derive(Debug)]
@@ -61,7 +63,7 @@ impl Operator {
         let (graceful_shutdown_event, _) = tokio::sync::watch::channel(());
         let forceful_shutdown = self.forceful_shutdown(&graceful_shutdown_event);
         let (status_tx, status_rx) = flume::unbounded();
-        let registry = Registry::new();
+        let registry = prometheus::Registry::new();
 
         self.start_sidecar_monitor(
             status_rx,
@@ -71,11 +73,16 @@ impl Operator {
         );
         self.start_controller(
             kube_client,
-            cluster_api,
+            cluster_api.clone(),
             &registry,
             graceful_shutdown_event.subscribe(),
         )?;
-        self.start_web_server(status_tx, registry, graceful_shutdown_event.subscribe())?;
+        self.start_web_server(
+            status_tx,
+            cluster_api,
+            registry,
+            graceful_shutdown_event.subscribe(),
+        )?;
 
         tokio::pin!(forceful_shutdown);
 
@@ -108,7 +115,7 @@ impl Operator {
         &self,
         kube_client: Client,
         cluster_api: Api<Cluster>,
-        registry: &Registry,
+        registry: &prometheus::Registry,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
         let metrics = ClusterMetrics::new();
@@ -178,15 +185,21 @@ impl Operator {
     fn start_web_server(
         &self,
         status_tx: flume::Sender<HeartbeatStatus>,
-        registry: Registry,
+        cluster_api: Api<Cluster>,
+        metrics_registry: prometheus::Registry,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
         let status = Router::new()
             .route(OPERATOR_MONITOR_ROUTE, post(sidecar_monitor))
+            .route(OPERATOR_REGISTRY_ROUTE, get(sidecar_registry))
             .route("/metrics", any(metrics))
             .route("/healthz", any(healthz))
             .layer(Extension(status_tx))
-            .layer(Extension(registry));
+            .layer(Extension(metrics_registry))
+            .layer(Extension(Arc::new(Mutex::new(Registry::new(
+                Duration::from_secs(self.config.registry_ttl),
+                cluster_api,
+            )))));
         let server = axum::Server::bind(&self.config.listen_addr.parse()?);
 
         let _ig = tokio::spawn(async move {
diff --git a/operator-k8s/src/registry.rs b/operator-k8s/src/registry.rs
new file mode 100644
index 00000000..d254e718
--- /dev/null
+++ b/operator-k8s/src/registry.rs
@@ -0,0 +1,76 @@
+use anyhow::Result;
+use crd_api::Cluster;
+use kube::Api;
+use operator_api::registry::Config;
+
+use std::collections::HashMap;
+use std::time::{Duration, Instant};
+
+use crate::SidecarClusterOwned;
+
+/// Member status
+#[derive(Debug, Clone)]
+struct MemberStatus {
+    /// Host
+    host: String,
+    /// Last active
+    last_active: Instant,
+}
+
+/// Http server registry
+#[derive(Debug)]
+pub(crate) struct Registry {
+    /// Member status, [cluster_name] => [sidecar cluster members]
+    members: HashMap<String, SidecarClusterOwned<MemberStatus>>,
+    /// Time to live
+    ttl: Duration,
+    /// Api for Cluster
+    cluster_api: Api<Cluster>,
+}
+
+impl Registry {
+    /// Create a new registry
+    pub(crate) fn new(ttl: Duration, cluster_api: Api<Cluster>) -> Self {
+        Self {
+            members: HashMap::new(),
+            ttl,
+            cluster_api,
+        }
+    }
+
+    /// State update inner
+    pub(crate) async fn receive(
+        &mut self,
+        cluster: String,
+        name: String,
+        host: String,
+    ) -> Result<Config> {
+        let cluster_size = self.get_spec_size(&cluster).await?;
+        let now = Instant::now();
+
+        let sidecars = self.members.entry(cluster).or_default();
+
+        let sidecar = sidecars.entry(name).or_insert_with(|| MemberStatus {
+            host: host.clone(),
+            last_active: now,
+        });
+        sidecar.last_active = now;
+        sidecar.host = host;
+
+        sidecars.retain(|_, status| now.duration_since(status.last_active) < self.ttl);
+
+        Ok(Config {
+            members: sidecars
+                .iter()
+                .map(|(k, v)| (k.clone(), v.host.clone()))
+                .collect(),
+            cluster_size,
+        })
+    }
+
+    /// Get the certain cluster size in the specification
+    async fn get_spec_size(&self, cluster: &str) -> Result<usize> {
+        let cluster = self.cluster_api.get(cluster).await?;
+        Ok(cluster.spec.size)
+    }
+}
diff --git a/operator-k8s/src/router.rs b/operator-k8s/src/router.rs
index 3887e186..040295ec 100644
--- a/operator-k8s/src/router.rs
+++ b/operator-k8s/src/router.rs
@@ -1,12 +1,20 @@
+use axum::extract::Query;
 use axum::{Extension, Json};
 use flume::Sender;
+use operator_api::registry::{Config, RegisterQuery};
 use operator_api::HeartbeatStatus;
-use prometheus::{Encoder, Registry};
+use prometheus::Encoder;
+use tokio::sync::Mutex;
 use tracing::error;
 
+use axum::http::StatusCode;
+use std::sync::Arc;
+
+use crate::registry::Registry;
+
 /// metrics handler
 #[allow(clippy::unused_async)] // require by axum
-pub(crate) async fn metrics(Extension(registry): Extension<Registry>) -> String {
+pub(crate) async fn metrics(Extension(registry): Extension<prometheus::Registry>) -> String {
     let mut buf1 = Vec::new();
     let encoder = prometheus::TextEncoder::new();
     let metric_families = registry.gather();
@@ -31,12 +39,29 @@ pub(crate) async fn healthz() -> &'static str {
 }
 
 /// sidecar monitor handler
-#[allow(clippy::unused_async)] // require by axum
 pub(crate) async fn sidecar_monitor(
     Extension(status_tx): Extension<Sender<HeartbeatStatus>>,
     Json(status): Json<HeartbeatStatus>,
 ) {
-    if let Err(e) = status_tx.send(status) {
+    if let Err(e) = status_tx.send_async(status).await {
         error!("channel send error: {e}");
     }
 }
+
+/// sidecar registry handler
+pub(crate) async fn sidecar_registry(
+    Query(RegisterQuery {
+        cluster,
+        name,
+        host,
+    }): Query<RegisterQuery>,
+    Extension(registry): Extension<Arc<Mutex<Registry>>>,
+) -> Result<Json<Config>, StatusCode> {
+    registry
+        .lock()
+        .await
+        .receive(cluster, name, host)
+        .await
+        .map_err(|_e| StatusCode::INTERNAL_SERVER_ERROR)
+        .map(Json)
+}
diff --git a/sidecar/src/controller.rs b/sidecar/src/controller.rs
index 2b5b358f..cfd3fa9e 100644
--- a/sidecar/src/controller.rs
+++ b/sidecar/src/controller.rs
@@ -1,10 +1,12 @@
 #![allow(dead_code)] // TODO remove when it is implemented
 
+use std::fmt::{Debug, Formatter};
 use std::future::Future;
 use std::sync::Arc;
 use std::time::Duration;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
+use operator_api::registry::Registry;
 use tokio::select;
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::{interval, MissedTickBehavior};
@@ -14,7 +16,6 @@ use crate::types::{MemberConfig, State, StatePayload, StateStatus};
 use crate::xline::XlineHandle;
 
 /// Sidecar operator controller
-#[derive(Debug)]
 pub(crate) struct Controller {
     /// The name of this sidecar
     name: String,
@@ -24,6 +25,18 @@ pub(crate) struct Controller {
     handle: Arc<RwLock<XlineHandle>>,
     /// Check interval
     reconcile_interval: Duration,
+    /// Configuration Registry
+    registry: Arc<dyn Registry + Sync + Send>,
+}
+
+impl Debug for Controller {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Controller")
+            .field("name", &self.name)
+            .field("state", &self.state)
+            .field("reconcile_interval", &self.reconcile_interval)
+            .finish()
+    }
 }
 
 impl Controller {
@@ -33,12 +46,14 @@ impl Controller {
         state: Arc<Mutex<StatePayload>>,
         handle: Arc<RwLock<XlineHandle>>,
         reconcile_interval: Duration,
+        registry: Arc<dyn Registry + Sync + Send>,
     ) -> Self {
         Self {
             name,
             state,
             handle,
             reconcile_interval,
+            registry,
         }
     }
 
@@ -63,14 +78,42 @@ impl Controller {
     pub(crate) async fn run_reconcile(self, init_member_config: MemberConfig) -> Result<()> {
         let mut tick = interval(self.reconcile_interval);
         tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
-        let member_config = init_member_config;
+
+        let self_host = init_member_config
+            .get_host(&self.name)
+            .ok_or_else(|| anyhow!("node name {} not found in initial members", &self.name))?
+            .clone();
+        let init_config = self
+            .registry
+            .wait_full_fetch(self.name.clone(), self_host.clone()) // wait for all nodes to register config
+            .await?;
+        let mut member_config = MemberConfig {
+            members: init_config.members,
+            ..init_member_config
+        };
+
         self.handle
             .write()
             .await
             .start(&member_config.xline_members())
             .await?;
+
         loop {
             let instant = tick.tick().await;
+
+            let config = match self
+                .registry
+                .wait_full_fetch(self.name.clone(), self_host.clone())
+                .await
+            {
+                Ok(c) => c,
+                Err(err) => {
+                    error!("fetch config failed, error {err}");
+                    continue;
+                }
+            };
+            member_config.members = config.members;
+
             if let Err(err) = self.reconcile_once(&member_config).await {
                 error!("reconcile failed, error: {err}");
                 continue;
@@ -91,6 +134,8 @@ impl Controller {
         let cluster_size = member_config.members.len();
         let majority = member_config.majority_cnt();
 
+        handle.apply_members(&xline_members).await?;
+
         let cluster_health = handle.is_healthy().await;
         let xline_running = handle.is_running().await;
         let states = StateStatus::gather(&sidecar_members).await?;
diff --git a/sidecar/src/main.rs b/sidecar/src/main.rs
index df0dcc79..3728f38e 100644
--- a/sidecar/src/main.rs
+++ b/sidecar/src/main.rs
@@ -146,7 +146,9 @@ use operator_api::consts::DEFAULT_DATA_DIR;
 use operator_api::XlineConfig;
 use tracing::debug;
 use xline_sidecar::sidecar::Sidecar;
-use xline_sidecar::types::{BackendConfig, BackupConfig, Config, MemberConfig, MonitorConfig};
+use xline_sidecar::types::{
+    BackendConfig, BackupConfig, Config, MemberConfig, MonitorConfig, RegistryConfig,
+};
 
 /// `DEFAULT_DATA_DIR` to String
 fn default_data_dir() -> String {
@@ -209,6 +211,12 @@ struct Cli {
     /// Heartbeat interval, it is enabled if --monitor_addr is set.
     #[arg(long, alias = "operator_heartbeat_interval", default_value = "10")]
     heartbeat_interval: u64,
+    /// Set registry to enable configuration discovery
+    /// e.g:
+    ///    sts:name:namespace           for k8s statefulset
+    ///    http:register_server_addr    for http registry
+    #[arg(long, value_parser = parse_registry)]
+    registry: Option<RegistryConfig>,
 }
 
 impl From<Cli> for Config {
@@ -236,6 +244,7 @@ impl From<Cli> for Config {
                 monitor_addr: addr,
                 heartbeat_interval: Duration::from_secs(value.heartbeat_interval),
             }),
+            registry: value.registry,
         }
     }
 }
@@ -328,6 +337,41 @@ fn parse_members(s: &str) -> Result<HashMap<String, String>, String> {
     Ok(map)
 }
 
+/// parse registry from string
+/// # Errors
+/// Return error when pass wrong args
+fn parse_registry(s: &str) -> Result<RegistryConfig, String> {
+    if s.is_empty() {
+        return Err("registry type is empty".to_owned());
+    }
+    let mut items: Vec<_> = s.split([':', ' ', ',', '-']).collect();
+    let kind = items.remove(0);
+    match kind {
+        "sts" => {
+            if items.len() != 2 {
+                return Err(format!(
+                    "sts registry type requires 2 argument, got {}",
+                    items.len()
+                ));
+            }
+            let name = items.remove(0).to_owned();
+            let namespace = items.remove(0).to_owned();
+            Ok(RegistryConfig::Sts { name, namespace })
+        }
+        "http" => {
+            if items.len() != 1 {
+                return Err(format!(
+                    "http registry type requires 1 argument, got {}",
+                    items.len()
+                ));
+            }
+            let server_addr = items.remove(0).to_owned();
+            Ok(RegistryConfig::Http { server_addr })
+        }
+        _ => Err(format!("unknown registry type: {kind}")),
+    }
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     tracing_subscriber::fmt::init();
@@ -340,11 +384,11 @@ async fn main() -> Result<()> {
 
 #[cfg(test)]
 mod test {
-    use crate::{parse_backend, parse_backup_type, parse_members, Cli};
+    use crate::{parse_backend, parse_backup_type, parse_members, parse_registry, Cli};
     use clap::Parser;
     use std::collections::HashMap;
     use std::path::PathBuf;
-    use xline_sidecar::types::{BackendConfig, BackupConfig};
+    use xline_sidecar::types::{BackendConfig, BackupConfig, RegistryConfig};
 
     fn full_parameter() -> Vec<&'static str> {
         vec![
@@ -371,6 +415,8 @@ mod test {
             "--operator-addr=xline-operator.svc.default.cluster.local:8080",
 
             "--heartbeat-interval=10",
+
+            "--registry=http:server_addr"
         ]
     }
 
@@ -487,6 +533,43 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_parse_registry() {
+        let test_cases = [
+            (
+                "sts",
+                Err("sts registry type requires 2 argument, got 0".to_owned()),
+            ),
+            (
+                "http",
+                Err("http registry type requires 1 argument, got 0".to_owned()),
+            ),
+            ("", Err("registry type is empty".to_owned())),
+            (
+                "sts:sts_name",
+                Err("sts registry type requires 2 argument, got 1".to_owned()),
+            ),
+            (
+                "sts:sts_name:sts_namespace",
+                Ok(RegistryConfig::Sts {
+                    name: "sts_name".to_owned(),
+                    namespace: "sts_namespace".to_owned(),
+                }),
+            ),
+            (
+                "http:server_addr",
+                Ok(RegistryConfig::Http {
+                    server_addr: "server_addr".to_owned(),
+                }),
+            ),
+            ("unknown", Err("unknown registry type: unknown".to_owned())),
+        ];
+        for (input, expected) in test_cases {
+            let result = parse_registry(input);
+            assert_eq!(result, expected);
+        }
+    }
+
     #[test]
     fn test_parse_cli_should_success() {
         let cli = Cli::parse_from(full_parameter());
@@ -532,5 +615,11 @@ mod test {
                 namespace: "default".to_owned(),
             }
         );
+        assert_eq!(
+            cli.registry,
+            Some(RegistryConfig::Http {
+                server_addr: "server_addr".to_owned()
+            })
+        );
     }
 }
diff --git a/sidecar/src/sidecar.rs b/sidecar/src/sidecar.rs
index 14694243..6a512459 100644
--- a/sidecar/src/sidecar.rs
+++ b/sidecar/src/sidecar.rs
@@ -6,6 +6,7 @@ use axum::routing::{get, post};
 use axum::{Extension, Router};
 use futures::{FutureExt, TryFutureExt};
 use operator_api::consts::{SIDECAR_BACKUP_ROUTE, SIDECAR_HEALTH_ROUTE, SIDECAR_STATE_ROUTE};
+use operator_api::registry::{DummyRegistry, HttpRegistry, K8sStsRegistry, Registry};
 use operator_api::{HeartbeatStatus, K8sXlineHandle, LocalXlineHandle};
 use tokio::sync::watch::{Receiver, Sender};
 use tokio::sync::{Mutex, RwLock};
@@ -16,7 +17,7 @@ use crate::backup::pv::Pv;
 use crate::backup::Provider;
 use crate::controller::Controller;
 use crate::routers;
-use crate::types::{BackendConfig, BackupConfig, Config, State, StatePayload};
+use crate::types::{BackendConfig, BackupConfig, Config, RegistryConfig, State, StatePayload};
 use crate::xline::XlineHandle;
 
 /// Sidecar
@@ -48,9 +49,20 @@ impl Sidecar {
             state: State::Start,
             revision,
         }));
+        let registry: Arc<dyn Registry + Send + Sync> = match self.config.registry.clone() {
+            None => Arc::new(DummyRegistry::new(self.config.init_member.members.clone())),
+            Some(RegistryConfig::Sts { name, namespace }) => Arc::new(
+                K8sStsRegistry::new_with_default(name, namespace, "cluster.local".to_owned()).await,
+            ),
+            Some(RegistryConfig::Http { server_addr }) => Arc::new(HttpRegistry::new(
+                server_addr,
+                self.config.cluster_name.clone(),
+            )),
+        };
 
         self.start_controller(
             Arc::clone(&handle),
+            Arc::clone(&registry),
             Arc::clone(&state),
             graceful_shutdown_event.subscribe(),
         );
@@ -59,7 +71,7 @@ impl Sidecar {
             Arc::clone(&state),
             graceful_shutdown_event.subscribe(),
         )?;
-        self.start_heartbeat(graceful_shutdown_event.subscribe());
+        self.start_heartbeat(registry, graceful_shutdown_event.subscribe())?;
 
         tokio::pin!(forceful_shutdown);
 
@@ -131,14 +143,24 @@ impl Sidecar {
     }
 
     /// Start heartbeat
-    fn start_heartbeat(&self, graceful_shutdown: Receiver<()>) {
+    fn start_heartbeat(
+        &self,
+        registry: Arc<dyn Registry + Send + Sync>,
+        graceful_shutdown: Receiver<()>,
+    ) -> Result<()> {
         let Some(monitor) = self.config.monitor.clone() else {
             info!("monitor did not set, disable heartbeat");
-            return;
+            return Ok(());
         };
         let cluster_name = self.config.cluster_name.clone();
         let name = self.config.name.clone();
-        let init_members = self.config.init_member.sidecar_members();
+        let self_host = self
+            .config
+            .init_member
+            .get_host(&name)
+            .ok_or_else(|| anyhow!("node name {} not found in initial members", &name))?
+            .clone();
+        let mut member_config = self.config.init_member.clone();
 
         #[allow(clippy::integer_arithmetic)] // this error originates in the macro `tokio::select`
         let _ig = tokio::spawn(async move {
@@ -150,10 +172,27 @@ impl Sidecar {
                 tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
                 loop {
                     let instant = tick.tick().await;
+
+                    let config = match registry
+                        .wait_full_fetch(name.clone(), self_host.clone())
+                        .await
+                    {
+                        Ok(c) => c,
+                        Err(err) => {
+                            error!("fetch config failed, error {err}");
+                            continue;
+                        }
+                    };
+                    member_config.members = config.members;
+
                     debug!("send heartbeat at {instant:?}");
-                    let status =
-                        HeartbeatStatus::gather(cluster_name.clone(), name.clone(), &init_members)
-                            .await;
+                    let status = HeartbeatStatus::gather(
+                        cluster_name.clone(),
+                        name.clone(),
+                        &member_config.sidecar_members(),
+                    )
+                    .await;
+
                     debug!("sidecar gathered status: {status:?}");
                     if let Err(e) = status.report(&monitor.monitor_addr).await {
                         error!("heartbeat report failed, error {e}");
@@ -168,12 +207,15 @@ impl Sidecar {
                 _ = heartbeat_task => {}
             }
         });
+
+        Ok(())
     }
 
     /// Start controller
     fn start_controller(
         &self,
         handle: Arc<RwLock<XlineHandle>>,
+        registry: Arc<dyn Registry + Sync + Send>,
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
     ) {
@@ -182,6 +224,7 @@ impl Sidecar {
             state,
             handle,
             self.config.reconcile_interval,
+            registry,
         );
         let init_member_config = self.config.init_member.clone();
         let _ig = tokio::spawn(async move {
diff --git a/sidecar/src/types.rs b/sidecar/src/types.rs
index 2a5bcc5c..07e377d0 100644
--- a/sidecar/src/types.rs
+++ b/sidecar/src/types.rs
@@ -24,11 +24,12 @@ pub struct Config {
     pub xline: XlineConfig,
     /// The backend to run xline
     pub backend: BackendConfig,
-    /// The sidecar monitor (operator) config, set to enable
-    /// heartbeat and configuration discovery
+    /// The sidecar monitor (operator) config, set to enable heartbeat
     pub monitor: Option<MonitorConfig>,
     /// Backup storage config
     pub backup: Option<BackupConfig>,
+    /// The sidecar registry config, set to enable configuration discovery
+    pub registry: Option<RegistryConfig>,
 }
 
 /// Monitor(Operator) config
@@ -41,6 +42,24 @@ pub struct MonitorConfig {
     pub heartbeat_interval: Duration,
 }
 
+/// Registry config
+#[derive(Debug, Clone, Eq, PartialEq)]
+#[non_exhaustive]
+pub enum RegistryConfig {
+    /// Statefulset mode
+    Sts {
+        /// Statefulset name
+        name: String,
+        /// Statefulset namespace
+        namespace: String,
+    },
+    /// Http mode
+    Http {
+        /// Http server address
+        server_addr: String,
+    },
+}
+
 /// Member config
 #[derive(Debug, Clone)]
 #[allow(clippy::exhaustive_structs)] // It is only constructed once
@@ -106,10 +125,14 @@ impl MemberConfig {
     }
 
     /// Get the majority count
-
     pub(crate) fn majority_cnt(&self) -> usize {
         self.members.len().div(2).add(1)
     }
+
+    /// Get the host
+    pub(crate) fn get_host(&self, name: &str) -> Option<&String> {
+        self.members.get(name)
+    }
 }
 
 /// Sidecar operator state
diff --git a/sidecar/src/xline.rs b/sidecar/src/xline.rs
index 1d97e415..3d2aee93 100644
--- a/sidecar/src/xline.rs
+++ b/sidecar/src/xline.rs
@@ -69,6 +69,8 @@ pub(crate) struct XlineHandle {
     is_healthy_retries: usize,
     /// The detailed xline process handle
     inner: Box<dyn operator_api::XlineHandle>,
+    /// The cached member status
+    cached_members: HashMap<String, String>,
 }
 
 impl XlineHandle {
@@ -96,6 +98,7 @@ impl XlineHandle {
             server_id: None,
             is_healthy_retries: 5,
             inner,
+            cached_members: HashMap::new(),
         })
     }
 
@@ -112,11 +115,46 @@ impl XlineHandle {
         Ok(())
     }
 
-    /// Update member update the client
+    /// Apply new members and clean the dead node in the cluster
     /// TODO: should xline client automatically discovery xlines?
-    pub(crate) async fn update_member(&mut self, xlines: &HashMap<String, String>) -> Result<()> {
+    pub(crate) async fn apply_members(&mut self, xlines: &HashMap<String, String>) -> Result<()> {
+        if &self.cached_members == xlines {
+            return Ok(());
+        }
+
+        let mut to_remove = vec![];
+        for name in self.cached_members.keys() {
+            if !xlines.contains_key(name) {
+                to_remove.push(name.clone());
+            }
+        }
+
         let new_client = Client::connect(xlines.values(), ClientOptions::default()).await?;
         _ = self.client.replace(new_client);
+        self.cached_members = xlines.clone();
+
+        if to_remove.is_empty() {
+            return Ok(());
+        }
+
+        // TODO: hold a distributed lock here.
+        let mut cluster_client = self.client().cluster_client();
+        let members: HashMap<String, _> = cluster_client
+            .member_list(MemberListRequest::new(true))
+            .await?
+            .members
+            .into_iter()
+            .map(|member| (member.name.clone(), member))
+            .collect();
+
+        for name in to_remove {
+            if let Some(member) = members.get(&name) {
+                let _ig = cluster_client
+                    .member_remove(MemberRemoveRequest::new(member.id))
+                    .await?;
+            }
+        }
+
         Ok(())
     }
 
@@ -194,6 +232,7 @@ impl XlineHandle {
         debug!("xline server started, member: {:?}", member);
         _ = self.server_id.replace(member.id);
         _ = self.client.replace(client);
+        self.cached_members = xlines.clone();
         Ok(())
     }
 

From d7339d06e20a7ac48203ce72761ecb7892cda03d Mon Sep 17 00:00:00 2001
From: iGxnon <igxnon@gmail.com>
Date: Wed, 1 Nov 2023 23:03:58 +0800
Subject: [PATCH 19/19] feat: add metrics for sidecar

Signed-off-by: iGxnon <igxnon@gmail.com>
---
 Cargo.lock                |  2 ++
 sidecar/Cargo.toml        |  2 ++
 sidecar/src/controller.rs | 72 +++++++++++++++++++++++++++++++++++++++
 sidecar/src/routers.rs    | 22 ++++++++++++
 sidecar/src/sidecar.rs    | 38 ++++++++++++++++++---
 sidecar/src/utils.rs      | 13 +++++++
 6 files changed, 144 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7fa61d0e..97ee62fd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4132,9 +4132,11 @@ dependencies = [
  "axum",
  "bytes",
  "clap 4.4.6",
+ "clippy-utilities 0.2.0",
  "engine",
  "futures",
  "operator-api",
+ "prometheus",
  "reqwest",
  "serde",
  "tokio",
diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml
index 94bba3d3..036c7da9 100644
--- a/sidecar/Cargo.toml
+++ b/sidecar/Cargo.toml
@@ -17,9 +17,11 @@ async-trait = "0.1.68"
 axum = "0.6.18"
 bytes = "1.4.0"
 clap = { version = "4.3.4", features = ["derive"] }
+clippy-utilities = "0.2.0"
 engine = { git = "https://github.com/xline-kv/Xline.git", package = "engine" }
 futures = "0.3.28"
 operator-api = { path = "../operator-api" }
+prometheus = "0.13.3"
 reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0.130", features = ["derive"] }
 tokio = { version = "1.0", features = [
diff --git a/sidecar/src/controller.rs b/sidecar/src/controller.rs
index cfd3fa9e..5e671d1c 100644
--- a/sidecar/src/controller.rs
+++ b/sidecar/src/controller.rs
@@ -7,12 +7,14 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Result};
 use operator_api::registry::Registry;
+use prometheus::{Histogram, HistogramOpts, IntCounter, IntCounterVec, Opts};
 use tokio::select;
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::{interval, MissedTickBehavior};
 use tracing::{debug, error, info};
 
 use crate::types::{MemberConfig, State, StatePayload, StateStatus};
+use crate::utils::exponential_time_bucket;
 use crate::xline::XlineHandle;
 
 /// Sidecar operator controller
@@ -25,10 +27,68 @@ pub(crate) struct Controller {
     handle: Arc<RwLock<XlineHandle>>,
     /// Check interval
     reconcile_interval: Duration,
+    /// Controller metrics
+    metrics: ControllerMetrics,
     /// Configuration Registry
     registry: Arc<dyn Registry + Sync + Send>,
 }
 
+/// Controller metrics
+pub(crate) struct ControllerMetrics {
+    /// Reconcile duration histogram
+    reconcile_duration: Histogram,
+    /// Reconcile failed count
+    reconcile_failed_count: IntCounterVec,
+    /// Xline restart count
+    restart_count: IntCounter,
+    /// Seed cluster count
+    seed_count: IntCounter,
+}
+
+impl ControllerMetrics {
+    /// New a controller metrics
+    #[allow(clippy::expect_used)]
+    pub(crate) fn new() -> Self {
+        Self {
+            reconcile_duration: Histogram::with_opts(
+                HistogramOpts::new(
+                    "sidecar_reconcile_duration_seconds",
+                    "Duration of sidecar reconcile loop in seconds",
+                )
+                .buckets(exponential_time_bucket(0.1, 2.0, 10)),
+            )
+            .expect("failed to create sidecar_reconcile_duration_seconds histogram"),
+            reconcile_failed_count: IntCounterVec::new(
+                Opts::new(
+                    "sidecar_reconcile_failed_count",
+                    "Number of failed times the sidecar reconcile loop has run",
+                ),
+                &["reason"],
+            )
+            .expect("failed to create sidecar_reconcile_failed_count counter"),
+            restart_count: IntCounter::new(
+                "sidecar_restart_xline_count",
+                "Number of how many times the xline restarts by this sidecar",
+            )
+            .expect("failed to create sidecar_restart_xline_count counter"),
+            seed_count: IntCounter::new(
+                "sidecar_seed_count",
+                "Number of how many times the sidecar seeds the cluster",
+            )
+            .expect("failed to create sidecar_seed_count counter"),
+        }
+    }
+
+    /// Register the metrics into registry
+    pub(crate) fn register(&self, registry: &prometheus::Registry) -> Result<()> {
+        registry.register(Box::new(self.reconcile_duration.clone()))?;
+        registry.register(Box::new(self.reconcile_failed_count.clone()))?;
+        registry.register(Box::new(self.restart_count.clone()))?;
+        registry.register(Box::new(self.seed_count.clone()))?;
+        Ok(())
+    }
+}
+
 impl Debug for Controller {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Controller")
@@ -46,6 +106,7 @@ impl Controller {
         state: Arc<Mutex<StatePayload>>,
         handle: Arc<RwLock<XlineHandle>>,
         reconcile_interval: Duration,
+        metrics: ControllerMetrics,
         registry: Arc<dyn Registry + Sync + Send>,
     ) -> Self {
         Self {
@@ -53,6 +114,7 @@ impl Controller {
             state,
             handle,
             reconcile_interval,
+            metrics,
             registry,
         }
     }
@@ -92,6 +154,7 @@ impl Controller {
             ..init_member_config
         };
 
+        self.metrics.restart_count.inc();
         self.handle
             .write()
             .await
@@ -99,6 +162,7 @@ impl Controller {
             .await?;
 
         loop {
+            let timer = self.metrics.reconcile_duration.start_timer();
             let instant = tick.tick().await;
 
             let config = match self
@@ -116,12 +180,17 @@ impl Controller {
 
             if let Err(err) = self.reconcile_once(&member_config).await {
                 error!("reconcile failed, error: {err}");
+                self.metrics
+                    .reconcile_failed_count
+                    .with_label_values(&[&err.to_string()])
+                    .inc();
                 continue;
             }
             debug!(
                 "successfully reconcile the cluster states within {:?}",
                 instant.elapsed()
             );
+            drop(timer);
         }
     }
 
@@ -149,6 +218,7 @@ impl Controller {
             (true, false) => {
                 self.set_state(State::Pending).await;
 
+                self.metrics.restart_count.inc();
                 info!("status: cluster healthy + xline not running, joining the cluster");
                 handle.start(&xline_members).await?;
             }
@@ -191,6 +261,8 @@ impl Controller {
                     info!(
                         "status: cluster unhealthy + xline not running + all start + seeder, seed cluster"
                     );
+                    self.metrics.restart_count.inc();
+                    self.metrics.seed_count.inc();
                     handle.start(&xline_members).await?;
                 }
             }
diff --git a/sidecar/src/routers.rs b/sidecar/src/routers.rs
index fdcb3261..4156bf97 100644
--- a/sidecar/src/routers.rs
+++ b/sidecar/src/routers.rs
@@ -2,12 +2,34 @@ use std::sync::Arc;
 
 use axum::http::StatusCode;
 use axum::{Extension, Json};
+use prometheus::Encoder;
 use tokio::sync::{Mutex, RwLock};
+use tracing::error;
 
 use crate::types::{MembershipChange, StatePayload};
 use crate::utils::{check_backup_volume, check_data_volume};
 use crate::xline::XlineHandle;
 
+/// metrics handler
+#[allow(clippy::unused_async)] // require by axum
+pub(crate) async fn metrics(Extension(registry): Extension<prometheus::Registry>) -> String {
+    let mut buf1 = Vec::new();
+    let encoder = prometheus::TextEncoder::new();
+    let metric_families = registry.gather();
+    if let Err(err) = encoder.encode(&metric_families, &mut buf1) {
+        error!("failed to encode custom metrics: {}", err);
+        return String::new();
+    }
+    let mut res = String::from_utf8(buf1).unwrap_or_default();
+    let mut buf2 = Vec::new();
+    if let Err(err) = encoder.encode(&prometheus::gather(), &mut buf2) {
+        error!("failed to encode prometheus metrics: {}", err);
+        return String::new();
+    }
+    res.push_str(&String::from_utf8_lossy(&buf2));
+    res
+}
+
 /// Return the current health condition according to the current node's storage volume and network status
 /// The network status is verified upon returning the HTTP response.
 #[allow(clippy::unused_async)] // This is required in axum
diff --git a/sidecar/src/sidecar.rs b/sidecar/src/sidecar.rs
index 6a512459..1576ca61 100644
--- a/sidecar/src/sidecar.rs
+++ b/sidecar/src/sidecar.rs
@@ -2,12 +2,13 @@ use std::net::ToSocketAddrs;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Result};
-use axum::routing::{get, post};
+use axum::routing::{any, get, post};
 use axum::{Extension, Router};
 use futures::{FutureExt, TryFutureExt};
 use operator_api::consts::{SIDECAR_BACKUP_ROUTE, SIDECAR_HEALTH_ROUTE, SIDECAR_STATE_ROUTE};
 use operator_api::registry::{DummyRegistry, HttpRegistry, K8sStsRegistry, Registry};
 use operator_api::{HeartbeatStatus, K8sXlineHandle, LocalXlineHandle};
+use prometheus::{Histogram, HistogramOpts};
 use tokio::sync::watch::{Receiver, Sender};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::{interval, MissedTickBehavior};
@@ -15,9 +16,10 @@ use tracing::{debug, error, info, warn};
 
 use crate::backup::pv::Pv;
 use crate::backup::Provider;
-use crate::controller::Controller;
+use crate::controller::{Controller, ControllerMetrics};
 use crate::routers;
 use crate::types::{BackendConfig, BackupConfig, Config, RegistryConfig, State, StatePayload};
+use crate::utils::exponential_time_bucket;
 use crate::xline::XlineHandle;
 
 /// Sidecar
@@ -59,19 +61,26 @@ impl Sidecar {
                 self.config.cluster_name.clone(),
             )),
         };
+        let metrics_registry = prometheus::Registry::new();
 
         self.start_controller(
             Arc::clone(&handle),
+            &metrics_registry,
             Arc::clone(&registry),
             Arc::clone(&state),
             graceful_shutdown_event.subscribe(),
-        );
+        )?;
+        self.start_heartbeat(
+            &metrics_registry,
+            registry,
+            graceful_shutdown_event.subscribe(),
+        )?;
         self.start_web_server(
             Arc::clone(&handle),
             Arc::clone(&state),
+            metrics_registry,
             graceful_shutdown_event.subscribe(),
         )?;
-        self.start_heartbeat(registry, graceful_shutdown_event.subscribe())?;
 
         tokio::pin!(forceful_shutdown);
 
@@ -145,9 +154,18 @@ impl Sidecar {
     /// Start heartbeat
     fn start_heartbeat(
         &self,
+        metrics_registry: &prometheus::Registry,
         registry: Arc<dyn Registry + Send + Sync>,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
+        let heartbeat_histogram = Histogram::with_opts(
+            HistogramOpts::new(
+                "sidecar_heartbeat_duration_histogram",
+                "Duration of sidecar heartbeat loop in seconds",
+            )
+            .buckets(exponential_time_bucket(0.1, 2.0, 10)),
+        )?;
+        metrics_registry.register(Box::new(heartbeat_histogram.clone()))?;
         let Some(monitor) = self.config.monitor.clone() else {
             info!("monitor did not set, disable heartbeat");
             return Ok(());
@@ -171,6 +189,7 @@ impl Sidecar {
                 // ensure a fixed heartbeat interval
                 tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
                 loop {
+                    let timer = heartbeat_histogram.start_timer();
                     let instant = tick.tick().await;
 
                     let config = match registry
@@ -197,6 +216,7 @@ impl Sidecar {
                     if let Err(e) = status.report(&monitor.monitor_addr).await {
                         error!("heartbeat report failed, error {e}");
                     }
+                    drop(timer);
                 }
             };
 
@@ -215,15 +235,19 @@ impl Sidecar {
     fn start_controller(
         &self,
         handle: Arc<RwLock<XlineHandle>>,
+        metrics_registry: &prometheus::Registry,
         registry: Arc<dyn Registry + Sync + Send>,
         state: Arc<Mutex<StatePayload>>,
         graceful_shutdown: Receiver<()>,
-    ) {
+    ) -> Result<()> {
+        let metrics = ControllerMetrics::new();
+        metrics.register(metrics_registry)?;
         let controller = Controller::new(
             self.config.name.clone(),
             state,
             handle,
             self.config.reconcile_interval,
+            metrics,
             registry,
         );
         let init_member_config = self.config.init_member.clone();
@@ -238,6 +262,7 @@ impl Sidecar {
                 info!("controller shutdown");
             }
         });
+        Ok(())
     }
 
     /// Run a web server to expose current state to other sidecar operators and k8s
@@ -245,6 +270,7 @@ impl Sidecar {
         &self,
         handle: Arc<RwLock<XlineHandle>>,
         state: Arc<Mutex<StatePayload>>,
+        metrics_registry: prometheus::Registry,
         graceful_shutdown: Receiver<()>,
     ) -> Result<()> {
         let members = self.config.init_member.sidecar_members();
@@ -260,8 +286,10 @@ impl Sidecar {
             .route(SIDECAR_HEALTH_ROUTE, get(routers::health))
             .route(SIDECAR_BACKUP_ROUTE, get(routers::backup))
             .route(SIDECAR_STATE_ROUTE, get(routers::state))
+            .route("/metrics", any(routers::metrics))
             .route("/membership", post(routers::membership))
             .layer(Extension(handle))
+            .layer(Extension(metrics_registry))
             .layer(Extension(state));
 
         debug!("web server listen addr: {addr}");
diff --git a/sidecar/src/utils.rs b/sidecar/src/utils.rs
index fa08cf1e..8b38560f 100644
--- a/sidecar/src/utils.rs
+++ b/sidecar/src/utils.rs
@@ -1,6 +1,9 @@
 use std::fs::{read_to_string, remove_file, write};
+use std::iter::repeat;
+use std::ops::Mul;
 use std::path::Path;
 
+use clippy_utilities::NumericCast;
 use operator_api::consts::{DEFAULT_BACKUP_DIR, DEFAULT_DATA_DIR};
 
 /// Check if the volume under the path is working fine
@@ -50,6 +53,16 @@ pub(crate) fn check_backup_volume() -> bool {
     check_volume(backup_volume)
 }
 
+/// Returns a vector of time buckets for the reconcile duration histogram.
+#[allow(clippy::as_conversions)] // count will not too large
+pub(crate) fn exponential_time_bucket(start: f64, factor: f64, count: usize) -> Vec<f64> {
+    repeat(factor)
+        .enumerate()
+        .take(count)
+        .map(|(i, f)| start.mul(f.powi(i.numeric_cast())))
+        .collect::<Vec<_>>()
+}
+
 #[cfg(test)]
 mod test {
     use super::*;