[reconfigurator] Call clickhouse-admin API from SMF services (#6533)

## Overview

This commit replaces the old replicated ClickHouse server and keeper
configuration templates with calls to the `clickhouse-admin` API that
generate said configuration files.

## Purpose

While the end goal is to have Nexus make the API calls to generate the
configuration files, we'd like to have a working implementation of the
`clickhouse-admin` API via the SMF services. Using `curl` is not what
the finished work will look like, but rather it is the simplest way to
have a working implementation in the mean time.

## Testing

Deployed this branch on a Helios machine with the following results

Replica 1
```console
root@oxz_clickhouse_server_a9d02cd3:~# /opt/oxide/clickhouse_server/clickhouse client --host fd00:1122:3344:101::f
ClickHouse client version 23.8.7.1.
Connecting to fd00:1122:3344:101::f:9000 as user default.
Connected to ClickHouse server version 23.8.7 revision 54465.

oximeter_cluster_1 :) SHOW TABLES FROM oximeter

SHOW TABLES FROM oximeter

Query id: 06867649-f49e-451f-b9f1-5a574e12ce5b

┌─name─────────────────────────────┐
│ fields_bool                      │
│ fields_bool_local                │
│ fields_i16                       │
│ fields_i16_local                 │
│ fields_i32                       │
│ fields_i32_local                 │
│ fields_i64                       │
│ fields_i64_local                 │
│ fields_i8                        │
│ fields_i8_local                  │
│ fields_ipaddr                    │
│ fields_ipaddr_local              │
│ fields_string                    │
│ fields_string_local              │
│ fields_u16                       │
│ fields_u16_local                 │
│ fields_u32                       │
│ fields_u32_local                 │
│ fields_u64                       │
│ fields_u64_local                 │
│ fields_u8                        │
│ fields_u8_local                  │
│ fields_uuid                      │
│ fields_uuid_local                │
│ measurements_bool                │
│ measurements_bool_local          │
│ measurements_bytes               │
│ measurements_bytes_local         │
│ measurements_cumulativef32       │
│ measurements_cumulativef32_local │
│ measurements_cumulativef64       │
│ measurements_cumulativef64_local │
│ measurements_cumulativei64       │
│ measurements_cumulativei64_local │
│ measurements_cumulativeu64       │
│ measurements_cumulativeu64_local │
│ measurements_f32                 │
│ measurements_f32_local           │
│ measurements_f64                 │
│ measurements_f64_local           │
│ measurements_histogramf32        │
│ measurements_histogramf32_local  │
│ measurements_histogramf64        │
│ measurements_histogramf64_local  │
│ measurements_histogrami16        │
│ measurements_histogrami16_local  │
│ measurements_histogrami32        │
│ measurements_histogrami32_local  │
│ measurements_histogrami64        │
│ measurements_histogrami64_local  │
│ measurements_histogrami8         │
│ measurements_histogrami8_local   │
│ measurements_histogramu16        │
│ measurements_histogramu16_local  │
│ measurements_histogramu32        │
│ measurements_histogramu32_local  │
│ measurements_histogramu64        │
│ measurements_histogramu64_local  │
│ measurements_histogramu8         │
│ measurements_histogramu8_local   │
│ measurements_i16                 │
│ measurements_i16_local           │
│ measurements_i32                 │
│ measurements_i32_local           │
│ measurements_i64                 │
│ measurements_i64_local           │
│ measurements_i8                  │
│ measurements_i8_local            │
│ measurements_string              │
│ measurements_string_local        │
│ measurements_u16                 │
│ measurements_u16_local           │
│ measurements_u32                 │
│ measurements_u32_local           │
│ measurements_u64                 │
│ measurements_u64_local           │
│ measurements_u8                  │
│ measurements_u8_local            │
│ timeseries_schema                │
│ timeseries_schema_local          │
│ version                          │
└──────────────────────────────────┘

81 rows in set. Elapsed: 0.005 sec. 

oximeter_cluster_1 :) SELECT * FROM oximeter.measurements_u64

SELECT *
FROM oximeter.measurements_u64

Query id: 2e13d330-8f0b-4346-afc0-ba3c21ea7674

┌─timeseries_name─────────────────────────┬───────timeseries_key─┬─────────────────────timestamp─┬─datum─┐
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:16:47.241835734 │     0 │
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:16:48.241091831 │     0 │
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:16:49.241294398 │     0 │
<...>
```

Replica 2
```console
root@oxz_clickhouse_server_ba1601d3:~# /opt/oxide/clickhouse_server/clickhouse client --host fd00:1122:3344:101::e
ClickHouse client version 23.8.7.1.
Connecting to fd00:1122:3344:101::e:9000 as user default.
Connected to ClickHouse server version 23.8.7 revision 54465.

oximeter_cluster_2 :) SHOW TABLES FROM oximeter

SHOW TABLES FROM oximeter

Query id: 33dd1d4d-1596-44e3-90ea-c755a1e3ae24

┌─name─────────────────────────────┐
│ fields_bool                      │
│ fields_bool_local                │
│ fields_i16                       │
│ fields_i16_local                 │
│ fields_i32                       │
│ fields_i32_local                 │
│ fields_i64                       │
│ fields_i64_local                 │
│ fields_i8                        │
│ fields_i8_local                  │
│ fields_ipaddr                    │
│ fields_ipaddr_local              │
│ fields_string                    │
│ fields_string_local              │
│ fields_u16                       │
│ fields_u16_local                 │
│ fields_u32                       │
│ fields_u32_local                 │
│ fields_u64                       │
│ fields_u64_local                 │
│ fields_u8                        │
│ fields_u8_local                  │
│ fields_uuid                      │
│ fields_uuid_local                │
│ measurements_bool                │
│ measurements_bool_local          │
│ measurements_bytes               │
│ measurements_bytes_local         │
│ measurements_cumulativef32       │
│ measurements_cumulativef32_local │
│ measurements_cumulativef64       │
│ measurements_cumulativef64_local │
│ measurements_cumulativei64       │
│ measurements_cumulativei64_local │
│ measurements_cumulativeu64       │
│ measurements_cumulativeu64_local │
│ measurements_f32                 │
│ measurements_f32_local           │
│ measurements_f64                 │
│ measurements_f64_local           │
│ measurements_histogramf32        │
│ measurements_histogramf32_local  │
│ measurements_histogramf64        │
│ measurements_histogramf64_local  │
│ measurements_histogrami16        │
│ measurements_histogrami16_local  │
│ measurements_histogrami32        │
│ measurements_histogrami32_local  │
│ measurements_histogrami64        │
│ measurements_histogrami64_local  │
│ measurements_histogrami8         │
│ measurements_histogrami8_local   │
│ measurements_histogramu16        │
│ measurements_histogramu16_local  │
│ measurements_histogramu32        │
│ measurements_histogramu32_local  │
│ measurements_histogramu64        │
│ measurements_histogramu64_local  │
│ measurements_histogramu8         │
│ measurements_histogramu8_local   │
│ measurements_i16                 │
│ measurements_i16_local           │
│ measurements_i32                 │
│ measurements_i32_local           │
│ measurements_i64                 │
│ measurements_i64_local           │
│ measurements_i8                  │
│ measurements_i8_local            │
│ measurements_string              │
│ measurements_string_local        │
│ measurements_u16                 │
│ measurements_u16_local           │
│ measurements_u32                 │
│ measurements_u32_local           │
│ measurements_u64                 │
│ measurements_u64_local           │
│ measurements_u8                  │
│ measurements_u8_local            │
│ timeseries_schema                │
│ timeseries_schema_local          │
│ version                          │
└──────────────────────────────────┘

81 rows in set. Elapsed: 0.010 sec. 

oximeter_cluster_2 :) SELECT * FROM oximeter.measurements_u64

SELECT *
FROM oximeter.measurements_u64

Query id: 06da0f16-3055-47cb-9984-94dc78f99afc

┌─timeseries_name─────────────────────────┬───────timeseries_key─┬─────────────────────timestamp─┬─datum─┐
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:22:02.443983562 │     0 │
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:22:03.444346219 │     0 │
│ ddm_router:originated_tunnel_endpoints  │  2085026407707057203 │ 2024-09-09 07:22:04.444356384 │     0 │
<...>
```

Keeper 1

```console
root@oxz_clickhouse_keeper_8cb0de91:~# echo mntr | nc fd00:1122:3344:101::12 9181
zk_version      v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64
zk_avg_latency  4
zk_max_latency  103
zk_min_latency  0
zk_packets_received     27769
zk_packets_sent 29290
zk_num_alive_connections        1
zk_outstanding_requests 0
zk_server_state leader
zk_znode_count  6535
zk_watch_count  83
zk_ephemerals_count     82
zk_approximate_data_size        2330794
zk_key_arena_size       1044480
zk_latest_snapshot_size 0
zk_followers    2
zk_synced_followers     2
```

Keeper 2

```console
root@oxz_clickhouse_keeper_a6c18bd2:~# echo mntr | nc fd00:1122:3344:101::10 9181
zk_version      v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64
zk_avg_latency  10
zk_max_latency  139
zk_min_latency  0
zk_packets_received     22278
zk_packets_sent 23922
zk_num_alive_connections        1
zk_outstanding_requests 0
zk_server_state follower
zk_znode_count  7015
zk_watch_count  83
zk_ephemerals_count     82
zk_approximate_data_size        2512980
zk_key_arena_size       1044480
zk_latest_snapshot_size 0
```

Keeper 3

```console
root@oxz_clickhouse_keeper_45d3e6ef:~# echo mntr | nc fd00:1122:3344:101::11 9181
zk_version      v23.8.7.1-lts-077df679bed122ad45c8b105d8916ccfec85ae64
zk_avg_latency  0
zk_max_latency  0
zk_min_latency  0
zk_packets_received     0
zk_packets_sent 0
zk_num_alive_connections        0
zk_outstanding_requests 0
zk_server_state follower
zk_znode_count  7188
zk_watch_count  0
zk_ephemerals_count     82
zk_approximate_data_size        2575631
zk_key_arena_size       1044480
zk_latest_snapshot_size 0
```

Related: #5999
Closes: #3824

clickhouse-admin/types/Cargo.toml

@@ -9,6 +9,7 @@ workspace = true
 
 [dependencies]
 anyhow.workspace = true
+atomicwrites.workspace = true
 camino.workspace = true
 camino-tempfile.workspace = true
 derive_more.workspace = true

clickhouse-admin/types/src/config.rs

@@ -97,6 +97,7 @@ impl ReplicaConfig {
         let keepers = keepers.to_xml();
         let remote_servers = remote_servers.to_xml();
         let user_files_path = data_path.clone().join("user_files");
+        let temp_files_path = data_path.clone().join("tmp");
         let format_schema_path = data_path.clone().join("format_schemas");
         format!(
             "
@@ -135,6 +136,7 @@ impl ReplicaConfig {
         </default>
     </quotas>
 
+    <tmp_path>{temp_files_path}</tmp_path>
     <user_files_path>{user_files_path}</user_files_path>
     <default_profile>default</default_profile>
     <format_schema_path>{format_schema_path}</format_schema_path>
@@ -472,6 +474,9 @@ pub struct KeeperConfig {
     pub coordination_settings: KeeperCoordinationSettings,
     /// Settings for each server in the keeper cluster
     pub raft_config: RaftServers,
+    /// Directory for all files generated by ClickHouse itself
+    #[schemars(schema_with = "path_schema")]
+    pub datastore_path: Utf8PathBuf,
 }
 
 impl KeeperConfig {
@@ -496,6 +501,7 @@ impl KeeperConfig {
             snapshot_storage_path,
             coordination_settings,
             raft_config,
+            datastore_path,
         }
     }
 
@@ -509,6 +515,7 @@ impl KeeperConfig {
             snapshot_storage_path,
             coordination_settings,
             raft_config,
+            datastore_path,
         } = self;
         let logger = logger.to_xml();
         let KeeperCoordinationSettings {
@@ -522,6 +529,7 @@ impl KeeperConfig {
 <clickhouse>
 {logger}
     <listen_host>{listen_host}</listen_host>
+    <path>{datastore_path}</path>
     <keeper_server>
         <enable_reconfiguration>false</enable_reconfiguration>
         <tcp_port>{tcp_port}</tcp_port>

clickhouse-admin/types/src/lib.rs

@@ -2,14 +2,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use anyhow::Result;
+use anyhow::{Context, Result};
+use atomicwrites::AtomicFile;
 use camino::Utf8PathBuf;
-use camino_tempfile::NamedUtf8TempFile;
 use derive_more::{Add, AddAssign, Display, From};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use std::fs::rename;
-use std::io::Write;
+use std::fs::create_dir;
+use std::io::{ErrorKind, Write};
 use std::net::Ipv6Addr;
 
 pub mod config;
@@ -121,12 +121,20 @@ impl ServerSettings {
             self.datastore_path.clone(),
         );
 
-        // Writing to a temporary file and then renaming it will ensure we
-        // don't end up with a partially written file after a crash
-        let mut f = NamedUtf8TempFile::new()?;
-        f.write_all(config.to_xml().as_bytes())?;
-        f.flush()?;
-        rename(f.path(), self.config_dir.join("replica-server-config.xml"))?;
+        match create_dir(self.config_dir.clone()) {
+            Ok(_) => (),
+            Err(e) if e.kind() == ErrorKind::AlreadyExists => (),
+            Err(e) => return Err(e.into()),
+        };
+
+        let path = self.config_dir.join("replica-server-config.xml");
+        AtomicFile::new(
+            path.clone(),
+            atomicwrites::OverwriteBehavior::AllowOverwrite,
+        )
+        .write(|f| f.write_all(config.to_xml().as_bytes()))
+        .with_context(|| format!("failed to write to `{}`", path))?;
+
         Ok(config)
     }
 }
@@ -180,12 +188,20 @@ impl KeeperSettings {
             raft_config,
         );
 
-        // Writing to a temporary file and then renaming it will ensure we
-        // don't end up with a partially written file after a crash
-        let mut f = NamedUtf8TempFile::new()?;
-        f.write_all(config.to_xml().as_bytes())?;
-        f.flush()?;
-        rename(f.path(), self.config_dir.join("keeper-config.xml"))?;
+        match create_dir(self.config_dir.clone()) {
+            Ok(_) => (),
+            Err(e) if e.kind() == ErrorKind::AlreadyExists => (),
+            Err(e) => return Err(e.into()),
+        };
+
+        let path = self.config_dir.join("keeper_config.xml");
+        AtomicFile::new(
+            path.clone(),
+            atomicwrites::OverwriteBehavior::AllowOverwrite,
+        )
+        .write(|f| f.write_all(config.to_xml().as_bytes()))
+        .with_context(|| format!("failed to write to `{}`", path))?;
+
         Ok(config)
     }
 }
@@ -244,9 +260,9 @@ mod tests {
 
         let expected_file = Utf8PathBuf::from_str("./testutils")
             .unwrap()
-            .join("keeper-config.xml");
+            .join("keeper_config.xml");
         let generated_file =
-            Utf8PathBuf::from(config_dir.path()).join("keeper-config.xml");
+            Utf8PathBuf::from(config_dir.path()).join("keeper_config.xml");
         let generated_content = std::fs::read_to_string(generated_file)
             .expect("Failed to read from generated ClickHouse keeper file");
 

clickhouse-admin/types/testutils/keeper-config.xml → clickhouse-admin/types/testutils/keeper_config.xml

@@ -10,6 +10,7 @@
     </logger>
 
     <listen_host>ff::8</listen_host>
+    <path>./</path>
     <keeper_server>
         <enable_reconfiguration>false</enable_reconfiguration>
         <tcp_port>9181</tcp_port>

clickhouse-admin/types/testutils/replica-server-config.xml

@@ -42,6 +42,7 @@
         </default>
     </quotas>
 
+    <tmp_path>./data/tmp</tmp_path>
     <user_files_path>./data/user_files</user_files_path>
     <default_profile>default</default_profile>
     <format_schema_path>./data/format_schemas</format_schema_path>

openapi/clickhouse-admin.json

@@ -162,6 +162,11 @@
               }
             ]
           },
+          "datastore_path": {
+            "description": "Directory for all files generated by ClickHouse itself",
+            "type": "string",
+            "format": "Utf8PathBuf"
+          },
           "listen_host": {
             "description": "Address the keeper is listening on",
             "type": "string",
@@ -210,6 +215,7 @@
         },
         "required": [
           "coordination_settings",
+          "datastore_path",
           "listen_host",
           "log_storage_path",
           "logger",

package-manifest.toml

@@ -200,7 +200,6 @@ source.paths = [
   { from = "out/clickhouse", to = "/opt/oxide/clickhouse_server" },
   { from = "smf/clickhouse_server/manifest.xml", to = "/var/svc/manifest/site/clickhouse_server/manifest.xml" },
   { from = "smf/clickhouse_server/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse_server.sh" },
-  { from = "smf/clickhouse_server/config_replica.xml", to = "/opt/oxide/clickhouse_server/config.d/config_replica.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -230,7 +229,6 @@ source.paths = [
   { from = "out/clickhouse", to = "/opt/oxide/clickhouse_keeper" },
   { from = "smf/clickhouse_keeper/manifest.xml", to = "/var/svc/manifest/site/clickhouse_keeper/manifest.xml" },
   { from = "smf/clickhouse_keeper/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse_keeper.sh" },
-  { from = "smf/clickhouse_keeper/keeper_config.xml", to = "/opt/oxide/clickhouse_keeper/keeper_config.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true

smf/clickhouse_keeper/manifest.xml

@@ -16,6 +16,11 @@
   <service_fmri value='svc:/oxide/zone-network-setup:default' />
   </dependency>
 
+  <dependency name='clickhouse-admin' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/clickhouse-admin:default' />
+  </dependency>
+
   <exec_method type='method' name='start'
     exec='/opt/oxide/lib/svc/manifest/clickhouse_keeper.sh'
     timeout_seconds='0' />

smf/clickhouse_keeper/method_script.sh

@@ -70,40 +70,37 @@ else
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
-# Setting environment variables this way is best practice, but has the downside
-# of obscuring the field values to anyone ssh=ing into the zone.  To mitigate
-# this, we will be saving them to ${DATASTORE}/config_env_vars
-export CH_LOG="${DATASTORE}/clickhouse-keeper.log"
-export CH_ERROR_LOG="${DATASTORE}/clickhouse-keeper.err.log"
-export CH_LISTEN_ADDR=${LISTEN_ADDR}
-export CH_DATASTORE=${DATASTORE}
-export CH_LISTEN_PORT=${LISTEN_PORT}
-export CH_KEEPER_ID_CURRENT=${KEEPER_ID_CURRENT}
-export CH_LOG_STORAGE_PATH="${DATASTORE}/log"
-export CH_SNAPSHOT_STORAGE_PATH="${DATASTORE}/snapshots"
-export CH_KEEPER_ID_01=${KEEPER_ID_01}
-export CH_KEEPER_ID_02=${KEEPER_ID_02}
-export CH_KEEPER_ID_03=${KEEPER_ID_03}
-export CH_KEEPER_HOST_01=${KEEPER_HOST_01}
-export CH_KEEPER_HOST_02=${KEEPER_HOST_02}
-export CH_KEEPER_HOST_03=${KEEPER_HOST_03}
-
-content="CH_LOG="${CH_LOG}"\n\
-CH_ERROR_LOG="${CH_ERROR_LOG}"\n\
-CH_LISTEN_ADDR="${CH_LISTEN_ADDR}"\n\
-CH_DATASTORE="${CH_DATASTORE}"\n\
-CH_LISTEN_PORT="${CH_LISTEN_PORT}"\n\
-CH_KEEPER_ID_CURRENT="${CH_KEEPER_ID_CURRENT}"\n\
-CH_LOG_STORAGE_PATH="${CH_LOG_STORAGE_PATH}"\n\
-CH_SNAPSHOT_STORAGE_PATH="${CH_SNAPSHOT_STORAGE_PATH}"\n\
-CH_KEEPER_ID_01="${CH_KEEPER_ID_01}"\n\
-CH_KEEPER_ID_02="${CH_KEEPER_ID_02}"\n\
-CH_KEEPER_ID_03="${CH_KEEPER_ID_03}"\n\
-CH_KEEPER_HOST_01="${CH_KEEPER_HOST_01}"\n\
-CH_KEEPER_HOST_02="${CH_KEEPER_HOST_02}"\n\
-CH_KEEPER_HOST_03="${CH_KEEPER_HOST_03}""
-
-echo $content >> "${DATASTORE}/config_env_vars"
+curl -X put http://[${LISTEN_ADDR}]:8888/keeper/config \
+-H "Content-Type: application/json" \
+-d '{
+  "generation": 0,
+  "settings": {
+    "id": '${KEEPER_ID_CURRENT}',
+    "raft_servers": [
+      {
+        "id": '${KEEPER_ID_01}',
+        "host": {
+          "domain_name": "'${KEEPER_HOST_01}'"
+        }
+      },
+      {
+        "id": '${KEEPER_ID_02}',
+        "host": {
+          "domain_name": "'${KEEPER_HOST_02}'"
+        }
+      },
+      {
+        "id": '${KEEPER_ID_03}',
+        "host": {
+          "domain_name": "'${KEEPER_HOST_03}'"
+        }
+      }
+    ],
+    "config_dir": "/opt/oxide/clickhouse_keeper",
+    "datastore_path": "'${DATASTORE}'",
+    "listen_addr": "'${LISTEN_ADDR}'"
+  }
+}'
 
 # The clickhouse binary must be run from within the directory that contains it.
 # Otherwise, it does not automatically detect the configuration files, nor does