Add 3 node zookeeper and keeper clusters to all suites

Altinity · May 17, 2024 · f1e723d · f1e723d
1 parent 6b1408b
commit f1e723d
Show file tree

Hide file tree

Showing 230 changed files with 4,984 additions and 2,220 deletions.
diff --git a/aes_encryption/aes_encryption_env/docker-compose.yml b/aes_encryption/aes_encryption_env/docker-compose.yml
@@ -1,14 +1,39 @@
 version: '2.3'
 
 services:
-  zookeeper:
+  zookeeper1:
     extends:
-      file: zookeeper-service.yml
-      service: zookeeper
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper1
+
+  zookeeper2:
+    extends:
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper2
+
+  zookeeper3:
+    extends:
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper3
+
+  keeper1:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper1
+
+  keeper2:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper2
+
+  keeper3:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper3
 
   mysql1:
     extends:
-      file: mysql-service.yml
+      file: ../../helpers/docker/mysql-service.yml
       service: mysql
     hostname: mysql1
     volumes:
@@ -22,14 +47,8 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
-      zookeeper:
+      zookeeper1:
         condition: service_healthy
 
   clickhouse2:
@@ -40,14 +59,8 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
-      zookeeper:
+      zookeeper1:
         condition: service_healthy
 
   clickhouse3:
@@ -58,14 +71,8 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
-      zookeeper:
+      zookeeper1:
         condition: service_healthy
 
   # dummy service which does nothing, but allows to postpone
@@ -81,5 +88,15 @@ services:
         condition: service_healthy
       clickhouse3:
         condition: service_healthy
-      zookeeper:
+      zookeeper1:
+        condition: service_healthy
+      zookeeper2:
+        condition: service_healthy
+      zookeeper3:
+        condition: service_healthy
+      keeper1:
+        condition: service_healthy
+      keeper2:
+        condition: service_healthy
+      keeper3:
         condition: service_healthy
diff --git a/aes_encryption/aes_encryption_env/zookeeper-service.yml b/aes_encryption/aes_encryption_env/zookeeper-service.yml
diff --git a/aes_encryption/aes_encryption_env_arm64/docker-compose.yml b/aes_encryption/aes_encryption_env_arm64/docker-compose.yml
@@ -1,14 +1,39 @@
 version: '2.3'
 
 services:
-  zookeeper:
+  zookeeper1:
     extends:
-      file: zookeeper-service.yml
-      service: zookeeper
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper1
+
+  zookeeper2:
+    extends:
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper2
+
+  zookeeper3:
+    extends:
+      file: ../../helpers/docker/zookeeper-service.yml
+      service: zookeeper3
+
+  keeper1:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper1
+
+  keeper2:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper2
+
+  keeper3:
+    extends:
+      file: ../../helpers/docker/keeper-service.yml
+      service: keeper3
 
   mysql1:
     extends:
-      file: mysql-service.yml
+      file: ../../helpers/docker/mysql-service.yml
       service: mysql
     hostname: mysql1
     volumes:
@@ -22,12 +47,6 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
       zookeeper:
         condition: service_healthy
@@ -40,12 +59,6 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
       zookeeper:
         condition: service_healthy
@@ -58,12 +71,6 @@ services:
     volumes:
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/"
       - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml"
-      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
     depends_on:
       zookeeper:
         condition: service_healthy
@@ -81,5 +88,15 @@ services:
         condition: service_healthy
       clickhouse3:
         condition: service_healthy
-      zookeeper:
+      zookeeper1:
+        condition: service_healthy
+      zookeeper2:
+        condition: service_healthy
+      zookeeper3:
+        condition: service_healthy
+      keeper1:
+        condition: service_healthy
+      keeper2:
+        condition: service_healthy
+      keeper3:
         condition: service_healthy
diff --git a/aes_encryption/aes_encryption_env_arm64/mysql-service.yml b/aes_encryption/aes_encryption_env_arm64/mysql-service.yml
diff --git a/aes_encryption/aes_encryption_env_arm64/zookeeper-service.yml b/aes_encryption/aes_encryption_env_arm64/zookeeper-service.yml
diff --git a/aes_encryption/configs/clickhouse/config.d/zookeeper.xml b/aes_encryption/configs/clickhouse/config.d/zookeeper.xml
@@ -1,10 +1,18 @@
 <?xml version="1.0"?>
-<clickhouse>
+<yandex>
     <zookeeper>
-	    <node index="1">
-		    <host>zookeeper</host>
+	    <node>
+		    <host from_env="COORDINATOR_HOST_1"/>
+		    <port>2181</port>
+	    </node>
+		<node>
+		    <host from_env="COORDINATOR_HOST_2"/>
+		    <port>2181</port>
+	    </node>
+		<node>
+		    <host from_env="COORDINATOR_HOST_3"/>
 		    <port>2181</port>
 	    </node>
         <session_timeout_ms>15000</session_timeout_ms>
     </zookeeper>
-</clickhouse>
+</yandex>
diff --git a/aes_encryption/configs/keeper/keeper_config.xml b/aes_encryption/configs/keeper/keeper_config.xml
@@ -0,0 +1,58 @@
+<clickhouse>
+    <logger>
+        <level>trace</level>
+		<log>/var/log/clickhouse-server/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-keeper.err.log</errorlog>
+        <size>100M</size>
+        <count>5</count>
+	    <console>true</console>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    <keeper_server>
+        <tcp_port>2181</tcp_port>
+        <server_id from_env="KEEPER_SERVER_ID"/>
+	    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+	    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms> <!-- defaults -->
+            <min_session_timeout_ms>10000</min_session_timeout_ms> <!-- defaults -->
+            <session_timeout_ms>30000</session_timeout_ms> <!-- defaults to 30sec in clickhouse -->
+            <raft_logs_level>information</raft_logs_level>
+            <compress_logs>false</compress_logs>
+            <!-- All settings listed in https://github.com/ClickHouse/ClickHouse/blob/master/src/Coordination/CoordinationSettings.h -->
+        </coordination_settings>
+
+        <!-- dynamic reconfiguration -->
+        <enable_reconfiguration>true</enable_reconfiguration>
+
+        <!-- enable sanity hostname checks for cluster configuration (if localhost is used with remote endpoints)
+        <hostname_checks_enabled>true</hostname_checks_enabled> -->
+
+        <http_control>
+            <port>9182</port>
+            <readiness>
+                <endpoint>/ready</endpoint>
+            </readiness>
+        </http_control>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+
+    </keeper_server>
+</clickhouse>
diff --git a/aes_encryption/regression.py b/aes_encryption/regression.py
@@ -99,6 +99,7 @@ def regression(
     collect_service_logs,
     keeper_binary_path=None,
     zookeeper_version=None,
+    use_keeper=False,
     stress=None,
     allow_vfs=False,
     with_analyzer=False,
@@ -119,6 +120,7 @@ def regression(
             clickhouse_binary_path=clickhouse_binary_path,
             keeper_binary_path=keeper_binary_path,
             zookeeper_version=zookeeper_version,
+            use_keeper=use_keeper,
             collect_service_logs=collect_service_logs,
             nodes=nodes,
             configs_dir=current_dir(),