adding experimental replicated function to clickhouse #40

raoptimus · Dec 3, 2024 · d9058ca · d9058ca
1 parent 301284d
commit d9058ca
Show file tree

Hide file tree

Showing 33 changed files with 279 additions and 53 deletions.
diff --git a/.env b/.env
@@ -6,10 +6,11 @@ POSTGRES_MIGRATIONS_PATH=./fixtures/postgres
 
 CLICKHOUSE_DSN=clickhouse://default:@clickhouse:9000/default?compress=true&debug=false
 CLICKHOUSE_MIGRATIONS_PATH=./fixtures/clickhouse
-CLICKHOUSE_CLUSTER_DSN1=clickhouse://default:@clickhouse1:9000,clickhouse2:9000/default?compress=true&debug=false
-CLICKHOUSE_CLUSTER_DSN2=clickhouse://default:@clickhouse2:9000/default?compress=true&debug=false
+CLICKHOUSE_CLUSTER_DSN=clickhouse://default:@clickhouse1:9000,clickhouse2:9000/default?compress=true&debug=false
+CLICKHOUSE_CLUSTER_R_DSN=clickhouse://default:@clickhouse-r-1:9000,clickhouse-r-2:9000/default?compress=true&debug=false&allow_experimental_database_replicated=1
 CLICKHOUSE_CLUSTER_NAME=test_cluster
 CLICKHOUSE_CLUSTER_MIGRATIONS_PATH=./fixtures/clickhouse_cluster
+CLICKHOUSE_CLUSTER_R_MIGRATIONS_PATH=./fixtures/clickhouse_cluster_replicated
 
 MYSQL_DSN=mysql://docker:docker@tcp(mysql:3306)/docker
 MYSQL_ROOT_PASSWORD=docker-pw

diff --git a/Makefile b/Makefile
@@ -125,3 +125,6 @@ gen-mocks-dry-run: install-mockery ## Run mockery --dry-run=true
 
 start:
 	@docker-compose -f "docker-compose.yml" -f "docker-compose.dev.yml" up -d
+
+stop:
+	@docker-compose -f "docker-compose.yml" down --remove-orphans
diff --git a/cmd/db-migrator/main.go b/cmd/db-migrator/main.go
@@ -151,6 +151,14 @@ func flags(options *migrator.Options) []cli.Flag {
 			Usage:       "Cluster name for history of migrates",
 			Destination: &options.ClusterName,
 		},
+		&cli.BoolFlag{
+			Name:        "migrationReplicated",
+			Sources:     cli.EnvVars("MIGRATION_REPLICATED"),
+			Aliases:     []string{"cr"},
+			Value:       false,
+			Usage:       "Using replicated experimental function to clickhouse for history table of migrates",
+			Destination: &options.Replicated,
+		},
 		&cli.BoolFlag{
 			Name:        "compact",
 			Sources:     cli.EnvVars("COMPACT"),

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -3,3 +3,7 @@ services:
     ports:
       - 9000:9000
       - 8123:8123
+  clickhouse-r-1:
+    ports:
+      - 19000:9000
+      - 18123:8123
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -26,12 +26,16 @@ services:
       - clickhouse
       - clickhouse1
       - clickhouse2
+      - clickhouse-r-1
+      - clickhouse-r-2
       - postgres
       - mysql
     links:
       - clickhouse
       - clickhouse1
       - clickhouse2
+      - clickhouse-r-1
+      - clickhouse-r-2
       - postgres
       - mysql
     volumes:
@@ -62,3 +66,27 @@ services:
       - "./docker/volume/clickhouse-cluster/config/clickhouse2:/etc/clickhouse-server/config.d/"
     depends_on:
       - clickhouse-keeper
+
+  clickhouse-r-keeper:
+    image: clickhouse/clickhouse-keeper:23.11-alpine
+    restart: on-failure
+    volumes:
+      - "./docker/volume/clickhouse-cluster/config/keeper1/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml"
+
+  clickhouse-r-1:
+    image: clickhouse/clickhouse-server:23.11-alpine
+    restart: on-failure
+    volumes:
+      - "./docker/volume/clickhouse-cluster-replicated/config/clickhouse1:/etc/clickhouse-server/config.d/"
+      - "./docker/volume/clickhouse-cluster-replicated/config/clickhouse1/users.xml:/etc/clickhouse-server/users.d/users.xml"
+    depends_on:
+      - clickhouse-r-keeper
+
+  clickhouse-r-2:
+    image: clickhouse/clickhouse-server:23.11-alpine
+    restart: on-failure
+    volumes:
+      - "./docker/volume/clickhouse-cluster-replicated/config/clickhouse2:/etc/clickhouse-server/config.d/"
+      - "./docker/volume/clickhouse-cluster-replicated/config/clickhouse2/users.xml:/etc/clickhouse-server/users.d/users.xml"
+    depends_on:
+      - clickhouse-r-keeper
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/config.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/config.xml
@@ -0,0 +1,15 @@
+<clickhouse>
+<logger>
+    <level>error</level>
+    <console>1</console>
+    <size>10M</size>
+    <count>3</count>
+</logger>
+<listen_host>0.0.0.0</listen_host>
+<http_port>8123</http_port>
+<tcp_port>9000</tcp_port>
+
+<distributed_ddl>
+    <path>/clickhouse/task_queue/ddl</path>
+</distributed_ddl>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/macros.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/macros.xml
@@ -0,0 +1,10 @@
+<clickhouse>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <macros>
+        <shard>01</shard>
+        <replica>clickhouse1</replica>
+        <cluster>test_cluster</cluster>
+    </macros>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/remote_servers.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/remote_servers.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/use_keeper.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/use_keeper.xml
@@ -0,0 +1,8 @@
+<clickhouse>
+    <zookeeper>
+        <node>
+            <host>clickhouse-keeper</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/users.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse1/users.xml
@@ -0,0 +1,8 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <allow_experimental_database_replicated>1</allow_experimental_database_replicated>
+            <distributed_ddl_output_mode>null_status_on_timeout</distributed_ddl_output_mode>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/config.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/config.xml
@@ -0,0 +1,15 @@
+<clickhouse>
+    <logger>
+        <level>error</level>
+        <console>1</console>
+        <size>10M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/macros.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/macros.xml
@@ -0,0 +1,10 @@
+<clickhouse>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <macros>
+        <shard>01</shard>
+        <replica>clickhouse2</replica>
+        <cluster>test_cluster</cluster>
+    </macros>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/remote_servers.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/remote_servers.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/use_keeper.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/use_keeper.xml
@@ -0,0 +1,8 @@
+<clickhouse>
+    <zookeeper>
+        <node>
+            <host>clickhouse-keeper</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/users.xml b/docker/volume/clickhouse-cluster-replicated/config/clickhouse2/users.xml
@@ -0,0 +1,8 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <allow_experimental_database_replicated>1</allow_experimental_database_replicated>
+            <distributed_ddl_output_mode>null_status_on_timeout</distributed_ddl_output_mode>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster-replicated/config/keeper1/keeper_config.xml b/docker/volume/clickhouse-cluster-replicated/config/keeper1/keeper_config.xml
@@ -0,0 +1,28 @@
+<clickhouse replace="true">
+    <logger>
+        <level>information</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>
diff --git a/docker/volume/clickhouse-cluster/config/clickhouse1/config.xml b/docker/volume/clickhouse-cluster/config/clickhouse1/config.xml
@@ -1,9 +1,8 @@
 <clickhouse>
     <logger>
-        <level>debug</level>
-        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-        <size>1000M</size>
+        <level>error</level>
+        <console>1</console>
+        <size>10M</size>
         <count>3</count>
     </logger>
     <listen_host>0.0.0.0</listen_host>

diff --git a/docker/volume/clickhouse-cluster/config/clickhouse2/config.xml b/docker/volume/clickhouse-cluster/config/clickhouse2/config.xml
@@ -1,9 +1,8 @@
 <clickhouse>
     <logger>
-        <level>debug</level>
-        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-        <size>1000M</size>
+        <level>error</level>
+        <console>1</console>
+        <size>10M</size>
         <count>3</count>
     </logger>
     <listen_host>0.0.0.0</listen_host>

diff --git a/fixtures/clickhouse_cluster/200905_192800_create_test_table.up.sql b/fixtures/clickhouse_cluster/200905_192800_create_test_table.up.sql
@@ -1,10 +1,11 @@
-CREATE TABLE test ON CLUSTER test_cluster (
+CREATE DATABASE IF NOT EXISTS raw ON CLUSTER test_cluster;
+
+CREATE TABLE raw.test ON CLUSTER test_cluster (
     time DateTime DEFAULT now(),
     value UInt32
-
 )
 ENGINE = ReplicatedMergeTree (
-    '/clickhouse/tables/{shard}/test_cluster_test',
+    '/clickhouse/tables/{shard}/raw_test_cluster_test',
     '{replica}'
 )
 PARTITION BY toYYYYMM(time)

diff --git a/fixtures/clickhouse_cluster/200922_210000_add_column_to_test_table.down.sql b/fixtures/clickhouse_cluster/200922_210000_add_column_to_test_table.down.sql
@@ -1,3 +1,3 @@
-ALTER TABLE test ON CLUSTER test_cluster
+ALTER TABLE raw.test ON CLUSTER test_cluster
 	DROP COLUMN text;
 
diff --git a/fixtures/clickhouse_cluster/200922_210000_add_column_to_test_table.up.sql b/fixtures/clickhouse_cluster/200922_210000_add_column_to_test_table.up.sql
@@ -1,4 +1,4 @@
-ALTER TABLE test ON CLUSTER test_cluster
+ALTER TABLE raw.test ON CLUSTER test_cluster
 	ADD COLUMN text String;
 
-INSERT INTO test (value, text) VALUES (1, 'Hello');
+INSERT INTO raw.test (value, text) VALUES (1, 'Hello');
diff --git a/fixtures/clickhouse_cluster/210327_230201_broken_migration.safe.up.sql b/fixtures/clickhouse_cluster/210327_230201_broken_migration.safe.up.sql
@@ -1 +1 @@
-INSERT INTO test2 (time) VALUES (now());
+INSERT INTO raw.test2 (time) VALUES (now());
diff --git a/fixtures/clickhouse_cluster_replicated/200905_192800_create_test_table.down.sql b/fixtures/clickhouse_cluster_replicated/200905_192800_create_test_table.down.sql
@@ -0,0 +1 @@
+DROP TABLE raw.test NO DELAY;
diff --git a/fixtures/clickhouse_cluster_replicated/200905_192800_create_test_table.up.sql b/fixtures/clickhouse_cluster_replicated/200905_192800_create_test_table.up.sql
@@ -0,0 +1,13 @@
+CREATE DATABASE IF NOT EXISTS raw
+ENGINE = Replicated('/clickhouse/databases/{shard}/raw', '{shard}', '{replica}');
+
+CREATE TABLE IF NOT EXISTS raw.test (
+    time DateTime DEFAULT now(),
+    value UInt32
+)
+ENGINE = ReplicatedMergeTree (
+    '/clickhouse/tables/{shard}/raw_test_cluster_test',
+    '{replica}'
+)
+PARTITION BY toYYYYMM(time)
+ORDER BY (time, value);
diff --git a/fixtures/clickhouse_cluster_replicated/200922_210000_add_column_to_test_table.down.sql b/fixtures/clickhouse_cluster_replicated/200922_210000_add_column_to_test_table.down.sql
@@ -0,0 +1 @@
+ALTER TABLE raw.test DROP COLUMN text;
diff --git a/fixtures/clickhouse_cluster_replicated/200922_210000_add_column_to_test_table.up.sql b/fixtures/clickhouse_cluster_replicated/200922_210000_add_column_to_test_table.up.sql
@@ -0,0 +1,3 @@
+ALTER TABLE raw.test ADD COLUMN text String;
+
+INSERT INTO raw.test (value, text) VALUES (1, 'Hello');
diff --git a/fixtures/clickhouse_cluster_replicated/210327_230201_broken_migration.safe.down.sql b/fixtures/clickhouse_cluster_replicated/210327_230201_broken_migration.safe.down.sql
diff --git a/fixtures/clickhouse_cluster_replicated/210327_230201_broken_migration.safe.up.sql b/fixtures/clickhouse_cluster_replicated/210327_230201_broken_migration.safe.up.sql
@@ -0,0 +1 @@
+INSERT INTO raw.test2 (time) VALUES (now());
diff --git a/internal/bak/migrator_clickhouse_cluster_integration_test.go.bak b/internal/bak/migrator_clickhouse_cluster_integration_test.go.bak
@@ -42,7 +42,7 @@ func TestMigrateService_ClickHouseCluster_UpDown(t *testing.T) {
 
 func createClickhouse1ClusterMigrator() (*Service, error) {
 	return New(&Options{
-		DSN:         os.Getenv("CLICKHOUSE_CLUSTER_DSN1"),
+		DSN:         os.Getenv("CLICKHOUSE_CLUSTER_DSN"),
 		Directory:   os.Getenv("CLICKHOUSE_CLUSTER_MIGRATIONS_PATH"),
 		TableName:   "migration",
 		ClusterName: os.Getenv("CLICKHOUSE_CLUSTER_NAME"),
-Original file line number
+Diff line change
@@ Expand Up @@
     start:
     	@docker-compose -f "docker-compose.yml" -f "docker-compose.dev.yml" up -d
+    stop:
+    	@docker-compose -f "docker-compose.yml" down --remove-orphans
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		INSERT INTO test2 (time) VALUES (now());
		INSERT INTO raw.test2 (time) VALUES (now());
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		ALTER TABLE raw.test ADD COLUMN text String;

		INSERT INTO raw.test (value, text) VALUES (1, 'Hello');