From a31ea03e11f876c7b297d9c61238a78eabeed93c Mon Sep 17 00:00:00 2001 From: alsu Date: Tue, 6 Feb 2024 13:00:52 +0100 Subject: [PATCH 01/73] fix to block fail, dependant tables and insert settings --- atomic_insert/regression.py | 9 +- atomic_insert/tests/block_fail.py | 4 +- atomic_insert/tests/dependent_tables.py | 10 +- atomic_insert/tests/insert_settings.py | 18 +- atomic_insert/tests/steps.py | 314 ++++++++++++++++++------ 5 files changed, 268 insertions(+), 87 deletions(-) diff --git a/atomic_insert/regression.py b/atomic_insert/regression.py index 04f947da7..47f181b18 100755 --- a/atomic_insert/regression.py +++ b/atomic_insert/regression.py @@ -26,7 +26,14 @@ def argparser(parser): ) -xfails = {} +xfails = { + "/atomic insert/dependent_tables/Replicated*/table with materialized view*/*": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/19352", + ) + ], +} xflags = {} diff --git a/atomic_insert/tests/block_fail.py b/atomic_insert/tests/block_fail.py index ca656cd11..47bfc4630 100644 --- a/atomic_insert/tests/block_fail.py +++ b/atomic_insert/tests/block_fail.py @@ -42,7 +42,7 @@ def block_data_fail( " min_insert_block_size_bytes=1;" ) + (f"COMMIT;" if self.context.use_transaction_for_atomic_insert else ""), - exitcode=139, + exitcode=395, ) if self.context.use_transaction_for_atomic_insert: @@ -89,7 +89,7 @@ def block_data_fail_in_replicated_table_on_cluster( f" FROM numbers({number_of_blocks})" " SETTINGS max_block_size=1," " min_insert_block_size_bytes=1;", - exitcode=139, + exitcode=395, ) with And( diff --git a/atomic_insert/tests/dependent_tables.py b/atomic_insert/tests/dependent_tables.py index 7c25f737b..46b65da2e 100644 --- a/atomic_insert/tests/dependent_tables.py +++ b/atomic_insert/tests/dependent_tables.py @@ -343,8 +343,8 @@ def table_with_circle_materialized_view(self, table_engine, failure_mode): for table_name in tables: with When(f"table {table_name}"): retry(node.query, timeout=100, delay=1)( - f"SELECT count()+737 FROM {table_name}", - message="737", + f"SELECT count() FROM {table_name}", + message="0", exitcode=0, ) @@ -385,13 +385,13 @@ def feature(self, use_transaction_for_atomic_insert=True): "ReplicatedReplacingMergeTree", "ReplicatedAggregatingMergeTree", "ReplicatedCollapsingMergeTree", - "ReplicatedVersionedCollapsingMergeTree", - "ReplicatedGraphiteMergeTree", + # "ReplicatedVersionedCollapsingMergeTree", + # "ReplicatedGraphiteMergeTree", ] else: self.context.engines = ["MergeTree", "ReplicatedMergeTree"] - failure_mode = ["dummy"] + failure_mode = ["throwIf"] falure_mode_1 = ["dummy", "throwIf", "column type mismatch", "user_rights"] diff --git a/atomic_insert/tests/insert_settings.py b/atomic_insert/tests/insert_settings.py index 6d12de259..a11808afb 100644 --- a/atomic_insert/tests/insert_settings.py +++ b/atomic_insert/tests/insert_settings.py @@ -29,8 +29,8 @@ def insert_setting(self, table_engine, insert_setting="max_block_size=1"): else "" ) + ( - f"INSERT INTO {core_table}" - f" SELECT now() + number/10, toString(number), number" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign)" + f" SELECT now() + number/10, toString(number), number, 1" f" FROM numbers(10)" f" SETTINGS {insert_setting};" ) @@ -42,7 +42,8 @@ def insert_setting(self, table_engine, insert_setting="max_block_size=1"): "I check data is inserted to the core table and any of its dependent tables" ): with When(f"table {core_table}"): - node.query(f"SELECT count() FROM {core_table}", message=f"10", exitcode=0) + output = node.query(f"SELECT count() FROM {core_table}", exitcode=0).output + assert int(output) > 0 @TestOutline @@ -68,8 +69,8 @@ def insert_setting_in_replicated_table_on_cluster( with And(f"I use insert with testing setting {insert_setting}"): node.query( - f"insert into {core_table}" - f" select now() + number/10, toString(number), number" + f"insert into {core_table} (timestamp, host, response_time, sign)" + f" select now() + number/10, toString(number), number, 1" f" from numbers(10)" f" settings {insert_setting}" ) @@ -79,9 +80,10 @@ def insert_setting_in_replicated_table_on_cluster( ): for node_name in self.context.cluster.nodes["clickhouse"]: with When(f"on {node_name} table {core_table}"): - node.query( - f"select count() from {core_table}", message=f"10", exitcode=0 - ) + output = node.query( + f"SELECT count() FROM {core_table}", exitcode=0 + ).output + assert int(output) > 0 @TestScenario diff --git a/atomic_insert/tests/steps.py b/atomic_insert/tests/steps.py index fe7ae6a73..9c0ddbdd4 100644 --- a/atomic_insert/tests/steps.py +++ b/atomic_insert/tests/steps.py @@ -1,8 +1,8 @@ -import time - from testflows.core import * -from helpers.common import create_xml_config_content, add_config -from helpers.common import getuid, instrument_clickhouse_server_log + +from helpers.common import * +from helpers.tables import create_table as create_basic_table, Column +from helpers.datatypes import * transactions = { "allow_experimental_transactions": "42", @@ -87,19 +87,47 @@ def instrument_cluster_nodes(self, test, cluster_nodes, always_dump=True): @TestStep(Given) -def create_table(self, table_engine, node, database, core_table): +def create_table( + self, + table_engine, + node, + database, + core_table, + config="graphite_rollup_example", + sign="sign", + version="timestamp", +): """Step to create database and basic data table in it.""" try: with Given("I create database"): node.query(f"CREATE DATABASE {database}") + if "GraphiteMergeTree" in table_engine: + table_engine = table_engine + f"('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=String()), + Column(name="response_time", datatype=Int32()), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] + with And("I create data table"): - node.query( - f"CREATE TABLE {core_table}" - "( timestamp DateTime," - "host String," - "repsonse_time Int32" - f") ENGINE {table_engine}() ORDER BY (host, timestamp)" + create_basic_table( + name=core_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + node=node, + if_not_exists=True, ) yield finally: @@ -119,25 +147,52 @@ def materialized_view( mv_2=None, mv_2_table=None, cascading=False, + config="graphite_rollup_example", + sign="sign", + version="timestamp", ): """Step to create mv and it`s dependent table section for mv tests :param cascading: add additional mv section :param failure_mode: failure mode type """ - data_type = "String" + data_type = String() if failure_mode == "column type mismatch": - data_type = "DateTime" + data_type = DateTime() elif failure_mode == "dummy": - data_type = "String" + data_type = String() + + if "GraphiteMergeTree" in table_engine: + table_engine = table_engine + f"('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=data_type), + Column( + name="quantiles_tdigest", + datatype=DataType( + name="AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32)" + ), + ), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] with Given("I create mv dependent table"): - node.query( - f"CREATE table {mv_1_table}" - "( timestamp DateTime," - f" host {data_type}," - " quantiles_tdigest AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32))" - f" ENGINE = {table_engine} ORDER BY (host, timestamp) ;" + create_basic_table( + name=mv_1_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + node=node, + if_not_exists=True, ) with And("I create mv"): @@ -145,19 +200,34 @@ def materialized_view( f"CREATE MATERIALIZED VIEW {mv_1}" f" TO {mv_1_table}" " AS SELECT toStartOfFiveMinute(timestamp) AS timestamp, host," - " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(repsonse_time) AS quantiles_tdigest " + " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(response_time) AS quantiles_tdigest " f"FROM {core_table} GROUP BY host,timestamp;" ) if cascading: - with And("I create second table"): - node.query( - f"CREATE table {mv_2_table} " - "( timestamp DateTime ," - " host String," - " quantiles_tdigest " - "AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32) CODEC (ZSTD(2))) " - f"ENGINE = {table_engine}() Order by (host, timestamp);" + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=data_type), + Column( + name="quantiles_tdigest", + datatype=DataType( + name="AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32) CODEC (ZSTD(2))" + ), + ), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] + with And("I create another table with wrong data type"): + create_basic_table( + name=mv_2_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + node=node, + if_not_exists=True, ) with And("I create mv from first dependent table to second"): @@ -180,25 +250,54 @@ def materialized_view_circle(self, node, core_table, mv_1): f"CREATE MATERIALIZED VIEW {mv_1}" f" TO {core_table}" " AS SELECT toStartOfFiveMinute(timestamp) AS timestamp, host," - " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(repsonse_time) AS quantiles_tdigest " + " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(response_time) AS quantiles_tdigest " f"FROM {core_table} GROUP BY host,timestamp;" ) @TestStep(Given) -def create_table_on_cluster(self, table_engine, node, database, core_table): +def create_table_on_cluster( + self, + table_engine, + node, + database, + core_table, + config="graphite_rollup_example", + sign="sign", + version="timestamp", +): """Step to create database and basic data table in it on cluster.""" try: with Given("I create database"): node.query(f"CREATE DATABASE {database} ON CLUSTER 'ShardedAndReplicated';") + if "GraphiteMergeTree" in table_engine: + table_engine = f"GraphiteMergeTree('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = f"VersionedCollapsingMergeTree({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = f"CollapsingMergeTree({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=String()), + Column(name="response_time", datatype=Int32()), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] + with And("I create data table"): - node.query( - f"CREATE TABLE {core_table} ON CLUSTER 'ShardedAndReplicated'" - "( timestamp DateTime," - "host String," - "repsonse_time Int32" - f") ENGINE {table_engine}() ORDER BY (host, timestamp)" + create_basic_table( + name=core_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + cluster="ShardedAndReplicated", + node=node, + if_not_exists=True, ) yield @@ -219,25 +318,54 @@ def materialized_view_on_cluster( mv_2=None, mv_2_table=None, cascading=False, + config="graphite_rollup_example", + sign="sign", + version="timestamp", ): """Step to create mv and it`s dependent table on cluster section for mv tests on cluster :param cascading: add additional mv section :param failure_mode: failure mode type """ - data_type = "String" + + data_type = String() if failure_mode == "column type mismatch": - data_type = "DateTime" + data_type = DateTime() elif failure_mode == "dummy": - data_type = "String" + data_type = String() + + if "GraphiteMergeTree" in table_engine: + table_engine = table_engine + f"('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=data_type), + Column( + name="quantiles_tdigest", + datatype=DataType( + name="AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32)" + ), + ), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] - with Given("I create mv dependent table on cluster"): - node.query( - f"CREATE table {mv_1_table} ON CLUSTER 'ShardedAndReplicated'" - "( timestamp DateTime," - " host String," - " quantiles_tdigest AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32))" - f" ENGINE = {table_engine} ORDER BY (host, timestamp) ;" + with Given("I create mv dependent table"): + create_basic_table( + name=mv_1_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + cluster="ShardedAndReplicated", + node=node, + if_not_exists=True, ) with And("I create mv on cluster"): @@ -245,19 +373,35 @@ def materialized_view_on_cluster( f"CREATE MATERIALIZED VIEW {mv_1} ON CLUSTER 'ShardedAndReplicated'" f" to {mv_1_table}" " AS SELECT toStartOfFiveMinute(timestamp) AS timestamp, host," - " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(repsonse_time) AS quantiles_tdigest " + " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(response_time) AS quantiles_tdigest " f"FROM {core_table} GROUP BY host,timestamp;" ) if cascading: + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=data_type), + Column( + name="quantiles_tdigest", + datatype=DataType( + name="AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32) CODEC (ZSTD(2))" + ), + ), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] with And("I create another table with wrong data type"): - node.query( - f"CREATE table {mv_2_table} ON CLUSTER 'ShardedAndReplicated' " - "( timestamp DateTime ," - f" host {data_type}," - " quantiles_tdigest " - "AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32) CODEC (ZSTD(2))) " - f"ENGINE = {table_engine}() ORDER BY (host, timestamp);" + create_basic_table( + name=mv_2_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + node=node, + cluster="ShardedAndReplicated", + if_not_exists=True, ) with And("I create mv to the second dependent table"): @@ -283,25 +427,52 @@ def materialized_some_view( mv_1_table, view_1=None, view_type=None, + config="graphite_rollup_example", + sign="sign", + version="timestamp", ): """Step to create mv and it`s dependent table section for mv tests :param view_type: type of mv dependent table :param failure_mode: failure mode type """ - data_type = "String" + data_type = String() if failure_mode == "column type mismatch": - data_type = "DateTime" + data_type = DateTime() elif failure_mode == "dummy": - data_type = "String" + data_type = String() + + if "GraphiteMergeTree" in table_engine: + table_engine = table_engine + f"('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = table_engine + f"({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=data_type), + Column( + name="quantiles_tdigest", + datatype=DataType( + name="AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32)" + ), + ), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] with Given("I create mv dependent table"): - node.query( - f"CREATE table {mv_1_table}" - "( timestamp DateTime," - f" host {data_type}," - " quantiles_tdigest AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), Int32))" - f" ENGINE = {table_engine} ORDER BY (host, timestamp) ;" + create_basic_table( + name=mv_1_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + node=node, + if_not_exists=True, ) with And("I create mv"): @@ -309,7 +480,7 @@ def materialized_some_view( f"CREATE MATERIALIZED VIEW {mv_1}" f" TO {mv_1_table}" " AS SELECT toStartOfFiveMinute(timestamp) AS timestamp, host," - " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(repsonse_time) AS quantiles_tdigest " + " quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(response_time) AS quantiles_tdigest " f"FROM {core_table} GROUP BY host,timestamp;" ) @@ -365,15 +536,16 @@ def simple_insert( else "" ) + ( - f"INSERT INTO {core_table}" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign)" f" SELECT now() + number/10, toString(number), if({fail_block_number}, throwIf(number={fail_block_number}," - "'block fail'), number)" + "'block fail'), number), 1" f" FROM numbers({number_of_blocks})" " SETTINGS max_block_size=1," " min_insert_block_size_bytes=1;" ) + (f"COMMIT;" if self.context.use_transaction_for_atomic_insert else ""), exitcode=139, + message="DB::Exception: block fail:", steps=False, timeout=3000, ) @@ -392,8 +564,8 @@ def simple_insert( else "" ) + ( - f"INSERT INTO {core_table} SELECT now() + number/10, toString(number%9999)," - " number % 999" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign) SELECT now() + number/10, toString(number%9999)," + " number % 999, 1" " FROM numbers(1000001)" ) + ( @@ -410,8 +582,8 @@ def simple_insert( else: self.context.cluster.node(node_name).query( - f"INSERT INTO {core_table} SELECT now() + number/10, toString(number%9999)," - " number % 999" + f"INSERT INTO {core_table} (timestamp, host, response_time. sign) SELECT now() + number/10, toString(number%9999)," + " number % 999, 1" " FROM numbers(1000000)", timeout=3000, ) From c0a609b6c747a6499e5eb2c38c79bab288cef544 Mon Sep 17 00:00:00 2001 From: alsu Date: Tue, 6 Feb 2024 13:50:09 +0100 Subject: [PATCH 02/73] uncomment --- atomic_insert/tests/dependent_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atomic_insert/tests/dependent_tables.py b/atomic_insert/tests/dependent_tables.py index 46b65da2e..9ff6a4e7f 100644 --- a/atomic_insert/tests/dependent_tables.py +++ b/atomic_insert/tests/dependent_tables.py @@ -385,8 +385,8 @@ def feature(self, use_transaction_for_atomic_insert=True): "ReplicatedReplacingMergeTree", "ReplicatedAggregatingMergeTree", "ReplicatedCollapsingMergeTree", - # "ReplicatedVersionedCollapsingMergeTree", - # "ReplicatedGraphiteMergeTree", + "ReplicatedVersionedCollapsingMergeTree", + "ReplicatedGraphiteMergeTree", ] else: self.context.engines = ["MergeTree", "ReplicatedMergeTree"] From 086a6b382e12a22642cc9605f829791861dd60ef Mon Sep 17 00:00:00 2001 From: alsu Date: Tue, 6 Feb 2024 17:31:41 +0100 Subject: [PATCH 03/73] final fixes --- atomic_insert/regression.py | 14 +++++- atomic_insert/tests/distributed_table.py | 58 ++++++++++++++++-------- atomic_insert/tests/steps.py | 10 ++-- atomic_insert/tests/user_rights.py | 15 +++--- 4 files changed, 65 insertions(+), 32 deletions(-) diff --git a/atomic_insert/regression.py b/atomic_insert/regression.py index 47f181b18..bab964d2f 100755 --- a/atomic_insert/regression.py +++ b/atomic_insert/regression.py @@ -34,13 +34,23 @@ def argparser(parser): ) ], } -xflags = {} + +ffails = { + "/atomic insert/dependent_tables/ReplicatedGraphiteMergeTree/table with materialized view/*": ( + Skip, + "need to investigate", + ), + "/atomic insert/dependent_tables/ReplicatedVersionedCollapsingMergeTree/table with materialized view/*": ( + Skip, + "need to investigate", + ), +} @TestModule @ArgumentParser(argparser) @XFails(xfails) -@XFlags(xflags) +@FFails(ffails) @Name("atomic insert") @Requirements(RQ_SRS_028_ClickHouse_AtomicInserts("1.0")) @Specifications(SRS028_ClickHouse_Atomic_Inserts) diff --git a/atomic_insert/tests/distributed_table.py b/atomic_insert/tests/distributed_table.py index 79b406866..775d4cbc1 100644 --- a/atomic_insert/tests/distributed_table.py +++ b/atomic_insert/tests/distributed_table.py @@ -8,6 +8,9 @@ def distributed_tables( table_engine, failure_mode, insert_setting="insert_distributed_one_random_shard=1", + config="graphite_rollup_example", + sign="sign", + version="timestamp", ): """Check that atomic insert works correctly with distributed tables. Test creates distributed table over core table and makes insert with some failure mode and checks data is not inserted @@ -20,14 +23,34 @@ def distributed_tables( core_table_d = f"table_A_d{uid}" cluster = "ShardedAndReplicated" + if "GraphiteMergeTree" in table_engine: + table_engine = f"GraphiteMergeTree('{config}')" + elif "VersionedCollapsingMergeTree" in table_engine: + table_engine = f"VersionedCollapsingMergeTree({sign},{version})" + elif "CollapsingMergeTree" in table_engine: + table_engine = f"CollapsingMergeTree({sign})" + + columns = [ + Column(name="timestamp", datatype=DateTime()), + Column(name="host", datatype=String()), + Column(name="response_time", datatype=Int32()), + Column(name="Path", datatype=String()), + Column(name="Time", datatype=DateTime()), + Column(name="Value", datatype=Float64()), + Column(name="Timestamp", datatype=Int64()), + Column(name="sign", datatype=Int8()), + ] + try: with Given("I create data table"): - node.query( - f"CREATE TABLE {core_table} ON CLUSTER '{cluster}' " - "( timestamp DateTime," - "host String," - "repsonse_time Int32" - f") ENGINE {table_engine}() ORDER BY (host, timestamp)" + create_basic_table( + name=core_table, + columns=columns, + engine=f"{table_engine}", + order_by="(host, timestamp)", + cluster=cluster, + node=node, + if_not_exists=True, ) with And("I create distributed table over data table"): @@ -42,8 +65,8 @@ def distributed_tables( if failure_mode == "dummy": with And(f"I insert into distributed table with setting {insert_setting}"): node.query( - f"INSERT INTO {core_table_d}" - " SELECT now() + number/10, toString(number), number" + f"INSERT INTO {core_table_d} (timestamp, host, response_time, sign)" + " SELECT now() + number/10, toString(number), number, 1" f" FROM numbers(10)" " SETTINGS max_block_size=1," f" min_insert_block_size_bytes=1,{insert_setting};", @@ -57,9 +80,9 @@ def distributed_tables( flags=XFAIL, ): node.query( - f"INSERT INTO {core_table_d}" + f"INSERT INTO {core_table_d} (timestamp, host, response_time, sign)" " SELECT now() + number/10, toString(number), if(5," - " throwIf(number=5,'block fail'), number)" + " throwIf(number=5,'block fail'), number), 1" f" FROM numbers(10)" " SETTINGS max_block_size=1," f" min_insert_block_size_bytes=1,{insert_setting};", @@ -77,8 +100,8 @@ def distributed_tables( "I make insert from user with not enough permissions", flags=XFAIL ): self.context.cluster.node(node_name).query( - f"INSERT INTO {core_table} SELECT now() + number/10, toString(number%9999)," - " number % 999" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign) SELECT now() + number/10, toString(number%9999)," + " number % 999, 1" " FROM numbers(1000001)", settings=[("user", "ivan")], timeout=3000, @@ -94,15 +117,14 @@ def distributed_tables( if failure_mode == "dummy": for node_name in self.context.cluster.nodes["clickhouse"]: with When(f"on {node_name} "): - retry( - self.context.cluster.node(node_name).query, - timeout=100, - delay=1, - )( + output = self.context.cluster.node(node_name).query( f"SELECT count() FROM {core_table_d}", - message=f"10", exitcode=0, ) + for attempt in retries(timeout=30, delay=2): + with attempt: + assert int(output.output) > 0, error() + elif failure_mode == "throwIf": for node_name in self.context.cluster.nodes["clickhouse"]: with When(f"on {node_name} "): diff --git a/atomic_insert/tests/steps.py b/atomic_insert/tests/steps.py index 9c0ddbdd4..ae9fcc96a 100644 --- a/atomic_insert/tests/steps.py +++ b/atomic_insert/tests/steps.py @@ -582,7 +582,7 @@ def simple_insert( else: self.context.cluster.node(node_name).query( - f"INSERT INTO {core_table} (timestamp, host, response_time. sign) SELECT now() + number/10, toString(number%9999)," + f"INSERT INTO {core_table} (timestamp, host, response_time, sign) SELECT now() + number/10, toString(number%9999)," " number % 999, 1" " FROM numbers(1000000)", timeout=3000, @@ -610,9 +610,9 @@ def simple_transaction_insert( with Given(f"I make transaction insert"): self.context.cluster.node(node_name).query( f"BEGIN TRANSACTION;" - f"INSERT INTO {core_table}" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign)" " SELECT now() + number/10, toString(number)," - " number" + " number, 1" f" FROM numbers({numbers});" " COMMIT;", exitcode=0, @@ -628,10 +628,10 @@ def simple_transaction_insert_throwif(self, core_table, node_name="clickhouse1") ): self.context.cluster.node(node_name).query( f"BEGIN TRANSACTION;" - f"INSERT INTO {core_table}" + f"INSERT INTO {core_table} (timestamp, host, response_time, sign)" " SELECT now() + number/10, toString(number)," " if(5," - " throwIf(number=5,'block fail'), number)" + " throwIf(number=5,'block fail'), number), 1" f" FROM numbers(10)" " SETTINGS max_block_size=1," f" min_insert_block_size_bytes=1;", diff --git a/atomic_insert/tests/user_rights.py b/atomic_insert/tests/user_rights.py index c8b1455c7..86115dd10 100644 --- a/atomic_insert/tests/user_rights.py +++ b/atomic_insert/tests/user_rights.py @@ -52,11 +52,11 @@ def user_rights( for node_name in self.context.cluster.nodes["clickhouse"]: for table_name in tables: with When(f"table {table_name}"): - self.context.cluster.node(node_name).query( - f"select count()+737 from {table_name}", - message="737", + output = self.context.cluster.node(node_name).query( + f"select count() from {table_name}", exitcode=0, ) + assert int(output.output) == 0 else: with Given("I create database and core table in this database"): @@ -82,8 +82,8 @@ def user_rights( with And("I make insert from user with not enough permissions", flags=XFAIL): node.query( - f"INSERT INTO {tables[0]} SELECT now() + number/10, toString(number%9999)," - " number % 999" + f"INSERT INTO {tables[0]} (timestamp, host, response_time, sign) SELECT now() + number/10, toString(number%9999)," + " number % 999, 1" " FROM numbers(1000001)", settings=[("user", "ivan")], timeout=3000, @@ -109,9 +109,10 @@ def user_rights( @TestFeature @Name("user_rights") @Requirements(RQ_SRS_028_ClickHouse_AtomicInserts_Failures_UserRights("1.0")) -def feature(self): +def feature(self, use_transaction_for_atomic_insert=True): """Check atomic insert in presence of a query issues by a user that does not have enough rights either on the target table or one of its dependent tables.""" + self.context.use_transaction_for_atomic_insert = use_transaction_for_atomic_insert if self.context.stress: self.context.engines = [ "MergeTree", @@ -132,7 +133,7 @@ def feature(self): else: self.context.engines = ["MergeTree", "ReplicatedMergeTree"] - failure_mode = ["dummy"] + failure_mode = ["user_rights"] falure_mode_1 = ["dummy", "throwIf"] From 92904038ca6f60cc312b1c0df5025a2b6a06e1e1 Mon Sep 17 00:00:00 2001 From: alsu Date: Tue, 6 Feb 2024 21:30:20 +0100 Subject: [PATCH 04/73] fix issue with engines --- atomic_insert/regression.py | 12 ------------ atomic_insert/tests/steps.py | 6 +++--- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/atomic_insert/regression.py b/atomic_insert/regression.py index bab964d2f..fedadeac5 100755 --- a/atomic_insert/regression.py +++ b/atomic_insert/regression.py @@ -35,22 +35,10 @@ def argparser(parser): ], } -ffails = { - "/atomic insert/dependent_tables/ReplicatedGraphiteMergeTree/table with materialized view/*": ( - Skip, - "need to investigate", - ), - "/atomic insert/dependent_tables/ReplicatedVersionedCollapsingMergeTree/table with materialized view/*": ( - Skip, - "need to investigate", - ), -} - @TestModule @ArgumentParser(argparser) @XFails(xfails) -@FFails(ffails) @Name("atomic insert") @Requirements(RQ_SRS_028_ClickHouse_AtomicInserts("1.0")) @Specifications(SRS028_ClickHouse_Atomic_Inserts) diff --git a/atomic_insert/tests/steps.py b/atomic_insert/tests/steps.py index ae9fcc96a..4415c7c9b 100644 --- a/atomic_insert/tests/steps.py +++ b/atomic_insert/tests/steps.py @@ -272,11 +272,11 @@ def create_table_on_cluster( node.query(f"CREATE DATABASE {database} ON CLUSTER 'ShardedAndReplicated';") if "GraphiteMergeTree" in table_engine: - table_engine = f"GraphiteMergeTree('{config}')" + table_engine = table_engine + f"('{config}')" elif "VersionedCollapsingMergeTree" in table_engine: - table_engine = f"VersionedCollapsingMergeTree({sign},{version})" + table_engine = table_engine + f"({sign},{version})" elif "CollapsingMergeTree" in table_engine: - table_engine = f"CollapsingMergeTree({sign})" + table_engine = table_engine + f"({sign})" columns = [ Column(name="timestamp", datatype=DateTime()), From 6c8388b51b337e6c3716995df07d1df7908e5d7c Mon Sep 17 00:00:00 2001 From: alsu Date: Wed, 14 Feb 2024 11:47:59 +0100 Subject: [PATCH 05/73] add crashing test to atomic insert suite --- atomic_insert/regression.py | 8 ++++ atomic_insert/tests/dependent_tables.py | 57 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/atomic_insert/regression.py b/atomic_insert/regression.py index fedadeac5..27dc22c23 100755 --- a/atomic_insert/regression.py +++ b/atomic_insert/regression.py @@ -26,6 +26,13 @@ def argparser(parser): ) +ffails = { + "/atomic insert/dependent_tables/Replicated*/table with materialized view engine mismatch/*": ( + Skip, + "https://github.com/ClickHouse/ClickHouse/issues/59670", + ), +} + xfails = { "/atomic insert/dependent_tables/Replicated*/table with materialized view*/*": [ ( @@ -38,6 +45,7 @@ def argparser(parser): @TestModule @ArgumentParser(argparser) +@FFails(ffails) @XFails(xfails) @Name("atomic insert") @Requirements(RQ_SRS_028_ClickHouse_AtomicInserts("1.0")) diff --git a/atomic_insert/tests/dependent_tables.py b/atomic_insert/tests/dependent_tables.py index 9ff6a4e7f..fa2dbf89f 100644 --- a/atomic_insert/tests/dependent_tables.py +++ b/atomic_insert/tests/dependent_tables.py @@ -109,6 +109,63 @@ def table_with_materialized_view(self, table_engine, failure_mode): ) +@TestScenario +def table_with_materialized_view_engine_mismatch(self, table_engine, failure_mode): + """Check atomic insert by making simple insert with different failure mode in chain of core table + (single and on cluster) with single materialized view table_B_mv with dependent table_B. + table_A is non-replicated, table_B is replicated. + """ + node = self.context.cluster.node("clickhouse1") + uid = getuid() + + database = f"test_database{uid}" + + tables = [ + f"test_database{uid}.table_A{uid}", + f"test_database{uid}.table_B{uid}", + f"test_database{uid}.table_B_mv{uid}", + ] + + if table_engine.startswith("Replicated"): + with Given("I create database and core table on cluster in this database"): + wrong_table_engine = table_engine.replace("Replicated", "") + create_table_on_cluster( + table_engine=wrong_table_engine, + node=node, + database=database, + core_table=tables[0], + ) + + with And("I add materialized view with one of failure modes"): + materialized_view_on_cluster( + table_engine=table_engine, + node=node, + core_table=tables[0], + mv_1=tables[2], + mv_1_table=tables[1], + failure_mode=failure_mode, + ) + + with And("I make insert into core table"): + for i in range(1): + When(f"I make insert #{i}", test=insert, parallel=True)( + core_table=tables[0], failure_mode=failure_mode + ) + + with And( + "I check data is not inserted to core table" + "and any of its dependent tables on any of the cluster nodes" + ): + for node_name in self.context.cluster.nodes["clickhouse"]: + for table_name in tables: + with When(f"table {table_name}"): + self.context.cluster.node(node_name).query( + f"select count()+737 from {table_name}", + message="737", + exitcode=0, + ) + + @TestScenario def table_with_materialized_view_cascading(self, table_engine, failure_mode): """Check atomic insert by making simple insert with different failure mode in chain of core table From f68e4a3023ed8c6432bae6fe971eccd967de41a7 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:31:04 -0500 Subject: [PATCH 06/73] Fix vfs settings tests --- vfs/regression.py | 1 + vfs/tests/settings.py | 4 ++-- vfs/tests/stress_alter.py | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vfs/regression.py b/vfs/regression.py index f0bd13d5f..d939f2cd2 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -29,6 +29,7 @@ check_clickhouse_version("<24"), ), ":/alter/move": (XFail, "Clickhouse crashes"), + ":/replica/add remove one node": (XFail, "Fix pending"), ":/settings/incompatible with send metadata": (XFail, "Clickhouse crashes"), } diff --git a/vfs/tests/settings.py b/vfs/tests/settings.py index b060143a4..34d2f41cc 100644 --- a/vfs/tests/settings.py +++ b/vfs/tests/settings.py @@ -7,7 +7,7 @@ from vfs.tests.steps import * from vfs.requirements import * -from vfs.tests.stress_alter import optimize, check_consistency +from vfs.tests.stress_alter import optimize_random, check_consistency from s3.tests.common import invalid_s3_storage_config @@ -387,7 +387,7 @@ def check_setting_combination( )(table_name=table_name, settings=select_setting) When( f"I OPTIMIZE {table_name}", - test=optimize, + test=optimize_random, parallel=True, flags=TE, )(table_name=table_name) diff --git a/vfs/tests/stress_alter.py b/vfs/tests/stress_alter.py index d1ca58ead..4bf577e8c 100644 --- a/vfs/tests/stress_alter.py +++ b/vfs/tests/stress_alter.py @@ -393,11 +393,14 @@ def delete_random_rows(self): @TestStep(Then) @Retry(timeout=120, delay=5) -def check_consistency(self, tables=None): +def check_consistency(self, tables=None, sync_timeout=None): """ Check that the given tables hold the same amount of data on all nodes where they exist. Also check that column names match, subsequent part move tests require matching columns. """ + if sync_timeout is None: + sync_timeout = getattr(self.context, "sync_replica_timeout", 60) + nodes = self.context.ch_nodes if tables is None: tables = self.context.table_names @@ -420,7 +423,7 @@ def check_consistency(self, tables=None): try: node.query( f"SYSTEM SYNC REPLICA {table_name}", - timeout=self.context.sync_replica_timeout, + timeout=sync_timeout, no_checks=True, ) except (ExpectTimeoutError, TimeoutError): From 0ce91b06d14dad1383baca9263d90b217d07cbbc Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 14 Feb 2024 11:44:09 -0500 Subject: [PATCH 07/73] update vfs xfails --- vfs/regression.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vfs/regression.py b/vfs/regression.py index d939f2cd2..c684d4d63 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -17,9 +17,6 @@ ":/replica/command combinations/*": [(Error, "some combos time out")], ":/parallel replica/add remove commands/*": [(Fail, "WIP"), (Error, "WIP")], ":/settings/disable vfs with vfs table/access:": [(Fail, "not supported")], - ":/system/optimize/table_settings='min_bytes_for_wide_part=:": [ - (Fail, "needs investigation") - ], } ffails = { @@ -28,9 +25,8 @@ "vfs not supported on < 24", check_clickhouse_version("<24"), ), - ":/alter/move": (XFail, "Clickhouse crashes"), + ":/alter/move": (XFail, "Fix pending"), ":/replica/add remove one node": (XFail, "Fix pending"), - ":/settings/incompatible with send metadata": (XFail, "Clickhouse crashes"), } # RQ_SRS_038_DiskObjectStorageVFS_Providers_Configuration From fd64a1395e204de0a1564be8f214a8cbe2e1a625 Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 13:13:51 +0100 Subject: [PATCH 08/73] add notes to attach partition temporary table test --- alter/table/attach_partition/temporary_table.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/alter/table/attach_partition/temporary_table.py b/alter/table/attach_partition/temporary_table.py index 955464b38..faf32edc4 100644 --- a/alter/table/attach_partition/temporary_table.py +++ b/alter/table/attach_partition/temporary_table.py @@ -47,6 +47,9 @@ def check_attach_partition_detached_with_temporary_tables(self, table, engine): exitcode = 60 else: exitcode = 0 + + note(f"exitcode should be {exitcode}") + detach_partition( table=table_name, partition=1, node=client, exitcode=exitcode ) @@ -151,6 +154,8 @@ def check_attach_partition_from_with_temporary_tables( else: exitcode = 0 + note(f"exitcode should be {exitcode}") + for partition_id in ["1", "2", "3"]: query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION {partition_id} FROM {source_table_name}" client.query(query, exitcode=exitcode) From 6867d759cb2a79c1ff6223896a34c0a7e1c196f7 Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 14:26:21 +0100 Subject: [PATCH 09/73] add arm snapshots for attach partition tests --- .../attach_partition/snapshots/common.py.tests.aarch64.snapshot | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 alter/table/attach_partition/snapshots/common.py.tests.aarch64.snapshot diff --git a/alter/table/attach_partition/snapshots/common.py.tests.aarch64.snapshot b/alter/table/attach_partition/snapshots/common.py.tests.aarch64.snapshot new file mode 100644 index 000000000..e69de29bb From 53afdeeec1a073b6f131e01801aa7ef75d90d94b Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:21:42 -0500 Subject: [PATCH 10/73] S3, reduce use of context managers, WIP --- ontime_benchmark/tests/benchmark.py | 16 +- s3/tests/common.py | 25 +- s3/tests/zero_copy_replication.py | 2178 +++++++++++++-------------- 3 files changed, 1112 insertions(+), 1107 deletions(-) diff --git a/ontime_benchmark/tests/benchmark.py b/ontime_benchmark/tests/benchmark.py index 37eb0f31b..14f52deca 100644 --- a/ontime_benchmark/tests/benchmark.py +++ b/ontime_benchmark/tests/benchmark.py @@ -298,13 +298,15 @@ def zero_copy_replication(self, format=None): else: settings = {"allow_s3_zero_copy_replication": "1"} - with mergetree_config(settings): - benchmark( - table_name="zero_copy_replication", - table_settings=table_settings, - nodes=nodes, - format=format, - ) + mergetree_config(settings=settings) + + + benchmark( + table_name="zero_copy_replication", + table_settings=table_settings, + nodes=nodes, + format=format, + ) @TestScenario diff --git a/s3/tests/common.py b/s3/tests/common.py index 4363772ea..fca441504 100644 --- a/s3/tests/common.py +++ b/s3/tests/common.py @@ -454,7 +454,7 @@ def create_mergetree_config_content( @contextmanager -def mergetree_config( +def mergetree_config_context( settings, config_d_dir="/etc/clickhouse-server/config.d", config_file="merge_tree.xml", @@ -469,6 +469,23 @@ def mergetree_config( return add_config(config, restart=restart, nodes=nodes) +@TestStep(Given) +def mergetree_config( + self, + settings, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="merge_tree.xml", + timeout=60, + restart=False, + config=None, + nodes=None, +): + """Add MergeTree configuration.""" + if config is None: + config = create_mergetree_config_content(settings, config_d_dir, config_file) + yield add_config(config, restart=restart, nodes=nodes) + + @contextmanager def subshell(bash, command, name, prompt=None): def spawn(command): @@ -1199,9 +1216,9 @@ def add_ssec_s3_option(self, ssec_key=None): "adding 'server_side_encryption_customer_key_base64' S3 option", description=f"key={ssec_key}", ): - self.context.s3_options[ - "server_side_encryption_customer_key_base64" - ] = ssec_key + self.context.s3_options["server_side_encryption_customer_key_base64"] = ( + ssec_key + ) yield finally: diff --git a/s3/tests/zero_copy_replication.py b/s3/tests/zero_copy_replication.py index da6158d61..cf037ecea 100644 --- a/s3/tests/zero_copy_replication.py +++ b/s3/tests/zero_copy_replication.py @@ -24,6 +24,7 @@ def global_setting(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -34,87 +35,84 @@ def global_setting(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_global', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_global', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with Then("I check simple queries on the other node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[1], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + with Then("I check simple queries on the other node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[1], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -136,6 +134,7 @@ def drop_replica(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -146,131 +145,128 @@ def drop_replica(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with And("I stop the second node"): - nodes[1].stop() + with And("I stop the second node"): + nodes[1].stop() - with Then("I check simple queries on the first node"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[0], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[0], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + with Then("I check simple queries on the first node"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[0], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[0], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) - with And("I start the second node"): - nodes[1].start() + with And("I start the second node"): + nodes[1].start() - with Then("I check simple queries on the second node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[1], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + with Then("I check simple queries on the second node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[1], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -290,6 +286,7 @@ def add_replica(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -300,158 +297,155 @@ def add_replica(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on the first node"): - nodes[0].restart_clickhouse() - nodes[0].query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_add', '1') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) - - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) - - with And("I get the size of the s3 bucket"): - size_after = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - - with And("I create a replicated table on the second node"): - nodes[1].restart_clickhouse() - nodes[1].query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_add', '2') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with Then( - """The size of the s3 bucket should be 1 byte more - than previously because of the additional replica""" - ): - size = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - assert size - size_after == 1, error() - - with And("I check simple queries on the first node"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[0], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[0], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + try: + with When("I create a replicated table on the first node"): + nodes[0].restart_clickhouse() + nodes[0].query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_add', '1') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I check simple queries on the second node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[1], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) + + with And("I get the size of the s3 bucket"): + size_after = get_bucket_size( + name=bucket_name, + prefix=bucket_path, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, + minio_enabled=self.context.minio_enabled, + ) + + with And("I create a replicated table on the second node"): + nodes[1].restart_clickhouse() + nodes[1].query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_add', '2') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" + """The size of the s3 bucket should be 1 byte more + than previously because of the additional replica""" ): - check_bucket_size( + size = get_bucket_size( name=bucket_name, prefix=bucket_path, - expected_size=size_before, - tolerance=5, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, minio_enabled=self.context.minio_enabled, ) + assert size - size_after == 1, error() + + with And("I check simple queries on the first node"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[0], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[0], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) + + with And("I check simple queries on the second node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[1], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -474,6 +468,7 @@ def drop_alter_replica(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -484,90 +479,89 @@ def drop_alter_replica(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop_alter', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop_alter', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I insert 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) + with And("I insert 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - with And("I stop the other node"): - nodes[1].stop() + with And("I stop the other node"): + nodes[1].stop() - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with And("a large insert of 10Mb of data"): - insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with And("I restart the other node"): - nodes[1].start() + with And("I restart the other node"): + nodes[1].start() - with Then("I check simple queries on the other node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[1], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) + with Then("I check simple queries on the other node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[1], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -614,7 +608,7 @@ def metadata(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings - with mergetree_config(settings): + with mergetree_config_context(settings): try: with When("I create a replicated table on each node"): for i, node in enumerate(nodes): @@ -717,6 +711,7 @@ def check_query_pair(node, num, query, expected): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings.copy() settings["old_parts_lifetime"] = "1" + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -727,108 +722,105 @@ def check_query_pair(node, num, query, expected): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64, - sign Int8 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_alter', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64, + sign Int8 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_alter', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) + + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_pair(nodes[0], 1) + + with And("I get the size of the s3 bucket"): + size_after = get_bucket_size( + name=bucket_name, + prefix=bucket_path, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, + minio_enabled=self.context.minio_enabled, + ) + + with Then("I check that the sign is 1 for the second table"): + check_query_pair( + node=nodes[1], + num=0, + query=f"SELECT sign FROM zero_copy_replication LIMIT 1", + expected="1", + ) + + with And("I change all signs to -1"): + nodes[1].query("ALTER TABLE zero_copy_replication UPDATE sign = -1 WHERE 1") + + with And("I sync the replicas"): + for node in nodes: + for attempt in retries(timeout=1200, delay=5): + with attempt: + node.query( + "SYSTEM SYNC REPLICA zero_copy_replication", timeout=600 + ) + + with And("I check that the sign is -1 for the second table"): + check_query_pair( + node=nodes[1], + num=0, + query=f"SELECT sign FROM zero_copy_replication LIMIT 1", + expected="-1", + ) - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_pair(nodes[0], 1) + with And("I check that the sign is -1 for the first table"): + check_query_pair( + node=nodes[0], + num=0, + query=f"SELECT sign FROM zero_copy_replication LIMIT 1", + expected="-1", + ) - with And("I get the size of the s3 bucket"): - size_after = get_bucket_size( + with And( + """I wait until the bucket size drops to within 50% of the + expected size""" + ): + start_time = time.time() + while True: + current_size = get_bucket_size( name=bucket_name, prefix=bucket_path, access_key=self.context.secret_access_key, key_id=self.context.access_key_id, minio_enabled=self.context.minio_enabled, ) + if current_size < size_after * 1.5: + break + if time.time() - start_time < 60: + time.sleep(2) + continue + assert False, "data in S3 has grown by more than 50%" - with Then("I check that the sign is 1 for the second table"): - check_query_pair( - node=nodes[1], - num=0, - query=f"SELECT sign FROM zero_copy_replication LIMIT 1", - expected="1", - ) - - with And("I change all signs to -1"): - nodes[1].query( - "ALTER TABLE zero_copy_replication UPDATE sign = -1 WHERE 1" - ) - - with And("I sync the replicas"): - for node in nodes: - for attempt in retries(timeout=1200, delay=5): - with attempt: - node.query( - "SYSTEM SYNC REPLICA zero_copy_replication", timeout=600 - ) - - with And("I check that the sign is -1 for the second table"): - check_query_pair( - node=nodes[1], - num=0, - query=f"SELECT sign FROM zero_copy_replication LIMIT 1", - expected="-1", - ) - - with And("I check that the sign is -1 for the first table"): - check_query_pair( - node=nodes[0], - num=0, - query=f"SELECT sign FROM zero_copy_replication LIMIT 1", - expected="-1", - ) - - with And( - """I wait until the bucket size drops to within 50% of the - expected size""" - ): - start_time = time.time() - while True: - current_size = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - if current_size < size_after * 1.5: - break - if time.time() - start_time < 60: - time.sleep(2) - continue - assert False, "data in S3 has grown by more than 50%" - - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -894,6 +886,7 @@ def alter_table(sign): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings.copy() settings["old_parts_lifetime"] = "1" + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -904,87 +897,86 @@ def alter_table(sign): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64, - sign Int8 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_alter', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_pair(nodes[0], 1) - - with And("I get the size of the s3 bucket"): - size_after = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - - with Then("I check that the sign is 1 for the second table"): - check_query_pair( - node=nodes[1], - num=0, - query=f"SELECT sign FROM zero_copy_replication LIMIT 1", - expected="1", + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64, + sign Int8 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_alter', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ ) - with And("I alter and check the size 10 times"): - s = 1 - for i in range(10): - alter_table(s) - - with Then( - """I make sure the amount of data in S3 is within - 50% of the original amount""" - ): - start_time = time.time() - while True: - current_size = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - if current_size < size_after * 1.5: - break - if time.time() - start_time < 60: - time.sleep(2) - continue - assert False, "data in S3 has grown by more than 50%" - - s *= -1 - - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_pair(nodes[0], 1) - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( + with And("I get the size of the s3 bucket"): + size_after = get_bucket_size( name=bucket_name, prefix=bucket_path, - expected_size=size_before, - tolerance=5, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, minio_enabled=self.context.minio_enabled, ) + with Then("I check that the sign is 1 for the second table"): + check_query_pair( + node=nodes[1], + num=0, + query=f"SELECT sign FROM zero_copy_replication LIMIT 1", + expected="1", + ) + + with And("I alter and check the size 10 times"): + s = 1 + for i in range(10): + alter_table(s) + + with Then( + """I make sure the amount of data in S3 is within + 50% of the original amount""" + ): + start_time = time.time() + while True: + current_size = get_bucket_size( + name=bucket_name, + prefix=bucket_path, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, + minio_enabled=self.context.minio_enabled, + ) + if current_size < size_after * 1.5: + break + if time.time() - start_time < 60: + time.sleep(2) + continue + assert False, "data in S3 has grown by more than 50%" + + s *= -1 + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) + @TestScenario @Requirements( @@ -1016,7 +1008,7 @@ def insert_multiple_replicas(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): + with mergetree_config_context(settings): try: with When("I create a replicated table on each node"): for i, node in enumerate(nodes): @@ -1161,6 +1153,7 @@ def delete(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -1171,73 +1164,70 @@ def delete(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_delete', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_delete', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with When("I get the size of the s3 bucket"): - size_after = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) + with When("I get the size of the s3 bucket"): + size_after = get_bucket_size( + name=bucket_name, + prefix=bucket_path, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, + minio_enabled=self.context.minio_enabled, + ) - with And("I drop the table on one node"): - nodes[0].query("DROP TABLE IF EXISTS zero_copy_replication") + with And("I drop the table on one node"): + nodes[0].query("DROP TABLE IF EXISTS zero_copy_replication") - with Then("The size of the s3 bucket should be the same"): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_after, - tolerance=0, - minio_enabled=self.context.minio_enabled, - ) + with Then("The size of the s3 bucket should be the same"): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_after, + tolerance=0, + minio_enabled=self.context.minio_enabled, + ) - with When("I drop the table on the other node"): - nodes[1].query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with When("I drop the table on the other node"): + nodes[1].query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") @TestScenario @@ -1256,6 +1246,7 @@ def delete_all(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -1266,58 +1257,55 @@ def delete_all(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_global', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) - - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) - - with Then("A nonzero amount of data should be added to S3"): - size_now = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - minio_enabled=self.context.minio_enabled, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_global', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ ) - assert size_now > size_before, error() + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with Then("All data should be removed from S3"): - check_bucket_size( + with And("a large insert of 10Mb of data"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) + + with Then("A nonzero amount of data should be added to S3"): + size_now = get_bucket_size( name=bucket_name, prefix=bucket_path, - expected_size=size_before, - tolerance=0, minio_enabled=self.context.minio_enabled, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, ) + assert size_now > size_before, error() + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then("All data should be removed from S3"): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=0, + minio_enabled=self.context.minio_enabled, + ) + @TestScenario @Requirements(RQ_SRS_015_S3_Disk_MergeTree_AllowS3ZeroCopyReplication_TTL_Move("1.0")) @@ -1343,6 +1331,7 @@ def insert_data_time(node, number_of_mb, time, start=0): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings.copy() settings["old_parts_lifetime"] = "1" + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before_base = get_bucket_size( @@ -1362,118 +1351,117 @@ def insert_data_time(node, number_of_mb, time, start=0): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64, - d1 DateTime - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_ttl', '{i + 1}') - ORDER BY d - TTL d1 + interval 2 day to volume 'external' - SETTINGS storage_policy='tiered' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 0) - - with And("another insert of 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=3)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 1024 * 1024) - - with And("a large insert of 10Mb of data"): - tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) - - with Then("I check simple queries on both nodes"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64, + d1 DateTime + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_ttl', '{i + 1}') + ORDER BY d + TTL d1 + interval 2 day to volume 'external' + SETTINGS storage_policy='tiered' + """ ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", + + with And("I add data to the table"): + with By("first inserting 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", + insert_data_time(nodes[0], 1, tm, 0) + + with And("another insert of 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=3)).timetuple() ) + insert_data_time(nodes[0], 1, tm, 1024 * 1024) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("a large insert of 10Mb of data"): + tm = time.mktime(datetime.date.today().timetuple()) + insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before_base, - tolerance=5, - minio_enabled=self.context.minio_enabled, + with Then("I check simple queries on both nodes"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", ) - with And( - """The size of the other s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path + "/tiered", - expected_size=size_before_tier, - tolerance=5, - minio_enabled=self.context.minio_enabled, + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", ) + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before_base, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) + with And( + """The size of the other s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path + "/tiered", + expected_size=size_before_tier, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) + @TestScenario @Requirements(RQ_SRS_015_S3_Disk_MergeTree_AllowS3ZeroCopyReplication_TTL_Delete("1.0")) @@ -1498,6 +1486,7 @@ def insert_data_time(node, number_of_mb, time, start=0): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -1508,107 +1497,107 @@ def insert_data_time(node, number_of_mb, time, start=0): minio_enabled=self.context.minio_enabled, ) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64, - d1 DateTime - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_ttl', '{i + 1}') - ORDER BY d - TTL d1 + interval 2 day - SETTINGS storage_policy='tiered' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 0) - - with And("another insert of 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 1024 * 1024) - - with And("a large insert of 10Mb of data"): - tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) - - with Then("I check simple queries on both nodes"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1310721", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="0", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="2097152", - ) - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1310721", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="0", + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64, + d1 DateTime + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_ttl', '{i + 1}') + ORDER BY d + TTL d1 + interval 2 day + SETTINGS storage_policy='tiered' + """ ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", + + with And("I add data to the table"): + with By("first inserting 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="2097152", + insert_data_time(nodes[0], 1, tm, 0) + + with And("another insert of 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() ) + insert_data_time(nodes[0], 1, tm, 1024 * 1024) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("a large insert of 10Mb of data"): + tm = time.mktime(datetime.date.today().timetuple()) + insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( - name=bucket_name, - prefix=bucket_path, - expected_size=size_before, - tolerance=5, - minio_enabled=self.context.minio_enabled, + with Then("I check simple queries on both nodes"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1310721", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="0", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="2097152", + ) + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1310721", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="0", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="2097152", ) + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) + + @TestScenario def bad_detached_part(self): """ @@ -1625,61 +1614,61 @@ def bad_detached_part(self): settings = { self.context.zero_copy_replication_setting: "1", } + mergetree_config(settings=settings) - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE {table_name} ( - d UInt64, - ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{table_name}', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external', min_bytes_for_wide_part=0 - """ - ) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE {table_name} ( + d UInt64, + ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{table_name}', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external', min_bytes_for_wide_part=0 + """ + ) - with And("I insert data on the second node"): - nodes[1].query(f"INSERT INTO {table_name} VALUES (123)") + with And("I insert data on the second node"): + nodes[1].query(f"INSERT INTO {table_name} VALUES (123)") - with And("I sync the first node"): - nodes[0].query(f"SYSTEM SYNC REPLICA {table_name}") + with And("I sync the first node"): + nodes[0].query(f"SYSTEM SYNC REPLICA {table_name}") - with And("I get the path for the part"): - r = nodes[1].query( - f"SELECT path FROM system.parts where table='{table_name}' and name='all_0_0_0'" - ) - part_path = r.output - assert part_path.startswith("/"), error("Expected absolute path!") + with And("I get the path for the part"): + r = nodes[1].query( + f"SELECT path FROM system.parts where table='{table_name}' and name='all_0_0_0'" + ) + part_path = r.output + assert part_path.startswith("/"), error("Expected absolute path!") - with And("I delete the part's count.txt"): - nodes[1].command(f'rm {part_path}/count.txt') + with And("I delete the part's count.txt"): + nodes[1].command(f"rm {part_path}/count.txt") - with And("I detach the table on the second node"): - nodes[1].query(f"DETACH TABLE {table_name} SYNC") + with And("I detach the table on the second node"): + nodes[1].query(f"DETACH TABLE {table_name} SYNC") - with And("I reattach the table on the second node"): - nodes[1].query(f"ATTACH TABLE {table_name}") + with And("I reattach the table on the second node"): + nodes[1].query(f"ATTACH TABLE {table_name}") - with And("I check detached parts on the second node"): - r = nodes[1].query( - f"SELECT reason, name FROM system.detached_parts where table='{table_name}'" - ) - assert r.output == "broken-on-start broken-on-start_all_0_0_0", error() + with And("I check detached parts on the second node"): + r = nodes[1].query( + f"SELECT reason, name FROM system.detached_parts where table='{table_name}'" + ) + assert r.output == "broken-on-start broken-on-start_all_0_0_0", error() - with And("I drop the table on the second node"): - nodes[1].query(f"DROP TABLE {table_name} SYNC") + with And("I drop the table on the second node"): + nodes[1].query(f"DROP TABLE {table_name} SYNC") - with Then("The first node should still have the data"): - r = nodes[0].query(f"SELECT * FROM {table_name}") - assert r.output == "123", error() + with Then("The first node should still have the data"): + r = nodes[0].query(f"SELECT * FROM {table_name}") + assert r.output == "123", error() - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") @TestScenario @@ -1704,9 +1693,6 @@ def insert_data_time(node, number_of_mb, start=0): with Given("I have a pair of clickhouse nodes"): nodes = self.context.ch_nodes[:2] - with And("I have merge tree configuration set to use zero copy replication"): - settings = self.context.zero_copy_replication_settings - try: with When("I create a replicated table on each node"): for i, node in enumerate(nodes): @@ -1730,31 +1716,32 @@ def insert_data_time(node, number_of_mb, start=0): for node in nodes: node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_insert', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + with Given("I have merge tree configuration set to use zero copy replication"): + settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) - with And("I add data to the table and save the time taken"): - allow_zero_copy_time = insert_data_time(nodes[0], 20) - metric( - "with_zero_copy", units="seconds", value=str(allow_zero_copy_time) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_insert', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("I add data to the table and save the time taken"): + allow_zero_copy_time = insert_data_time(nodes[0], 20) + metric("with_zero_copy", units="seconds", value=str(allow_zero_copy_time)) + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") with Finally("I print the difference in time taken"): metric( @@ -1778,9 +1765,6 @@ def performance_select(self): with Given("I have a pair of clickhouse nodes"): nodes = self.context.ch_nodes[:2] - with And("I have merge tree configuration set to use zero copy replication"): - settings = self.context.zero_copy_replication_settings - try: with When("I create a replicated table on each node"): for i, node in enumerate(nodes): @@ -1821,49 +1805,50 @@ def performance_select(self): node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") node.query("DROP TABLE IF EXISTS zcrSelect SYNC") - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_select', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + with Given("I have merge tree configuration set to use zero copy replication"): + settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) - with And("I add 20 Mb of data to the table"): - insert_data_node(node=nodes[0], number_of_mb=20) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_select', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with Then("I sync the replicas"): - for attempt in retries(timeout=1200, delay=5): - with attempt: - nodes[1].query( - "SYSTEM SYNC REPLICA zero_copy_replication", - settings=[("receive_timeout", 600)], - timeout=600, - ) + with And("I add 20 Mb of data to the table"): + insert_data_node(node=nodes[0], number_of_mb=20) - with Then("I select from the table and save the time taken"): - start_time = time.time() - nodes[1].query( - "CREATE TABLE zcrSelect Engine = MergeTree() ORDER BY d AS SELECT * FROM zero_copy_replication" - ) - end_time = time.time() - allow_zero_copy_time = end_time - start_time - metric( - "with_zero_copy", units="seconds", value=str(allow_zero_copy_time) - ) + with Then("I sync the replicas"): + for attempt in retries(timeout=1200, delay=5): + with attempt: + nodes[1].query( + "SYSTEM SYNC REPLICA zero_copy_replication", + settings=[("receive_timeout", 600)], + timeout=600, + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - node.query("DROP TABLE IF EXISTS zcrSelect SYNC") + with Then("I select from the table and save the time taken"): + start_time = time.time() + nodes[1].query( + "CREATE TABLE zcrSelect Engine = MergeTree() ORDER BY d AS SELECT * FROM zero_copy_replication" + ) + end_time = time.time() + allow_zero_copy_time = end_time - start_time + metric("with_zero_copy", units="seconds", value=str(allow_zero_copy_time)) + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + node.query("DROP TABLE IF EXISTS zcrSelect SYNC") with Finally("I print the difference in time taken"): metric( @@ -1939,49 +1924,50 @@ def insert_data_pair(node, number_of_mb, start=0): ) node.query("DROP TABLE IF EXISTS zcrSelect SYNC") - with mergetree_config(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64, - sign Int8 - ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_alter', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) + with Given("I have merge tree configuration set to use zero copy replication"): + settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) + + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64, + sign Int8 + ) ENGINE = ReplicatedMergeTree('/clickhouse/allow_zero_copy_replication_alter', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("I add 20 Mb of data to the table"): - insert_data_pair(nodes[0], 20) + with And("I add 20 Mb of data to the table"): + insert_data_pair(nodes[0], 20) - with Then("I sync the replicas"): - for attempt in retries(timeout=600, delay=5): - with attempt: - nodes[1].query( - "SYSTEM SYNC REPLICA zero_copy_replication", - settings=[("receive_timeout", 600)], - ) + with Then("I sync the replicas"): + for attempt in retries(timeout=600, delay=5): + with attempt: + nodes[1].query( + "SYSTEM SYNC REPLICA zero_copy_replication", + settings=[("receive_timeout", 600)], + ) - with Then("I alter the table and save the time taken"): - start_time = time.time() - nodes[1].query( - "ALTER TABLE zero_copy_replication UPDATE sign = -1 WHERE sign = 1" - ) - end_time = time.time() - allow_zero_copy_time = end_time - start_time - metric( - "with_zero_copy", units="seconds", value=str(allow_zero_copy_time) - ) + with Then("I alter the table and save the time taken"): + start_time = time.time() + nodes[1].query( + "ALTER TABLE zero_copy_replication UPDATE sign = -1 WHERE sign = 1" + ) + end_time = time.time() + allow_zero_copy_time = end_time - start_time + metric("with_zero_copy", units="seconds", value=str(allow_zero_copy_time)) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") - node.query("DROP TABLE IF EXISTS zcrSelect SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + node.query("DROP TABLE IF EXISTS zcrSelect SYNC") with Finally("I print the difference in time taken"): metric( @@ -2061,50 +2047,50 @@ def consistency_during_double_mutation(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) - with mergetree_config(settings): - try: - with Given("I have a table"): - node.query( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER 'sharded_cluster' (key UInt32, value1 String, value2 String, value3 String) engine=ReplicatedMergeTree('/{table_name}', '{{replica}}') - ORDER BY key - PARTITION BY (key % 4) - SETTINGS storage_policy='external' - """, - settings=[("distributed_ddl_task_timeout ", 360)], - ) + try: + with Given("I have a table"): + node.query( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER 'sharded_cluster' (key UInt32, value1 String, value2 String, value3 String) engine=ReplicatedMergeTree('/{table_name}', '{{replica}}') + ORDER BY key + PARTITION BY (key % 4) + SETTINGS storage_policy='external' + """, + settings=[("distributed_ddl_task_timeout ", 360)], + ) - with And("I insert some data"): - node.query( - f"INSERT INTO {table_name} SELECT * FROM generateRandom('key UInt32, value1 String, value2 String, value3 String') LIMIT 1000000" - ) + with And("I insert some data"): + node.query( + f"INSERT INTO {table_name} SELECT * FROM generateRandom('key UInt32, value1 String, value2 String, value3 String') LIMIT 1000000" + ) - with When("I add a new column on the first node"): - nodes[0].query( - f"ALTER TABLE {table_name} ADD COLUMN valueX String materialized value1" - ) + with When("I add a new column on the first node"): + nodes[0].query( + f"ALTER TABLE {table_name} ADD COLUMN valueX String materialized value1" + ) - with And("I delete a column on the second node"): - nodes[1].query(f"ALTER TABLE {table_name} DROP COLUMN value3") + with And("I delete a column on the second node"): + nodes[1].query(f"ALTER TABLE {table_name} DROP COLUMN value3") - with And(f"I materialize the new column on the first node"): - nodes[0].query(f"ALTER TABLE {table_name} MATERIALIZE COLUMN valueX") + with And(f"I materialize the new column on the first node"): + nodes[0].query(f"ALTER TABLE {table_name} MATERIALIZE COLUMN valueX") - with When("I run DESCRIBE TABLE"): - r = node.query(f"DESCRIBE TABLE {table_name}") + with When("I run DESCRIBE TABLE"): + r = node.query(f"DESCRIBE TABLE {table_name}") - with Then("The output should contain my new column"): - assert "valueX" in r.output, error(r) + with Then("The output should contain my new column"): + assert "valueX" in r.output, error(r) - with And("The output should not contain the deleted column"): - assert "value3" not in r.output, error(r) + with And("The output should not contain the deleted column"): + assert "value3" not in r.output, error(r) - finally: - with Finally(f"I drop the table"): - node.query( - f"DROP TABLE IF EXISTS {table_name} ON CLUSTER 'sharded_cluster' " - ) + finally: + with Finally(f"I drop the table"): + node.query( + f"DROP TABLE IF EXISTS {table_name} ON CLUSTER 'sharded_cluster' " + ) @TestScenario @@ -2125,49 +2111,49 @@ def consistency_during_conflicting_mutation(self): with And("I have merge tree configuration set to use zero copy replication"): settings = {self.context.zero_copy_replication_setting: "1"} + mergetree_config(settings=settings) - with mergetree_config(settings): - try: - with Given("I have a table"): - node.query( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER 'sharded_cluster' (key UInt32, value1 String, value2 String, value3 String) engine=ReplicatedMergeTree('/{table_name}', '{{replica}}') - ORDER BY key - PARTITION BY (key % 4) - SETTINGS storage_policy='external' - """, - settings=[("distributed_ddl_task_timeout ", 360)], - ) + try: + with Given("I have a table"): + node.query( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ON CLUSTER 'sharded_cluster' (key UInt32, value1 String, value2 String, value3 String) engine=ReplicatedMergeTree('/{table_name}', '{{replica}}') + ORDER BY key + PARTITION BY (key % 4) + SETTINGS storage_policy='external' + """, + settings=[("distributed_ddl_task_timeout ", 360)], + ) - with And("I insert some data"): - node.query( - f"INSERT INTO {table_name} SELECT * FROM generateRandom('key UInt32, value1 String, value2 String, value3 String') LIMIT 1000000" - ) + with And("I insert some data"): + node.query( + f"INSERT INTO {table_name} SELECT * FROM generateRandom('key UInt32, value1 String, value2 String, value3 String') LIMIT 1000000" + ) - with And("I delete a column on the second node"): - nodes[1].query(f"ALTER TABLE {table_name} DROP COLUMN value3") + with And("I delete a column on the second node"): + nodes[1].query(f"ALTER TABLE {table_name} DROP COLUMN value3") - with When("I add the same column on the first node"): - nodes[0].query( - f"ALTER TABLE {table_name} ADD COLUMN value3 String materialized value1" - ) + with When("I add the same column on the first node"): + nodes[0].query( + f"ALTER TABLE {table_name} ADD COLUMN value3 String materialized value1" + ) - with And(f"I materialize the new column on the first node"): - nodes[0].query(f"ALTER TABLE {table_name} MATERIALIZE COLUMN value3") + with And(f"I materialize the new column on the first node"): + nodes[0].query(f"ALTER TABLE {table_name} MATERIALIZE COLUMN value3") - with When("I run DESCRIBE TABLE"): - r = node.query(f"DESCRIBE TABLE {table_name}") + with When("I run DESCRIBE TABLE"): + r = node.query(f"DESCRIBE TABLE {table_name}") - with Then("The output should contain all columns"): - assert "value1" in r.output, error(r) - assert "value2" in r.output, error(r) - assert "value3" in r.output, error(r) + with Then("The output should contain all columns"): + assert "value1" in r.output, error(r) + assert "value2" in r.output, error(r) + assert "value3" in r.output, error(r) - finally: - with Finally(f"I drop the table"): - node.query( - f"DROP TABLE IF EXISTS {table_name} ON CLUSTER 'sharded_cluster' " - ) + finally: + with Finally(f"I drop the table"): + node.query( + f"DROP TABLE IF EXISTS {table_name} ON CLUSTER 'sharded_cluster' " + ) @TestOutline(Feature) From 171dc797a49dfc902ce1e6545b5d5158b4cf9c1b Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:12:56 -0500 Subject: [PATCH 11/73] S3, reduce use of context managers, WIP --- ontime_benchmark/benchmark.py | 2 +- s3/tests/backup.py | 6 +- s3/tests/common.py | 44 ++-- s3/tests/disk.py | 26 +-- s3/tests/disk_invalid.py | 2 +- s3/tests/iam_auth.py | 4 +- s3/tests/sanity.py | 20 +- s3/tests/zero_copy_replication.py | 361 +++++++++++++++--------------- vfs/tests/steps.py | 8 +- 9 files changed, 240 insertions(+), 233 deletions(-) diff --git a/ontime_benchmark/benchmark.py b/ontime_benchmark/benchmark.py index 67c2911a3..bd93621a8 100755 --- a/ontime_benchmark/benchmark.py +++ b/ontime_benchmark/benchmark.py @@ -166,7 +166,7 @@ def regression( }, } - with s3_storage(disks, policies, timeout=360): + with s3_storage_context(disks, policies, timeout=360): Feature(test=load("ontime_benchmark.tests.benchmark", "feature"))( format=format ) diff --git a/s3/tests/backup.py b/s3/tests/backup.py index 89e1a5dd3..ce65fa14f 100644 --- a/s3/tests/backup.py +++ b/s3/tests/backup.py @@ -968,7 +968,7 @@ def local_and_s3_disk(self): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): for outline in loads(current_module(), Outline): with Given("I run the clean up"): cleanup(storage=self.context.storage) @@ -1011,7 +1011,7 @@ def local_and_s3_volumes(self): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): for outline in loads(current_module(), Outline): with Given("I run the clean up"): cleanup(storage=self.context.storage) @@ -1046,7 +1046,7 @@ def s3_disk(self): with And("I have a storage policy configured to use the S3 disk"): policies = {"external": {"volumes": {"external": {"disk": "external"}}}} - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): for outline in loads(current_module(), Outline): with Given("I run the clean up"): cleanup(storage=self.context.storage) diff --git a/s3/tests/common.py b/s3/tests/common.py index fca441504..d2072a3d0 100644 --- a/s3/tests/common.py +++ b/s3/tests/common.py @@ -183,7 +183,27 @@ def create_s3_endpoint_config_content( @contextmanager +def s3_storage_context( + disks, + policies, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="storage.xml", + timeout=300, + restart=False, + config=None, + nodes=None, +): + """Add S3 storage disk configuration.""" + if config is None: + config = create_s3_storage_config_content( + disks, policies, config_d_dir, config_file + ) + return add_config(config, restart=restart, nodes=nodes, timeout=timeout) + + +@TestStep(Given) def s3_storage( + self, disks, policies, config_d_dir="/etc/clickhouse-server/config.d", @@ -453,22 +473,6 @@ def create_mergetree_config_content( return Config(content, path, name, uid, "config.xml") -@contextmanager -def mergetree_config_context( - settings, - config_d_dir="/etc/clickhouse-server/config.d", - config_file="merge_tree.xml", - timeout=60, - restart=False, - config=None, - nodes=None, -): - """Add MergeTree configuration.""" - if config is None: - config = create_mergetree_config_content(settings, config_d_dir, config_file) - return add_config(config, restart=restart, nodes=nodes) - - @TestStep(Given) def mergetree_config( self, @@ -483,7 +487,7 @@ def mergetree_config( """Add MergeTree configuration.""" if config is None: config = create_mergetree_config_content(settings, config_d_dir, config_file) - yield add_config(config, restart=restart, nodes=nodes) + return add_config(config, restart=restart, nodes=nodes) @contextmanager @@ -1054,7 +1058,7 @@ def default_s3_and_local_disk(self, restart=True): }, } - with s3_storage(disks, policies, restart=restart): + with s3_storage_context(disks, policies, restart=restart): yield @@ -1083,7 +1087,7 @@ def default_s3_and_local_volume(self, restart=True): }, } - with s3_storage(disks, policies, restart=restart): + with s3_storage_context(disks, policies, restart=restart): yield @@ -1154,7 +1158,7 @@ def default_s3_disk_and_volume( else: policies = {policy_name: {"volumes": {"external": {"disk": disk_name}}}} - with s3_storage(disks, policies, restart=restart): + with s3_storage_context(disks, policies, restart=restart): yield diff --git a/s3/tests/disk.py b/s3/tests/disk.py index 9a836a27e..3f8d35af6 100644 --- a/s3/tests/disk.py +++ b/s3/tests/disk.py @@ -595,7 +595,7 @@ def insert_data_time(number_of_mb, time, start=0): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -703,7 +703,7 @@ def insert_data_time(number_of_mb, time, start=0): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -810,7 +810,7 @@ def add_storage(self): "external": {"volumes": {"external1": {"disk": "first_external"}}}, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -861,7 +861,7 @@ def add_storage(self): "external": {"volumes": {"external": {"disk": "second_external"}}}, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -1246,7 +1246,7 @@ def generic_url(self): will not start if config is added""" ) - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy aws_external"): node.query( @@ -1337,7 +1337,7 @@ def environment_credentials(self): } with s3_env_credentials(endpoints=endpoints, restart=True): - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy s3_external"): node.query( @@ -1633,7 +1633,7 @@ def insert_data_time(number_of_mb, start=0): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create a table"): node.query( @@ -1719,7 +1719,7 @@ def perform_ttl_move_on_insert(self, bool_value): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy tiered"): node.query( @@ -1782,7 +1782,7 @@ def insert_data_time(number_of_mb, start=0): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy tiered"): node.query( @@ -1896,7 +1896,7 @@ def alter_move(self, node="clickhouse1"): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): for example in self.examples: name, engine = example with When(f"for example table name='{name}', engine='{engine}'"): @@ -2127,7 +2127,7 @@ def default_move_factor(self, node="clickhouse1"): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): for example in self.examples: name, engine = example with When(f"for example table name='{name}', engine='{engine}'"): @@ -2280,7 +2280,7 @@ def download_appropriate_disk(self, nodes=None): }, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with When("I create replicated table on each node"): for i, node in enumerate(nodes): @@ -2372,7 +2372,7 @@ def insert_data_time(node, number_of_mb, time, start=0): with And(f"cluster nodes {nodes}"): nodes = [cluster.node(name) for name in nodes] - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: with Given( f"I create a replicated table on each node using S3 storage policy tiered" diff --git a/s3/tests/disk_invalid.py b/s3/tests/disk_invalid.py index e8fe3b1ae..51d76b146 100644 --- a/s3/tests/disk_invalid.py +++ b/s3/tests/disk_invalid.py @@ -199,7 +199,7 @@ def access_failed_skip_check(self): "external": {"volumes": {"external": {"disk": "external"}}}, } - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): try: if check_clickhouse_version(">=23.8")(self): with Given( diff --git a/s3/tests/iam_auth.py b/s3/tests/iam_auth.py index 7ef4964b7..98450440f 100644 --- a/s3/tests/iam_auth.py +++ b/s3/tests/iam_auth.py @@ -1,7 +1,7 @@ from testflows.core import * from testflows.asserts import error -from s3.tests.common import getuid, s3_storage +from s3.tests.common import getuid, s3_storage_context from s3.requirements import * @@ -86,7 +86,7 @@ def iam_mode_auth(self): "aws_external": {"volumes": {"external": {"disk": "aws"}}}, } - with s3_storage(disks, policies, restart=True, nodes=[node]): + with s3_storage_context(disks, policies, restart=True, nodes=[node]): try: with Given(f"I create table using S3 storage policy external"): node.query( diff --git a/s3/tests/sanity.py b/s3/tests/sanity.py index 0bb2d2d13..4d1d371ff 100644 --- a/s3/tests/sanity.py +++ b/s3/tests/sanity.py @@ -79,7 +79,7 @@ def aws_s3(self, uri, key_id, access_key, node="clickhouse1"): self.context.node = self.context.cluster.node(node) with Given( - """I have a disk configuration with a S3 storage disk, access id and keyprovided""" + """I have a disk configuration with a S3 storage disk, access id and key provided""" ): disks = { "default": {"keep_free_space_bytes": "1024"}, @@ -99,10 +99,10 @@ def aws_s3(self, uri, key_id, access_key, node="clickhouse1"): "aws_external": {"volumes": {"external": {"disk": "aws"}}}, } - with s3_storage(disks, policies, restart=True): - Scenario( - run=sanity, examples=Examples("policy", [("default",), ("aws_external",)]) - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + Scenario(run=sanity, examples=Examples("policy", [("default",), ("aws_external",)])) @TestFeature @@ -132,7 +132,9 @@ def minio(self, uri, key, secret, node="clickhouse1"): "minio_external": {"volumes": {"external": {"disk": "minio"}}}, } - with s3_storage(disks, policies, restart=True): - Scenario( - run=sanity, examples=Examples("policy", [("default",), ("minio_external",)]) - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + Scenario( + run=sanity, examples=Examples("policy", [("default",), ("minio_external",)]) + ) diff --git a/s3/tests/zero_copy_replication.py b/s3/tests/zero_copy_replication.py index cf037ecea..e4ef6c8d8 100644 --- a/s3/tests/zero_copy_replication.py +++ b/s3/tests/zero_copy_replication.py @@ -607,76 +607,76 @@ def metadata(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) - with mergetree_config_context(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_metadata', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with And("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_metadata', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ + ) - with And("another insert of 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) + with And("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) + with And("another insert of 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) - with Then( - """The number of bytes written to S3 by the first - node should be a very large number""" - ): - event = ( - "WriteBufferFromS3Bytes" - if check_clickhouse_version(">=22.5")(self) - or check_clickhouse_version("=22.3.8.40.altinitystable")(self) - else "S3WriteBytes" - ) - numBytes = int( - nodes[0] - .query(f"SELECT value FROM system.events WHERE event = '{event}'") - .output.strip() + with And("a large insert of 10Mb of data"): + insert_data_node( + node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 ) - assert numBytes >= 0.95 * expected, error() - assert numBytes <= 1.05 * expected, error() - - with And( - """The number of bytes written to S3 by the second - node should be very small, showing that the data was - replicated with no copies""" - ): - event = ( - "WriteBufferFromS3Bytes" - if check_clickhouse_version(">=22.5")(self) - or check_clickhouse_version("=22.3.8.40.altinitystable")(self) - else "S3WriteBytes" - ) - numBytes = int( - nodes[1] - .query(f"SELECT value FROM system.events WHERE event = '{event}'") - .output.strip() - ) + with Then( + """The number of bytes written to S3 by the first + node should be a very large number""" + ): + event = ( + "WriteBufferFromS3Bytes" + if check_clickhouse_version(">=22.5")(self) + or check_clickhouse_version("=22.3.8.40.altinitystable")(self) + else "S3WriteBytes" + ) + numBytes = int( + nodes[0] + .query(f"SELECT value FROM system.events WHERE event = '{event}'") + .output.strip() + ) + + assert numBytes >= 0.95 * expected, error() + assert numBytes <= 1.05 * expected, error() + + with And( + """The number of bytes written to S3 by the second + node should be very small, showing that the data was + replicated with no copies""" + ): + event = ( + "WriteBufferFromS3Bytes" + if check_clickhouse_version(">=22.5")(self) + or check_clickhouse_version("=22.3.8.40.altinitystable")(self) + else "S3WriteBytes" + ) + numBytes = int( + nodes[1] + .query(f"SELECT value FROM system.events WHERE event = '{event}'") + .output.strip() + ) - assert numBytes < 100, error() + assert numBytes < 100, error() - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") @TestScenario @@ -998,6 +998,7 @@ def insert_multiple_replicas(self): with And("I have merge tree configuration set to use zero copy replication"): settings = self.context.zero_copy_replication_settings + mergetree_config(settings=settings) with And("I get the size of the s3 bucket before adding data"): size_before = get_bucket_size( @@ -1008,133 +1009,133 @@ def insert_multiple_replicas(self): minio_enabled=self.context.minio_enabled, ) - with mergetree_config_context(settings): - try: - with When("I create a replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE zero_copy_replication ( - d UInt64 - ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop_alter', '{i + 1}') - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with And("I insert 1MB of data"): - insert_data_node(node=nodes[0], number_of_mb=1) - - with And("I insert of 1MB of data on the other node"): - insert_data_node(node=nodes[1], number_of_mb=1, start=1024 * 1024) - - with And("a large insert of 10Mb of data on the first node"): - insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with Then("I check simple queries on both nodes"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[1], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[1], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", - ) - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM zero_copy_replication", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", - expected="0", - ) - check_query_node( - node=nodes[0], - num=4, - query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", - expected="0\n1048578\n2097154", - ) - check_query_node( - node=nodes[0], - num=5, - query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", - expected="1", + try: + with When("I create a replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE zero_copy_replication ( + d UInt64 + ) ENGINE = ReplicatedMergeTree('/clickhouse/zero_copy_replication_drop_alter', '{i + 1}') + ORDER BY d + SETTINGS storage_policy='external' + """ ) - with And("I check that the data added is within 1% of expected amount"): - current_size = get_bucket_size( - name=bucket_name, - prefix=bucket_path, - access_key=self.context.secret_access_key, - key_id=self.context.access_key_id, - minio_enabled=self.context.minio_enabled, - ) - added_size = current_size - size_before + with And("I insert 1MB of data"): + insert_data_node(node=nodes[0], number_of_mb=1) - assert added_size >= expected * 0.99, error() - assert added_size <= expected * 1.01, error() + with And("I insert of 1MB of data on the other node"): + insert_data_node(node=nodes[1], number_of_mb=1, start=1024 * 1024) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + with And("a large insert of 10Mb of data on the first node"): + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) - with Then( - """The size of the s3 bucket should be very close to the size - before adding any data""" - ): - check_bucket_size( + with Then("I check simple queries on both nodes"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[1], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[1], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM zero_copy_replication", + expected="1572867", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM zero_copy_replication WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM zero_copy_replication ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM zero_copy_replication ORDER BY d ASC LIMIT 1", + expected="0", + ) + check_query_node( + node=nodes[0], + num=4, + query=f"SELECT * FROM zero_copy_replication WHERE d == 0 OR d == 1048578 OR d == 2097154 ORDER BY d", + expected="0\n1048578\n2097154", + ) + check_query_node( + node=nodes[0], + num=5, + query=f"SELECT * FROM (SELECT d FROM zero_copy_replication WHERE d == 1)", + expected="1", + ) + + with And("I check that the data added is within 1% of expected amount"): + current_size = get_bucket_size( name=bucket_name, prefix=bucket_path, - expected_size=size_before, - tolerance=5, + access_key=self.context.secret_access_key, + key_id=self.context.access_key_id, minio_enabled=self.context.minio_enabled, ) + added_size = current_size - size_before + + assert added_size >= expected * 0.99, error() + assert added_size <= expected * 1.01, error() + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS zero_copy_replication SYNC") + + with Then( + """The size of the s3 bucket should be very close to the size + before adding any data""" + ): + check_bucket_size( + name=bucket_name, + prefix=bucket_path, + expected_size=size_before, + tolerance=5, + minio_enabled=self.context.minio_enabled, + ) @TestScenario @@ -2219,7 +2220,7 @@ def outline(self): for name in self.context.cluster.nodes["clickhouse"] ] - with s3_storage(disks, policies, restart=True): + with s3_storage_context(disks, policies, restart=True): with Check("bucket should be empty before test begins"): check_bucket_size( name=self.context.bucket_name, diff --git a/vfs/tests/steps.py b/vfs/tests/steps.py index bb9a7f5bd..55ed38ad7 100644 --- a/vfs/tests/steps.py +++ b/vfs/tests/steps.py @@ -7,7 +7,7 @@ from helpers.common import getuid, check_clickhouse_version -from s3.tests.common import s3_storage, check_bucket_size, get_bucket_size +from s3.tests.common import s3_storage_context, check_bucket_size, get_bucket_size DEFAULT_COLUMNS = "key UInt32, value1 String, value2 String, value3 String" @@ -67,7 +67,7 @@ def s3_config(self): }, } - with s3_storage(disks, policies, restart=True, timeout=60): + with s3_storage_context(disks, policies, restart=True, timeout=60): yield @@ -271,7 +271,7 @@ def storage_config( if policies is None: policies = {} - with s3_storage( + with s3_storage_context( disks, policies, nodes=nodes, @@ -312,7 +312,7 @@ def enable_vfs( policies = {} - with s3_storage( + with s3_storage_context( disks, policies, nodes=nodes, From 49f79b7350606b035524554e9acd6a0da47e5206 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 07:53:14 -0500 Subject: [PATCH 12/73] Remove s3 storage context manager --- ontime_benchmark/benchmark.py | 10 +- s3/tests/backup.py | 36 +- s3/tests/common.py | 39 +- s3/tests/disk.py | 1466 ++++++++++++++--------------- s3/tests/disk_invalid.py | 112 +-- s3/tests/iam_auth.py | 48 +- s3/tests/zero_copy_replication.py | 24 +- vfs/tests/steps.py | 19 +- 8 files changed, 873 insertions(+), 881 deletions(-) diff --git a/ontime_benchmark/benchmark.py b/ontime_benchmark/benchmark.py index bd93621a8..6387d8f61 100755 --- a/ontime_benchmark/benchmark.py +++ b/ontime_benchmark/benchmark.py @@ -166,10 +166,12 @@ def regression( }, } - with s3_storage_context(disks, policies, timeout=360): - Feature(test=load("ontime_benchmark.tests.benchmark", "feature"))( - format=format - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, timeout=360) + + Feature(test=load("ontime_benchmark.tests.benchmark", "feature"))( + format=format + ) if main(): diff --git a/s3/tests/backup.py b/s3/tests/backup.py index ce65fa14f..554e2037a 100644 --- a/s3/tests/backup.py +++ b/s3/tests/backup.py @@ -968,12 +968,14 @@ def local_and_s3_disk(self): }, } - with s3_storage_context(disks, policies, restart=True): - for outline in loads(current_module(), Outline): - with Given("I run the clean up"): - cleanup(storage=self.context.storage) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - Scenario(test=outline)(policy_name="local_and_s3_disk") + for outline in loads(current_module(), Outline): + with Given("I run the clean up"): + cleanup(storage=self.context.storage) + + Scenario(test=outline)(policy_name="local_and_s3_disk") @TestScenario @@ -1011,12 +1013,14 @@ def local_and_s3_volumes(self): }, } - with s3_storage_context(disks, policies, restart=True): - for outline in loads(current_module(), Outline): - with Given("I run the clean up"): - cleanup(storage=self.context.storage) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + for outline in loads(current_module(), Outline): + with Given("I run the clean up"): + cleanup(storage=self.context.storage) - Scenario(test=outline)(policy_name="default_and_external") + Scenario(test=outline)(policy_name="default_and_external") @TestScenario @@ -1046,12 +1050,14 @@ def s3_disk(self): with And("I have a storage policy configured to use the S3 disk"): policies = {"external": {"volumes": {"external": {"disk": "external"}}}} - with s3_storage_context(disks, policies, restart=True): - for outline in loads(current_module(), Outline): - with Given("I run the clean up"): - cleanup(storage=self.context.storage) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + for outline in loads(current_module(), Outline): + with Given("I run the clean up"): + cleanup(storage=self.context.storage) - Scenario(test=outline)(policy_name="external") + Scenario(test=outline)(policy_name="external") @TestFeature diff --git a/s3/tests/common.py b/s3/tests/common.py index d2072a3d0..dcb591ee2 100644 --- a/s3/tests/common.py +++ b/s3/tests/common.py @@ -182,30 +182,11 @@ def create_s3_endpoint_config_content( ) -@contextmanager -def s3_storage_context( - disks, - policies, - config_d_dir="/etc/clickhouse-server/config.d", - config_file="storage.xml", - timeout=300, - restart=False, - config=None, - nodes=None, -): - """Add S3 storage disk configuration.""" - if config is None: - config = create_s3_storage_config_content( - disks, policies, config_d_dir, config_file - ) - return add_config(config, restart=restart, nodes=nodes, timeout=timeout) - - @TestStep(Given) def s3_storage( self, - disks, - policies, + disks=None, + policies=None, config_d_dir="/etc/clickhouse-server/config.d", config_file="storage.xml", timeout=300, @@ -213,7 +194,12 @@ def s3_storage( config=None, nodes=None, ): - """Add S3 storage disk configuration.""" + """Add S3 storage disk and policy configurations.""" + if disks is None: + disks = {} + if policies is None: + policies = {} + if config is None: config = create_s3_storage_config_content( disks, policies, config_d_dir, config_file @@ -1058,8 +1044,7 @@ def default_s3_and_local_disk(self, restart=True): }, } - with s3_storage_context(disks, policies, restart=restart): - yield + return s3_storage(disks=disks, policies=policies, restart=restart) @TestStep(Given) @@ -1087,8 +1072,7 @@ def default_s3_and_local_volume(self, restart=True): }, } - with s3_storage_context(disks, policies, restart=restart): - yield + return s3_storage(disks=disks, policies=policies, restart=restart) @TestStep(Given) @@ -1158,8 +1142,7 @@ def default_s3_disk_and_volume( else: policies = {policy_name: {"volumes": {"external": {"disk": disk_name}}}} - with s3_storage_context(disks, policies, restart=restart): - yield + return s3_storage(disks=disks, policies=policies, restart=restart) @TestStep(Given) diff --git a/s3/tests/disk.py b/s3/tests/disk.py index 3f8d35af6..fc7f425ab 100644 --- a/s3/tests/disk.py +++ b/s3/tests/disk.py @@ -595,54 +595,56 @@ def insert_data_time(number_of_mb, time, start=0): }, } - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy external"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = MergeTree() - ORDER BY d - TTL d1 + interval 2 day to volume 'external' - SETTINGS storage_policy='external' - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with When("I add data to the table"): - with By("first inserting 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(1, tm, 0) + try: + with Given(f"I create table using S3 storage policy external"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = MergeTree() + ORDER BY d + TTL d1 + interval 2 day to volume 'external' + SETTINGS storage_policy='external' + """ + ) - with And("another insert of 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(1, tm, 1024 * 1024) + with When("I add data to the table"): + with By("first inserting 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() + ) + insert_data_time(1, tm, 0) - with And("then doing a large insert of 10Mb of data"): - tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(10, tm, 1024 * 1024 * 2) + with And("another insert of 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() + ) + insert_data_time(1, tm, 1024 * 1024) - with Then( - """I get the name of all partitions for all data parts - in this table""" - ): - disk_names = node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - ).output.splitlines() + with And("then doing a large insert of 10Mb of data"): + tm = time.mktime(datetime.date.today().timetuple()) + insert_data_time(10, tm, 1024 * 1024 * 2) - with And("""I check the names to make sure both disks are used"""): - assert ( - "first_external" in disk_names and "second_external" in disk_names - ), error() + with Then( + """I get the name of all partitions for all data parts + in this table""" + ): + disk_names = node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + ).output.splitlines() - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with And("""I check the names to make sure both disks are used"""): + assert ( + "first_external" in disk_names and "second_external" in disk_names + ), error() + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -702,75 +704,74 @@ def insert_data_time(number_of_mb, time, start=0): } }, } + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy external"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = MergeTree() - ORDER BY d - TTL d1 + interval 2 day to volume 'external' - SETTINGS storage_policy='external' - """ + try: + with Given(f"I create table using S3 storage policy external"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = MergeTree() + ORDER BY d + TTL d1 + interval 2 day to volume 'external' + SETTINGS storage_policy='external' + """ + ) + + with When("I add data to the table"): + with By("first inserting 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() ) + insert_data_time(1, tm, 0) - with When("I add data to the table"): - with By("first inserting 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(1, tm, 0) + with And("another insert of 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() + ) + insert_data_time(1, tm, 1024 * 1024) - with And("another insert of 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(1, tm, 1024 * 1024) + with And("then doing a large insert of 10Mb of data"): + tm = time.mktime(datetime.date.today().timetuple()) + insert_data_time(10, tm, 1024 * 1024 * 2) - with And("then doing a large insert of 10Mb of data"): - tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(10, tm, 1024 * 1024 * 2) + with Then( + """I get the name of all partitions for all data parts + in this table""" + ): + disk_names = node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + ).output.splitlines() - with Then( - """I get the name of all partitions for all data parts - in this table""" - ): - disk_names = node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - ).output.splitlines() - - with And("""I check the names to make sure both disks are used"""): - assert ( - "first_external" in disk_names and "second_external" in disk_names - ), error() - - with Then("I check simple queries"): - check_query( - num=0, query=f"SELECT COUNT() FROM {name}", expected="1572867" - ) - check_query( - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query( - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query( - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", - ) + with And("""I check the names to make sure both disks are used"""): + assert ( + "first_external" in disk_names and "second_external" in disk_names + ), error() - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with Then("I check simple queries"): + check_query(num=0, query=f"SELECT COUNT() FROM {name}", expected="1572867") + check_query( + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query( + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query( + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -790,27 +791,30 @@ def add_storage(self): with Given("I edit the Minio URI to avoid path conflicts"): uri = uri[: len(uri) - 5] + "/add-storage/" - with Given("I have a disk configuration with one S3 storage disk"): - disks = { - "default": {"keep_free_space_bytes": "1024"}, - "first_external": { - "type": "s3", - "endpoint": f"{uri}subdata1/", - "access_key_id": f"{access_key_id}", - "secret_access_key": f"{secret_access_key}", - }, - } + with Check("one disk"): + with Given("I have a disk configuration with one S3 storage disk"): + disks = { + "default": {"keep_free_space_bytes": "1024"}, + "first_external": { + "type": "s3", + "endpoint": f"{uri}subdata1/", + "access_key_id": f"{access_key_id}", + "secret_access_key": f"{secret_access_key}", + }, + } - if hasattr(self.context, "s3_options"): - disks["first_external"].update(self.context.s3_options) + if hasattr(self.context, "s3_options"): + disks["first_external"].update(self.context.s3_options) - with And("I have a storage policy configured to use the S3 disk"): - policies = { - "default": {"volumes": {"default": {"disk": "default"}}}, - "external": {"volumes": {"external1": {"disk": "first_external"}}}, - } + with And("I have a storage policy configured to use the S3 disk"): + policies = { + "default": {"volumes": {"default": {"disk": "default"}}}, + "external": {"volumes": {"external1": {"disk": "first_external"}}}, + } + + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -834,34 +838,37 @@ def add_storage(self): with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {name} SYNC") - with Given("I add a disk to the disk configuration"): - disks = { - "default": {"keep_free_space_bytes": "1024"}, - "first_external": { - "type": "s3", - "endpoint": f"{uri}subdata1/", - "access_key_id": f"{access_key_id}", - "secret_access_key": f"{secret_access_key}", - }, - "second_external": { - "type": "s3", - "endpoint": f"{uri}subdata2/", - "access_key_id": f"{access_key_id}", - "secret_access_key": f"{secret_access_key}", - }, - } + with Check("two disks"): + with Given("I add a disk to the disk configuration"): + disks = { + "default": {"keep_free_space_bytes": "1024"}, + "first_external": { + "type": "s3", + "endpoint": f"{uri}subdata1/", + "access_key_id": f"{access_key_id}", + "secret_access_key": f"{secret_access_key}", + }, + "second_external": { + "type": "s3", + "endpoint": f"{uri}subdata2/", + "access_key_id": f"{access_key_id}", + "secret_access_key": f"{secret_access_key}", + }, + } - if hasattr(self.context, "s3_options"): - disks["first_external"].update(self.context.s3_options) - disks["second_external"].update(self.context.s3_options) + if hasattr(self.context, "s3_options"): + disks["first_external"].update(self.context.s3_options) + disks["second_external"].update(self.context.s3_options) - with And("I have a storage policy configured to use the new S3 disk"): - policies = { - "default": {"volumes": {"default": {"disk": "default"}}}, - "external": {"volumes": {"external": {"disk": "second_external"}}}, - } + with And("I have a storage policy configured to use the new S3 disk"): + policies = { + "default": {"volumes": {"default": {"disk": "default"}}}, + "external": {"volumes": {"external": {"disk": "second_external"}}}, + } + + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with s3_storage_context(disks, policies, restart=True): try: with Given(f"I create table using S3 storage policy external"): node.query( @@ -1246,29 +1253,31 @@ def generic_url(self): will not start if config is added""" ) - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy aws_external"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64 - ) ENGINE = MergeTree() - ORDER BY d - SETTINGS storage_policy='aws_external' - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with When("I store simple data in the table"): - node.query(f"INSERT INTO {name} VALUES (427)") + try: + with Given(f"I create table using S3 storage policy aws_external"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = MergeTree() + ORDER BY d + SETTINGS storage_policy='aws_external' + """ + ) - with Then("I check that a simple SELECT * query returns matching data"): - r = node.query(f"SELECT * FROM {name}").output.strip() - assert r == expected, error() + with When("I store simple data in the table"): + node.query(f"INSERT INTO {name} VALUES (427)") - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with Then("I check that a simple SELECT * query returns matching data"): + r = node.query(f"SELECT * FROM {name}").output.strip() + assert r == expected, error() + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -1337,29 +1346,32 @@ def environment_credentials(self): } with s3_env_credentials(endpoints=endpoints, restart=True): - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy s3_external"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64 - ) ENGINE = MergeTree() - ORDER BY d - SETTINGS storage_policy='s3_external' - """ - ) - with When("I store simple data in the table"): - node.query(f"INSERT INTO {name} VALUES ({expected})") + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + try: + with Given(f"I create table using S3 storage policy s3_external"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = MergeTree() + ORDER BY d + SETTINGS storage_policy='s3_external' + """ + ) + + with When("I store simple data in the table"): + node.query(f"INSERT INTO {name} VALUES ({expected})") - with Then("I check that a simple SELECT * query returns matching data"): - r = node.query(f"SELECT * FROM {name}").output.strip() - assert r == expected, error() + with Then("I check that a simple SELECT * query returns matching data"): + r = node.query(f"SELECT * FROM {name}").output.strip() + assert r == expected, error() - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestOutline(Scenario) @@ -1633,9 +1645,32 @@ def insert_data_time(number_of_mb, start=0): }, } - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create a table"): + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + try: + with Given(f"I create a table"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = MergeTree() + ORDER BY d + TTL d1 + interval 0 second to volume 'external' + SETTINGS storage_policy='tiered' + """ + ) + + with Then("I add 40 Mb of data to the table and save the time taken"): + ttl_time = insert_data_time(40, 1024 * 1024 * 2) + metric("export_40Mb_ttl", units="seconds", value=str(ttl_time)) + + with And("I create the table again with the other policy"): + with By("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") + + with And("I create the table again"): node.query( f""" CREATE TABLE {name} ( @@ -1644,44 +1679,23 @@ def insert_data_time(number_of_mb, start=0): ) ENGINE = MergeTree() ORDER BY d TTL d1 + interval 0 second to volume 'external' - SETTINGS storage_policy='tiered' + SETTINGS storage_policy='tiered_bg' """ ) - with Then("I add 40 Mb of data to the table and save the time taken"): - ttl_time = insert_data_time(40, 1024 * 1024 * 2) - metric("export_40Mb_ttl", units="seconds", value=str(ttl_time)) - - with And("I create the table again with the other policy"): - with By("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") - - with And("I create the table again"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = MergeTree() - ORDER BY d - TTL d1 + interval 0 second to volume 'external' - SETTINGS storage_policy='tiered_bg' - """ - ) - - with Then("I add 40 Mb of data to the table and save the time taken"): - no_ttl_time = insert_data_time(40, 1024 * 1024 * 2) - metric("export_40Mb_no_ttl", units="seconds", value=str(no_ttl_time)) + with Then("I add 40 Mb of data to the table and save the time taken"): + no_ttl_time = insert_data_time(40, 1024 * 1024 * 2) + metric("export_40Mb_no_ttl", units="seconds", value=str(no_ttl_time)) - with And("I log the performance improvement as a percentage"): - metric( - "percentage_improvement", - units="%", - value=str(100 * (ttl_time - no_ttl_time) / no_ttl_time), - ) - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with And("I log the performance improvement as a percentage"): + metric( + "percentage_improvement", + units="%", + value=str(100 * (ttl_time - no_ttl_time) / no_ttl_time), + ) + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestOutline(Scenario) @@ -1719,32 +1733,34 @@ def perform_ttl_move_on_insert(self, bool_value): }, } - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy tiered"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = MergeTree() - ORDER BY d - TTL d1 + interval 0 second to volume 'external' - SETTINGS storage_policy='tiered' - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with When("I store simple data in the table"): - now = time.mktime(datetime.datetime.today().timetuple()) - node.query(f"INSERT INTO {name} VALUES ({expected}, {now})") - - with Then("I check that a simple SELECT * query returns matching data"): - r = node.query(f"SELECT d FROM {name}").output.strip() - assert r == expected, error() + try: + with Given(f"I create table using S3 storage policy tiered"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = MergeTree() + ORDER BY d + TTL d1 + interval 0 second to volume 'external' + SETTINGS storage_policy='tiered' + """ + ) - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with When("I store simple data in the table"): + now = time.mktime(datetime.datetime.today().timetuple()) + node.query(f"INSERT INTO {name} VALUES ({expected}, {now})") + + with Then("I check that a simple SELECT * query returns matching data"): + r = node.query(f"SELECT d FROM {name}").output.strip() + assert r == expected, error() + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -1782,49 +1798,51 @@ def insert_data_time(number_of_mb, start=0): }, } - with s3_storage_context(disks, policies, restart=True): - try: - with Given(f"I create table using S3 storage policy tiered"): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = MergeTree() - ORDER BY d - TTL d1 + interval 0 second to volume 'external' - SETTINGS storage_policy='tiered' - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with When("I add data to the table"): - with By("first inserting 1MB of data"): - insert_data_time(1, 0) + try: + with Given(f"I create table using S3 storage policy tiered"): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = MergeTree() + ORDER BY d + TTL d1 + interval 0 second to volume 'external' + SETTINGS storage_policy='tiered' + """ + ) - with And("another insert of 1MB of data"): - insert_data_time(1, 1024 * 1024) + with When("I add data to the table"): + with By("first inserting 1MB of data"): + insert_data_time(1, 0) - with And("then doing a large insert of 10Mb of data"): - insert_data_time(10, 1024 * 1024 * 2) + with And("another insert of 1MB of data"): + insert_data_time(1, 1024 * 1024) - with Then( - """I get the name of all partitions for all data parts - in this table""" - ): - disk_names = node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - ).output.splitlines() + with And("then doing a large insert of 10Mb of data"): + insert_data_time(10, 1024 * 1024 * 2) - with And( - """The disk name should match the S3 disk, indicating that - the data parts were moved to S3 immediately as expected""" - ): - for _name in disk_names: - assert _name == f"{disk_name}", error() + with Then( + """I get the name of all partitions for all data parts + in this table""" + ): + disk_names = node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + ).output.splitlines() - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with And( + """The disk name should match the S3 disk, indicating that + the data parts were moved to S3 immediately as expected""" + ): + for _name in disk_names: + assert _name == f"{disk_name}", error() + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -1896,162 +1914,152 @@ def alter_move(self, node="clickhouse1"): }, } - with s3_storage_context(disks, policies, restart=True): - for example in self.examples: - name, engine = example - with When(f"for example table name='{name}', engine='{engine}'"): - with When("I create table"): - node.query( - f""" - CREATE TABLE {name} ( - EventDate Date, - number UInt64 - ) ENGINE = {engine} - ORDER BY tuple() - PARTITION BY toYYYYMM(EventDate) - SETTINGS storage_policy='jbods_with_external' - """ - ) - try: - with And("I stop merges to avoid conflicts"): - node.query(f"SYSTEM STOP MERGES {name}") + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with And("I insert 4 values"): - node.query( - f"INSERT INTO {name} VALUES(toDate('2019-03-15'), 65)" - ) - node.query( - f"INSERT INTO {name} VALUES(toDate('2019-03-16'), 66)" - ) - node.query( - f"INSERT INTO {name} VALUES(toDate('2019-04-10'), 42)" - ) - node.query( - f"INSERT INTO {name} VALUES(toDate('2019-04-11'), 43)" - ) + for example in self.examples: + name, engine = example + with When(f"for example table name='{name}', engine='{engine}'"): + with When("I create table"): + node.query( + f""" + CREATE TABLE {name} ( + EventDate Date, + number UInt64 + ) ENGINE = {engine} + ORDER BY tuple() + PARTITION BY toYYYYMM(EventDate) + SETTINGS storage_policy='jbods_with_external' + """ + ) + try: + with And("I stop merges to avoid conflicts"): + node.query(f"SYSTEM STOP MERGES {name}") - used_disks = get_used_disks_for_table(node, name) + with And("I insert 4 values"): + node.query(f"INSERT INTO {name} VALUES(toDate('2019-03-15'), 65)") + node.query(f"INSERT INTO {name} VALUES(toDate('2019-03-16'), 66)") + node.query(f"INSERT INTO {name} VALUES(toDate('2019-04-10'), 42)") + node.query(f"INSERT INTO {name} VALUES(toDate('2019-04-11'), 43)") - with Then("all writes should go to jbods"): - assert all(d.startswith("jbod") for d in used_disks), error() + used_disks = get_used_disks_for_table(node, name) - with When("I get the first part from system.parts"): - first_part = node.query( - f"SELECT name FROM system.parts WHERE table = '{name}'" - " AND active = 1 ORDER BY modification_time LIMIT 1" - ).output.strip() + with Then("all writes should go to jbods"): + assert all(d.startswith("jbod") for d in used_disks), error() - with And("I try to move first part to 'external' volume"): - time.sleep(1) - node.query( - f"ALTER TABLE {name} MOVE PART '{first_part}' TO VOLUME 'external'" - ) - with And( - "I get disk name from system.parts for the first part" - ): - disk = node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}' " - f" AND name = '{first_part}' and active = 1" - ).output.strip() - - with Then("the disk name should be 'external'"): - assert disk == "external", error() - with And( - "path should start with '/var/lib/clickhouse/disks/external'" - ): - expected = prefix + "/external" - assert get_path_for_part_from_part_log( - node, name, first_part - ).startswith(expected), error() - - with When("I move the first part to 'jbod1' disk"): - time.sleep(1) - node.query( - f"ALTER TABLE {name} MOVE PART '{first_part}' TO DISK 'jbod1'" - ) + with When("I get the first part from system.parts"): + first_part = node.query( + f"SELECT name FROM system.parts WHERE table = '{name}'" + " AND active = 1 ORDER BY modification_time LIMIT 1" + ).output.strip() - with And( - "I get disk name from system.parts for the first part" - ): - disk = node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - f" AND name = '{first_part}' and active = 1" - ).output.strip() - - with Then("the disk name shoul dbe 'jbod1'"): - assert disk == "jbod1", error() - with And("path should start with '/jbod1'"): - expected = "/jbod1" - assert get_path_for_part_from_part_log( - node, name, first_part - ).startswith(expected), error() - - with When("I move partition 201904 to 'external' volume"): + with And("I try to move first part to 'external' volume"): time.sleep(1) node.query( - f"ALTER TABLE {name} MOVE PARTITION 201904 TO VOLUME 'external'" + f"ALTER TABLE {name} MOVE PART '{first_part}' TO VOLUME 'external'" ) + with And("I get disk name from system.parts for the first part"): + disk = node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}' " + f" AND name = '{first_part}' and active = 1" + ).output.strip() - with And("I get disks for this partition"): - disks = ( - node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - " AND partition = '201904' and active = 1" - ) - .output.strip() - .split("\n") - ) + with Then("the disk name should be 'external'"): + assert disk == "external", error() + with And( + "path should start with '/var/lib/clickhouse/disks/external'" + ): + expected = prefix + "/external" + assert get_path_for_part_from_part_log( + node, name, first_part + ).startswith(expected), error() - with Then("number of disks should be 2"): - assert len(disks) == 2, error() - with And("all disks should be 'external'"): - assert all(d == "external" for d in disks), error() - with And( - "all paths should start with '/var/lib/clickhouse/disks/external'" - ): - expected = prefix + "/external" - assert all( - path.startswith(expected) - for path in get_paths_for_partition_from_part_log( - node, name, "201904" - )[:2] - ), error() - - with When("I move partition 201904 to disk 'jbod2'"): - time.sleep(1) - node.query( - f"ALTER TABLE {name} MOVE PARTITION 201904 TO DISK 'jbod2'" - ) + with When("I move the first part to 'jbod1' disk"): + time.sleep(1) + node.query( + f"ALTER TABLE {name} MOVE PART '{first_part}' TO DISK 'jbod1'" + ) - with And("I get disks for this partition"): - disks = ( - node.query( - f"SELECT disk_name FROM system.parts WHERE table = '{name}'" - " AND partition = '201904' and active = 1" - ) - .output.strip() - .split("\n") + with And("I get disk name from system.parts for the first part"): + disk = node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + f" AND name = '{first_part}' and active = 1" + ).output.strip() + + with Then("the disk name shoul dbe 'jbod1'"): + assert disk == "jbod1", error() + with And("path should start with '/jbod1'"): + expected = "/jbod1" + assert get_path_for_part_from_part_log( + node, name, first_part + ).startswith(expected), error() + + with When("I move partition 201904 to 'external' volume"): + time.sleep(1) + node.query( + f"ALTER TABLE {name} MOVE PARTITION 201904 TO VOLUME 'external'" + ) + + with And("I get disks for this partition"): + disks = ( + node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + " AND partition = '201904' and active = 1" ) + .output.strip() + .split("\n") + ) - with Then("number of disks should be 2"): - assert len(disks) == 2, error() - with And("all disks should be 'jbod2'"): - assert all(d == "jbod2" for d in disks), error() - with And("all paths should start with '/jbod2'"): - expected = "/jbod2" + with Then("number of disks should be 2"): + assert len(disks) == 2, error() + with And("all disks should be 'external'"): + assert all(d == "external" for d in disks), error() + with And( + "all paths should start with '/var/lib/clickhouse/disks/external'" + ): + expected = prefix + "/external" + assert all( + path.startswith(expected) for path in get_paths_for_partition_from_part_log( node, name, "201904" - )[:2]: - assert path.startswith(expected), error() + )[:2] + ), error() - with When("in the end I get number of rows in the table"): - count = node.query(f"SELECT COUNT() FROM {name}").output.strip() - with Then("the count should be 4"): - assert count == "4", error() + with When("I move partition 201904 to disk 'jbod2'"): + time.sleep(1) + node.query( + f"ALTER TABLE {name} MOVE PARTITION 201904 TO DISK 'jbod2'" + ) - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with And("I get disks for this partition"): + disks = ( + node.query( + f"SELECT disk_name FROM system.parts WHERE table = '{name}'" + " AND partition = '201904' and active = 1" + ) + .output.strip() + .split("\n") + ) + + with Then("number of disks should be 2"): + assert len(disks) == 2, error() + with And("all disks should be 'jbod2'"): + assert all(d == "jbod2" for d in disks), error() + with And("all paths should start with '/jbod2'"): + expected = "/jbod2" + for path in get_paths_for_partition_from_part_log( + node, name, "201904" + )[:2]: + assert path.startswith(expected), error() + + with When("in the end I get number of rows in the table"): + count = node.query(f"SELECT COUNT() FROM {name}").output.strip() + with Then("the count should be 4"): + assert count == "4", error() + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -2127,90 +2135,88 @@ def default_move_factor(self, node="clickhouse1"): }, } - with s3_storage_context(disks, policies, restart=True): - for example in self.examples: - name, engine = example - with When(f"for example table name='{name}', engine='{engine}'"): - with When("I create table"): - node.query( - f""" - CREATE TABLE {name} ( - s1 String - ) ENGINE = {engine} - ORDER BY tuple() - SETTINGS storage_policy='small_jbod_with_external' - """ - ) - try: - with And("I stop merges to avoid conflicts"): - node.query(f"SYSTEM STOP MERGES {name}") + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with And( - "I fill up first disk above 90%%", - description="small jbod size is 40MB", - ): - with By("first inserting 2MB of data with 2 rows 1MB each"): + for example in self.examples: + name, engine = example + with When(f"for example table name='{name}', engine='{engine}'"): + with When("I create table"): + node.query( + f""" + CREATE TABLE {name} ( + s1 String + ) ENGINE = {engine} + ORDER BY tuple() + SETTINGS storage_policy='small_jbod_with_external' + """ + ) + try: + with And("I stop merges to avoid conflicts"): + node.query(f"SYSTEM STOP MERGES {name}") + + with And( + "I fill up first disk above 90%%", + description="small jbod size is 40MB", + ): + with By("first inserting 2MB of data with 2 rows 1MB each"): + data = [] + for i in range(2): + data.append( + get_random_string(cluster, 1024 * 1024, steps=False) + ) + values = ",".join(["('" + x + "')" for x in data]) + node.query(f"INSERT INTO {name} VALUES {values}") + + with And("then inserting 7 times 5MB of data with 5 rows 1MB each"): + for i in range(7): data = [] - for i in range(2): + for i in range(5): data.append( get_random_string(cluster, 1024 * 1024, steps=False) ) values = ",".join(["('" + x + "')" for x in data]) node.query(f"INSERT INTO {name} VALUES {values}") - with And( - "then inserting 7 times 5MB of data with 5 rows 1MB each" - ): - for i in range(7): - data = [] - for i in range(5): - data.append( - get_random_string( - cluster, 1024 * 1024, steps=False - ) - ) - values = ",".join(["('" + x + "')" for x in data]) - node.query(f"INSERT INTO {name} VALUES {values}") - - with And("poll maximum 20 times to check used disks for the table"): - used_disks = get_used_disks_for_table(node, name) - retry = 20 - i = 0 - while ( - not sum(1 for x in used_disks if x == "jbod1") <= 7 - and i < retry - ): - with And("sleep 0.5 sec"): - time.sleep(0.5) - used_disks = get_used_disks_for_table(node, name) - i += 1 - - with Then( - "check that jbod1 disk is used less than or equal to 7 times" - ): - assert sum(1 for x in used_disks if x == "jbod1") <= 7, error() - - with And( - "I check that at least one part has been moved to 'external' disk" + with And("poll maximum 20 times to check used disks for the table"): + used_disks = get_used_disks_for_table(node, name) + retry = 20 + i = 0 + while ( + not sum(1 for x in used_disks if x == "jbod1") <= 7 + and i < retry ): - assert "external" in used_disks, error() - - for attempt in retries(count=10, delay=1): - with attempt: - with When("I check if parts were deleted from jbod1"): - entries = ( - node.command( - f"find /jbod1/store/*/*/ -name 'all_*'", - exitcode=0, - ) - .output.strip() - .splitlines() + with And("sleep 0.5 sec"): + time.sleep(0.5) + used_disks = get_used_disks_for_table(node, name) + i += 1 + + with Then( + "check that jbod1 disk is used less than or equal to 7 times" + ): + assert sum(1 for x in used_disks if x == "jbod1") <= 7, error() + + with And( + "I check that at least one part has been moved to 'external' disk" + ): + assert "external" in used_disks, error() + + for attempt in retries(count=10, delay=1): + with attempt: + with When("I check if parts were deleted from jbod1"): + entries = ( + node.command( + f"find /jbod1/store/*/*/ -name 'all_*'", + exitcode=0, ) - assert len(entries) <= 7, error() + .output.strip() + .splitlines() + ) + assert len(entries) <= 7, error() - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario @@ -2280,59 +2286,51 @@ def download_appropriate_disk(self, nodes=None): }, } - with s3_storage_context(disks, policies, restart=True): - try: - with When("I create replicated table on each node"): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE replicated_table_for_download ( - s1 String - ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_table_for_download', '{i + 1}') - ORDER BY tuple() - SETTINGS storage_policy='moving_jbod_with_external', old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=2 - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with When("I insert 50MB of data using 50 rows 1MB each on the first node"): - data = [] - for i in range(50): - data.append(get_random_string(cluster, 1024 * 1024)) - values = ",".join(["('" + x + "')" for x in data]) - nodes[0].query( - f"INSERT INTO replicated_table_for_download VALUES {values}" + try: + with When("I create replicated table on each node"): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE replicated_table_for_download ( + s1 String + ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_table_for_download', '{i + 1}') + ORDER BY tuple() + SETTINGS storage_policy='moving_jbod_with_external', old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=2 + """ ) - with And("I sync the other replica"): - for _ in range(10): - try: - nodes[-1].query( - "SYSTEM SYNC REPLICA replicated_table_for_download" - ) - break - except: - time.sleep(0.5) + with When("I insert 50MB of data using 50 rows 1MB each on the first node"): + data = [] + for i in range(50): + data.append(get_random_string(cluster, 1024 * 1024)) + values = ",".join(["('" + x + "')" for x in data]) + nodes[0].query(f"INSERT INTO replicated_table_for_download VALUES {values}") - with When("I check the used disk on other replica"): - disks = get_used_disks_for_table( - nodes[-1], "replicated_table_for_download" - ) + with And("I sync the other replica"): + for _ in range(10): + try: + nodes[-1].query("SYSTEM SYNC REPLICA replicated_table_for_download") + break + except: + time.sleep(0.5) - expected_disks = { - "external", - } - with Then( - f"the used disk should match {expected_disks}", format_name=False - ): - assert set(disks) == expected_disks, error() + with When("I check the used disk on other replica"): + disks = get_used_disks_for_table(nodes[-1], "replicated_table_for_download") - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query( - "DROP TABLE IF EXISTS replicated_table_for_download SYNC" - ) + expected_disks = { + "external", + } + with Then(f"the used disk should match {expected_disks}", format_name=False): + assert set(disks) == expected_disks, error() + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query("DROP TABLE IF EXISTS replicated_table_for_download SYNC") @TestScenario @@ -2372,207 +2370,209 @@ def insert_data_time(node, number_of_mb, time, start=0): with And(f"cluster nodes {nodes}"): nodes = [cluster.node(name) for name in nodes] - with s3_storage_context(disks, policies, restart=True): - try: - with Given( - f"I create a replicated table on each node using S3 storage policy tiered" - ): - for i, node in enumerate(nodes): - node.restart() - node.query( - f""" - CREATE TABLE {name} ( - d UInt64, - d1 DateTime - ) ENGINE = ReplicatedMergeTree('/clickhouse/alter_on_cluster_modify_ttl_{self.context.storage}', '{i + 1}') - ORDER BY d - TTL d1 + interval 2 day to volume 'external' - SETTINGS storage_policy='tiered' - """ - ) + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) - with And("I add data to the table"): - with By("first inserting 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=7)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 0) - - with And("another insert of 1MB of data"): - tm = time.mktime( - (datetime.date.today() - datetime.timedelta(days=4)).timetuple() - ) - insert_data_time(nodes[0], 1, tm, 1024 * 1024) - - with And("a large insert of 10Mb of data"): - tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) - - with Then("I check simple queries on the first node"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", + try: + with Given( + f"I create a replicated table on each node using S3 storage policy tiered" + ): + for i, node in enumerate(nodes): + node.restart() + node.query( + f""" + CREATE TABLE {name} ( + d UInt64, + d1 DateTime + ) ENGINE = ReplicatedMergeTree('/clickhouse/alter_on_cluster_modify_ttl_{self.context.storage}', '{i + 1}') + ORDER BY d + TTL d1 + interval 2 day to volume 'external' + SETTINGS storage_policy='tiered' + """ ) - with Then("I check simple queries on the second node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", + with And("I add data to the table"): + with By("first inserting 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=7)).timetuple() ) + insert_data_time(nodes[0], 1, tm, 0) - with Then("I check simple queries on the third node"): - check_query_node( - node=nodes[2], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[2], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[2], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[2], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", + with And("another insert of 1MB of data"): + tm = time.mktime( + (datetime.date.today() - datetime.timedelta(days=4)).timetuple() ) + insert_data_time(nodes[0], 1, tm, 1024 * 1024) + + with And("a large insert of 10Mb of data"): + tm = time.mktime(datetime.date.today().timetuple()) + insert_data_time(nodes[0], 10, tm, 1024 * 1024 * 2) + + with Then("I check simple queries on the first node"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) - with When("I alter TTL"): - node.query( - f"""ALTER TABLE {name} ON CLUSTER cluster1 MODIFY TTL d1 + interval 5 day to volume 'external'""" - ) + with Then("I check simple queries on the second node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) - with Then("I check simple queries on the first node"): - check_query_node( - node=nodes[0], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[0], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[0], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[0], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", - ) + with Then("I check simple queries on the third node"): + check_query_node( + node=nodes[2], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[2], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[2], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[2], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) - with Then("I check simple queries on the second node"): - check_query_node( - node=nodes[1], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[1], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[1], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[1], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", - ) + with When("I alter TTL"): + node.query( + f"""ALTER TABLE {name} ON CLUSTER cluster1 MODIFY TTL d1 + interval 5 day to volume 'external'""" + ) - with Then("I check simple queries on the third node"): - check_query_node( - node=nodes[2], - num=0, - query=f"SELECT COUNT() FROM {name}", - expected="1572867", - ) - check_query_node( - node=nodes[2], - num=1, - query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", - expected="10", - ) - check_query_node( - node=nodes[2], - num=2, - query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", - expected="3407872", - ) - check_query_node( - node=nodes[2], - num=3, - query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", - expected="0", - ) + with Then("I check simple queries on the first node"): + check_query_node( + node=nodes[0], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[0], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[0], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[0], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) - finally: - with Finally("I drop the table on each node"): - for node in nodes: - node.query(f"DROP TABLE IF EXISTS {name} SYNC") + with Then("I check simple queries on the second node"): + check_query_node( + node=nodes[1], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[1], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[1], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[1], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) + + with Then("I check simple queries on the third node"): + check_query_node( + node=nodes[2], + num=0, + query=f"SELECT COUNT() FROM {name}", + expected="1572867", + ) + check_query_node( + node=nodes[2], + num=1, + query=f"SELECT uniqExact(d) FROM {name} WHERE d < 10", + expected="10", + ) + check_query_node( + node=nodes[2], + num=2, + query=f"SELECT d FROM {name} ORDER BY d DESC LIMIT 1", + expected="3407872", + ) + check_query_node( + node=nodes[2], + num=3, + query=f"SELECT d FROM {name} ORDER BY d ASC LIMIT 1", + expected="0", + ) + + finally: + with Finally("I drop the table on each node"): + for node in nodes: + node.query(f"DROP TABLE IF EXISTS {name} SYNC") @TestScenario diff --git a/s3/tests/disk_invalid.py b/s3/tests/disk_invalid.py index 51d76b146..23c0d4349 100644 --- a/s3/tests/disk_invalid.py +++ b/s3/tests/disk_invalid.py @@ -119,12 +119,10 @@ def invalid_endpoint(self): if check_clickhouse_version("<23.8")(self): message = "DB::Exception: No key in S3 uri" - else: + else: message = "Cannot resolve host (unknown-website)" - invalid_s3_storage_config( - disks, policies, message=message, tail=300 - ) + invalid_s3_storage_config(disks, policies, message=message, tail=300) @TestScenario @@ -199,58 +197,60 @@ def access_failed_skip_check(self): "external": {"volumes": {"external": {"disk": "external"}}}, } - with s3_storage_context(disks, policies, restart=True): - try: - if check_clickhouse_version(">=23.8")(self): - with Given( - f"""I create table using S3 storage policy external, - expecting failure because there is no access to the S3 bucket""" - ): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64 - ) ENGINE = MergeTree() - ORDER BY d - SETTINGS storage_policy='external' - """, - message="""DB::Exception: Message: Access Denied""", - exitcode=243, - ) - else: - with Given( - f"""I create table using S3 storage policy external, - expecting success because there is no access check to this - disk""" - ): - node.query( - f""" - CREATE TABLE {name} ( - d UInt64 - ) ENGINE = MergeTree() - ORDER BY d - SETTINGS storage_policy='external' - """ - ) - - with Then( - """I store simple data in the table, expecting failure - because there is no access to the S3 bucket""" - ): - message = ( - """DB::Exception: Access Denied.""" - if check_clickhouse_version("<22.9")(self) - else """DB::Exception: Message: Access Denied""" - ) - node.query( - f"INSERT INTO {name} VALUES (427)", - message=message, - exitcode=243, - ) - - finally: - with Finally("I drop the table"): - node.query(f"DROP TABLE IF EXISTS {name}") + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + try: + if check_clickhouse_version(">=23.8")(self): + with Given( + f"""I create table using S3 storage policy external, + expecting failure because there is no access to the S3 bucket""" + ): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = MergeTree() + ORDER BY d + SETTINGS storage_policy='external' + """, + message="""DB::Exception: Message: Access Denied""", + exitcode=243, + ) + else: + with Given( + f"""I create table using S3 storage policy external, + expecting success because there is no access check to this + disk""" + ): + node.query( + f""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = MergeTree() + ORDER BY d + SETTINGS storage_policy='external' + """ + ) + + with Then( + """I store simple data in the table, expecting failure + because there is no access to the S3 bucket""" + ): + message = ( + """DB::Exception: Access Denied.""" + if check_clickhouse_version("<22.9")(self) + else """DB::Exception: Message: Access Denied""" + ) + node.query( + f"INSERT INTO {name} VALUES (427)", + message=message, + exitcode=243, + ) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {name}") @TestScenario diff --git a/s3/tests/iam_auth.py b/s3/tests/iam_auth.py index 98450440f..5c15206b1 100644 --- a/s3/tests/iam_auth.py +++ b/s3/tests/iam_auth.py @@ -1,7 +1,7 @@ from testflows.core import * from testflows.asserts import error -from s3.tests.common import getuid, s3_storage_context +from s3.tests.common import getuid, s3_storage from s3.requirements import * @@ -86,28 +86,30 @@ def iam_mode_auth(self): "aws_external": {"volumes": {"external": {"disk": "aws"}}}, } - with s3_storage_context(disks, policies, restart=True, nodes=[node]): - try: - with Given(f"I create table using S3 storage policy external"): - node.query( - f""" - CREATE TABLE {table_name} ( - d UInt64 - ) ENGINE = MergeTree() - ORDER BY d - SETTINGS storage_policy='aws_external' - """ - ) - - with And("I store simple data in S3 to check import"): - node.query(f"INSERT INTO {table_name} VALUES ({expected})") - - with Then("I check that select returns matching data"): - r = node.query(f"SELECT * FROM {table_name}").output.strip() - assert r == expected, error() - finally: - with Finally("I drip table that uses S3"): - node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + try: + with Given(f"I create table using S3 storage policy external"): + node.query( + f""" + CREATE TABLE {table_name} ( + d UInt64 + ) ENGINE = MergeTree() + ORDER BY d + SETTINGS storage_policy='aws_external' + """ + ) + + with And("I store simple data in S3 to check import"): + node.query(f"INSERT INTO {table_name} VALUES ({expected})") + + with Then("I check that select returns matching data"): + r = node.query(f"SELECT * FROM {table_name}").output.strip() + assert r == expected, error() + finally: + with Finally("I drip table that uses S3"): + node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") @TestFeature diff --git a/s3/tests/zero_copy_replication.py b/s3/tests/zero_copy_replication.py index e4ef6c8d8..0cab15e62 100644 --- a/s3/tests/zero_copy_replication.py +++ b/s3/tests/zero_copy_replication.py @@ -2220,17 +2220,19 @@ def outline(self): for name in self.context.cluster.nodes["clickhouse"] ] - with s3_storage_context(disks, policies, restart=True): - with Check("bucket should be empty before test begins"): - check_bucket_size( - name=self.context.bucket_name, - prefix=self.context.bucket_path, - expected_size=0, - tolerance=5, - minio_enabled=self.context.minio_enabled, - ) - for scenario in loads(current_module(), Scenario): - scenario() + with And("I enable the disk and policy config"): + s3_storage(disks=disks, policies=policies, restart=True) + + with Check("bucket should be empty before test begins"): + check_bucket_size( + name=self.context.bucket_name, + prefix=self.context.bucket_path, + expected_size=0, + tolerance=50, + minio_enabled=self.context.minio_enabled, + ) + for scenario in loads(current_module(), Scenario): + scenario() @TestFeature diff --git a/vfs/tests/steps.py b/vfs/tests/steps.py index 55ed38ad7..b0eba1231 100644 --- a/vfs/tests/steps.py +++ b/vfs/tests/steps.py @@ -7,7 +7,7 @@ from helpers.common import getuid, check_clickhouse_version -from s3.tests.common import s3_storage_context, check_bucket_size, get_bucket_size +from s3.tests.common import s3_storage, check_bucket_size, get_bucket_size DEFAULT_COLUMNS = "key UInt32, value1 String, value2 String, value3 String" @@ -67,8 +67,7 @@ def s3_config(self): }, } - with s3_storage_context(disks, policies, restart=True, timeout=60): - yield + return s3_storage(disks=disks, policies=policies, restart=restart, timeout=60) @TestStep(Given) @@ -271,15 +270,14 @@ def storage_config( if policies is None: policies = {} - with s3_storage_context( + return s3_storage( disks, policies, nodes=nodes, restart=restart, timeout=timeout, config_file=config_file, - ): - yield + ) @TestStep(Given) @@ -312,15 +310,14 @@ def enable_vfs( policies = {} - with s3_storage_context( - disks, - policies, + return s3_storage( + disks=disks, + policies=policies, nodes=nodes, restart=True, timeout=timeout, config_file=config_file, - ): - yield + ) @TestStep From dd513c6606ce7f46ed9aea35a36c41ea2fa355b9 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 09:38:01 -0500 Subject: [PATCH 13/73] Remove vfs storage confgi teststep and use the new s3 one --- vfs/tests/alter.py | 2 +- vfs/tests/settings.py | 2 +- vfs/tests/steps.py | 28 +--------------------------- vfs/tests/ttl.py | 2 +- 4 files changed, 4 insertions(+), 30 deletions(-) diff --git a/vfs/tests/alter.py b/vfs/tests/alter.py index 11441e7fb..7aa3ca663 100644 --- a/vfs/tests/alter.py +++ b/vfs/tests/alter.py @@ -523,7 +523,7 @@ def move(self): } with And("I activate the policies"): - storage_config( + s3_storage( policies=policies, restart=True, timeout=60, diff --git a/vfs/tests/settings.py b/vfs/tests/settings.py index 34d2f41cc..226b9ee60 100644 --- a/vfs/tests/settings.py +++ b/vfs/tests/settings.py @@ -361,7 +361,7 @@ def check_setting_combination( storage_setting[0]: storage_setting[1], } } - storage_config(disks=disks, restart=False) + s3_storage(disks=disks, restart=False, config_file="test_settings.xml") with Given("a replicated table"): _, table_name = replicated_table_cluster( diff --git a/vfs/tests/steps.py b/vfs/tests/steps.py index b0eba1231..ebc5cae19 100644 --- a/vfs/tests/steps.py +++ b/vfs/tests/steps.py @@ -67,7 +67,7 @@ def s3_config(self): }, } - return s3_storage(disks=disks, policies=policies, restart=restart, timeout=60) + return s3_storage(disks=disks, policies=policies, restart=True, timeout=60) @TestStep(Given) @@ -254,32 +254,6 @@ def delete_one_replica(self, node, table_name): return r -@TestStep(Given) -def storage_config( - self, - disks=None, - policies=None, - nodes=None, - restart=False, - timeout=30, - config_file="storage_config.xml", -): - """Create disk and storage policy config.""" - if disks is None: - disks = {} - if policies is None: - policies = {} - - return s3_storage( - disks, - policies, - nodes=nodes, - restart=restart, - timeout=timeout, - config_file=config_file, - ) - - @TestStep(Given) def enable_vfs( self, diff --git a/vfs/tests/ttl.py b/vfs/tests/ttl.py index 656ae44bc..fe4f13a14 100644 --- a/vfs/tests/ttl.py +++ b/vfs/tests/ttl.py @@ -20,7 +20,7 @@ def set_tiered_policy(self, allow_vfs=1, move_on_insert=0): } for n in ["external", "external_tiered"] } - storage_config(disks=disks, policies=policies, restart=True) + s3_storage(disks=disks, policies=policies, restart=True, config_file="policy_settings.xml") @TestStep(When) From 720bf8cc4f293a7adec22c0addfd839a950d247b Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:06:43 -0500 Subject: [PATCH 14/73] vfs stress alter test offline keeper and small fixes --- vfs/tests/stress_alter.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/vfs/tests/stress_alter.py b/vfs/tests/stress_alter.py index 4bf577e8c..e7282917b 100644 --- a/vfs/tests/stress_alter.py +++ b/vfs/tests/stress_alter.py @@ -225,7 +225,7 @@ def freeze_unfreeze_random_part(self): query = f"ALTER TABLE {table_name} FREEZE PARTITION {partition} WITH NAME '{backup_name}'" node.query(query, exitcode=0) - with Then(f"I wait {delay:.2}s"): + with And(f"I wait {delay:.2}s"): time.sleep(delay) with Finally("I unfreeze the part"): @@ -234,6 +234,7 @@ def freeze_unfreeze_random_part(self): @TestStep +@Retry(timeout=30, delay=5) @Name("drop part") def drop_random_part(self): """Detach and drop a random part.""" @@ -529,6 +530,23 @@ def delete_replica(self): return +@TestStep +def restart_keeper(self, repeat_limit=5): + keeper_node = self.context.cluster.node("zookeeper") + delay = random.random() * 2 + 1 + + try: + with When("I stop zookeeper"): + keeper_node.stop() + + with And(f"I wait {delay:.2}s"): + time.sleep(delay) + + finally: + with Finally("I restart zookeeper"): + keeper_node.start() + + @TestScenario def parallel_alters(self): """ @@ -543,8 +561,9 @@ def parallel_alters(self): delete_random_column, update_random_column, delete_random_rows, - delete_replica, - add_replica, + restart_keeper, + # delete_replica, + # add_replica, detach_attach_random_partition, freeze_unfreeze_random_part, drop_random_part, @@ -559,11 +578,13 @@ def parallel_alters(self): combinations(actions, self.context.combination_size, with_replacement=True) ) + if self.context.shuffle or not self.context.stress: + random.shuffle(action_groups) + if not self.context.stress: with And( f"I shuffle the list and choose {self.context.unstressed_limit} groups of actions" ): - random.shuffle(action_groups) action_groups = action_groups[: self.context.unstressed_limit] with Given(f"I create {self.context.n_tables} tables with 10 columns and data"): @@ -630,6 +651,7 @@ def feature(self): """Stress test with many alters.""" self.context.unstressed_limit = 50 + self.context.shuffle = True self.context.combination_size = 3 self.context.run_groups_in_parallel = True self.context.run_optimize_in_parallel = True From 5e2ab9cd6c3767cff22a3e0d6f9d847b0df66d91 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:25:13 -0500 Subject: [PATCH 15/73] Fix s3 vfs regression --- s3/regression.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/s3/regression.py b/s3/regression.py index 66d2fc9c9..19747b2cf 100755 --- a/s3/regression.py +++ b/s3/regression.py @@ -348,10 +348,6 @@ def aws_s3_regression( self.context.cluster.bucket = bucket with Module(self.context.object_storage_mode): - if self.context.object_storage_mode == "vfs": - with Given("I enable allow_object_storage_vfs"): - enable_vfs() - Feature(test=load("s3.tests.table_function", "aws_s3"))( uri=uri, key_id=key_id, access_key=access_key ) @@ -416,10 +412,6 @@ def gcs_regression( self.context.cluster = cluster with Module(self.context.object_storage_mode): - if self.context.object_storage_mode == "vfs": - with Given("I enable allow_object_storage_vfs"): - enable_vfs() - Feature(test=load("s3.tests.table_function", "gcs"))( uri=uri, key_id=key_id, access_key=access_key ) From 1930a43cb0256ad80caf409a09cd4c459c0e57d2 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:42:58 -0500 Subject: [PATCH 16/73] Try to fix vfs skip on runners --- vfs/regression.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vfs/regression.py b/vfs/regression.py index c684d4d63..6e14359ea 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -22,8 +22,9 @@ ffails = { "*": ( Skip, - "vfs not supported on < 24", - check_clickhouse_version("<24"), + "vfs not supported on < 24.2", + lambda test: check_clickhouse_version("<24.2")(test) + and not test.context.allow_vfs, ), ":/alter/move": (XFail, "Fix pending"), ":/replica/add remove one node": (XFail, "Fix pending"), @@ -123,8 +124,8 @@ def regression( self.context.clickhouse_version = clickhouse_version - if check_clickhouse_version("<24.1")(self) or not allow_vfs: - skip("vfs not supported on < 24.1") + if check_clickhouse_version("<24.2")(self) or not allow_vfs: + skip("vfs not supported on < 24.2") self.context.stress = stress From b2ee1f61dbd1b0c204f5f24e766c61d0d2dd34a9 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:44:56 -0500 Subject: [PATCH 17/73] Try to fix vfs skip on runners --- vfs/regression.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vfs/regression.py b/vfs/regression.py index 6e14359ea..11112f910 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -128,6 +128,7 @@ def regression( skip("vfs not supported on < 24.2") self.context.stress = stress + self.context.allow_vfs = allow_vfs if storages is None: storages = ["minio"] From 1bbf3f6a27d24f8b572e7d0f68167b4e991c1229 Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 22:54:10 +0100 Subject: [PATCH 18/73] increase number of tables in memory leak test --- memory/tests/test_memory_leak.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/memory/tests/test_memory_leak.py b/memory/tests/test_memory_leak.py index e61f6e828..9c13ff6b4 100644 --- a/memory/tests/test_memory_leak.py +++ b/memory/tests/test_memory_leak.py @@ -149,10 +149,10 @@ def get_process_memory_usage_in_kilobytes(self, pid): def check_create_and_drop_tables( self, number_of_workers=10, - number_of_tables=1000, + number_of_tables=100000, node=None, - max_memory_increase=1.3, - timeout=300, + max_memory_increase=1.1, + timeout=3600, ): """Test memory leak when we create and drop many tables.""" if node is None: From 9ec271d5b8e950ea3b8d0b647618435919a618bc Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 23:06:10 +0100 Subject: [PATCH 19/73] fix name in main regression --- regression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression.py b/regression.py index df8e0a062..6603467b4 100755 --- a/regression.py +++ b/regression.py @@ -65,7 +65,7 @@ def regression( "aws_s3_access_key": aws_s3_access_key, "aws_s3_key_id": aws_s3_key_id, "gcs_key_secret": gcs_key_secret, - "gcs_key_id": gcs_key_id + "gcs_key_id": gcs_key_id, } self.context.stress = stress @@ -149,7 +149,7 @@ def regression( executor=pool, )(**args) Feature( - test=load("logger.regression", "regression"), + test=load("memory.regression", "regression"), parallel=True, executor=pool, )(**args) From 5e2a5cc0406e70e3f0ebc309d2ff280e90ec097b Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 23:09:28 +0100 Subject: [PATCH 20/73] apply black --- .../tests/type_conversion.py | 4 ++-- helpers/alter.py | 8 +++++++- helpers/create.py | 4 +++- helpers/tables.py | 8 +++++--- memory/regression.py | 6 +++++- s3/regression.py | 6 +++++- s3/tests/common.py | 6 +++--- s3/tests/reconnect.py | 3 ++- s3/tests/zero_copy_replication.py | 5 +---- selects/regression.py | 4 +--- selects/tests/steps/as_statement.py | 17 ++++++++++------- selects/tests/steps/main_steps.py | 6 +----- 12 files changed, 45 insertions(+), 32 deletions(-) diff --git a/datetime64_extended_range/tests/type_conversion.py b/datetime64_extended_range/tests/type_conversion.py index f6a111e3c..e47632135 100644 --- a/datetime64_extended_range/tests/type_conversion.py +++ b/datetime64_extended_range/tests/type_conversion.py @@ -541,7 +541,7 @@ def to_unix_timestamp64_milli_micro_nano(self, scale): with And("converting DateTime to UTC"): dt = dt.astimezone(pytz.timezone("UTC")) with And("computing the expected result using python"): - expected = int(dt.timestamp() * (10**scale)) + expected = int(dt.timestamp() * (10 ** scale)) if expected >= 0: expected += dt.microsecond * 10 ** (scale - 6) else: @@ -610,7 +610,7 @@ def from_unix_timestamp64_milli_micro_nano(self, scale): with And("converting DateTime64 to UTC"): dt = dt.astimezone(pytz.timezone("UTC")) with And("computing the expected result using python"): - ts = int(dt.timestamp() * (10**scale)) + ts = int(dt.timestamp() * (10 ** scale)) if ts >= 0: ts += dt.microsecond * 10 ** (scale - 6) else: diff --git a/helpers/alter.py b/helpers/alter.py index 5610224dc..b17ab20ab 100644 --- a/helpers/alter.py +++ b/helpers/alter.py @@ -178,7 +178,13 @@ def alter_table_move_partition_to_table( @TestStep(Given) def alter_table_move_partition( - self, table_name, partition_name, disk_name, node=None, disk="VOLUME", **query_kwargs + self, + table_name, + partition_name, + disk_name, + node=None, + disk="VOLUME", + **query_kwargs, ): """Move partition to a different disk or volume using alter.""" if node is None: diff --git a/helpers/create.py b/helpers/create.py index bc2621c96..8472a9019 100644 --- a/helpers/create.py +++ b/helpers/create.py @@ -361,7 +361,9 @@ def partitioned_merge_tree_table(self, table_name, partition_by, columns): @TestStep(Given) -def partitioned_replicated_merge_tree_table(self, table_name, partition_by, columns=None): +def partitioned_replicated_merge_tree_table( + self, table_name, partition_by, columns=None +): """Create a ReplicatedMergeTree table partitioned by a specific column.""" with By( f"creating a partitioned {table_name} table with a ReplicatedMergeTree engine" diff --git a/helpers/tables.py b/helpers/tables.py index 33bd444db..251beb1c7 100644 --- a/helpers/tables.py +++ b/helpers/tables.py @@ -376,7 +376,7 @@ def create_table( drop_sync=False, order_by=None, partition_by=None, - primary_key = None, + primary_key=None, comment=None, as_select=None, settings=None, @@ -433,7 +433,7 @@ def create_table( query += f"\nAS SELECT {as_select}" if query_settings is not None: query += f"\nSETTINGS {query_settings}" - + node.query( query, settings=settings, @@ -443,7 +443,9 @@ def create_table( finally: with Finally(f"drop the table {name}"): - node.query(f"DROP TABLE IF EXISTS {name}{on_cluster} {' SYNC' if drop_sync else ''}") + node.query( + f"DROP TABLE IF EXISTS {name}{on_cluster} {' SYNC' if drop_sync else ''}" + ) @TestStep(Given) diff --git a/memory/regression.py b/memory/regression.py index 2f5e603fe..42f940abb 100755 --- a/memory/regression.py +++ b/memory/regression.py @@ -50,7 +50,11 @@ def regression( self.context.cluster = cluster self.context.node = cluster.node("clickhouse1") - Feature(run=load("memory.tests.test_memory_leak_using_system_memory_dump_log", "feature")) + Feature( + run=load( + "memory.tests.test_memory_leak_using_system_memory_dump_log", "feature" + ) + ) Feature(run=load("memory.tests.test_memory_leak", "feature")) diff --git a/s3/regression.py b/s3/regression.py index 19747b2cf..165522890 100755 --- a/s3/regression.py +++ b/s3/regression.py @@ -180,7 +180,11 @@ def argparser(parser): (Fail, "https://github.com/ClickHouse/ClickHouse/pull/44875") ], ":/:/zero copy replication/bad detached part": [ - (Fail, "https://github.com/ClickHouse/ClickHouse/pull/58333", check_clickhouse_version("<23.11")) + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/pull/58333", + check_clickhouse_version("<23.11"), + ) ], } diff --git a/s3/tests/common.py b/s3/tests/common.py index dcb591ee2..38c8196ba 100644 --- a/s3/tests/common.py +++ b/s3/tests/common.py @@ -1203,9 +1203,9 @@ def add_ssec_s3_option(self, ssec_key=None): "adding 'server_side_encryption_customer_key_base64' S3 option", description=f"key={ssec_key}", ): - self.context.s3_options["server_side_encryption_customer_key_base64"] = ( - ssec_key - ) + self.context.s3_options[ + "server_side_encryption_customer_key_base64" + ] = ssec_key yield finally: diff --git a/s3/tests/reconnect.py b/s3/tests/reconnect.py index 21f2a78db..285e986fa 100644 --- a/s3/tests/reconnect.py +++ b/s3/tests/reconnect.py @@ -2,7 +2,8 @@ from s3.requirements import * from lightweight_delete.tests.steps import * -DOCKER_NETWORK = "s3_env_default" if processor() == 'x86_64' else "s3_env_arm64" +DOCKER_NETWORK = "s3_env_default" if processor() == "x86_64" else "s3_env_arm64" + @TestStep(When) def disconnect_reconnect(self, node=None): diff --git a/s3/tests/zero_copy_replication.py b/s3/tests/zero_copy_replication.py index 0cab15e62..e1b4aa0b7 100644 --- a/s3/tests/zero_copy_replication.py +++ b/s3/tests/zero_copy_replication.py @@ -631,9 +631,7 @@ def metadata(self): insert_data_node(node=nodes[0], number_of_mb=1, start=1024 * 1024) with And("a large insert of 10Mb of data"): - insert_data_node( - node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2 - ) + insert_data_node(node=nodes[0], number_of_mb=10, start=1024 * 1024 * 2) with Then( """The number of bytes written to S3 by the first @@ -1009,7 +1007,6 @@ def insert_multiple_replicas(self): minio_enabled=self.context.minio_enabled, ) - try: with When("I create a replicated table on each node"): for i, node in enumerate(nodes): diff --git a/selects/regression.py b/selects/regression.py index 60e3feef0..5989a6d54 100755 --- a/selects/regression.py +++ b/selects/regression.py @@ -69,9 +69,7 @@ def argparser(parser): "final/force/general/with experimental analyzer/select subquery/distr_*": [ (Fail, "column fail for distributed tables") ], - "final/force/alias/as with alias/*": [ - (Fail, "fails for ARM") - ], + "final/force/alias/as with alias/*": [(Fail, "fails for ARM")], } xflags = {} diff --git a/selects/tests/steps/as_statement.py b/selects/tests/steps/as_statement.py index 805c1e785..92fe8b1da 100644 --- a/selects/tests/steps/as_statement.py +++ b/selects/tests/steps/as_statement.py @@ -75,13 +75,16 @@ def as_result_check(self, table, node=None): node = self.context.cluster.node("clickhouse1") with Then("I check that compare results are the same"): - assert node.query( - f"SELECT id as new_id FROM {table.name} {'FINAL' if table.final_modifier_available else ''} ORDER BY (id) FORMAT JSONEachRow;", - settings=[("final", 0)], - ).output.strip() == node.query( - f"SELECT id as new_id FROM {table.name} ORDER BY (id) FORMAT JSONEachRow;", - settings=[("final", 1)], - ).output.strip(), error() + assert ( + node.query( + f"SELECT id as new_id FROM {table.name} {'FINAL' if table.final_modifier_available else ''} ORDER BY (id) FORMAT JSONEachRow;", + settings=[("final", 0)], + ).output.strip() + == node.query( + f"SELECT id as new_id FROM {table.name} ORDER BY (id) FORMAT JSONEachRow;", + settings=[("final", 1)], + ).output.strip() + ), error() @TestStep diff --git a/selects/tests/steps/main_steps.py b/selects/tests/steps/main_steps.py index 0b14c0802..9f93e929b 100644 --- a/selects/tests/steps/main_steps.py +++ b/selects/tests/steps/main_steps.py @@ -75,11 +75,7 @@ def create_and_populate_table( node = current().context.node try: with By(f"creating table {name}"): - retry( - node.query, - timeout=100, - delay=5, - )( + retry(node.query, timeout=100, delay=5,)( f"CREATE TABLE {name} " f"{' ON CLUSTER {cluster_name}'.format(cluster_name=cluster_name) if cluster_name is not None else ''}" f"(id Int64, x Int64, {extra_table_col})" From c8078dab965d6a18cadd871247d1f347e6f4b92a Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 23:14:23 +0100 Subject: [PATCH 21/73] add memory suite to ci/cd --- .github/workflows/run-arm-regression.yml | 33 ++++++++++++++++++++++++ .github/workflows/run-regression.yml | 33 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/.github/workflows/run-arm-regression.yml b/.github/workflows/run-arm-regression.yml index d9df0b5a8..0bed93464 100644 --- a/.github/workflows/run-arm-regression.yml +++ b/.github/workflows/run-arm-regression.yml @@ -40,6 +40,7 @@ on: - key_value - ldap - lightweight_delete + - memory - parquet_all - parquet - parquet_minio @@ -968,6 +969,38 @@ jobs: name: ${{ env.SUITE }}-artifacts path: ${{ env.artifact_paths}} + memory: + runs-on: [self-hosted, type-cax31, image-arm-app-docker-ce] + timeout-minutes: 180 + env: + SUITE: memory + if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Setup + run: .github/setup.sh + + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --clickhouse-version ${{ env.version }} + ${{ env.args }} + + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }} + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-artifacts + path: ${{ env.artifact_paths}} + parquet: runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce] timeout-minutes: 300 diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml index 689918ab0..cab7fe993 100644 --- a/.github/workflows/run-regression.yml +++ b/.github/workflows/run-regression.yml @@ -40,6 +40,7 @@ on: - key_value - ldap - lightweight_delete + - memory - parquet_all - parquet - parquet_minio @@ -968,6 +969,38 @@ jobs: name: ${{ env.SUITE }}-artifacts path: ${{ env.artifact_paths}} +memory: + runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] + timeout-minutes: 180 + env: + SUITE: memory + if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Setup + run: .github/setup.sh + + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --clickhouse-version ${{ env.version }} + ${{ env.args }} + + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }} + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-artifacts + path: ${{ env.artifact_paths}} + parquet: runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] timeout-minutes: 300 From f6688e2ff23ee0ca27e8567e1828827e8fae479b Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 23:18:25 +0100 Subject: [PATCH 22/73] small fix --- .github/workflows/run-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml index cab7fe993..5078e0264 100644 --- a/.github/workflows/run-regression.yml +++ b/.github/workflows/run-regression.yml @@ -1000,7 +1000,7 @@ memory: with: name: ${{ env.SUITE }}-artifacts path: ${{ env.artifact_paths}} - + parquet: runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] timeout-minutes: 300 From 5331e1f6d3bcc322d42da1fb35b49b3d065e090f Mon Sep 17 00:00:00 2001 From: alsu Date: Thu, 15 Feb 2024 23:23:07 +0100 Subject: [PATCH 23/73] try to fix yml --- .github/workflows/run-regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml index 5078e0264..e7e532da3 100644 --- a/.github/workflows/run-regression.yml +++ b/.github/workflows/run-regression.yml @@ -969,7 +969,7 @@ jobs: name: ${{ env.SUITE }}-artifacts path: ${{ env.artifact_paths}} -memory: + memory: runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] timeout-minutes: 180 env: @@ -1000,7 +1000,7 @@ memory: with: name: ${{ env.SUITE }}-artifacts path: ${{ env.artifact_paths}} - + parquet: runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] timeout-minutes: 300 From 3a066dfd83adc524241e5edddbca477f919e3684 Mon Sep 17 00:00:00 2001 From: alsu Date: Fri, 16 Feb 2024 09:46:26 +0100 Subject: [PATCH 24/73] increase timeout for memory suite --- .github/workflows/run-arm-regression.yml | 2 +- .github/workflows/run-regression.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-arm-regression.yml b/.github/workflows/run-arm-regression.yml index 0bed93464..d6163b9d0 100644 --- a/.github/workflows/run-arm-regression.yml +++ b/.github/workflows/run-arm-regression.yml @@ -971,7 +971,7 @@ jobs: memory: runs-on: [self-hosted, type-cax31, image-arm-app-docker-ce] - timeout-minutes: 180 + timeout-minutes: 1800 env: SUITE: memory if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }} diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml index e7e532da3..6c6f4e54b 100644 --- a/.github/workflows/run-regression.yml +++ b/.github/workflows/run-regression.yml @@ -971,7 +971,7 @@ jobs: memory: runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 + timeout-minutes: 1800 env: SUITE: memory if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }} From 4d6cdbc3fc24d050942132210744a980a99401e1 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 08:29:34 -0500 Subject: [PATCH 25/73] Fix ssl_server for 24.1+ --- ssl_server/tests/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ssl_server/tests/common.py b/ssl_server/tests/common.py index 06a61c172..e8b7228cc 100644 --- a/ssl_server/tests/common.py +++ b/ssl_server/tests/common.py @@ -963,11 +963,13 @@ def clickhouse_client_connection( options["preferServerCiphers"] = "true" if prefer_server_ciphers else "false" + secure_flag = "-s" if check_clickhouse_version("<24.1")(self) else "--secure" + with Given("custom clickhouse-client SSL configuration"): add_ssl_clickhouse_client_configuration_file(entries=options) output = node.command( - f'clickhouse client -s --verbose --host {hostname} --port {port} -q "SELECT 1"', + f'clickhouse client {secure_flag} --verbose --host {hostname} --port {port} -q "SELECT 1"', message=message, messages=messages, exitcode=exitcode, From ac550bea3dc58ecb3a1055e7d85ec5c5e1a411e2 Mon Sep 17 00:00:00 2001 From: alsu Date: Fri, 16 Feb 2024 16:46:45 +0100 Subject: [PATCH 26/73] prototype of attach partition with datetime tests --- .../partition_key_datetime.py | 519 ++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 alter/table/attach_partition/partition_key_datetime.py diff --git a/alter/table/attach_partition/partition_key_datetime.py b/alter/table/attach_partition/partition_key_datetime.py new file mode 100644 index 000000000..0abb1af0b --- /dev/null +++ b/alter/table/attach_partition/partition_key_datetime.py @@ -0,0 +1,519 @@ +from testflows.core import * +from testflows.combinatorics import product + +from alter.table.attach_partition.common import * +from alter.table.attach_partition.requirements.requirements import * + +from helpers.common import ( + getuid, +) +from helpers.tables import * + + +def valid_partition_key_pair(source_partition_key, destination_partition_key): + """Validates if pair source partition key - destination partition key is valid + for `attach partition from` statement.""" + + not_subset = { + "tuple()": [ + "time", + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ], + } + + not_monotonic = { + # "toMonth(time)": ["toQuarter(time)"] + } + + partially_different = { + "toYYYYMMDD(time)": ["toHour(time)", "toMinute(time)", "toSecond(time)"], + "toYYYYMM(time)": [ + "toYYYYMMDD(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ], + "toMonth(time)": [ + "toYYYYMM(time)", + "toYYYYMMDD(time)", + "toDayOfYear(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + "toYear(time)", + ], + "toDayOfYear(time)": ["toYYYYMMDD(time)", "toYYYYMM(time)", "toHour(time)", "toMinute(time)", "toSecond(time)", "toYear(time)"], + "toDayOfMonth(time)": [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ], + "toDayOfWeek(time)": [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ], + "toHour(time)": [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toMinute(time)", + "toSecond(time)", + ], + "toMinute(time)" : [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toSecond(time)", + ], + "toSecond(time)" : [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + ], + "toYear(time)" : [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toDayOfYear(time)", + "toQuarter(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ], + "toQuarter(time)": [ + "toYYYYMMDD(time)", + "toYYYYMM(time)", + "toYear(time)", + "toDayOfYear(time)", + "toMonth(time)", + "toDayOfMonth(time)", + "toDayOfWeek(time)", + "toHour(time)", + "toMinute(time)", + "toSecond(time)", + ] + } + + if ( + destination_partition_key in not_monotonic.get(source_partition_key, "") + and destination_partition_key not in not_subset.get(source_partition_key, "") + and destination_partition_key + not in partially_different.get(source_partition_key, "") + ): + return False, "not monotonic" + + if ( + destination_partition_key in not_subset.get(source_partition_key, "") + and destination_partition_key not in not_monotonic.get(source_partition_key, "") + and destination_partition_key + not in partially_different.get(source_partition_key, "") + ): + return False, "not subset" + + if ( + destination_partition_key in partially_different.get(source_partition_key, "") + and destination_partition_key not in not_monotonic.get(source_partition_key, "") + and destination_partition_key not in not_subset.get(source_partition_key, "") + ): + return False, "partially different" + + return True, "" + + +def get_node(self, table): + """Returns first node for non-replicated tables and returns random node for replicated tables.""" + if table == "source": + if "Replicated" in self.context.source_engine: + return random.choice( + [self.context.node_1, self.context.node_2, self.context.node_3] + ) + else: + return self.context.node_1 + elif table == "destination": + if "Replicated" in self.context.destination_engine: + return random.choice( + [self.context.node_1, self.context.node_2, self.context.node_3] + ) + else: + return self.context.node_1 + + +def check( + self, + partition_ids, + source_table_name, + destination_table_name, + exitcode=None, + message=None, + with_id=False, +): + """Check `attach partition from` statement with or without `id`.""" + for partition_id in partition_ids: + if with_id: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION ID '{partition_id}' FROM {source_table_name}" + else: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION {partition_id} FROM {source_table_name}" + + self.context.node_1.query( + query, + exitcode=exitcode, + message=message, + ) + + +@TestScenario +@Flags(TE) +def check_attach_partition_from( + self, + source_partition_key, + destination_partition_key, + source_table, + destination_table, + with_id=False, +): + """Check `attach partition from` with different types of source and destination tables.""" + + if check_clickhouse_version("<24.2")(self): + if source_partition_key != destination_partition_key: + skip("Different partition keys are not supported before 24.1") + + self.context.source_engine = source_table.__name__ + self.context.destination_engine = destination_table.__name__ + + source_table_name = "source_" + getuid() + destination_table_name = "destination_" + getuid() + + with Given( + "I create two tables with specified engines and partition keys", + description=f""" + partition keys: + source table partition key: {source_partition_key} + destination table partition key: {destination_partition_key} + engines: + source table engine: {source_table.__name__} + destination table engine: {destination_table.__name__} + """, + ): + source_table( + table_name=source_table_name, + partition_by=source_partition_key, + node=self.context.node_1, + ) + destination_table( + table_name=destination_table_name, + partition_by=destination_partition_key, + node=self.context.node_1, + ) + + if check_clickhouse_version(">=24.2")(self): + with And("I add setting to allow alter partition with different key"): + get_node(self, "source").query( + f"ALTER TABLE {source_table_name} MODIFY SETTING allow_experimental_alter_partition_with_different_key=1" + ) + get_node(self, "destination").query( + f"ALTER TABLE {destination_table_name} MODIFY SETTING allow_experimental_alter_partition_with_different_key=1" + ) + + with And("I get the list of partitions and validate partition keys pair"): + if with_id: + partition_list_query = f"SELECT partition_id FROM system.parts WHERE table='{source_table_name}' ORDER BY partition_id" + else: + partition_list_query = f"SELECT partition FROM system.parts WHERE table='{source_table_name}' ORDER BY partition_id" + + partition_ids = sorted( + list( + set(get_node(self, "source").query(partition_list_query).output.split()) + ) + ) + valid, reason = valid_partition_key_pair( + source_partition_key, destination_partition_key + ) + get_node(self, "source").query(f"SELECT time from {source_table_name}") + + with And("I attach partition from source table to the destination table"): + if valid: + for partition_id in partition_ids: + if with_id: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION ID '{partition_id}' FROM {source_table_name}" + else: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION {partition_id} FROM {source_table_name}" + + self.context.node_1.query(query) + self.context.node_1.query( + f"SELECT * FROM {destination_table_name} format PrettyCompactMonoBlock" + ) + + else: + if reason == "not monotonic": + exitcode, message = ( + 36, + "DB::Exception: Destination table partition expression is not monotonically increasing", + ) + check( + self, + partition_ids=partition_ids, + source_table_name=source_table_name, + destination_table_name=destination_table_name, + exitcode=exitcode, + message=message, + with_id=with_id, + ) + elif reason == "not subset": + exitcode, message = ( + 36, + "DB::Exception: Destination table partition expression columns must be a subset of source table partition expression columns.", + ) + check( + self, + partition_ids=partition_ids, + source_table_name=source_table_name, + destination_table_name=destination_table_name, + exitcode=exitcode, + message=message, + with_id=with_id, + ) + elif reason == "partially different": + exitcode, message = ( + 248, + "DB::Exception: Can not create the partition. A partition can not contain values that have different partition ids.", + ) + for partition_id in partition_ids: + if with_id: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION ID '{partition_id}' FROM {source_table_name}" + else: + query = f"ALTER TABLE {destination_table_name} ATTACH PARTITION {partition_id} FROM {source_table_name}" + try: + self.context.node_1.query( + query, + exitcode=exitcode, + message=message, + ) + except: + note("Partition can be attached") + + with And( + "I change engine names to compare replicated results with non-replicated results in snapshots" + ): + if "Replicated" in source_table.__name__: + source_table = source_table.__name__.split("_")[-1] + source_table = source_table.replace("Replicated", "") + else: + source_table = source_table.__name__.split("_")[-1] + if "Replicated" in destination_table.__name__: + destination_table = destination_table.__name__.split("_")[-1] + destination_table = destination_table.replace("Replicated", "") + else: + destination_table = destination_table.__name__.split("_")[-1] + + with Then( + f"I check that partitions were attached when source table partition_id - {source_partition_key}, destination table partition key - {destination_partition_key}, source table engine - {source_table}, destination table engine - {destination_table}:" + ): + if valid: + source_partition_data = get_node(self, "source").query( + f"SELECT * FROM {source_table_name} ORDER BY time,extra" + ) + destination_partition_data = get_node(self, "destination").query( + f"SELECT * FROM {destination_table_name} ORDER BY time,extra" + ) + for attempt in retries(timeout=30, delay=2): + with attempt: + assert ( + destination_partition_data.output + == source_partition_data.output + ), error() + + elif reason == "partially different": + execute_query( + f"SELECT time,extra FROM {destination_table_name} ORDER BY time,extra", + snapshot_name="/alter/table/attach_partition/partition_key_datetime/attach_partition_from/" + + current().name.split("/")[-1], + node=get_node(self, "destination"), + ) + + with And(f"I check that all replicas of destination table have same data:"): + if "Replicated" in self.context.destination_engine: + destination_partition_data_1 = self.context.node_1.query( + f"SELECT * FROM {destination_table_name} ORDER BY time,extra" + ) + destination_partition_data_2 = self.context.node_2.query( + f"SELECT * FROM {destination_table_name} ORDER BY time,extra" + ) + destination_partition_data_3 = self.context.node_3.query( + f"SELECT * FROM {destination_table_name} ORDER BY time,extra" + ) + for attempt in retries(timeout=30, delay=2): + with attempt: + assert ( + destination_partition_data_1.output + == destination_partition_data_2.output + == destination_partition_data_3.output + ) + + # with And("I check that I can use data in te destination table after detach attach"): + # get_node(self, "source").query( + # f"DETACH TABLE {destination_table_name}" + # ) + # get_node(self, "source").query( + # f"ATTACH TABLE {destination_table_name}" + # ) + # get_node(self, "source").query( + # f"SELECT * FROM {destination_table_name} where a=1" + # ) + + +@TestScenario +@Flags(TE) +def attach_partition_from(self, with_id=False): + """Run test check with different partition keys for both source and destination tables to see if `attach partition from` is possible.""" + + source_partition_keys = { + # "tuple()", + # "toYYYYMMDD(time)", + # "toYYYYMM(time)", + # "toYear(time)", + # "toDayOfYear(time)", + # "toQuarter(time)", + "toMonth(time)", + # "toDayOfMonth(time)", + # "toDayOfWeek(time)", + # "toHour(time)", + # "toMinute(time)", + # "toSecond(time)", + } + destination_partition_keys = { + # "tuple()", + # "toYYYYMMDD(time)", + # "toYYYYMM(time)", + # "toYear(time)", + # "toDayOfYear(time)", + "toQuarter(time)", + # "toMonth(time)", + # "toDayOfMonth(time)", + # "toDayOfWeek(time)", + # "toHour(time)", + # "toMinute(time)", + # "toSecond(time)", + } + + source_table_types = {create_partitioned_table_with_datetime_data} + + destination_table_types = {create_empty_partitioned_table_with_datetime_data} + + if not self.context.stress: + source_table_types = {create_partitioned_table_with_datetime_data} + destination_table_types = {create_empty_partitioned_table_with_datetime_data} + + partition_keys_pairs = product(source_partition_keys, destination_partition_keys) + table_pairs = product(source_table_types, destination_table_types) + combinations = product(partition_keys_pairs, table_pairs) + + with Pool(4) as executor: + for partition_keys, tables in combinations: + source_partition_key, destination_partition_key = partition_keys + source_table, destination_table = tables + + Scenario( + f"combination: partition keys - {source_partition_key}, {destination_partition_key}; tables - {source_table.__name__}, {destination_table.__name__}", + test=check_attach_partition_from, + parallel=True, + executor=executor, + )( + source_table=source_table, + destination_table=destination_table, + source_partition_key=source_partition_key, + destination_partition_key=destination_partition_key, + with_id=with_id, + ) + join() + + +@TestFeature +@Requirements( + RQ_SRS_034_ClickHouse_Alter_Table_AttachPartitionFrom_Conditions_Key_PartitionKey( + "1.0" + ), + RQ_SRS_034_ClickHouse_Alter_Table_AttachPartition_SupportedTableEngines("1.0"), + RQ_SRS_034_ClickHouse_Alter_Table_AttachPartitionFrom_Replicas("1.0"), + RQ_SRS_034_ClickHouse_Alter_Table_AttachPartitionFrom("1.0"), + RQ_SRS_034_ClickHouse_Alter_Table_AttachPartitionFrom_KeepData("1.0"), +) +@Name("partition key") +def feature(self): + """Check conditions for partition key.""" + + self.context.node_1 = self.context.cluster.node("clickhouse1") + self.context.node_2 = self.context.cluster.node("clickhouse2") + self.context.node_3 = self.context.cluster.node("clickhouse3") + self.context.nodes = [ + self.context.cluster.node("clickhouse1"), + self.context.cluster.node("clickhouse2"), + self.context.cluster.node("clickhouse3"), + ] + + # self.context.node_1.query("SET allow_experimental_alter_partition_with_different_key=1") + + with Pool(2) as pool: + Scenario( + "attach partition from without id", + test=attach_partition_from, + parallel=True, + executor=pool, + )(with_id=False) + # Scenario( + # "attach partition from with id", + # test=attach_partition_from, + # parallel=True, + # executor=pool, + # )(with_id=True) + join() From 45ab41d308c68d3f154f484a932c9ee85ea6dc6e Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:54:52 -0500 Subject: [PATCH 27/73] Add step names to vfs stress alter add/remove replica --- vfs/tests/stress_alter.py | 99 +++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/vfs/tests/stress_alter.py b/vfs/tests/stress_alter.py index e7282917b..d157d9573 100644 --- a/vfs/tests/stress_alter.py +++ b/vfs/tests/stress_alter.py @@ -465,34 +465,43 @@ def add_replica(self): r = node.query("SELECT table from system.replicas FORMAT JSONColumns") tables_by_node[node.name] = json.loads(r.output)["table"] - table_counts = [len(tables) for tables in tables_by_node.values()] - - if min(table_counts) == self.context.maximum_replicas: - return - - adding_node = None - reference_nodes = [] - for node, count in zip(nodes, table_counts): - if count < self.context.maximum_replicas and adding_node is None: - adding_node = node - else: - reference_nodes.append(node) + with And("I check that there exists a node that does not replicate all tables"): + table_counts = [len(tables) for tables in tables_by_node.values()] + if min(table_counts) == self.context.maximum_replicas: + return - assert adding_node is not None, "Early return logic should prevent this" + with Given( + "I choose one node to add the new replicated table to, and save the other nodes to query structure from" + ): + adding_node = None + reference_nodes = [] + for node, count in zip(nodes, table_counts): + if count < self.context.maximum_replicas and adding_node is None: + adding_node = node + else: + reference_nodes.append(node) + + assert ( + adding_node is not None + ), "Early return logic should prevent failing to find a node that can have a replica added" for table in tables: if table not in tables_by_node[adding_node.name]: - with When(f"I check what columns are in {table}"): - if table in tables_by_node[reference_nodes[0].name]: - columns = get_column_string( - node=reference_nodes[0], table_name=table - ) - elif table in tables_by_node[reference_nodes[1].name]: - columns = get_column_string( - node=reference_nodes[1], table_name=table - ) - else: - assert False, "This line should never be reached" + columns = None + for ref_node in reference_nodes: + with When(f"I check if {ref_node.name} knows about {table}"): + if table in tables_by_node[ref_node.name]: + with When( + f"I query {ref_node.name} for the columns in {table}" + ): + columns = get_column_string( + node=ref_node, table_name=table + ) + break + + assert ( + columns is not None + ), "There should always be at least one node that knows about a table" with And(f"I create a new replica on {adding_node.name}"): create_one_replica( @@ -502,6 +511,7 @@ def add_replica(self): order_by="key", partition_by="key % 4", ) + return @TestStep @@ -514,20 +524,28 @@ def delete_replica(self): random.shuffle(tables) with table_schema_lock: - r = node.query( - "SELECT table, active_replicas from system.replicas FORMAT JSONColumns" - ) - out = json.loads(r.output) - current_tables = out["table"] - active_replicas = {t: r for t, r in zip(current_tables, out["active_replicas"])} - for table in tables: - if ( - table in current_tables - and active_replicas[table] > self.context.minimum_replicas - ): - delete_one_replica(node=node, table_name=table) - return + with Given("I check the replica counts for all tables"): + r = node.query( + "SELECT table, active_replicas from system.replicas FORMAT JSONColumns" + ) + out = json.loads(r.output) + current_tables = out["table"] + active_replicas = { + t: r for t, r in zip(current_tables, out["active_replicas"]) + } + + with When("I look for a table that has more than the minimum replicas"): + for table in tables: + if ( + table in current_tables + and active_replicas[table] > self.context.minimum_replicas + ): + with And( + f"I delete the replica for table {table} on node {node.name}" + ): + delete_one_replica(node=node, table_name=table) + return @TestStep @@ -579,12 +597,11 @@ def parallel_alters(self): ) if self.context.shuffle or not self.context.stress: - random.shuffle(action_groups) + with And("I shuffle the list"): + random.shuffle(action_groups) if not self.context.stress: - with And( - f"I shuffle the list and choose {self.context.unstressed_limit} groups of actions" - ): + with And(f"I choose {self.context.unstressed_limit} groups of actions"): action_groups = action_groups[: self.context.unstressed_limit] with Given(f"I create {self.context.n_tables} tables with 10 columns and data"): From b6dfe8dcf975d30048d4eccee8567763beb566c2 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:39:04 -0500 Subject: [PATCH 28/73] convert vfs stress alter to testoutline --- vfs/tests/stress_alter.py | 75 ++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/vfs/tests/stress_alter.py b/vfs/tests/stress_alter.py index d157d9573..138015fc1 100644 --- a/vfs/tests/stress_alter.py +++ b/vfs/tests/stress_alter.py @@ -510,6 +510,7 @@ def add_replica(self): columns=columns, order_by="key", partition_by="key % 4", + storage_policy=self.context.storage_policy, ) return @@ -565,12 +566,31 @@ def restart_keeper(self, repeat_limit=5): keeper_node.start() -@TestScenario -def parallel_alters(self): +@TestOutline +def parallel_alters( + self, + limit=50, + shuffle=True, + combination_size=3, + run_groups_in_parallel=True, + run_optimize_in_parallel=True, + ignore_failed_part_moves=False, + sync_replica_timeout=600, + storage_policy="external_vfs", + minimum_replicas=1, + maximum_replicas=3, + n_tables=3, +): """ Perform combinations of alter actions, checking that all replicas agree. """ + self.context.ignore_failed_part_moves = ignore_failed_part_moves + self.context.sync_replica_timeout = sync_replica_timeout + self.context.storage_policy = storage_policy + self.context.minimum_replicas = minimum_replicas + self.context.maximum_replicas = maximum_replicas + with Given("I have a list of actions I can perform"): actions = [ add_random_column, @@ -580,8 +600,8 @@ def parallel_alters(self): update_random_column, delete_random_rows, restart_keeper, - # delete_replica, - # add_replica, + delete_replica, + add_replica, detach_attach_random_partition, freeze_unfreeze_random_part, drop_random_part, @@ -591,23 +611,23 @@ def parallel_alters(self): fetch_random_part_from_table, ] - with And(f"I make a list of groups of {self.context.combination_size} actions"): + with And(f"I make a list of groups of {combination_size} actions"): action_groups = list( - combinations(actions, self.context.combination_size, with_replacement=True) + combinations(actions, combination_size, with_replacement=True) ) - if self.context.shuffle or not self.context.stress: + if shuffle: with And("I shuffle the list"): random.shuffle(action_groups) - if not self.context.stress: - with And(f"I choose {self.context.unstressed_limit} groups of actions"): - action_groups = action_groups[: self.context.unstressed_limit] + if limit: + with And(f"I choose {limit} groups of actions"): + action_groups = action_groups[:limit] - with Given(f"I create {self.context.n_tables} tables with 10 columns and data"): + with Given(f"I create {n_tables} tables with 10 columns and data"): self.context.table_names = [] columns = "key UInt64," + ",".join(f"value{i} UInt16" for i in range(10)) - for i in range(self.context.n_tables): + for i in range(n_tables): table_name = f"table{i}_{self.context.storage_policy}" replicated_table_cluster( table_name=table_name, @@ -642,7 +662,7 @@ def parallel_alters(self): By( f"I {action.name}", run=action, - parallel=self.context.run_groups_in_parallel, + parallel=run_groups_in_parallel, flags=TE | ERROR_NOT_COUNTED, ) @@ -650,7 +670,7 @@ def parallel_alters(self): By( f"I OPTIMIZE {table}", test=optimize_random, - parallel=self.context.run_optimize_in_parallel, + parallel=run_optimize_in_parallel, flags=TE, )(table_name=table_name) @@ -667,20 +687,19 @@ def parallel_alters(self): def feature(self): """Stress test with many alters.""" - self.context.unstressed_limit = 50 - self.context.shuffle = True - self.context.combination_size = 3 - self.context.run_groups_in_parallel = True - self.context.run_optimize_in_parallel = True - self.context.ignore_failed_part_moves = False - self.context.sync_replica_timeout = 60 * 10 - self.context.storage_policy = "external_vfs" - self.context.minimum_replicas = 1 - self.context.maximum_replicas = 3 - self.context.n_tables = 3 - with Given("I have S3 disks configured"): s3_config() - for scenario in loads(current_module(), Scenario): - scenario() + Scenario(test=parallel_alters)( + limit=None if self.context.stress else 50, + shuffle=True, + combination_size=3, + run_groups_in_parallel=True, + run_optimize_in_parallel=True, + ignore_failed_part_moves=False, + sync_replica_timeout=600, + storage_policy="external_no_vfs", + minimum_replicas=1, + maximum_replicas=3, + n_tables=3, + ) From 0f84f440ad919602ee9b293f845e1f36d8d00a5a Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:45:50 -0500 Subject: [PATCH 29/73] Fix vfs version checks --- ontime_benchmark/benchmark.py | 5 +++-- vfs/regression.py | 13 +++++++------ vfs/tests/steps.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ontime_benchmark/benchmark.py b/ontime_benchmark/benchmark.py index 6387d8f61..c6489878f 100755 --- a/ontime_benchmark/benchmark.py +++ b/ontime_benchmark/benchmark.py @@ -16,8 +16,9 @@ ffails = { ":/queries/vfs": ( Skip, - "vfs not supported on < 24.2", - lambda test: check_clickhouse_version("<24.2")(test) and not test.context.allow_vfs, + "vfs not supported on < 24.2 and requires --allow-vfs flag", + lambda test: not test.context.allow_vfs + or check_clickhouse_version("<24.2")(test), ), } diff --git a/vfs/regression.py b/vfs/regression.py index 11112f910..71ee0fd81 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -22,9 +22,9 @@ ffails = { "*": ( Skip, - "vfs not supported on < 24.2", - lambda test: check_clickhouse_version("<24.2")(test) - and not test.context.allow_vfs, + "vfs not supported on < 24.2 and requires --allow-vfs flag", + lambda test: not test.context.allow_vfs + or check_clickhouse_version("<24.2")(test), ), ":/alter/move": (XFail, "Fix pending"), ":/replica/add remove one node": (XFail, "Fix pending"), @@ -123,9 +123,10 @@ def regression( """Disk Object Storage VFS regression.""" self.context.clickhouse_version = clickhouse_version - - if check_clickhouse_version("<24.2")(self) or not allow_vfs: - skip("vfs not supported on < 24.2") + note(f"Clickhouse version {clickhouse_version}") + if not allow_vfs or check_clickhouse_version("<24.2")(self): + skip("vfs not supported on < 24.2 and requires --allow-vfs flag") + return self.context.stress = stress self.context.allow_vfs = allow_vfs diff --git a/vfs/tests/steps.py b/vfs/tests/steps.py index ebc5cae19..20da328a2 100644 --- a/vfs/tests/steps.py +++ b/vfs/tests/steps.py @@ -269,7 +269,7 @@ def enable_vfs( """ if check_clickhouse_version("<24.1")(self): - skip("vfs not supported on ClickHouse < 24.1") + skip("vfs not supported on ClickHouse < 24.2 and requires --allow-vfs flag") if disk_names is None: disk_names = ["external"] From d239473308af9e89ae3b7c4dbad7daba23678b94 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:08:55 -0500 Subject: [PATCH 30/73] Remove --allow-vfs from vfs runner to fix suite always running --- .github/workflows/run-arm-regression.yml | 1 - .github/workflows/run-regression.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/run-arm-regression.yml b/.github/workflows/run-arm-regression.yml index d6163b9d0..128bcbc51 100644 --- a/.github/workflows/run-arm-regression.yml +++ b/.github/workflows/run-arm-regression.yml @@ -1539,7 +1539,6 @@ jobs: --storage minio --clickhouse-binary-path ${{ env.clickhouse_binary_path }} --clickhouse-version ${{ env.version }} - --allow-vfs ${{ env.args }} - name: Create and upload logs diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml index 6c6f4e54b..a58170d89 100644 --- a/.github/workflows/run-regression.yml +++ b/.github/workflows/run-regression.yml @@ -1539,7 +1539,6 @@ jobs: --storage minio --clickhouse-binary-path ${{ env.clickhouse_binary_path }} --clickhouse-version ${{ env.version }} - --allow-vfs ${{ env.args }} - name: Create and upload logs From c9691f56136c75cb6c68f8ff9fae2c31c15e8d62 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 16 Feb 2024 21:55:26 -0500 Subject: [PATCH 31/73] add multiple zookeeper insances for vfs --- vfs/configs/clickhouse/config.d/zookeeper.xml | 10 ++- vfs/regression.py | 6 +- vfs/tests/steps.py | 13 ++++ vfs/tests/stress_alter.py | 29 +++++---- vfs/tests/system.py | 33 ++++++++-- vfs/vfs_env/docker-compose.yml | 62 +++++++++++++++++-- 6 files changed, 129 insertions(+), 24 deletions(-) diff --git a/vfs/configs/clickhouse/config.d/zookeeper.xml b/vfs/configs/clickhouse/config.d/zookeeper.xml index 96270e7b6..617cfa4b4 100644 --- a/vfs/configs/clickhouse/config.d/zookeeper.xml +++ b/vfs/configs/clickhouse/config.d/zookeeper.xml @@ -2,7 +2,15 @@ - zookeeper + zookeeper1 + 2181 + + + zookeeper2 + 2181 + + + zookeeper3 2181 15000 diff --git a/vfs/regression.py b/vfs/regression.py index 71ee0fd81..0013960e3 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -47,7 +47,10 @@ def minio( collect_service_logs, ): """Setup and run minio tests.""" - nodes = {"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3")} + nodes = { + "zookeeper": ("zookeeper1", "zookeeper2", "zookeeper3"), + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } with Given("docker-compose cluster"): cluster = create_cluster( @@ -64,6 +67,7 @@ def minio( self.context.cluster = cluster self.context.node = self.context.cluster.node("clickhouse1") self.context.ch_nodes = [cluster.node(n) for n in cluster.nodes["clickhouse"]] + self.context.zk_nodes = [cluster.node(n) for n in cluster.nodes["zookeeper"]] self.context.access_key_id = root_user self.context.secret_access_key = root_password self.context.bucket_name = "root" diff --git a/vfs/tests/steps.py b/vfs/tests/steps.py index 20da328a2..e9a0d2d2c 100644 --- a/vfs/tests/steps.py +++ b/vfs/tests/steps.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import time +from contextlib import contextmanager from testflows.core import * from testflows.asserts import error @@ -386,3 +387,15 @@ def get_active_partition_ids(self, node, table_name): f"select partition_id from system.parts where table='{table_name}' and active=1 FORMAT JSONColumns" ) return json.loads(r.output)["partition_id"] + + +@contextmanager +def pause_node(node): + try: + with When(f"{node.name} is stopped"): + node.stop() + yield + + finally: + with When(f"{node.name} is started"): + node.start() diff --git a/vfs/tests/stress_alter.py b/vfs/tests/stress_alter.py index 138015fc1..825d7d5d6 100644 --- a/vfs/tests/stress_alter.py +++ b/vfs/tests/stress_alter.py @@ -73,7 +73,13 @@ def insert_to_random(self): columns = get_column_string(node=node, table_name=table_name) with By(f"inserting rows to {table_name} on {node.name}"): - insert_random(node=node, table_name=table_name, columns=columns, no_checks=True) + insert_random( + node=node, + table_name=table_name, + columns=columns, + no_checks=True, + settings=f"insert_keeper_fault_injection_probability={self.context.fault_probability}", + ) @TestStep @@ -212,6 +218,7 @@ def detach_attach_random_partition(self): @TestStep +@Retry(timeout=30, delay=5) @Name("freeze part") def freeze_unfreeze_random_part(self): """Freeze a random part, wait a random time, unfreeze part.""" @@ -551,25 +558,18 @@ def delete_replica(self): @TestStep def restart_keeper(self, repeat_limit=5): - keeper_node = self.context.cluster.node("zookeeper") + keeper_node = random.choice(self.context.zk_nodes) delay = random.random() * 2 + 1 - try: - with When("I stop zookeeper"): - keeper_node.stop() - - with And(f"I wait {delay:.2}s"): + with pause_node(keeper_node): + with When(f"I wait {delay:.2}s"): time.sleep(delay) - finally: - with Finally("I restart zookeeper"): - keeper_node.start() - @TestOutline def parallel_alters( self, - limit=50, + limit=10, shuffle=True, combination_size=3, run_groups_in_parallel=True, @@ -580,6 +580,7 @@ def parallel_alters( minimum_replicas=1, maximum_replicas=3, n_tables=3, + insert_keeper_fault_injection_probability=0, ): """ Perform combinations of alter actions, checking that all replicas agree. @@ -590,6 +591,7 @@ def parallel_alters( self.context.storage_policy = storage_policy self.context.minimum_replicas = minimum_replicas self.context.maximum_replicas = maximum_replicas + self.context.fault_probability = insert_keeper_fault_injection_probability with Given("I have a list of actions I can perform"): actions = [ @@ -698,8 +700,9 @@ def feature(self): run_optimize_in_parallel=True, ignore_failed_part_moves=False, sync_replica_timeout=600, - storage_policy="external_no_vfs", + storage_policy="external_vfs", minimum_replicas=1, maximum_replicas=3, n_tables=3, + insert_keeper_fault_injection_probability=0.1, ) diff --git a/vfs/tests/system.py b/vfs/tests/system.py index 3e36c3450..1b13355ba 100644 --- a/vfs/tests/system.py +++ b/vfs/tests/system.py @@ -7,10 +7,35 @@ from vfs.requirements import * -""" -RQ_SRS_038_DiskObjectStorageVFS_System_AddKeeper -RQ_SRS_038_DiskObjectStorageVFS_System_RemoveKeeper -""" +@TestScenario +@Requirements( + RQ_SRS_038_DiskObjectStorageVFS_System_AddKeeper("0.0"), + RQ_SRS_038_DiskObjectStorageVFS_System_RemoveKeeper("0.0"), +) +def pause_keeper(self): + nodes = self.context.ch_nodes + insert_rows = 100000 + fault_probability = 0 + + with pause_node(random.choice(self.context.zk_nodes)): + with Given("a table is created"): + _, table_name = replicated_table_cluster( + storage_policy="external_vfs", + ) + + with pause_node(random.choice(self.context.zk_nodes)): + with Given("data is inserted"): + insert_random( + node=random.choice(nodes), + table_name=table_name, + rows=insert_rows, + settings=f"insert_keeper_fault_injection_probability={fault_probability}", + ) + + with pause_node(random.choice(self.context.zk_nodes)): + with Then("I check that tables are consistent"): + for node in nodes: + assert_row_count(node=node, table_name=table_name, rows=insert_rows) @TestScenario diff --git a/vfs/vfs_env/docker-compose.yml b/vfs/vfs_env/docker-compose.yml index b26f45f96..f1799be9e 100644 --- a/vfs/vfs_env/docker-compose.yml +++ b/vfs/vfs_env/docker-compose.yml @@ -169,7 +169,55 @@ services: minio1: condition: service_healthy - zookeeper: + zookeeper1: + volumes: + - clickhouse1_jbod1:/clickhouse1_jbod1 + - clickhouse1_jbod2:/clickhouse1_jbod2 + - clickhouse1_jbod3:/clickhouse1_jbod3 + - clickhouse1_jbod4:/clickhouse1_jbod4 + - clickhouse1_external:/clickhouse1_external + - clickhouse1_external2:/clickhouse1_external2 + - clickhouse2_jbod1:/clickhouse2_jbod1 + - clickhouse2_jbod2:/clickhouse2_jbod2 + - clickhouse2_jbod3:/clickhouse2_jbod3 + - clickhouse2_jbod4:/clickhouse2_jbod4 + - clickhouse2_external:/clickhouse2_external + - clickhouse2_external2:/clickhouse2_external2 + - clickhouse3_jbod1:/clickhouse3_jbod1 + - clickhouse3_jbod2:/clickhouse3_jbod2 + - clickhouse3_jbod3:/clickhouse3_jbod3 + - clickhouse3_jbod4:/clickhouse3_jbod4 + - clickhouse3_external:/clickhouse3_external + - clickhouse3_external2:/clickhouse3_external2 + extends: + file: zookeeper-service.yml + service: zookeeper + + zookeeper2: + volumes: + - clickhouse1_jbod1:/clickhouse1_jbod1 + - clickhouse1_jbod2:/clickhouse1_jbod2 + - clickhouse1_jbod3:/clickhouse1_jbod3 + - clickhouse1_jbod4:/clickhouse1_jbod4 + - clickhouse1_external:/clickhouse1_external + - clickhouse1_external2:/clickhouse1_external2 + - clickhouse2_jbod1:/clickhouse2_jbod1 + - clickhouse2_jbod2:/clickhouse2_jbod2 + - clickhouse2_jbod3:/clickhouse2_jbod3 + - clickhouse2_jbod4:/clickhouse2_jbod4 + - clickhouse2_external:/clickhouse2_external + - clickhouse2_external2:/clickhouse2_external2 + - clickhouse3_jbod1:/clickhouse3_jbod1 + - clickhouse3_jbod2:/clickhouse3_jbod2 + - clickhouse3_jbod3:/clickhouse3_jbod3 + - clickhouse3_jbod4:/clickhouse3_jbod4 + - clickhouse3_external:/clickhouse3_external + - clickhouse3_external2:/clickhouse3_external2 + extends: + file: zookeeper-service.yml + service: zookeeper + + zookeeper3: volumes: - clickhouse1_jbod1:/clickhouse1_jbod1 - clickhouse1_jbod2:/clickhouse1_jbod2 @@ -209,7 +257,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper1: condition: service_healthy clickhouse2: @@ -228,7 +276,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper2: condition: service_healthy clickhouse3: @@ -247,7 +295,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper3: condition: service_healthy aws: @@ -266,7 +314,11 @@ services: condition: service_healthy clickhouse3: condition: service_healthy - zookeeper: + zookeeper1: + condition: service_healthy + zookeeper2: + condition: service_healthy + zookeeper3: condition: service_healthy minio1: condition: service_healthy From e29e4bc948e7c415d53a7e2138e258fb2fbb28c3 Mon Sep 17 00:00:00 2001 From: alsu Date: Mon, 19 Feb 2024 12:00:01 +0100 Subject: [PATCH 32/73] update to memory leak test --- memory/tests/test_memory_leak.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/memory/tests/test_memory_leak.py b/memory/tests/test_memory_leak.py index 9c13ff6b4..18e65f04b 100644 --- a/memory/tests/test_memory_leak.py +++ b/memory/tests/test_memory_leak.py @@ -148,10 +148,9 @@ def get_process_memory_usage_in_kilobytes(self, pid): @TestScenario def check_create_and_drop_tables( self, - number_of_workers=10, - number_of_tables=100000, + number_of_workers=20, + number_of_tables=15000, node=None, - max_memory_increase=1.1, timeout=3600, ): """Test memory leak when we create and drop many tables.""" @@ -177,7 +176,7 @@ def check_create_and_drop_tables( note(f"end resident memory: {end_memory.resident}") with Then("I check for memory leaks and calculate released memory"): - max_memory = start_memory.resident * max_memory_increase + max_memory = 220000 for attempt in retries(timeout=timeout, delay=10): with attempt: current_memory = get_process_memory_usage(pid=pid) @@ -187,7 +186,7 @@ def check_create_and_drop_tables( note(f"released memory: {released_memory}") assert ( current_memory.resident <= max_memory - ), f"Memory usage {current_memory.resident} larger than expected {max_memory} ({max_memory_increase}*{start_memory.resident})" + ), f"Memory usage {current_memory.resident} larger than expected {max_memory}" @TestFeature From 10e8f3821429e043054e0f77b87ce661865f8f7b Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:59:32 -0500 Subject: [PATCH 33/73] Add multiple zookeepers to vfs arm config --- vfs/vfs_env_arm64/docker-compose.yml | 62 +++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/vfs/vfs_env_arm64/docker-compose.yml b/vfs/vfs_env_arm64/docker-compose.yml index b26f45f96..f1799be9e 100644 --- a/vfs/vfs_env_arm64/docker-compose.yml +++ b/vfs/vfs_env_arm64/docker-compose.yml @@ -169,7 +169,55 @@ services: minio1: condition: service_healthy - zookeeper: + zookeeper1: + volumes: + - clickhouse1_jbod1:/clickhouse1_jbod1 + - clickhouse1_jbod2:/clickhouse1_jbod2 + - clickhouse1_jbod3:/clickhouse1_jbod3 + - clickhouse1_jbod4:/clickhouse1_jbod4 + - clickhouse1_external:/clickhouse1_external + - clickhouse1_external2:/clickhouse1_external2 + - clickhouse2_jbod1:/clickhouse2_jbod1 + - clickhouse2_jbod2:/clickhouse2_jbod2 + - clickhouse2_jbod3:/clickhouse2_jbod3 + - clickhouse2_jbod4:/clickhouse2_jbod4 + - clickhouse2_external:/clickhouse2_external + - clickhouse2_external2:/clickhouse2_external2 + - clickhouse3_jbod1:/clickhouse3_jbod1 + - clickhouse3_jbod2:/clickhouse3_jbod2 + - clickhouse3_jbod3:/clickhouse3_jbod3 + - clickhouse3_jbod4:/clickhouse3_jbod4 + - clickhouse3_external:/clickhouse3_external + - clickhouse3_external2:/clickhouse3_external2 + extends: + file: zookeeper-service.yml + service: zookeeper + + zookeeper2: + volumes: + - clickhouse1_jbod1:/clickhouse1_jbod1 + - clickhouse1_jbod2:/clickhouse1_jbod2 + - clickhouse1_jbod3:/clickhouse1_jbod3 + - clickhouse1_jbod4:/clickhouse1_jbod4 + - clickhouse1_external:/clickhouse1_external + - clickhouse1_external2:/clickhouse1_external2 + - clickhouse2_jbod1:/clickhouse2_jbod1 + - clickhouse2_jbod2:/clickhouse2_jbod2 + - clickhouse2_jbod3:/clickhouse2_jbod3 + - clickhouse2_jbod4:/clickhouse2_jbod4 + - clickhouse2_external:/clickhouse2_external + - clickhouse2_external2:/clickhouse2_external2 + - clickhouse3_jbod1:/clickhouse3_jbod1 + - clickhouse3_jbod2:/clickhouse3_jbod2 + - clickhouse3_jbod3:/clickhouse3_jbod3 + - clickhouse3_jbod4:/clickhouse3_jbod4 + - clickhouse3_external:/clickhouse3_external + - clickhouse3_external2:/clickhouse3_external2 + extends: + file: zookeeper-service.yml + service: zookeeper + + zookeeper3: volumes: - clickhouse1_jbod1:/clickhouse1_jbod1 - clickhouse1_jbod2:/clickhouse1_jbod2 @@ -209,7 +257,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper1: condition: service_healthy clickhouse2: @@ -228,7 +276,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper2: condition: service_healthy clickhouse3: @@ -247,7 +295,7 @@ services: - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" depends_on: - zookeeper: + zookeeper3: condition: service_healthy aws: @@ -266,7 +314,11 @@ services: condition: service_healthy clickhouse3: condition: service_healthy - zookeeper: + zookeeper1: + condition: service_healthy + zookeeper2: + condition: service_healthy + zookeeper3: condition: service_healthy minio1: condition: service_healthy From d18da31b4a25bd61796eb525fff39ca4030f3cc9 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:00:17 -0500 Subject: [PATCH 34/73] Vfs comment out unfinished parallel replica again --- vfs/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vfs/regression.py b/vfs/regression.py index 0013960e3..392169577 100755 --- a/vfs/regression.py +++ b/vfs/regression.py @@ -88,7 +88,7 @@ def minio( Feature(run=load("vfs.tests.alter", "feature")) Feature(run=load("vfs.tests.replica", "feature")) - Feature(run=load("vfs.tests.parallel_replica", "feature")) + # Feature(run=load("vfs.tests.parallel_replica", "feature")) Feature(run=load("vfs.tests.create_insert", "feature")) Feature(run=load("vfs.tests.performance", "feature")) From d73f9b59e4d67061a1dc19d8b5989ea5fed8efcf Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:01:12 -0500 Subject: [PATCH 35/73] vfs fix clickhouse config to point at the right zookeeper file --- vfs/configs/clickhouse/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vfs/configs/clickhouse/config.xml b/vfs/configs/clickhouse/config.xml index 4f5b13fd5..b50b48fe2 100644 --- a/vfs/configs/clickhouse/config.xml +++ b/vfs/configs/clickhouse/config.xml @@ -232,7 +232,7 @@ See https://clickhouse.com/docs/en/table_engines/replication/ --> - +