From 12d0addb51a01492b5a4197597e487701b3c6878 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 24 Oct 2024 01:26:53 +0530 Subject: [PATCH 01/10] fix: SQL COUNT with GROUP BY to prevent incorrect row returns --- .../dql/groupby/GroupByMemoryMergedResult.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java index 6c456bfca7b63..fb24c0020aac1 100644 --- a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java +++ b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java @@ -143,22 +143,10 @@ private boolean getValueCaseSensitiveFromTables(final QueryResult queryResult, private List getMemoryResultSetRows(final SelectStatementContext selectStatementContext, final Map dataMap, final List valueCaseSensitive) { if (dataMap.isEmpty()) { - Object[] data = generateReturnData(selectStatementContext); - return Arrays.stream(data).anyMatch(Objects::nonNull) ? Collections.singletonList(new MemoryQueryResultRow(data)) : Collections.emptyList(); + return Collections.emptyList(); } List result = new ArrayList<>(dataMap.values()); result.sort(new GroupByRowComparator(selectStatementContext, valueCaseSensitive)); return result; } - - private Object[] generateReturnData(final SelectStatementContext selectStatementContext) { - List projections = new LinkedList<>(selectStatementContext.getProjectionsContext().getExpandProjections()); - Object[] result = new Object[projections.size()]; - for (int i = 0; i < projections.size(); i++) { - if (projections.get(i) instanceof AggregationProjection && AggregationType.COUNT == ((AggregationProjection) projections.get(i)).getType()) { - result[i] = 0; - } - } - return result; - } } From 42ff1976a36fa5a64f238751fd0a1387b2a802d3 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 24 Oct 2024 16:25:08 +0530 Subject: [PATCH 02/10] test: Add test cases for empty result with GROUP BY and ORDER BY --- .../groupby/GroupByMemoryMergedResult.java | 9 +-- .../GroupByMemoryMergedResultTest.java | 55 ++++++++++++++++++- .../cases/dql/e2e-dql-select-group-by.xml | 5 ++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java index fb24c0020aac1..f928c611dc0c4 100644 --- a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java +++ b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java @@ -17,35 +17,30 @@ package org.apache.shardingsphere.sharding.merge.dql.groupby; -import org.apache.shardingsphere.infra.exception.dialect.exception.syntax.table.NoSuchTableException; -import org.apache.shardingsphere.infra.binder.context.segment.select.projection.Projection; import org.apache.shardingsphere.infra.binder.context.segment.select.projection.impl.AggregationDistinctProjection; import org.apache.shardingsphere.infra.binder.context.segment.select.projection.impl.AggregationProjection; import org.apache.shardingsphere.infra.binder.context.statement.SQLStatementContext; import org.apache.shardingsphere.infra.binder.context.statement.dml.SelectStatementContext; +import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; +import org.apache.shardingsphere.infra.exception.dialect.exception.syntax.table.NoSuchTableException; import org.apache.shardingsphere.infra.executor.sql.execute.result.query.QueryResult; import org.apache.shardingsphere.infra.merge.result.impl.memory.MemoryMergedResult; import org.apache.shardingsphere.infra.merge.result.impl.memory.MemoryQueryResultRow; import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereSchema; import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereTable; -import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; import org.apache.shardingsphere.sharding.exception.data.NotImplementComparableValueException; import org.apache.shardingsphere.sharding.merge.dql.groupby.aggregation.AggregationUnit; import org.apache.shardingsphere.sharding.merge.dql.groupby.aggregation.AggregationUnitFactory; import org.apache.shardingsphere.sharding.rule.ShardingRule; -import org.apache.shardingsphere.sql.parser.statement.core.enums.AggregationType; import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; diff --git a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java index 0692069ad9a1f..5aad2b216c70c 100644 --- a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java +++ b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java @@ -20,6 +20,7 @@ import org.apache.shardingsphere.infra.binder.context.statement.dml.SelectStatementContext; import org.apache.shardingsphere.infra.config.props.ConfigurationProperties; import org.apache.shardingsphere.infra.database.core.DefaultDatabase; +import org.apache.shardingsphere.infra.database.core.metadata.database.enums.NullsOrderType; import org.apache.shardingsphere.infra.database.core.type.DatabaseType; import org.apache.shardingsphere.infra.executor.sql.execute.result.query.QueryResult; import org.apache.shardingsphere.infra.merge.result.MergedResult; @@ -33,7 +34,6 @@ import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.apache.shardingsphere.sharding.merge.dql.ShardingDQLResultMerger; import org.apache.shardingsphere.sql.parser.statement.core.enums.AggregationType; -import org.apache.shardingsphere.infra.database.core.metadata.database.enums.NullsOrderType; import org.apache.shardingsphere.sql.parser.statement.core.enums.OrderDirection; import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.AggregationProjectionSegment; import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionsSegment; @@ -217,4 +217,57 @@ void assertNextForDistinctShorthandResultSetsEmpty() throws SQLException { MergedResult actual = merger.merge(Arrays.asList(queryResult, queryResult, queryResult), createSelectStatementContext(database), database, mock(ConnectionContext.class)); assertFalse(actual.next()); } + + @Test + void assertNextForEmptyResultWithCountAndGroupBy() throws SQLException { + when(database.getName()).thenReturn("db_schema"); + QueryResult queryResult1 = createEmptyQueryResultWithCountGroupBy(); + QueryResult queryResult2 = createEmptyQueryResultWithCountGroupBy(); + ShardingDQLResultMerger resultMerger = new ShardingDQLResultMerger(TypedSPILoader.getService(DatabaseType.class, "MySQL")); + MergedResult actual = resultMerger.merge(Arrays.asList(queryResult1, queryResult2), createSelectStatementContextForCountGroupBy(), database, mock(ConnectionContext.class)); + assertFalse(actual.next()); + } + + @Test + void assertNextForEmptyResultWithCountGroupByDifferentOrderBy() throws SQLException { + when(database.getName()).thenReturn("db_schema"); + QueryResult queryResult = createEmptyQueryResultWithCountGroupBy(); + ShardingDQLResultMerger resultMerger = new ShardingDQLResultMerger(TypedSPILoader.getService(DatabaseType.class, "MySQL")); + MergedResult actual = resultMerger.merge(Collections.singletonList(queryResult), createSelectStatementContextForCountGroupByDifferentOrderBy(), database, mock(ConnectionContext.class)); + assertFalse(actual.next()); + } + + private QueryResult createEmptyQueryResultWithCountGroupBy() throws SQLException { + QueryResult result = mock(QueryResult.class, RETURNS_DEEP_STUBS); + when(result.getMetaData().getColumnCount()).thenReturn(3); + when(result.getMetaData().getColumnLabel(1)).thenReturn("COUNT(*)"); + when(result.getMetaData().getColumnLabel(2)).thenReturn("user_id"); + when(result.getMetaData().getColumnLabel(3)).thenReturn("order_id"); + when(result.next()).thenReturn(false); + return result; + } + + private SelectStatementContext createSelectStatementContextForCountGroupBy() { + SelectStatement selectStatement = new MySQLSelectStatement(); + ProjectionsSegment projectionsSegment = new ProjectionsSegment(0, 0); + projectionsSegment.getProjections().add(new AggregationProjectionSegment(0, 0, AggregationType.COUNT, "COUNT(*)")); + selectStatement.setGroupBy(new GroupBySegment(0, 0, Collections.singletonList(new IndexOrderByItemSegment(0, 0, 2, OrderDirection.ASC, NullsOrderType.FIRST)))); + selectStatement.setOrderBy(new OrderBySegment(0, 0, Collections.singletonList(new IndexOrderByItemSegment(0, 0, 2, OrderDirection.ASC, NullsOrderType.FIRST)))); + selectStatement.setProjections(projectionsSegment); + ShardingSphereDatabase database = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); + when(database.getSchema(DefaultDatabase.LOGIC_NAME)).thenReturn(mock(ShardingSphereSchema.class)); + return new SelectStatementContext(createShardingSphereMetaData(database), Collections.emptyList(), selectStatement, DefaultDatabase.LOGIC_NAME, Collections.emptyList()); + } + + private SelectStatementContext createSelectStatementContextForCountGroupByDifferentOrderBy() { + SelectStatement selectStatement = new MySQLSelectStatement(); + ProjectionsSegment projectionsSegment = new ProjectionsSegment(0, 0); + projectionsSegment.getProjections().add(new AggregationProjectionSegment(0, 0, AggregationType.COUNT, "COUNT(*)")); + selectStatement.setGroupBy(new GroupBySegment(0, 0, Collections.singletonList(new IndexOrderByItemSegment(0, 0, 2, OrderDirection.ASC, NullsOrderType.FIRST)))); + selectStatement.setOrderBy(new OrderBySegment(0, 0, Collections.singletonList(new IndexOrderByItemSegment(0, 0, 3, OrderDirection.ASC, NullsOrderType.FIRST)))); + selectStatement.setProjections(projectionsSegment); + ShardingSphereDatabase database = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS); + when(database.getSchema(DefaultDatabase.LOGIC_NAME)).thenReturn(mock(ShardingSphereSchema.class)); + return new SelectStatementContext(createShardingSphereMetaData(database), Collections.emptyList(), selectStatement, DefaultDatabase.LOGIC_NAME, Collections.emptyList()); + } } diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index aa525aeb344f5..683f6e811a6be 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -52,4 +52,9 @@ scenario-comments="Test single table's LIKE operator underscore wildcard in select group by statement when use sharding feature.|Test encrypt table's LIKE operator underscore wildcard in select group by statement when use encrypt feature."> + + + + From 759053cf55fb3077195174cdc5d7d4691b9c8fbd Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 24 Oct 2024 17:55:26 +0530 Subject: [PATCH 03/10] fix: update db types and scenario type for e2e test case --- .../src/test/resources/cases/dql/e2e-dql-select-group-by.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index 683f6e811a6be..76c8fac6e03d1 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -53,7 +53,7 @@ - From 8a931ab57913608780b1416a52a6b71f50519980 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 09:25:42 +0530 Subject: [PATCH 04/10] fix: update column names for e2e test --- .asf.yaml | 16 +- RELEASE-NOTES.md | 2 + .../preconditions/AgentPreconditionsTest.java | 3 +- .../core}/yaml/AgentYamlEngineTest.java | 3 +- .../EncryptIndexColumnTokenGenerator.java | 3 +- .../EncryptBinaryConditionTest.java | 3 +- ...ShardingTableRuleStatementCheckerTest.java | 3 +- ...ardingTableRuleStatementConverterTest.java | 3 +- .../data/model/TableMetaDataTest.java | 0 .../context/ExecutionContextBuilderTest.java | 13 - .../infra/route/context/RouteUnit.java | 10 +- .../infra/route/context/RouteUnitTest.java | 11 + .../InventoryColumnValueReaderEngine.java | 6 +- .../pipeline/core/task/IncrementalTask.java | 9 +- .../pipeline/core/task/InventoryTask.java | 4 - .../data/pipeline/core/task/PipelineTask.java | 8 +- .../core/task/TaskExecuteCallback.java | 2 - .../task/runner/TransmissionTasksRunner.java | 11 +- .../QPSJobRateLimitAlgorithmTest.java | 3 +- .../TPSJobRateLimitAlgorithmTest.java | 3 +- .../task/IncrementalTaskAckCallbackTest.java | 67 ++++ .../cdc/core/task/CDCIncrementalTask.java | 4 - .../cdc/core/task/CDCInventoryTask.java | 4 - .../cdc/core/task/CDCTasksRunner.java | 5 +- .../DistSQLQueryBackendHandlerTest.java | 3 +- .../DistSQLUpdateBackendHandlerTest.java | 3 +- .../cases/dql/e2e-dql-select-group-by.xml | 2 +- .../InventoryColumnValueReaderEngineTest.java | 320 ++++++++++++++++-- .../pipeline/core/task/InventoryTaskTest.java | 1 - 29 files changed, 411 insertions(+), 114 deletions(-) rename agent/core/src/test/java/org/apache/shardingsphere/{ => agent/core}/preconditions/AgentPreconditionsTest.java (93%) rename agent/core/src/test/java/org/apache/shardingsphere/{ => agent/core}/yaml/AgentYamlEngineTest.java (98%) rename features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/{ => impl}/EncryptBinaryConditionTest.java (90%) rename features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/{ => handler}/checker/ShardingTableRuleStatementCheckerTest.java (99%) rename features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/{ => handler}/converter/ShardingTableRuleStatementConverterTest.java (98%) rename infra/{common => database/core}/src/test/java/org/apache/shardingsphere/infra/database/core/metadata/data/model/TableMetaDataTest.java (100%) rename kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/{ => type}/QPSJobRateLimitAlgorithmTest.java (94%) rename kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/{ => type}/TPSJobRateLimitAlgorithmTest.java (94%) create mode 100644 kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTaskAckCallbackTest.java rename proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/{ral => }/DistSQLQueryBackendHandlerTest.java (97%) rename proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/{ral/updatable => }/DistSQLUpdateBackendHandlerTest.java (96%) diff --git a/.asf.yaml b/.asf.yaml index 2bcdaf1d37e4b..fdcaf2c2e42f7 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -22,24 +22,24 @@ notifications: jobs: notifications@shardingsphere.apache.org github: - description: Distributed SQL transaction & query engine for data sharding, scaling, encryption, and more - on any database. + description: Empowering Data Intelligence with Distributed SQL for Sharding, Scalability, and Security Across All Databases. labels: - - database-plus - database + - database-cluster - distributed-database - distributed-sql-database - - distributed-transactions - - database-cluster - - rdbms + - distributed-transaction + - database-gateway + - database-middleware - sql - mysql - postgresql - shard + - read-write-splitting - encrypt - - dba - - oltp + - data-encryption + - data-pipeline - bigdata - - hacktoberfest features: issues: true projects: true diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index cf217ef3f7682..7fa6c77e94c8a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -11,6 +11,8 @@ 1. Proxy: Add query parameters and check for mysql kill processId - [#33274](https://github.com/apache/shardingsphere/pull/33274) 1. SQL Parser: Support parsing Doris INSTR - [#33289](https://github.com/apache/shardingsphere/pull/33289) 1. Agent: Simplify the use of Agent's Docker Image - [#33356](https://github.com/apache/shardingsphere/pull/33356) +1. Add arguments not null check when creating RouteUnit - [#33382](https://github.com/apache/shardingsphere/pull/33382) +1. Add index columns not empty judgement for IndexColumnTokenGenerator - [#33384](https://github.com/apache/shardingsphere/pull/33384) ### Bug Fixes diff --git a/agent/core/src/test/java/org/apache/shardingsphere/preconditions/AgentPreconditionsTest.java b/agent/core/src/test/java/org/apache/shardingsphere/agent/core/preconditions/AgentPreconditionsTest.java similarity index 93% rename from agent/core/src/test/java/org/apache/shardingsphere/preconditions/AgentPreconditionsTest.java rename to agent/core/src/test/java/org/apache/shardingsphere/agent/core/preconditions/AgentPreconditionsTest.java index b1b665e6e5a5e..0d8f79e34c53c 100644 --- a/agent/core/src/test/java/org/apache/shardingsphere/preconditions/AgentPreconditionsTest.java +++ b/agent/core/src/test/java/org/apache/shardingsphere/agent/core/preconditions/AgentPreconditionsTest.java @@ -15,9 +15,8 @@ * limitations under the License. */ -package org.apache.shardingsphere.preconditions; +package org.apache.shardingsphere.agent.core.preconditions; -import org.apache.shardingsphere.agent.core.preconditions.AgentPreconditions; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; diff --git a/agent/core/src/test/java/org/apache/shardingsphere/yaml/AgentYamlEngineTest.java b/agent/core/src/test/java/org/apache/shardingsphere/agent/core/yaml/AgentYamlEngineTest.java similarity index 98% rename from agent/core/src/test/java/org/apache/shardingsphere/yaml/AgentYamlEngineTest.java rename to agent/core/src/test/java/org/apache/shardingsphere/agent/core/yaml/AgentYamlEngineTest.java index 42e911fc420b0..77834f2e28433 100644 --- a/agent/core/src/test/java/org/apache/shardingsphere/yaml/AgentYamlEngineTest.java +++ b/agent/core/src/test/java/org/apache/shardingsphere/agent/core/yaml/AgentYamlEngineTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.yaml; +package org.apache.shardingsphere.agent.core.yaml; import org.apache.shardingsphere.agent.api.PluginConfiguration; import org.apache.shardingsphere.agent.core.advisor.config.yaml.entity.YamlAdvisorConfiguration; @@ -26,7 +26,6 @@ import org.apache.shardingsphere.agent.core.advisor.config.yaml.fixture.YamlTargetObjectFixture; import org.apache.shardingsphere.agent.core.plugin.config.yaml.entity.YamlAgentConfiguration; import org.apache.shardingsphere.agent.core.plugin.config.yaml.swapper.YamlPluginsConfigurationSwapper; -import org.apache.shardingsphere.agent.core.yaml.AgentYamlEngine; import org.junit.jupiter.api.Test; import java.io.File; diff --git a/features/encrypt/core/src/main/java/org/apache/shardingsphere/encrypt/rewrite/token/generator/select/EncryptIndexColumnTokenGenerator.java b/features/encrypt/core/src/main/java/org/apache/shardingsphere/encrypt/rewrite/token/generator/select/EncryptIndexColumnTokenGenerator.java index d6d07f01a7666..17c6e230b1c76 100644 --- a/features/encrypt/core/src/main/java/org/apache/shardingsphere/encrypt/rewrite/token/generator/select/EncryptIndexColumnTokenGenerator.java +++ b/features/encrypt/core/src/main/java/org/apache/shardingsphere/encrypt/rewrite/token/generator/select/EncryptIndexColumnTokenGenerator.java @@ -56,7 +56,8 @@ public final class EncryptIndexColumnTokenGenerator implements CollectionSQLToke @Override public boolean isGenerateSQLToken(final SQLStatementContext sqlStatementContext) { - return sqlStatementContext instanceof IndexAvailable && sqlStatementContext instanceof TableAvailable && !((TableAvailable) sqlStatementContext).getTablesContext().getTableNames().isEmpty(); + return sqlStatementContext instanceof IndexAvailable && sqlStatementContext instanceof TableAvailable && !((TableAvailable) sqlStatementContext).getTablesContext().getTableNames().isEmpty() + && !((IndexAvailable) sqlStatementContext).getIndexColumns().isEmpty(); } @Override diff --git a/features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/EncryptBinaryConditionTest.java b/features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/impl/EncryptBinaryConditionTest.java similarity index 90% rename from features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/EncryptBinaryConditionTest.java rename to features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/impl/EncryptBinaryConditionTest.java index b9e7b5caef14a..7376cda197549 100644 --- a/features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/EncryptBinaryConditionTest.java +++ b/features/encrypt/core/src/test/java/org/apache/shardingsphere/encrypt/rewrite/condition/impl/EncryptBinaryConditionTest.java @@ -15,9 +15,8 @@ * limitations under the License. */ -package org.apache.shardingsphere.encrypt.rewrite.condition; +package org.apache.shardingsphere.encrypt.rewrite.condition.impl; -import org.apache.shardingsphere.encrypt.rewrite.condition.impl.EncryptBinaryCondition; import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.simple.LiteralExpressionSegment; import org.junit.jupiter.api.Test; diff --git a/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/checker/ShardingTableRuleStatementCheckerTest.java b/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/checker/ShardingTableRuleStatementCheckerTest.java similarity index 99% rename from features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/checker/ShardingTableRuleStatementCheckerTest.java rename to features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/checker/ShardingTableRuleStatementCheckerTest.java index 14d5ec98a4e32..f41d1e0a6d997 100644 --- a/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/checker/ShardingTableRuleStatementCheckerTest.java +++ b/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/checker/ShardingTableRuleStatementCheckerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.sharding.distsql.checker; +package org.apache.shardingsphere.sharding.distsql.handler.checker; import org.apache.shardingsphere.distsql.segment.AlgorithmSegment; import org.apache.shardingsphere.infra.algorithm.core.config.AlgorithmConfiguration; @@ -35,7 +35,6 @@ import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableRuleConfiguration; import org.apache.shardingsphere.sharding.api.config.strategy.keygen.KeyGenerateStrategyConfiguration; import org.apache.shardingsphere.sharding.api.config.strategy.sharding.StandardShardingStrategyConfiguration; -import org.apache.shardingsphere.sharding.distsql.handler.checker.ShardingTableRuleStatementChecker; import org.apache.shardingsphere.sharding.distsql.segment.strategy.AuditStrategySegment; import org.apache.shardingsphere.sharding.distsql.segment.strategy.KeyGenerateStrategySegment; import org.apache.shardingsphere.sharding.distsql.segment.strategy.ShardingAuditorSegment; diff --git a/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/converter/ShardingTableRuleStatementConverterTest.java b/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/converter/ShardingTableRuleStatementConverterTest.java similarity index 98% rename from features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/converter/ShardingTableRuleStatementConverterTest.java rename to features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/converter/ShardingTableRuleStatementConverterTest.java index 9b0147bbabcfa..07984d6022d64 100644 --- a/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/converter/ShardingTableRuleStatementConverterTest.java +++ b/features/sharding/distsql/handler/src/test/java/org/apache/shardingsphere/sharding/distsql/handler/converter/ShardingTableRuleStatementConverterTest.java @@ -15,13 +15,12 @@ * limitations under the License. */ -package org.apache.shardingsphere.sharding.distsql.converter; +package org.apache.shardingsphere.sharding.distsql.handler.converter; import org.apache.shardingsphere.distsql.segment.AlgorithmSegment; import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration; import org.apache.shardingsphere.sharding.api.config.rule.ShardingAutoTableRuleConfiguration; import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableRuleConfiguration; -import org.apache.shardingsphere.sharding.distsql.handler.converter.ShardingTableRuleStatementConverter; import org.apache.shardingsphere.sharding.distsql.segment.strategy.AuditStrategySegment; import org.apache.shardingsphere.sharding.distsql.segment.strategy.KeyGenerateStrategySegment; import org.apache.shardingsphere.sharding.distsql.segment.strategy.ShardingAuditorSegment; diff --git a/infra/common/src/test/java/org/apache/shardingsphere/infra/database/core/metadata/data/model/TableMetaDataTest.java b/infra/database/core/src/test/java/org/apache/shardingsphere/infra/database/core/metadata/data/model/TableMetaDataTest.java similarity index 100% rename from infra/common/src/test/java/org/apache/shardingsphere/infra/database/core/metadata/data/model/TableMetaDataTest.java rename to infra/database/core/src/test/java/org/apache/shardingsphere/infra/database/core/metadata/data/model/TableMetaDataTest.java diff --git a/infra/executor/src/test/java/org/apache/shardingsphere/infra/executor/sql/context/ExecutionContextBuilderTest.java b/infra/executor/src/test/java/org/apache/shardingsphere/infra/executor/sql/context/ExecutionContextBuilderTest.java index cf0870dfe6b8b..ca4ff31c6039e 100644 --- a/infra/executor/src/test/java/org/apache/shardingsphere/infra/executor/sql/context/ExecutionContextBuilderTest.java +++ b/infra/executor/src/test/java/org/apache/shardingsphere/infra/executor/sql/context/ExecutionContextBuilderTest.java @@ -89,19 +89,6 @@ void assertBuildGenericSQLRewriteResultWithTableAvailableSQLStatement() { assertThat(actual, is(expected)); } - @Test - void assertBuildRouteSQLRewriteResultWithNullTableMappers() { - RouteUnit routeUnit = new RouteUnit(new RouteMapper("foo_db", "actual_db"), null); - SQLRewriteUnit sqlRewriteUnit = new SQLRewriteUnit("sql", Collections.singletonList("parameter")); - Map sqlRewriteUnits = Collections.singletonMap(routeUnit, sqlRewriteUnit); - ResourceMetaData resourceMetaData = new ResourceMetaData(Collections.emptyMap()); - RuleMetaData ruleMetaData = new RuleMetaData(Collections.emptyList()); - ShardingSphereDatabase database = new ShardingSphereDatabase(DefaultDatabase.LOGIC_NAME, mock(DatabaseType.class), resourceMetaData, ruleMetaData, buildDatabase()); - Collection actual = ExecutionContextBuilder.build(database, new RouteSQLRewriteResult(sqlRewriteUnits), mock(SQLStatementContext.class)); - ExecutionUnit expectedUnit = new ExecutionUnit("actual_db", new SQLUnit("sql", Collections.singletonList("parameter"))); - assertThat(actual, is(Collections.singleton(expectedUnit))); - } - @Test void assertBuildRouteSQLRewriteResult() { RouteUnit routeUnit1 = new RouteUnit(new RouteMapper("foo_db_1", "actual_db_1"), Collections.singletonList(new RouteMapper("foo_tbl", "actual_tbl"))); diff --git a/infra/route/src/main/java/org/apache/shardingsphere/infra/route/context/RouteUnit.java b/infra/route/src/main/java/org/apache/shardingsphere/infra/route/context/RouteUnit.java index f16d01fe8ec47..ed8699177037b 100644 --- a/infra/route/src/main/java/org/apache/shardingsphere/infra/route/context/RouteUnit.java +++ b/infra/route/src/main/java/org/apache/shardingsphere/infra/route/context/RouteUnit.java @@ -19,8 +19,8 @@ import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; import java.util.Collection; import java.util.HashSet; @@ -31,7 +31,6 @@ /** * Route unit. */ -@RequiredArgsConstructor @Getter @EqualsAndHashCode @ToString @@ -41,6 +40,13 @@ public final class RouteUnit { private final Collection tableMappers; + public RouteUnit(final RouteMapper dataSourceMapper, final Collection tableMappers) { + ShardingSpherePreconditions.checkNotNull(dataSourceMapper, () -> new IllegalArgumentException("`dataSourceMapper` is required")); + ShardingSpherePreconditions.checkNotNull(tableMappers, () -> new IllegalArgumentException("`tableMappers` is required")); + this.dataSourceMapper = dataSourceMapper; + this.tableMappers = tableMappers; + } + /** * Get logic table names. * diff --git a/infra/route/src/test/java/org/apache/shardingsphere/infra/route/context/RouteUnitTest.java b/infra/route/src/test/java/org/apache/shardingsphere/infra/route/context/RouteUnitTest.java index 0db1557137724..397fbe4226615 100644 --- a/infra/route/src/test/java/org/apache/shardingsphere/infra/route/context/RouteUnitTest.java +++ b/infra/route/src/test/java/org/apache/shardingsphere/infra/route/context/RouteUnitTest.java @@ -26,6 +26,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; class RouteUnitTest { @@ -70,4 +71,14 @@ void assertFindTableMapper() { void assertTableMapperNotFound() { assertFalse(routeUnit.findTableMapper("invalid_ds", "invalid_tbl").isPresent()); } + + @Test + void assertTableMapperIsNull() { + assertThrows(IllegalArgumentException.class, () -> new RouteUnit(new RouteMapper(LOGIC_DATA_SOURCE, ACTUAL_DATA_SOURCE), null)); + } + + @Test + void assertDataSourceMapperIsNull() { + assertThrows(IllegalArgumentException.class, () -> new RouteUnit(null, Arrays.asList(new RouteMapper(LOGIC_TABLE, ACTUAL_TABLE_0), new RouteMapper(LOGIC_TABLE, ACTUAL_TABLE_1)))); + } } diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngine.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngine.java index 338ffb9450f31..3387066793c55 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngine.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngine.java @@ -112,15 +112,15 @@ private Object readStandardValue(final ResultSet resultSet, final ResultSetMetaD case Types.VARBINARY: case Types.LONGVARBINARY: return resultSet.getBytes(columnIndex); + case Types.BLOB: + Blob blob = resultSet.getBlob(columnIndex); + return null == blob ? null : blob.getBytes(1L, (int) blob.length()); case Types.CLOB: Clob clob = resultSet.getClob(columnIndex); return null == clob ? null : clob.getSubString(1L, (int) clob.length()); case Types.NCLOB: NClob nClob = resultSet.getNClob(columnIndex); return null == nClob ? null : nClob.getSubString(1L, (int) nClob.length()); - case Types.BLOB: - Blob blob = resultSet.getBlob(columnIndex); - return null == blob ? null : blob.getBytes(1L, (int) blob.length()); case Types.ARRAY: return resultSet.getArray(columnIndex); default: diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTask.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTask.java index 1fe7258f6a625..2d736760f22af 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTask.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTask.java @@ -21,6 +21,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.apache.shardingsphere.data.pipeline.core.execute.PipelineExecuteEngine; +import org.apache.shardingsphere.data.pipeline.core.execute.PipelineLifecycleRunnable; import org.apache.shardingsphere.data.pipeline.core.importer.Importer; import org.apache.shardingsphere.data.pipeline.core.ingest.dumper.Dumper; import org.apache.shardingsphere.data.pipeline.core.task.progress.IncrementalTaskProgress; @@ -62,12 +63,6 @@ public Collection> start() { @Override public void stop() { dumper.stop(); - for (Importer each : importers) { - each.stop(); - } - } - - @Override - public void close() { + importers.forEach(PipelineLifecycleRunnable::stop); } } diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTask.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTask.java index 613e47b69e4fe..8f177233abe1d 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTask.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTask.java @@ -71,8 +71,4 @@ public void stop() { public InventoryTaskProgress getTaskProgress() { return new InventoryTaskProgress(position.get()); } - - @Override - public void close() { - } } diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/PipelineTask.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/PipelineTask.java index 8438044489d50..1a1d09259db8e 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/PipelineTask.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/PipelineTask.java @@ -19,14 +19,13 @@ import org.apache.shardingsphere.data.pipeline.core.task.progress.TaskProgress; -import java.io.Closeable; import java.util.Collection; import java.util.concurrent.CompletableFuture; /** * Pipeline task interface. */ -public interface PipelineTask extends Closeable { +public interface PipelineTask { /** * Start task. @@ -53,9 +52,4 @@ public interface PipelineTask extends Closeable { * @return task progress */ TaskProgress getTaskProgress(); - - /** - * Close. - */ - void close(); } diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/TaskExecuteCallback.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/TaskExecuteCallback.java index 280c7345f10be..0de96b8b09851 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/TaskExecuteCallback.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/TaskExecuteCallback.java @@ -19,7 +19,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; import org.apache.shardingsphere.data.pipeline.core.execute.ExecuteCallback; /** @@ -39,6 +38,5 @@ public void onSuccess() { public void onFailure(final Throwable throwable) { log.error("onFailure, task ID={}", task.getTaskId(), throwable); task.stop(); - IOUtils.closeQuietly(task); } } diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/runner/TransmissionTasksRunner.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/runner/TransmissionTasksRunner.java index 062946f7e6052..48ba7523af37b 100644 --- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/runner/TransmissionTasksRunner.java +++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/task/runner/TransmissionTasksRunner.java @@ -35,7 +35,6 @@ import org.apache.shardingsphere.data.pipeline.core.task.PipelineTask; import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; -import org.apache.shardingsphere.infra.util.close.QuietlyCloser; import java.util.Collection; import java.util.LinkedList; @@ -120,14 +119,8 @@ private void updateJobItemStatus(final JobStatus jobStatus) { @Override public void stop() { jobItemContext.setStopping(true); - for (PipelineTask each : inventoryTasks) { - each.stop(); - QuietlyCloser.close(each); - } - for (PipelineTask each : incrementalTasks) { - each.stop(); - QuietlyCloser.close(each); - } + inventoryTasks.forEach(PipelineTask::stop); + incrementalTasks.forEach(PipelineTask::stop); } private final class InventoryTaskExecuteCallback implements ExecuteCallback { diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/QPSJobRateLimitAlgorithmTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/QPSJobRateLimitAlgorithmTest.java similarity index 94% rename from kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/QPSJobRateLimitAlgorithmTest.java rename to kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/QPSJobRateLimitAlgorithmTest.java index 673869cae31b7..12b5b74698362 100644 --- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/QPSJobRateLimitAlgorithmTest.java +++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/QPSJobRateLimitAlgorithmTest.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.shardingsphere.data.pipeline.core.ratelimit; +package org.apache.shardingsphere.data.pipeline.core.ratelimit.type; import org.apache.shardingsphere.data.pipeline.core.constant.PipelineSQLOperationType; +import org.apache.shardingsphere.data.pipeline.core.ratelimit.JobRateLimitAlgorithm; import org.apache.shardingsphere.infra.algorithm.core.exception.AlgorithmInitializationException; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.apache.shardingsphere.test.util.PropertiesBuilder; diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/TPSJobRateLimitAlgorithmTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/TPSJobRateLimitAlgorithmTest.java similarity index 94% rename from kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/TPSJobRateLimitAlgorithmTest.java rename to kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/TPSJobRateLimitAlgorithmTest.java index 344f2e270b870..56a84436685bc 100644 --- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/TPSJobRateLimitAlgorithmTest.java +++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ratelimit/type/TPSJobRateLimitAlgorithmTest.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.shardingsphere.data.pipeline.core.ratelimit; +package org.apache.shardingsphere.data.pipeline.core.ratelimit.type; import org.apache.shardingsphere.data.pipeline.core.constant.PipelineSQLOperationType; +import org.apache.shardingsphere.data.pipeline.core.ratelimit.JobRateLimitAlgorithm; import org.apache.shardingsphere.infra.algorithm.core.exception.AlgorithmInitializationException; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.apache.shardingsphere.test.util.PropertiesBuilder; diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTaskAckCallbackTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTaskAckCallbackTest.java new file mode 100644 index 0000000000000..6d6f391c64590 --- /dev/null +++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/IncrementalTaskAckCallbackTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.data.pipeline.core.task; + +import org.apache.shardingsphere.data.pipeline.core.ingest.position.IngestPosition; +import org.apache.shardingsphere.data.pipeline.core.ingest.position.type.finished.IngestFinishedPosition; +import org.apache.shardingsphere.data.pipeline.core.ingest.position.type.placeholder.IngestPlaceholderPosition; +import org.apache.shardingsphere.data.pipeline.core.ingest.record.FinishedRecord; +import org.apache.shardingsphere.data.pipeline.core.ingest.record.PlaceholderRecord; +import org.apache.shardingsphere.data.pipeline.core.task.progress.IncrementalTaskProgress; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +class IncrementalTaskAckCallbackTest { + + @Mock + private IngestPosition position; + + private final IncrementalTaskProgress taskProgress = new IncrementalTaskProgress(position); + + private final IncrementalTaskAckCallback callback = new IncrementalTaskAckCallback(taskProgress); + + @Test + void assertOnAckWithIngestPlaceholderPosition() { + PlaceholderRecord record = new PlaceholderRecord(new IngestPlaceholderPosition()); + callback.onAck(Collections.singletonList(record)); + assertNull(taskProgress.getPosition()); + assertThat(taskProgress.getIncrementalTaskDelay().getLastEventTimestamps(), is(0L)); + assertThat(System.currentTimeMillis() - taskProgress.getIncrementalTaskDelay().getLatestActiveTimeMillis(), lessThan(10000L)); + } + + @Test + void assertOnAckWithNotIngestPlaceholderPosition() { + IngestFinishedPosition position = new IngestFinishedPosition(); + FinishedRecord record = new FinishedRecord(position); + record.setCommitTime(1L); + callback.onAck(Collections.singletonList(record)); + assertThat(taskProgress.getPosition(), is(position)); + assertThat(taskProgress.getIncrementalTaskDelay().getLastEventTimestamps(), is(1L)); + assertThat(System.currentTimeMillis() - taskProgress.getIncrementalTaskDelay().getLatestActiveTimeMillis(), lessThan(10000L)); + } +} diff --git a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCIncrementalTask.java b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCIncrementalTask.java index f3700cbb2de4a..0d18ccf4b601d 100644 --- a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCIncrementalTask.java +++ b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCIncrementalTask.java @@ -70,8 +70,4 @@ public void stop() { importer.stop(); } } - - @Override - public void close() { - } } diff --git a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCInventoryTask.java b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCInventoryTask.java index 2226ae090445e..6e3eb9066ea2b 100644 --- a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCInventoryTask.java +++ b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCInventoryTask.java @@ -82,8 +82,4 @@ public void stop() { public InventoryTaskProgress getTaskProgress() { return new InventoryTaskProgress(position.get()); } - - @Override - public void close() { - } } diff --git a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCTasksRunner.java b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCTasksRunner.java index 0e8f7d52a931c..015d3e6666af8 100644 --- a/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCTasksRunner.java +++ b/kernel/data-pipeline/scenario/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/task/CDCTasksRunner.java @@ -17,9 +17,8 @@ package org.apache.shardingsphere.data.pipeline.cdc.core.task; -import org.apache.shardingsphere.data.pipeline.core.context.TransmissionJobItemContext; import org.apache.shardingsphere.data.pipeline.core.context.PipelineJobItemContext; -import org.apache.shardingsphere.infra.util.close.QuietlyCloser; +import org.apache.shardingsphere.data.pipeline.core.context.TransmissionJobItemContext; import org.apache.shardingsphere.data.pipeline.core.task.PipelineTask; import org.apache.shardingsphere.data.pipeline.core.task.runner.PipelineTasksRunner; @@ -56,11 +55,9 @@ public void stop() { jobItemContext.setStopping(true); for (PipelineTask each : inventoryTasks) { each.stop(); - QuietlyCloser.close(each); } for (PipelineTask each : incrementalTasks) { each.stop(); - QuietlyCloser.close(each); } } } diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/DistSQLQueryBackendHandlerTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLQueryBackendHandlerTest.java similarity index 97% rename from proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/DistSQLQueryBackendHandlerTest.java rename to proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLQueryBackendHandlerTest.java index 78b5e89d8ccc5..846aff29f59b3 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/DistSQLQueryBackendHandlerTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLQueryBackendHandlerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.proxy.backend.handler.distsql.ral; +package org.apache.shardingsphere.proxy.backend.handler.distsql; import org.apache.shardingsphere.distsql.statement.ral.queryable.QueryableRALStatement; import org.apache.shardingsphere.distsql.statement.ral.queryable.export.ExportDatabaseConfigurationStatement; @@ -40,7 +40,6 @@ import org.apache.shardingsphere.mode.metadata.MetaDataContextsFactory; import org.apache.shardingsphere.mode.spi.PersistRepository; import org.apache.shardingsphere.proxy.backend.context.ProxyContext; -import org.apache.shardingsphere.proxy.backend.handler.distsql.DistSQLQueryBackendHandler; import org.apache.shardingsphere.proxy.backend.session.ConnectionSession; import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.DatabaseSegment; import org.apache.shardingsphere.sql.parser.statement.core.value.identifier.IdentifierValue; diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/DistSQLUpdateBackendHandlerTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLUpdateBackendHandlerTest.java similarity index 96% rename from proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/DistSQLUpdateBackendHandlerTest.java rename to proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLUpdateBackendHandlerTest.java index a9f75dc909776..24522f507f674 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/DistSQLUpdateBackendHandlerTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/DistSQLUpdateBackendHandlerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.updatable; +package org.apache.shardingsphere.proxy.backend.handler.distsql; import org.apache.shardingsphere.infra.exception.kernel.metadata.resource.storageunit.EmptyStorageUnitException; import org.apache.shardingsphere.infra.exception.kernel.metadata.resource.storageunit.MissingRequiredStorageUnitsException; @@ -25,7 +25,6 @@ import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.apache.shardingsphere.mode.manager.ContextManager; import org.apache.shardingsphere.proxy.backend.context.ProxyContext; -import org.apache.shardingsphere.proxy.backend.handler.distsql.DistSQLUpdateBackendHandler; import org.apache.shardingsphere.proxy.backend.response.header.ResponseHeader; import org.apache.shardingsphere.proxy.backend.response.header.update.UpdateResponseHeader; import org.apache.shardingsphere.proxy.backend.session.ConnectionSession; diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index 76c8fac6e03d1..a030cbfe54021 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -53,7 +53,7 @@ - diff --git a/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngineTest.java b/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngineTest.java index c75cf52bd002a..f500b543e1239 100644 --- a/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngineTest.java +++ b/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/inventory/column/InventoryColumnValueReaderEngineTest.java @@ -17,46 +17,310 @@ package org.apache.shardingsphere.data.pipeline.core.ingest.dumper.inventory.column; -import com.zaxxer.hikari.HikariDataSource; -import org.apache.commons.lang3.RandomStringUtils; +import org.apache.shardingsphere.infra.database.core.spi.DatabaseTypedSPILoader; import org.apache.shardingsphere.infra.database.core.type.DatabaseType; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; +import org.apache.shardingsphere.test.mock.AutoMockExtension; +import org.apache.shardingsphere.test.mock.StaticMockSettings; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; -import java.sql.Connection; +import java.math.BigDecimal; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Optional; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@ExtendWith(AutoMockExtension.class) +@StaticMockSettings(DatabaseTypedSPILoader.class) class InventoryColumnValueReaderEngineTest { + private final DatabaseType databaseType = TypedSPILoader.getService(DatabaseType.class, "FIXTURE"); + + private final InventoryColumnValueReaderEngine engine = new InventoryColumnValueReaderEngine(databaseType); + + @Mock + private ResultSet resultSet; + + @Mock + private ResultSetMetaData metaData; + + @Test + void assertReadWithDialectInventoryColumnValueReader() throws SQLException { + DialectInventoryColumnValueReader dialectReader = mock(DialectInventoryColumnValueReader.class); + when(dialectReader.read(resultSet, metaData, 1)).thenReturn(Optional.of("foo")); + when(DatabaseTypedSPILoader.findService(DialectInventoryColumnValueReader.class, databaseType)).thenReturn(Optional.of(dialectReader)); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithBooleanValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.BOOLEAN); + when(resultSet.getBoolean(1)).thenReturn(true); + assertThat(engine.read(resultSet, metaData, 1), is(true)); + } + + @Test + void assertReadWithSingedTinyIntValue() throws SQLException { + when(metaData.isSigned(1)).thenReturn(true); + when(metaData.getColumnType(1)).thenReturn(Types.TINYINT); + when(resultSet.getByte(1)).thenReturn((byte) 1); + assertThat(engine.read(resultSet, metaData, 1), is((byte) 1)); + } + + @Test + void assertReadWithUnSingedTinyIntValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.TINYINT); + when(resultSet.getShort(1)).thenReturn((short) 1); + assertThat(engine.read(resultSet, metaData, 1), is((short) 1)); + } + + @Test + void assertReadWithSingedSmallIntValue() throws SQLException { + when(metaData.isSigned(1)).thenReturn(true); + when(metaData.getColumnType(1)).thenReturn(Types.SMALLINT); + when(resultSet.getShort(1)).thenReturn((short) 1); + assertThat(engine.read(resultSet, metaData, 1), is((short) 1)); + } + + @Test + void assertReadWithUnSingedSmallIntValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.SMALLINT); + when(resultSet.getInt(1)).thenReturn(1); + assertThat(engine.read(resultSet, metaData, 1), is(1)); + } + + @Test + void assertReadWithSingedIntegerValue() throws SQLException { + when(metaData.isSigned(1)).thenReturn(true); + when(metaData.getColumnType(1)).thenReturn(Types.INTEGER); + when(resultSet.getInt(1)).thenReturn(1); + assertThat(engine.read(resultSet, metaData, 1), is(1)); + } + + @Test + void assertReadWithUnSingedIntegerValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.INTEGER); + when(resultSet.getLong(1)).thenReturn(1L); + assertThat(engine.read(resultSet, metaData, 1), is(1L)); + } + + @Test + void assertReadWithSingedBigIntValue() throws SQLException { + when(metaData.isSigned(1)).thenReturn(true); + when(metaData.getColumnType(1)).thenReturn(Types.BIGINT); + when(resultSet.getLong(1)).thenReturn(1L); + assertThat(engine.read(resultSet, metaData, 1), is(1L)); + } + + @Test + void assertReadWithUnSingedBigIntValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.BIGINT); + when(resultSet.getBigDecimal(1)).thenReturn(new BigDecimal("1")); + assertThat(engine.read(resultSet, metaData, 1), is(new BigDecimal("1"))); + } + + @Test + void assertReadWithNumericValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.NUMERIC); + when(resultSet.getBigDecimal(1)).thenReturn(new BigDecimal("1")); + assertThat(engine.read(resultSet, metaData, 1), is(new BigDecimal("1"))); + } + + @Test + void assertReadWithDecimalValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.DECIMAL); + when(resultSet.getBigDecimal(1)).thenReturn(new BigDecimal("1")); + assertThat(engine.read(resultSet, metaData, 1), is(new BigDecimal("1"))); + } + + @Test + void assertReadWithRealValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.REAL); + when(resultSet.getFloat(1)).thenReturn(1F); + assertThat(engine.read(resultSet, metaData, 1), is(1F)); + } + + @Test + void assertReadWithFloatValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.FLOAT); + when(resultSet.getFloat(1)).thenReturn(1F); + assertThat(engine.read(resultSet, metaData, 1), is(1F)); + } + + @Test + void assertReadWithDoubleValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.DOUBLE); + when(resultSet.getDouble(1)).thenReturn(1D); + assertThat(engine.read(resultSet, metaData, 1), is(1D)); + } + + @Test + void assertReadWithTimeValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.TIME); + when(resultSet.getTime(1)).thenReturn(new Time(1L)); + assertThat(engine.read(resultSet, metaData, 1), is(new Time(1L))); + } + + @Test + void assertReadWithDateValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.DATE); + when(resultSet.getDate(1)).thenReturn(new Date(1L)); + assertThat(engine.read(resultSet, metaData, 1), is(new Date(1L))); + } + + @Test + void assertReadWithTimestampValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.TIMESTAMP); + when(resultSet.getTimestamp(1)).thenReturn(new Timestamp(1L)); + assertThat(engine.read(resultSet, metaData, 1), is(new Timestamp(1L))); + } + + @Test + void assertReadWithCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.CHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithVarCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.VARCHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithLongVarCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.LONGVARCHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithNCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.NCHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithNVarCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.NVARCHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithLongNVarCharValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.LONGNVARCHAR); + when(resultSet.getString(1)).thenReturn("foo"); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithBinaryValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.BINARY); + when(resultSet.getBytes(1)).thenReturn(new byte[]{1}); + assertThat(engine.read(resultSet, metaData, 1), is(new byte[]{1})); + } + + @Test + void assertReadWithVarBinaryValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.VARBINARY); + when(resultSet.getBytes(1)).thenReturn(new byte[]{1}); + assertThat(engine.read(resultSet, metaData, 1), is(new byte[]{1})); + } + + @Test + void assertReadWithLongVarBinaryValue() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.LONGVARBINARY); + when(resultSet.getBytes(1)).thenReturn(new byte[]{1}); + assertThat(engine.read(resultSet, metaData, 1), is(new byte[]{1})); + } + + @Test + void assertReadWithNullBlob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.BLOB); + assertNull(engine.read(resultSet, metaData, 1)); + } + + @Test + void assertReadWithBlob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.BLOB); + Blob blob = mock(Blob.class); + when(blob.length()).thenReturn(10L); + when(blob.getBytes(1, 10)).thenReturn(new byte[]{1}); + when(resultSet.getBlob(1)).thenReturn(blob); + assertThat(engine.read(resultSet, metaData, 1), is(new byte[]{1})); + } + + @Test + void assertReadWithNullClob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.CLOB); + assertNull(engine.read(resultSet, metaData, 1)); + } + + @Test + void assertReadWithClob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.CLOB); + Clob clob = mock(Clob.class); + when(clob.length()).thenReturn(10L); + when(clob.getSubString(1, 10)).thenReturn("foo"); + when(resultSet.getClob(1)).thenReturn(clob); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithNullNClob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.NCLOB); + assertNull(engine.read(resultSet, metaData, 1)); + } + + @Test + void assertReadWithNClob() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.NCLOB); + NClob nClob = mock(NClob.class); + when(nClob.length()).thenReturn(10L); + when(nClob.getSubString(1, 10)).thenReturn("foo"); + when(resultSet.getNClob(1)).thenReturn(nClob); + assertThat(engine.read(resultSet, metaData, 1), is("foo")); + } + + @Test + void assertReadWithArray() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.ARRAY); + Array array = mock(Array.class); + when(resultSet.getArray(1)).thenReturn(array); + assertThat(engine.read(resultSet, metaData, 1), is(array)); + } + + @Test + void assertReadWithObject() throws SQLException { + when(metaData.getColumnType(1)).thenReturn(Types.JAVA_OBJECT); + Object object = new Object(); + when(resultSet.getObject(1)).thenReturn(object); + assertThat(engine.read(resultSet, metaData, 1), is(object)); + } + @Test - void assertReadValue() throws SQLException { - InventoryColumnValueReaderEngine columnValueReaderEngine = new InventoryColumnValueReaderEngine(TypedSPILoader.getService(DatabaseType.class, "H2")); - try ( - HikariDataSource dataSource = createDataSource(RandomStringUtils.randomAlphanumeric(6)); - Connection connection = dataSource.getConnection()) { - connection.createStatement().execute("CREATE TABLE t_order (order_id INT PRIMARY KEY, user_id INT, status VARCHAR(12), c_year year)"); - connection.createStatement().executeUpdate("INSERT INTO t_order(order_id, user_id, status, c_year) VALUES (1, 2,'ok', null)"); - ResultSet resultSet = connection.createStatement().executeQuery("SELECT * FROM t_order"); - resultSet.next(); - assertThat(columnValueReaderEngine.read(resultSet, resultSet.getMetaData(), 1), is(1)); - assertThat(columnValueReaderEngine.read(resultSet, resultSet.getMetaData(), 2), is(2)); - assertThat(columnValueReaderEngine.read(resultSet, resultSet.getMetaData(), 3), is("ok")); - assertNull(columnValueReaderEngine.read(resultSet, resultSet.getMetaData(), 4)); - } - } - - private static HikariDataSource createDataSource(final String databaseName) { - HikariDataSource result = new HikariDataSource(); - result.setJdbcUrl(String.format("jdbc:h2:mem:%s;DATABASE_TO_UPPER=false;MODE=MySQL", databaseName)); - result.setUsername("root"); - result.setPassword("root"); - result.setMaximumPoolSize(10); - result.setMinimumIdle(2); - return result; + void assertReadWithNull() throws SQLException { + when(resultSet.wasNull()).thenReturn(true); + assertNull(engine.read(resultSet, metaData, 1)); } } diff --git a/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTaskTest.java b/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTaskTest.java index b7bfbf6942f3d..3dcf4159bcd48 100644 --- a/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTaskTest.java +++ b/test/it/pipeline/src/test/java/org/apache/shardingsphere/data/pipeline/core/task/InventoryTaskTest.java @@ -78,7 +78,6 @@ void assertGetProgress() throws SQLException, ExecutionException, InterruptedExc PipelineContextUtils.getExecuteEngine(), PipelineContextUtils.getExecuteEngine(), mock(Dumper.class), mock(Importer.class), position); CompletableFuture.allOf(inventoryTask.start().toArray(new CompletableFuture[0])).get(10L, TimeUnit.SECONDS); assertThat(inventoryTask.getTaskProgress().getPosition(), instanceOf(IntegerPrimaryKeyIngestPosition.class)); - inventoryTask.close(); } private void initTableData(final IncrementalDumperContext dumperContext) throws SQLException { From d8226c3ff310fcc0d9f39c6a45d07f1ddcf4981f Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 15:14:35 +0530 Subject: [PATCH 05/10] fix: fix unit tests for empty result set --- .../merge/dql/groupby/GroupByMemoryMergedResultTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java index 5aad2b216c70c..06dccc263e41b 100644 --- a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java +++ b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java @@ -62,7 +62,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -80,9 +79,6 @@ void assertNextForResultSetsAllEmpty() throws SQLException { when(database.getName()).thenReturn("db_schema"); ShardingDQLResultMerger resultMerger = new ShardingDQLResultMerger(TypedSPILoader.getService(DatabaseType.class, "MySQL")); MergedResult actual = resultMerger.merge(Arrays.asList(createQueryResult(), createQueryResult(), createQueryResult()), createSelectStatementContext(), database, mock(ConnectionContext.class)); - assertTrue(actual.next()); - assertThat(actual.getValue(1, Object.class), is(0)); - assertNull(actual.getValue(2, Object.class)); assertFalse(actual.next()); } From d2dc08b9af2bb3a549ecb5ec48e0caf6e6a9c5f0 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 27 Oct 2024 22:54:11 +0530 Subject: [PATCH 06/10] test: add e2e tests for issue #4680 --- .../cases/dql/e2e-dql-select-group-by.xml | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index a030cbfe54021..678de6acd1e4f 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -53,8 +53,33 @@ - + + + + + + + + + + + + + + + + + + + + From 3ef0f86dbdff3989fd43589022fc18fff2b5d290 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 28 Oct 2024 10:27:06 +0530 Subject: [PATCH 07/10] fix: fix e2e tests for issue #4680 --- .../resources/cases/dql/e2e-dql-select-group-by.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index 678de6acd1e4f..18d9894849c12 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -58,27 +58,27 @@ - - - - - From 7db48d2bfe3138a1f2ce8e47dce539f3767a9d9b Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 29 Oct 2024 02:58:22 +0530 Subject: [PATCH 08/10] update e2e tests for isssue #4680 --- .../resources/cases/dql/e2e-dql-select-group-by.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml index 18d9894849c12..2ac2d9b65f22e 100644 --- a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-group-by.xml @@ -58,27 +58,27 @@ - - - - - From ce608b4e9609a07e4eff5dc9f49f69256425b452 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 29 Oct 2024 17:30:47 +0530 Subject: [PATCH 09/10] fix: fix failing checks --- .../dql/groupby/GroupByMemoryMergedResult.java | 14 ++++++++++++++ .../dql/groupby/GroupByMemoryMergedResultTest.java | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java index 2f73e8c077cee..7f02a2344ce74 100644 --- a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java +++ b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java @@ -34,12 +34,14 @@ import org.apache.shardingsphere.sharding.merge.dql.groupby.aggregation.AggregationUnit; import org.apache.shardingsphere.sharding.merge.dql.groupby.aggregation.AggregationUnitFactory; import org.apache.shardingsphere.sharding.rule.ShardingRule; +import org.apache.shardingsphere.sql.parser.statement.core.enums.AggregationType; import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -139,6 +141,7 @@ private boolean getValueCaseSensitiveFromTables(final QueryResult queryResult, private List getMemoryResultSetRows(final SelectStatementContext selectStatementContext, final Map dataMap, final List valueCaseSensitive) { + Object[] data = generateReturnData(selectStatementContext); if (dataMap.isEmpty()) { return selectStatementContext.getProjectionsContext().getAggregationProjections().isEmpty() ? Collections.emptyList() : Collections.singletonList(new MemoryQueryResultRow(data)); } @@ -146,4 +149,15 @@ private List getMemoryResultSetRows(final SelectStatementC result.sort(new GroupByRowComparator(selectStatementContext, valueCaseSensitive)); return result; } + + private Object[] generateReturnData(final SelectStatementContext selectStatementContext) { + List projections = new LinkedList<>(selectStatementContext.getProjectionsContext().getExpandProjections()); + Object[] result = new Object[projections.size()]; + for (int i = 0; i < projections.size(); i++) { + if (projections.get(i) instanceof AggregationProjection && AggregationType.COUNT == ((AggregationProjection) projections.get(i)).getType()) { + result[i] = 0; + } + } + return result; + } } diff --git a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java index 06dccc263e41b..5aad2b216c70c 100644 --- a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java +++ b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java @@ -62,6 +62,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -79,6 +80,9 @@ void assertNextForResultSetsAllEmpty() throws SQLException { when(database.getName()).thenReturn("db_schema"); ShardingDQLResultMerger resultMerger = new ShardingDQLResultMerger(TypedSPILoader.getService(DatabaseType.class, "MySQL")); MergedResult actual = resultMerger.merge(Arrays.asList(createQueryResult(), createQueryResult(), createQueryResult()), createSelectStatementContext(), database, mock(ConnectionContext.class)); + assertTrue(actual.next()); + assertThat(actual.getValue(1, Object.class), is(0)); + assertNull(actual.getValue(2, Object.class)); assertFalse(actual.next()); } From dce50e82f1f4a01092438edc5f784514c08e6d23 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 30 Oct 2024 16:24:36 +0530 Subject: [PATCH 10/10] fix: update conditions for group by and aggregate functions --- .../merge/dql/groupby/GroupByMemoryMergedResult.java | 10 +++++++++- .../dql/groupby/GroupByMemoryMergedResultTest.java | 4 ---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java index 7f02a2344ce74..e1cf5eccca4e5 100644 --- a/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java +++ b/features/sharding/core/src/main/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResult.java @@ -142,9 +142,17 @@ private boolean getValueCaseSensitiveFromTables(final QueryResult queryResult, private List getMemoryResultSetRows(final SelectStatementContext selectStatementContext, final Map dataMap, final List valueCaseSensitive) { Object[] data = generateReturnData(selectStatementContext); + if (dataMap.isEmpty()) { - return selectStatementContext.getProjectionsContext().getAggregationProjections().isEmpty() ? Collections.emptyList() : Collections.singletonList(new MemoryQueryResultRow(data)); + boolean hasGroupBy = !selectStatementContext.getGroupByContext().getItems().isEmpty(); + boolean hasAggregations = !selectStatementContext.getProjectionsContext().getAggregationProjections().isEmpty(); + + if (hasGroupBy || !hasAggregations) { + return Collections.emptyList(); + } + return Collections.singletonList(new MemoryQueryResultRow(data)); } + List result = new ArrayList<>(dataMap.values()); result.sort(new GroupByRowComparator(selectStatementContext, valueCaseSensitive)); return result; diff --git a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java index 5aad2b216c70c..06dccc263e41b 100644 --- a/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java +++ b/features/sharding/core/src/test/java/org/apache/shardingsphere/sharding/merge/dql/groupby/GroupByMemoryMergedResultTest.java @@ -62,7 +62,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -80,9 +79,6 @@ void assertNextForResultSetsAllEmpty() throws SQLException { when(database.getName()).thenReturn("db_schema"); ShardingDQLResultMerger resultMerger = new ShardingDQLResultMerger(TypedSPILoader.getService(DatabaseType.class, "MySQL")); MergedResult actual = resultMerger.merge(Arrays.asList(createQueryResult(), createQueryResult(), createQueryResult()), createSelectStatementContext(), database, mock(ConnectionContext.class)); - assertTrue(actual.next()); - assertThat(actual.getValue(1, Object.class), is(0)); - assertNull(actual.getValue(2, Object.class)); assertFalse(actual.next()); }