From 4125433f4a6ca7c3ead1f678b33a1aca9de3465f Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Thu, 25 Apr 2024 11:04:46 +0800 Subject: [PATCH 1/6] dev parallel without stats --- .../antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 | 2 +- .../tsinghua/iginx/client/IginxClient.java | 2 + conf/config.properties | 38 +- .../cn/edu/tsinghua/iginx/IginxWorker.java | 14 +- .../cn/edu/tsinghua/iginx/conf/Config.java | 20 + .../tsinghua/iginx/conf/ConfigDescriptor.java | 63 +- .../tsinghua/iginx/engine/ContextBuilder.java | 6 + .../iginx/engine/StatementExecutor.java | 195 ++---- .../coordinator/ConnectManager.java | 48 ++ .../coordinator/Evaluator.java | 8 + .../coordinator/NaiveEvaluator.java | 34 + .../coordinator/NaiveSplitter.java | 114 ++++ .../distributedquery/coordinator/Plan.java | 29 + .../coordinator/PlanExecutor.java | 24 + .../coordinator/Splitter.java | 8 + .../worker/SubPlanExecutor.java | 63 ++ .../logical/optimizer/rules/MarkLoadRule.java | 51 ++ .../optimizer/rules/RuleCollection.java | 1 + .../engine/physical/PhysicalEngineImpl.java | 6 + .../naive/NaiveOperatorMemoryExecutor.java | 377 ++++++++--- .../memory/execute/utils/RowUtils.java | 184 +++++- .../naive/NaivePhysicalOptimizer.java | 2 + .../execute/StoragePhysicalTaskExecutor.java | 17 +- .../physical/task/IGinXPhysicalTask.java | 24 + .../task/UnaryMemoryPhysicalTask.java | 16 + .../engine/physical/task/utils/TaskUtils.java | 11 +- .../task/visitor/TaskInfoVisitor.java | 13 +- .../physical/task/visitor/TaskVisitor.java | 8 +- .../iginx/engine/shared/RequestContext.java | 4 + .../tsinghua/iginx/engine/shared/Result.java | 26 +- .../iginx/engine/shared/ResultUtils.java | 157 +++++ .../iginx/engine/shared/operator/Load.java | 36 ++ .../shared/operator/type/OperatorType.java | 3 +- .../operator/visitor/FragmentsVisitor.java | 32 + .../engine/shared/source/IGinXSource.java | 47 ++ .../engine/shared/source/SourceType.java | 1 + .../iginx/metadata/DefaultMetaManager.java | 4 +- .../sql/statement/ShowConfigStatement.java | 1 - core/src/main/resources/log4j.properties | 17 + .../java/cn/edu/tsinghua/iginx/MyCounter.java | 38 ++ .../java/cn/edu/tsinghua/iginx/MyTest.java | 119 ++++ .../tsinghua/iginx/iotdb/IoTDBStorage.java | 10 +- .../query/entity/IoTDBQueryRowStream.java | 7 +- pom.xml | 2 + .../edu/tsinghua/iginx/session/Session.java | 9 + .../session/SessionExecuteSqlResult.java | 79 ++- .../session/SessionExecuteSubPlanResult.java | 53 ++ .../iginx/utils/FastjsonSerializeUtils.java | 19 + .../integration/func/sql/SQLSessionIT.java | 123 ---- .../fileReadAndWrite/byteStream/test.s1 | Bin 0 -> 40 bytes .../fileReadAndWrite/byteStream/test.s2 | Bin 0 -> 40 bytes .../fileReadAndWrite/byteStream/test.s3 | Bin 0 -> 5 bytes .../fileReadAndWrite/byteStream/test.s4 | 1 + .../resources/fileReadAndWrite/csv/test.csv | 5 + thrift/src/main/proto/rpc.thrift | 19 +- tools-benchmark/pom.xml | 89 +++ .../resources/sbin/iginx-benchmark.sh | 118 ++++ tools-benchmark/src/assembly/tools.xml | 39 ++ .../tools/benchmark/MyQueryBenchmark.java | 596 ++++++++++++++++++ tools-tpch/pom.xml | 89 +++ .../src/assembly/resources/sbin/iginx-tpch.sh | 118 ++++ tools-tpch/src/assembly/tools.xml | 39 ++ .../tsinghua/iginx/tools/tpch/DataReader.java | 185 ++++++ .../iginx/tools/tpch/TableReader.java | 134 ++++ 64 files changed, 3116 insertions(+), 481 deletions(-) create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Evaluator.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveEvaluator.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveSplitter.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Plan.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/PlanExecutor.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Splitter.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/worker/SubPlanExecutor.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/MarkLoadRule.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/IGinXPhysicalTask.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Load.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/FragmentsVisitor.java create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/IGinXSource.java create mode 100644 core/src/main/resources/log4j.properties create mode 100644 core/src/test/java/cn/edu/tsinghua/iginx/MyCounter.java create mode 100644 core/src/test/java/cn/edu/tsinghua/iginx/MyTest.java create mode 100644 session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSubPlanResult.java create mode 100644 shared/src/main/java/cn/edu/tsinghua/iginx/utils/FastjsonSerializeUtils.java create mode 100644 test/src/test/resources/fileReadAndWrite/byteStream/test.s1 create mode 100644 test/src/test/resources/fileReadAndWrite/byteStream/test.s2 create mode 100644 test/src/test/resources/fileReadAndWrite/byteStream/test.s3 create mode 100644 test/src/test/resources/fileReadAndWrite/byteStream/test.s4 create mode 100644 test/src/test/resources/fileReadAndWrite/csv/test.csv create mode 100644 tools-benchmark/pom.xml create mode 100644 tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh create mode 100644 tools-benchmark/src/assembly/tools.xml create mode 100644 tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java create mode 100644 tools-tpch/pom.xml create mode 100644 tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh create mode 100644 tools-tpch/src/assembly/tools.xml create mode 100644 tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java create mode 100644 tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java diff --git a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 index 46e975418b..dae73749f4 100644 --- a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 +++ b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 @@ -25,7 +25,7 @@ statement | SHOW jobStatus TRANSFORM JOB # showEligibleJobStatement | REMOVE HISTORYDATASOURCE removedStorageEngine (COMMA removedStorageEngine)* # removeHistoryDataSourceStatement | SET CONFIG configName = stringLiteral configValue = stringLiteral # setConfigStatement - | SHOW CONFIG (configName = stringLiteral)? # showConfigStatement + | SHOW CONFIG configName = stringLiteral # showConfigStatement | SHOW SESSIONID # showSessionIDStatement | COMPACT # compactStatement | SHOW RULES # showRulesStatement diff --git a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java index eada7330d0..6f55302ed3 100644 --- a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java +++ b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java @@ -365,6 +365,7 @@ private static void processSql(String sql) { } catch (SessionException | ExecutionException e) { System.out.println(e.getMessage()); } catch (Exception e) { + e.printStackTrace(); System.out.println( "Execute Error: encounter error(s) when executing sql statement, " + "see server log for more details."); @@ -422,6 +423,7 @@ private static void processSqlWithStream(String sql) { } catch (SessionException | ExecutionException e) { System.out.println(e.getMessage()); } catch (Exception e) { + e.printStackTrace(); System.out.println( "Execute Error: encounter error(s) when executing sql statement, " + "see server log for more details."); diff --git a/conf/config.properties b/conf/config.properties index 53e41746ea..36479ee4d8 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -53,6 +53,8 @@ queryOptimizer=rbo # 优化器规则 ruleBasedOptimizer=RemoveNotRule=on,FilterFragmentRule=on +# 是否开启并行化算子 +enableParallelOperator=true # ParallelFilter触发行数 parallelFilterThreshold=10000 # ParallelGroupBy触发行数 @@ -62,8 +64,8 @@ parallelApplyFuncGroupsThreshold=1000 # ParallelGroupBy线程池大小 parallelGroupByPoolSize=5 # ParallelGroupBy线程池数量 -parallelGroupByPoolNum=5 -# ParallelGroupBy线程池数量 +parallelGroupByPoolNum=10 +# 流式ParallelGroupBy线程池数量 streamParallelGroupByWorkerNum=5 # 约束 @@ -91,8 +93,8 @@ minThriftWorkerThreadNum = 20 # thrift线程池最大线程数量 maxThriftWrokerThreadNum = 2147483647 -# 当前是否是UT测试环境 -utTestEnv = false +# 分布式查询的触发分片数 +distributedQueryTriggerThreshold = 3 #################### ### Migration 相关配置 @@ -148,10 +150,10 @@ zookeeperConnectionString=127.0.0.1:2181 #etcdEndpoints=http://localhost:2379 # 是否开启元数据内存管理 -enableMetaCacheControl=false +enable_meta_cache_control=false # 分片缓存最大内存限制，单位为 KB，默认 128 MB -fragmentCacheThreshold=131072 +fragment_cache_threshold=131072 ########################## ### 执行层配置 @@ -165,15 +167,15 @@ useStreamExecutor=false ### 内存控制 ########################## -enableMemoryControl=false +enable_memory_control=false -systemResourceMetrics=default +system_resource_metrics=default -heapMemoryThreshold=0.9 +heap_memory_threshold=0.9 -systemMemoryThreshold=0.9 +system_memory_threshold=0.9 -systemCpuThreshold=0.9 +system_cpu_threshold=0.9 #################### ### REST 服务配置 @@ -186,7 +188,7 @@ restIp=0.0.0.0 restPort=6666 # 是否启用 rest 服务 -enableRestService=true +enableRestService=false # 乱序数据 margin, 单位是秒 disorderMargin=10 @@ -218,17 +220,17 @@ transformMaxRetryTimes=3 ### MQTT 配置 #################### -enableMqtt=false +enable_mqtt=false -mqttHost=0.0.0.0 +mqtt_host=0.0.0.0 -mqttPort=1883 +mqtt_port=1883 -mqttHandlerPoolSize=1 +mqtt_handler_pool_size=1 -mqttPayloadFormatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter +mqtt_payload_formatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter -mqttMaxMessageSize=1048576 +mqtt_max_message_size=1048576 ########################## ### SimplePolicy 策略配置 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java index cf4196c19a..3b294c0f88 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java @@ -34,6 +34,7 @@ import cn.edu.tsinghua.iginx.conf.Constants; import cn.edu.tsinghua.iginx.engine.ContextBuilder; import cn.edu.tsinghua.iginx.engine.StatementExecutor; +import cn.edu.tsinghua.iginx.engine.distributedquery.worker.SubPlanExecutor; import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleCollection; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngineImpl; import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; @@ -542,7 +543,18 @@ public ExecuteSqlResp executeSql(ExecuteSqlReq req) { StatementExecutor executor = StatementExecutor.getInstance(); RequestContext ctx = contextBuilder.build(req); executor.execute(ctx); - return ctx.getResult().getExecuteSqlResp(); + logger.info("total cost time: " + (ctx.getEndTime() - ctx.getStartTime())); + ExecuteSqlResp resp = ctx.getResult().getExecuteSqlResp(); + resp.setCostTime(ctx.getEndTime() - ctx.getStartTime()); + return resp; + } + + @Override + public ExecuteSubPlanResp executeSubPlan(ExecuteSubPlanReq req) { + SubPlanExecutor executor = SubPlanExecutor.getInstance(); + RequestContext ctx = contextBuilder.build(req); + executor.execute(ctx); + return ctx.getResult().getExecuteSubPlanResp(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java index 81bebf6e39..bb0713a095 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java @@ -184,6 +184,8 @@ public class Config { private String ruleBasedOptimizer = "RemoveNotRule=on,FilterFragmentRule=on"; + private int distributedQueryTriggerThreshold = 3; + ////////////// public static final String tagNameAnnotation = TagKVUtils.tagNameAnnotation; @@ -194,6 +196,8 @@ public class Config { ///////////// + private boolean enableParallelOperator = true; + private int parallelFilterThreshold = 10000; private int parallelGroupByRowsThreshold = 10000; @@ -824,6 +828,14 @@ public void setMaxThriftWrokerThreadNum(int maxThriftWrokerThreadNum) { this.maxThriftWrokerThreadNum = maxThriftWrokerThreadNum; } + public boolean isEnableParallelOperator() { + return enableParallelOperator; + } + + public void setEnableParallelOperator(boolean enableParallelOperator) { + this.enableParallelOperator = enableParallelOperator; + } + public int getParallelFilterThreshold() { return parallelFilterThreshold; } @@ -895,4 +907,12 @@ public String getRuleBasedOptimizer() { public void setRuleBasedOptimizer(String ruleBasedOptimizer) { this.ruleBasedOptimizer = ruleBasedOptimizer; } + + public int getDistributedQueryTriggerThreshold() { + return distributedQueryTriggerThreshold; + } + + public void setDistributedQueryTriggerThreshold(int distributedQueryTriggerThreshold) { + this.distributedQueryTriggerThreshold = distributedQueryTriggerThreshold; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java index d164d40c5f..297609ab82 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java @@ -140,16 +140,16 @@ private void loadPropsFromFile() { config.setMetaStorage(properties.getProperty("metaStorage", "zookeeper")); config.setEtcdEndpoints(properties.getProperty("etcdEndpoints", "http://localhost:2379")); - config.setEnableMQTT(Boolean.parseBoolean(properties.getProperty("enableMqtt", "false"))); - config.setMqttHost(properties.getProperty("mqttHost", "0.0.0.0")); - config.setMqttPort(Integer.parseInt(properties.getProperty("mqttPort", "1883"))); + config.setEnableMQTT(Boolean.parseBoolean(properties.getProperty("enable_mqtt", "false"))); + config.setMqttHost(properties.getProperty("mqtt_host", "0.0.0.0")); + config.setMqttPort(Integer.parseInt(properties.getProperty("mqtt_port", "1883"))); config.setMqttHandlerPoolSize( - Integer.parseInt(properties.getProperty("mqttHandlerPoolSize", "1"))); + Integer.parseInt(properties.getProperty("mqtt_handler_pool_size", "1"))); config.setMqttPayloadFormatter( properties.getProperty( - "mqttPayloadFormatter", "cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter")); + "mqtt_payload_formatter", "cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter")); config.setMqttMaxMessageSize( - Integer.parseInt(properties.getProperty("mqttMaxMessageSize", "1048576"))); + Integer.parseInt(properties.getProperty("mqtt_max_message_size", "1048576"))); config.setClients(properties.getProperty("clients", "")); config.setInstancesNumPerClient( @@ -186,19 +186,19 @@ private void loadPropsFromFile() { Boolean.parseBoolean(properties.getProperty("useStreamExecutor", "true"))); config.setEnableMemoryControl( - Boolean.parseBoolean(properties.getProperty("enableMemoryControl", "true"))); - config.setSystemResourceMetrics(properties.getProperty("systemResourceMetrics", "default")); + Boolean.parseBoolean(properties.getProperty("enable_memory_control", "true"))); + config.setSystemResourceMetrics(properties.getProperty("system_resource_metrics", "default")); config.setHeapMemoryThreshold( - Double.parseDouble(properties.getProperty("heapMemoryThreshold", "0.9"))); + Double.parseDouble(properties.getProperty("heap_memory_threshold", "0.9"))); config.setSystemMemoryThreshold( - Double.parseDouble(properties.getProperty("systemMemoryThreshold", "0.9"))); + Double.parseDouble(properties.getProperty("system_memory_threshold", "0.9"))); config.setSystemCpuThreshold( - Double.parseDouble(properties.getProperty("systemCpuThreshold", "0.9"))); + Double.parseDouble(properties.getProperty("system_cpu_threshold", "0.9"))); config.setEnableMetaCacheControl( - Boolean.parseBoolean(properties.getProperty("enableMetaCacheControl", "false"))); + Boolean.parseBoolean(properties.getProperty("enable_meta_cache_control", "false"))); config.setFragmentCacheThreshold( - Long.parseLong(properties.getProperty("fragmentCacheThreshold", "131072"))); + Long.parseLong(properties.getProperty("fragment_cache_threshold", "131072"))); config.setBatchSize(Integer.parseInt(properties.getProperty("batchSize", "50"))); config.setPythonCMD(properties.getProperty("pythonCMD", "python3")); config.setTransformTaskThreadPoolSize( @@ -216,6 +216,8 @@ private void loadPropsFromFile() { Integer.parseInt(properties.getProperty("minThriftWorkerThreadNum", "20"))); config.setMaxThriftWrokerThreadNum( Integer.parseInt(properties.getProperty("maxThriftWorkerThreadNum", "2147483647"))); + config.setEnableParallelOperator( + Boolean.parseBoolean(properties.getProperty("enableParallelOperator", "true"))); config.setParallelFilterThreshold( Integer.parseInt(properties.getProperty("parallelFilterThreshold", "10000"))); config.setParallelGroupByRowsThreshold( @@ -232,6 +234,8 @@ private void loadPropsFromFile() { Integer.parseInt(properties.getProperty("batchSizeImportCsv", "10000"))); config.setRuleBasedOptimizer( properties.getProperty("ruleBasedOptimizer", "RemoveNotRule=on,FilterFragmentRule=on")); + config.setDistributedQueryTriggerThreshold( + Integer.parseInt(properties.getProperty("distributedQueryTriggerThreshold", "3"))); } catch (IOException e) { config.setUTTestEnv(true); config.setNeedInitBasicUDFFunctions(false); @@ -276,15 +280,15 @@ private void loadPropsFromEnv() { EnvUtils.loadEnv("enableRestService", config.isEnableRestService())); config.setMetaStorage(EnvUtils.loadEnv("metaStorage", config.getMetaStorage())); config.setEtcdEndpoints(EnvUtils.loadEnv("etcdEndpoints", config.getEtcdEndpoints())); - config.setEnableMQTT(EnvUtils.loadEnv("enableMqtt", config.isEnableMQTT())); - config.setMqttHost(EnvUtils.loadEnv("mqttHost", config.getMqttHost())); - config.setMqttPort(EnvUtils.loadEnv("mqttPort", config.getMqttPort())); + config.setEnableMQTT(EnvUtils.loadEnv("enable_mqtt", config.isEnableMQTT())); + config.setMqttHost(EnvUtils.loadEnv("mqtt_host", config.getMqttHost())); + config.setMqttPort(EnvUtils.loadEnv("mqtt_port", config.getMqttPort())); config.setMqttHandlerPoolSize( - EnvUtils.loadEnv("mqttHandlerPoolSize", config.getMqttHandlerPoolSize())); + EnvUtils.loadEnv("mqtt_handler_pool_size", config.getMqttHandlerPoolSize())); config.setMqttPayloadFormatter( - EnvUtils.loadEnv("mqttPayloadFormatter", config.getMqttPayloadFormatter())); + EnvUtils.loadEnv("mqtt_payload_formatter", config.getMqttPayloadFormatter())); config.setMqttMaxMessageSize( - EnvUtils.loadEnv("mqttMaxMessageSize", config.getMqttMaxMessageSize())); + EnvUtils.loadEnv("mqtt_max_message_size", config.getMqttMaxMessageSize())); config.setQueryOptimizer(EnvUtils.loadEnv("queryOptimizer", config.getQueryOptimizer())); config.setConstraintChecker( EnvUtils.loadEnv("constraintChecker", config.getConstraintChecker())); @@ -314,19 +318,19 @@ private void loadPropsFromEnv() { config.setUseStreamExecutor( EnvUtils.loadEnv("useStreamExecutor", config.isUseStreamExecutor())); config.setEnableMemoryControl( - EnvUtils.loadEnv("enableMemoryControl", config.isEnableMemoryControl())); + EnvUtils.loadEnv("enable_memory_control", config.isEnableMemoryControl())); config.setSystemResourceMetrics( - EnvUtils.loadEnv("systemResourceMetrics", config.getSystemResourceMetrics())); + EnvUtils.loadEnv("system_resource_metrics", config.getSystemResourceMetrics())); config.setHeapMemoryThreshold( - EnvUtils.loadEnv("heapMemoryThreshold", config.getHeapMemoryThreshold())); + EnvUtils.loadEnv("heap_memory_threshold", config.getHeapMemoryThreshold())); config.setSystemMemoryThreshold( - EnvUtils.loadEnv("systemMemoryThreshold", config.getSystemMemoryThreshold())); + EnvUtils.loadEnv("system_memory_threshold", config.getSystemMemoryThreshold())); config.setSystemCpuThreshold( - EnvUtils.loadEnv("systemCpuThreshold", config.getSystemCpuThreshold())); + EnvUtils.loadEnv("system_cpu_threshold", config.getSystemCpuThreshold())); config.setEnableMetaCacheControl( - EnvUtils.loadEnv("enableMetaCacheControl", config.isEnableMetaCacheControl())); + EnvUtils.loadEnv("enable_meta_cache_control", config.isEnableMetaCacheControl())); config.setFragmentCacheThreshold( - EnvUtils.loadEnv("fragmentCacheThreshold", config.getFragmentCacheThreshold())); + EnvUtils.loadEnv("fragment_cache_threshold", config.getFragmentCacheThreshold())); config.setBatchSize(EnvUtils.loadEnv("batchSize", config.getBatchSize())); config.setPythonCMD(EnvUtils.loadEnv("pythonCMD", config.getPythonCMD())); config.setTransformTaskThreadPoolSize( @@ -339,6 +343,8 @@ private void loadPropsFromEnv() { EnvUtils.loadEnv("historicalPrefixList", config.getHistoricalPrefixList())); config.setExpectedStorageUnitNum( EnvUtils.loadEnv("expectedStorageUnitNum", config.getExpectedStorageUnitNum())); + config.setEnableParallelOperator( + EnvUtils.loadEnv("enableParallelOperator", config.isEnableParallelOperator())); config.setParallelFilterThreshold( EnvUtils.loadEnv("parallelFilterThreshold", config.getParallelFilterThreshold())); config.setParallelGroupByRowsThreshold( @@ -355,9 +361,12 @@ private void loadPropsFromEnv() { "streamParallelGroupByWorkerNum", config.getStreamParallelGroupByWorkerNum())); config.setBatchSizeImportCsv( EnvUtils.loadEnv("batchSizeImportCsv", config.getBatchSizeImportCsv())); - config.setUTTestEnv(EnvUtils.loadEnv("utTestEnv", config.isUTTestEnv())); + config.setUTTestEnv(EnvUtils.loadEnv("ut_test_env", config.isUTTestEnv())); config.setRuleBasedOptimizer( EnvUtils.loadEnv("ruleBasedOptimizer", config.getRuleBasedOptimizer())); + config.setDistributedQueryTriggerThreshold( + EnvUtils.loadEnv( + "distributedQueryTriggerThreshold", config.getDistributedQueryTriggerThreshold())); } private void loadUDFListFromFile() { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java index cafd95dc3f..1e58e1e8be 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java @@ -206,6 +206,12 @@ public RequestContext build(ExecuteStatementReq req) { return new RequestContext(req.getSessionId(), req.getStatement(), true); } + public RequestContext build(ExecuteSubPlanReq req) { + RequestContext context = new RequestContext(req.getSessionId()); + context.setSubPlanMsg(req.getSubPlan()); + return context; + } + public RequestContext build(LoadCSVReq req) { return new RequestContext(req.getSessionId(), req.getStatement()); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java index 1b29a22899..7c88d2d320 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java @@ -5,6 +5,7 @@ import cn.edu.tsinghua.iginx.conf.Config; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.distributedquery.coordinator.*; import cn.edu.tsinghua.iginx.engine.logical.constraint.ConstraintChecker; import cn.edu.tsinghua.iginx.engine.logical.constraint.ConstraintCheckerManager; import cn.edu.tsinghua.iginx.engine.logical.generator.DeleteGenerator; @@ -21,6 +22,7 @@ import cn.edu.tsinghua.iginx.engine.physical.task.visitor.TaskInfoVisitor; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; import cn.edu.tsinghua.iginx.engine.shared.Result; +import cn.edu.tsinghua.iginx.engine.shared.ResultUtils; import cn.edu.tsinghua.iginx.engine.shared.constraint.ConstraintManager; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; @@ -68,14 +70,12 @@ import cn.edu.tsinghua.iginx.utils.Bitmap; import cn.edu.tsinghua.iginx.utils.ByteUtils; import cn.edu.tsinghua.iginx.utils.DataTypeInferenceUtils; -import cn.edu.tsinghua.iginx.utils.DataTypeUtils; import cn.edu.tsinghua.iginx.utils.RpcUtils; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -103,6 +103,12 @@ public class StatementExecutor { private static final PhysicalEngine engine = PhysicalEngineImpl.getInstance(); + private static final PlanExecutor planExecutor = PlanExecutor.getInstance(); + + private static final Evaluator evaluator = NaiveEvaluator.getInstance(); + + private static final Splitter splitter = NaiveSplitter.getInstance(); + private static final ConstraintChecker checker = ConstraintCheckerManager.getInstance().getChecker(config.getConstraintChecker()); private static final ConstraintManager constraintManager = engine.getConstraintManager(); @@ -349,14 +355,31 @@ private void process(RequestContext ctx) throws ExecutionException, PhysicalExce if (type == StatementType.SELECT) { SelectStatement selectStatement = (SelectStatement) ctx.getStatement(); if (selectStatement.isNeedLogicalExplain()) { - processExplainLogicalStatement(ctx, root); + if (evaluator.needDistributedQuery(root)) { + // if (true) { + Plan plan = splitter.split(root); + processExplainLogicalPlan(ctx, plan); + } else { + processExplainLogicalStatement(ctx, root); + } return; } } before(ctx, prePhysicalProcessors); - RowStream stream = engine.execute(ctx, root); + RowStream stream; + long startTime = System.currentTimeMillis(); + if (evaluator.needDistributedQuery(root)) { + Plan plan = splitter.split(root); + stream = planExecutor.execute(ctx, plan); + } else { + stream = engine.execute(ctx, root); + } + long endTime = System.currentTimeMillis(); + long engineCostTime = endTime - startTime; + logger.info("engine cost time: " + engineCostTime); after(ctx, postPhysicalProcessors); + ctx.setEngineCostTime(engineCostTime); if (type == StatementType.SELECT) { SelectStatement selectStatement = (SelectStatement) ctx.getStatement(); @@ -373,6 +396,33 @@ private void process(RequestContext ctx) throws ExecutionException, PhysicalExce throw new ExecutionException("Execute Error: can not construct a legal logical tree."); } + private void processExplainLogicalPlan(RequestContext ctx, Plan plan) + throws PhysicalException, ExecutionException { + List fields = + new ArrayList<>( + Arrays.asList( + new Field("Logical Tree", DataType.BINARY), + new Field("Operator Type", DataType.BINARY), + new Field("Operator Info", DataType.BINARY))); + Header header = new Header(fields); + + List roots = new ArrayList<>(); + roots.add(plan.getRoot()); + roots.addAll(plan.getSubPlans()); + + int maxLen = 0; + List cache = new ArrayList<>(); + for (int i = 0; i < roots.size(); i++) { + Operator root = roots.get(i); + cache.add(new Object[] {"[subplan" + i + "]: ", "".getBytes(), "".getBytes()}); + OperatorInfoVisitor visitor = new OperatorInfoVisitor(); + root.accept(visitor); + maxLen = Math.max(maxLen, visitor.getMaxLen()); + cache.addAll(visitor.getCache()); + } + formatTree(ctx, header, cache, maxLen); + } + private void processExplainLogicalStatement(RequestContext ctx, Operator root) throws PhysicalException, ExecutionException { List fields = @@ -433,7 +483,7 @@ private void processExportFileFromSelect(RequestContext ctx) RowStream stream = selectContext.getResult().getResultStream(); // step 2: export file - setResultFromRowStream(ctx, stream); + ResultUtils.setResultFromRowStream(ctx, stream); ExportFile exportFile = statement.getExportFile(); switch (exportFile.getType()) { case CSV: @@ -677,15 +727,6 @@ private void processClearData(RequestContext ctx) throws ExecutionException, Phy process(ctx); } - private void setEmptyQueryResp(RequestContext ctx, List paths) { - Result result = new Result(RpcUtils.SUCCESS); - result.setKeys(new Long[0]); - result.setValuesList(new ArrayList<>()); - result.setBitmapList(new ArrayList<>()); - result.setPaths(paths); - ctx.setResult(result); - } - private void setResult(RequestContext ctx, RowStream stream) throws PhysicalException, ExecutionException { Statement statement = ctx.getStatement(); @@ -702,10 +743,10 @@ private void setResult(RequestContext ctx, RowStream stream) } break; case SELECT: - setResultFromRowStream(ctx, stream); + ResultUtils.setResultFromRowStream(ctx, stream); break; case SHOW_COLUMNS: - setShowTSRowStreamResult(ctx, stream); + ResultUtils.setShowTSRowStreamResult(ctx, stream); break; default: throw new ExecutionException( @@ -713,128 +754,6 @@ private void setResult(RequestContext ctx, RowStream stream) } } - private void setResultFromRowStream(RequestContext ctx, RowStream stream) - throws PhysicalException { - Result result = null; - if (ctx.isUseStream()) { - Status status = RpcUtils.SUCCESS; - if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { - status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); - status.setMessage(ctx.getWarningMsg()); - } - result = new Result(status); - result.setResultStream(stream); - ctx.setResult(result); - return; - } - - if (stream == null) { - setEmptyQueryResp(ctx, new ArrayList<>()); - return; - } - - List paths = new ArrayList<>(); - List> tagsList = new ArrayList<>(); - List types = new ArrayList<>(); - stream - .getHeader() - .getFields() - .forEach( - field -> { - paths.add(field.getFullName()); - types.add(field.getType()); - if (field.getTags() == null) { - tagsList.add(new HashMap<>()); - } else { - tagsList.add(field.getTags()); - } - }); - - List timestampList = new ArrayList<>(); - List valuesList = new ArrayList<>(); - List bitmapList = new ArrayList<>(); - - boolean hasTimestamp = stream.getHeader().hasKey(); - while (stream.hasNext()) { - Row row = stream.next(); - - Object[] rowValues = row.getValues(); - valuesList.add(ByteUtils.getRowByteBuffer(rowValues, types)); - - Bitmap bitmap = new Bitmap(rowValues.length); - for (int i = 0; i < rowValues.length; i++) { - if (rowValues[i] != null) { - bitmap.mark(i); - } - } - bitmapList.add(ByteBuffer.wrap(bitmap.getBytes())); - - if (hasTimestamp) { - timestampList.add(row.getKey()); - } - } - - if (valuesList.isEmpty()) { // empty result - setEmptyQueryResp(ctx, paths); - return; - } - - Status status = RpcUtils.SUCCESS; - if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { - status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); - status.setMessage(ctx.getWarningMsg()); - } - result = new Result(status); - if (timestampList.size() != 0) { - Long[] timestamps = timestampList.toArray(new Long[timestampList.size()]); - result.setKeys(timestamps); - } - result.setValuesList(valuesList); - result.setBitmapList(bitmapList); - result.setPaths(paths); - result.setTagsList(tagsList); - result.setDataTypes(types); - ctx.setResult(result); - - stream.close(); - } - - private void setShowTSRowStreamResult(RequestContext ctx, RowStream stream) - throws PhysicalException { - if (ctx.isUseStream()) { - Result result = new Result(RpcUtils.SUCCESS); - result.setResultStream(stream); - ctx.setResult(result); - return; - } - List paths = new ArrayList<>(); - // todo:need physical layer to support. - List> tagsList = new ArrayList<>(); - List types = new ArrayList<>(); - - while (stream.hasNext()) { - Row row = stream.next(); - Object[] rowValues = row.getValues(); - - if (rowValues.length == 2) { - paths.add(new String((byte[]) rowValues[0])); - DataType type = DataTypeUtils.getDataTypeFromString(new String((byte[]) rowValues[1])); - if (type == null) { - logger.warn("unknown data type [{}]", rowValues[1]); - } - types.add(type); - } else { - logger.warn("show columns result col size = {}", rowValues.length); - } - } - - Result result = new Result(RpcUtils.SUCCESS); - result.setPaths(paths); - result.setTagsList(tagsList); - result.setDataTypes(types); - ctx.setResult(result); - } - private void parseOldTagsFromHeader(Header header, InsertStatement insertStatement) throws PhysicalException, ExecutionException { if (insertStatement.getPaths().size() != header.getFieldSize()) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java new file mode 100644 index 0000000000..3a443377fb --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java @@ -0,0 +1,48 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.engine.shared.source.IGinXSource; +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConnectManager { + + private static final Logger logger = LoggerFactory.getLogger(ConnectManager.class); + + private static class ConnectManagerHolder { + private static final ConnectManager INSTANCE = new ConnectManager(); + } + + public static ConnectManager getInstance() { + return ConnectManagerHolder.INSTANCE; + } + + private final Map connnectMap = new ConcurrentHashMap<>(); + + private void addConnect(IGinXSource source, Session session) { + connnectMap.put(source, session); + } + + private void remove(IGinXSource source) { + connnectMap.remove(source); + } + + public Session getSession(IGinXSource source) { + if (connnectMap.containsKey(source)) { + return connnectMap.get(source); + } + + Session session = new Session(source.getIp(), source.getPort()); + try { + session.openSession(); + } catch (SessionException e) { + logger.error("open session failed, because: ", e); + return null; + } + connnectMap.put(source, session); + return session; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Evaluator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Evaluator.java new file mode 100644 index 0000000000..5f12b4198d --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Evaluator.java @@ -0,0 +1,8 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; + +public interface Evaluator { + + boolean needDistributedQuery(Operator root); +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveEvaluator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveEvaluator.java new file mode 100644 index 0000000000..50eac60fa6 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveEvaluator.java @@ -0,0 +1,34 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.conf.Config; +import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.FragmentsVisitor; +import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; +import cn.edu.tsinghua.iginx.metadata.IMetaManager; + +public class NaiveEvaluator implements Evaluator { + + private static final Config config = ConfigDescriptor.getInstance().getConfig(); + + private final IMetaManager metaManager = DefaultMetaManager.getInstance(); + + private static class NaiveEvaluatorHolder { + private static final NaiveEvaluator INSTANCE = new NaiveEvaluator(); + } + + public static NaiveEvaluator getInstance() { + return NaiveEvaluatorHolder.INSTANCE; + } + + @Override + public boolean needDistributedQuery(Operator root) { + FragmentsVisitor visitor = new FragmentsVisitor(); + root.accept(visitor); + int fragmentSize = visitor.getFragmentCount(); + int iginxSize = metaManager.getIginxList().size(); + return fragmentSize > config.getDistributedQueryTriggerThreshold() + && fragmentSize >= iginxSize + && iginxSize > 1; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveSplitter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveSplitter.java new file mode 100644 index 0000000000..7040253d93 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/NaiveSplitter.java @@ -0,0 +1,114 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.conf.Config; +import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.shared.operator.*; +import cn.edu.tsinghua.iginx.engine.shared.source.IGinXSource; +import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; +import cn.edu.tsinghua.iginx.metadata.IMetaManager; +import cn.edu.tsinghua.iginx.metadata.entity.IginxMeta; +import cn.edu.tsinghua.iginx.utils.Pair; +import java.util.*; + +public class NaiveSplitter implements Splitter { + + private final IMetaManager metaManager = DefaultMetaManager.getInstance(); + + private static final Config config = ConfigDescriptor.getInstance().getConfig(); + + private final String selfIp = config.getIp(); + private final int selfPort = config.getPort(); + + private static class NaiveSplitterHolder { + private static final NaiveSplitter INSTANCE = new NaiveSplitter(); + } + + public static NaiveSplitter getInstance() { + return NaiveSplitterHolder.INSTANCE; + } + + @Override + public Plan split(Operator root) { + List availableIginx = new ArrayList<>(metaManager.getIginxList()); + // List availableIginx = new ArrayList<>(); + // availableIginx.add(new IginxMeta(1, "127.0.0.1", 1111, null)); + // availableIginx.add(new IginxMeta(1, "127.0.0.1", 2222, null)); + // availableIginx.add(new IginxMeta(1, "127.0.0.1", 3333, null)); + // availableIginx.add(new IginxMeta(1, "0.0.0.0", 6888, null)); + int k = availableIginx.size() - 1; + + Pair, Map> pair = split(root, k); + Queue splitPartQueue = pair.getK(); + Map parents = pair.getV(); + List subPlans = new ArrayList<>(); + + int index = 0; + while (!splitPartQueue.isEmpty()) { + Operator op = splitPartQueue.poll(); + IginxMeta iginxMeta = availableIginx.get(index++); + if (isSelf(iginxMeta)) { + iginxMeta = availableIginx.get(index++); + } + Load load = new Load(new IGinXSource(iginxMeta.getIp(), iginxMeta.getPort()), index, op); + subPlans.add(op); + + Operator parent = parents.get(op); + if (parent instanceof UnaryOperator) { + ((UnaryOperator) parent).setSource(new OperatorSource(load)); + } else if (parent instanceof BinaryOperator) { + Operator childA = ((OperatorSource) ((BinaryOperator) parent).getSourceA()).getOperator(); + if (childA.equals(op)) { + ((BinaryOperator) parent).setSourceA(new OperatorSource(load)); + } else { + ((BinaryOperator) parent).setSourceB(new OperatorSource(load)); + } + } + } + + return new Plan(root, subPlans); + } + + private Pair, Map> split(Operator root, int k) { + Queue queue = new ArrayDeque<>(); + Map parents = new HashMap<>(); + queue.offer(root); + while (queue.size() < k) { + Operator op = queue.poll(); + + if (op instanceof UnaryOperator) { + Source source = ((UnaryOperator) op).getSource(); + if (source.getType().equals(SourceType.Fragment)) { + queue.offer(op); + continue; + } + + Operator child = ((OperatorSource) source).getOperator(); + parents.put(child, op); + queue.offer(child); + } else if (op instanceof BinaryOperator) { + Operator childA = ((OperatorSource) ((BinaryOperator) op).getSourceA()).getOperator(); + Operator childB = ((OperatorSource) ((BinaryOperator) op).getSourceB()).getOperator(); + + parents.put(childA, op); + parents.put(childB, op); + queue.offer(childA); + queue.offer(childB); + } else { + List sources = ((MultipleOperator) op).getSources(); + for (Source source : sources) { + Operator child = ((OperatorSource) source).getOperator(); + parents.put(child, op); + queue.offer(child); + } + } + } + return new Pair<>(queue, parents); + } + + private boolean isSelf(IginxMeta iginxMeta) { + return iginxMeta.getIp().equals(selfIp) && iginxMeta.getPort() == selfPort; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Plan.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Plan.java new file mode 100644 index 0000000000..965dc97cd3 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Plan.java @@ -0,0 +1,29 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import java.util.List; + +public class Plan { + + private final Operator root; + + private final List subPlans; + + public Plan(Operator root, List subPlans) { + this.root = root; + this.subPlans = subPlans; + } + + public Operator getRoot() { + return root; + } + + private Operator getSubPlan(int id) { + assert id <= subPlans.size(); + return subPlans.get(id - 1); + } + + public List getSubPlans() { + return subPlans; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/PlanExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/PlanExecutor.java new file mode 100644 index 0000000000..48acf3d485 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/PlanExecutor.java @@ -0,0 +1,24 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; +import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngineImpl; +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.shared.RequestContext; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; + +public class PlanExecutor { + + private static final PhysicalEngine engine = PhysicalEngineImpl.getInstance(); + + private static class PlanExecutorHolder { + private static final PlanExecutor INSTANCE = new PlanExecutor(); + } + + public static PlanExecutor getInstance() { + return PlanExecutorHolder.INSTANCE; + } + + public RowStream execute(RequestContext ctx, Plan plan) throws PhysicalException { + return engine.execute(ctx, plan.getRoot()); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Splitter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Splitter.java new file mode 100644 index 0000000000..742530be9a --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/Splitter.java @@ -0,0 +1,8 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; + +public interface Splitter { + + Plan split(Operator root); +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/worker/SubPlanExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/worker/SubPlanExecutor.java new file mode 100644 index 0000000000..dc1be43381 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/worker/SubPlanExecutor.java @@ -0,0 +1,63 @@ +package cn.edu.tsinghua.iginx.engine.distributedquery.worker; + +import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; +import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngineImpl; +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.shared.RequestContext; +import cn.edu.tsinghua.iginx.engine.shared.Result; +import cn.edu.tsinghua.iginx.engine.shared.ResultUtils; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; +import cn.edu.tsinghua.iginx.metadata.IMetaManager; +import cn.edu.tsinghua.iginx.utils.FastjsonSerializeUtils; +import cn.edu.tsinghua.iginx.utils.RpcUtils; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SubPlanExecutor { + + private static final Logger logger = LoggerFactory.getLogger(SubPlanExecutor.class); + + private static final IMetaManager metaManager = DefaultMetaManager.getInstance(); + + private final AtomicBoolean hasLoadFragments = new AtomicBoolean(false); + + private static class SubPlanExecutorHolder { + private static final SubPlanExecutor INSTANCE = new SubPlanExecutor(); + } + + public static SubPlanExecutor getInstance() { + return SubPlanExecutorHolder.INSTANCE; + } + + public void execute(RequestContext context) { + if (!hasLoadFragments.get()) { + metaManager.createInitialFragmentsAndStorageUnits(null, null); + hasLoadFragments.set(true); + } + + Operator subPlan = FastjsonSerializeUtils.deserialize(context.getSubPlanMsg(), Operator.class); + PhysicalEngine engine = PhysicalEngineImpl.getInstance(); + + RowStream rowStream = null; + try { + rowStream = engine.execute(context, subPlan); + } catch (PhysicalException e) { + logger.error("execute sub plan failed, because: ", e); + } + + if (rowStream == null) { + context.setResult(new Result(RpcUtils.FAILURE)); + return; + } + + try { + ResultUtils.setResultFromRowStream(context, rowStream); + } catch (PhysicalException e) { + logger.error("read data from row stream error, because, ", e); + context.setResult(new Result(RpcUtils.FAILURE)); + } + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/MarkLoadRule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/MarkLoadRule.java new file mode 100644 index 0000000000..4d8173dc5b --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/MarkLoadRule.java @@ -0,0 +1,51 @@ +package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; + +public class MarkLoadRule extends Rule { + + public static final class InstanceHolder { + static final MarkLoadRule INSTANCE = new MarkLoadRule(); + } + + public static MarkLoadRule getInstance() { + return InstanceHolder.INSTANCE; + } + + private MarkLoadRule() { + super("MarkLoadRule", any()); + } + + @Override + public boolean matches(RuleCall call) { + switch (call.getMatchedRoot().getType()) { + case GroupBy: + case Join: + case InnerJoin: + case OuterJoin: + case CrossJoin: + case MarkJoin: + return true; + default: + return false; + } + } + + @Override + public void onMatch(RuleCall call) { + // Operator matchedRoot = call.getMatchedRoot(); + // if (matchedRoot instanceof UnaryOperator) { + // // deal with GroupBy + // UnaryOperator unaryOp = (UnaryOperator) matchedRoot; + // Source newSource = new OperatorSource(new Load(unaryOp.getSource())); + // unaryOp.setSource(newSource); + // } else if (matchedRoot instanceof BinaryOperator) { + // // deal with Join + // BinaryOperator binOp = (BinaryOperator) matchedRoot; + // Source newSourceA = new OperatorSource(new Load(binOp.getSourceA())); + // Source newSourceB = new OperatorSource(new Load(binOp.getSourceB())); + // binOp.setSourceA(newSourceA); + // binOp.setSourceB(newSourceB); + // } + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java index 358835a7a3..a9a5a295f9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java @@ -29,6 +29,7 @@ private RuleCollection() { // 在这里添加规则 addRule(RemoveNotRule.getInstance()); addRule(FilterFragmentRule.getInstance()); + addRule(MarkLoadRule.getInstance()); setRulesByConfig(); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java index c1be7ac965..ac74cedaeb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java @@ -97,14 +97,20 @@ public RowStream execute(RequestContext ctx, Operator root) throws PhysicalExcep private void commitBottomTasks(List bottomTasks) { List storageTasks = new ArrayList<>(); List globalTasks = new ArrayList<>(); + List iGinXTasks = new ArrayList<>(); for (PhysicalTask task : bottomTasks) { if (task.getType().equals(TaskType.Storage)) { storageTasks.add((StoragePhysicalTask) task); } else if (task.getType().equals(TaskType.Global)) { globalTasks.add((GlobalPhysicalTask) task); + } else if (task instanceof IGinXPhysicalTask) { + iGinXTasks.add((IGinXPhysicalTask) task); } } storageTaskExecutor.commit(storageTasks); + for (IGinXPhysicalTask iGinXTask : iGinXTasks) { + memoryTaskExecutor.addMemoryTask(iGinXTask); + } for (GlobalPhysicalTask globalTask : globalTasks) { storageTaskExecutor.executeGlobalTask(globalTask); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java index 8151966c1a..7ed6d706a0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java @@ -36,6 +36,7 @@ import cn.edu.tsinghua.iginx.conf.Config; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.distributedquery.coordinator.ConnectManager; import cn.edu.tsinghua.iginx.engine.physical.exception.InvalidOperatorParameterException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; @@ -58,38 +59,19 @@ import cn.edu.tsinghua.iginx.engine.shared.function.SetMappingFunction; import cn.edu.tsinghua.iginx.engine.shared.function.system.Max; import cn.edu.tsinghua.iginx.engine.shared.function.system.Min; -import cn.edu.tsinghua.iginx.engine.shared.operator.AddSchemaPrefix; -import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; -import cn.edu.tsinghua.iginx.engine.shared.operator.CrossJoin; -import cn.edu.tsinghua.iginx.engine.shared.operator.Distinct; -import cn.edu.tsinghua.iginx.engine.shared.operator.Downsample; -import cn.edu.tsinghua.iginx.engine.shared.operator.Except; -import cn.edu.tsinghua.iginx.engine.shared.operator.GroupBy; -import cn.edu.tsinghua.iginx.engine.shared.operator.InnerJoin; -import cn.edu.tsinghua.iginx.engine.shared.operator.Intersect; -import cn.edu.tsinghua.iginx.engine.shared.operator.Join; -import cn.edu.tsinghua.iginx.engine.shared.operator.Limit; -import cn.edu.tsinghua.iginx.engine.shared.operator.MappingTransform; -import cn.edu.tsinghua.iginx.engine.shared.operator.MarkJoin; -import cn.edu.tsinghua.iginx.engine.shared.operator.OuterJoin; -import cn.edu.tsinghua.iginx.engine.shared.operator.PathUnion; -import cn.edu.tsinghua.iginx.engine.shared.operator.Project; -import cn.edu.tsinghua.iginx.engine.shared.operator.Rename; -import cn.edu.tsinghua.iginx.engine.shared.operator.Reorder; -import cn.edu.tsinghua.iginx.engine.shared.operator.RowTransform; -import cn.edu.tsinghua.iginx.engine.shared.operator.Select; -import cn.edu.tsinghua.iginx.engine.shared.operator.SetTransform; -import cn.edu.tsinghua.iginx.engine.shared.operator.SingleJoin; -import cn.edu.tsinghua.iginx.engine.shared.operator.Sort; +import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.Sort.SortType; -import cn.edu.tsinghua.iginx.engine.shared.operator.UnaryOperator; -import cn.edu.tsinghua.iginx.engine.shared.operator.Union; -import cn.edu.tsinghua.iginx.engine.shared.operator.ValueToSelectedPath; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OuterJoinType; import cn.edu.tsinghua.iginx.engine.shared.source.EmptySource; +import cn.edu.tsinghua.iginx.engine.shared.source.IGinXSource; +import cn.edu.tsinghua.iginx.exceptions.ExecutionException; +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.session.SessionExecuteSubPlanResult; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Bitmap; +import cn.edu.tsinghua.iginx.utils.FastjsonSerializeUtils; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; import java.nio.charset.StandardCharsets; @@ -103,6 +85,8 @@ import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ForkJoinPool; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,7 +107,9 @@ public static NaiveOperatorMemoryExecutor getInstance() { public RowStream executeUnaryOperator( UnaryOperator operator, RowStream stream, RequestContext context) throws PhysicalException { Table table = transformToTable(stream); - table.setContext(context); + if (table != null) { + table.setContext(context); + } switch (operator.getType()) { case Project: return executeProject((Project) operator, table); @@ -153,6 +139,8 @@ public RowStream executeUnaryOperator( return executeDistinct((Distinct) operator, table); case ValueToSelectedPath: return executeValueToSelectedPath((ValueToSelectedPath) operator, table); + case Load: + return executeLoad((Load) operator); default: throw new UnexpectedOperatorException("unknown unary operator: " + operator.getType()); } @@ -193,6 +181,9 @@ public RowStream executeBinaryOperator( } private Table transformToTable(RowStream stream) throws PhysicalException { + if (stream == null) { + return null; + } if (stream instanceof Table) { return (Table) stream; } @@ -648,6 +639,46 @@ private RowStream executeValueToSelectedPath(ValueToSelectedPath operator, Table return new Table(targetHeader, targetRows); } + private RowStream executeLoad(Load load) throws PhysicalException { + IGinXSource source = (IGinXSource) load.getSource(); + Operator subPlan = load.getOperator(); + String subPlanMsg = FastjsonSerializeUtils.serialize(subPlan); + + Session session = ConnectManager.getInstance().getSession(source); + try { + SessionExecuteSubPlanResult result = session.executeSubPlan(subPlanMsg); + return constructRowStream(result); + } catch (SessionException | ExecutionException e) { + logger.error("execute load fail, because: ", e); + throw new PhysicalException(e); + } + } + + private RowStream constructRowStream(SessionExecuteSubPlanResult result) { + List paths = result.getPaths(); + List dataTypes = result.getDataTypeList(); + List> values = result.getValues(); + long[] keys = result.getKeys(); + boolean hasKey = keys != null; + + assert paths.size() == dataTypes.size(); + List fields = new ArrayList<>(); + for (int i = 0; i < paths.size(); i++) { + fields.add(new Field(paths.get(i), dataTypes.get(i))); + } + Header header = hasKey ? new Header(Field.KEY, fields) : new Header(fields); + + List rows = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + Row row = + hasKey + ? new Row(header, keys[i], values.get(i).toArray(new Object[0])) + : new Row(header, values.get(i).toArray(new Object[0])); + rows.add(row); + } + return new Table(header, rows); + } + private RowStream executeJoin(Join join, Table tableA, Table tableB) throws PhysicalException { boolean hasIntersect = false; Header headerA = tableA.getHeader(); @@ -898,11 +929,13 @@ private RowStream executeHashInnerJoin(InnerJoin innerJoin, Table tableA, Table innerJoin.getPrefixB()); // 检查左右两表需要进行额外连接的path - List extraJoinPaths = new ArrayList<>(); + List extraJoinPaths; if (!innerJoin.getExtraJoinPrefix().isEmpty()) { extraJoinPaths = getSamePathWithSpecificPrefix( tableA.getHeader(), tableB.getHeader(), innerJoin.getExtraJoinPrefix()); + } else { + extraJoinPaths = new ArrayList<>(); } // 计算建立和访问哈希表所用的path @@ -923,7 +956,7 @@ private RowStream executeHashInnerJoin(InnerJoin innerJoin, Table tableA, Table checkNeedTypeCast(tableA.getRows(), tableB.getRows(), joinPathA, joinPathB); // 扫描右表建立哈希表 - HashMap> rowsBHashMap = + Map> rowsBHashMap = establishHashMap(tableB.getRows(), joinPathB, needTypeCast); // 计算连接之后的header @@ -937,41 +970,131 @@ private RowStream executeHashInnerJoin(InnerJoin innerJoin, Table tableA, Table joinColumns, extraJoinPaths); - List transformedRows = new ArrayList<>(); - for (Row rowA : tableA.getRows()) { - Value value = rowA.getAsValue(joinPathA); - if (value.isNull()) { - continue; + List transformedRows = Collections.synchronizedList(new ArrayList<>()); + + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() + && tableA.getRowSize() > config.getParallelGroupByRowsThreshold()) { + use = "parallel"; + CountDownLatch latch = new CountDownLatch(1); + ForkJoinPool pool = null; + try { + pool = RowUtils.poolQueue.take(); + pool.submit( + () -> { + Collections.synchronizedList(tableA.getRows()) + .parallelStream() + .forEach( + rowA -> { + Value value = rowA.getAsValue(joinPathA); + if (value.isNull()) { + return; + } + int hash = getHash(value, needTypeCast); + + if (rowsBHashMap.containsKey(hash)) { + for (Row rowB : rowsBHashMap.get(hash)) { + try { + if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { + continue; + } else { + try { + if (!equalOnSpecificPaths( + rowA, + rowB, + innerJoin.getPrefixA(), + innerJoin.getPrefixB(), + joinColumns)) { + continue; + } + } catch (PhysicalException e) { + throw new RuntimeException(e); + } + } + } catch (PhysicalException e) { + throw new RuntimeException(e); + } + Row joinedRow = + RowUtils.constructNewRow( + newHeader, + rowA, + rowB, + innerJoin.getPrefixA(), + innerJoin.getPrefixB(), + true, + joinColumns, + extraJoinPaths); + if (innerJoin.getFilter() != null) { + try { + if (!FilterUtils.validate(innerJoin.getFilter(), joinedRow)) { + continue; + } + } catch (PhysicalException e) { + throw new RuntimeException(e); + } + } + transformedRows.add(joinedRow); + } + } + }); + latch.countDown(); + }); + + try { + latch.await(); + } catch (InterruptedException e) { + throw new PhysicalException("Interrupt when latch await ", e); + } + } catch (InterruptedException e) { + throw new PhysicalException("Interrupt when parallel apply func", e); + } finally { + if (pool != null) { + RowUtils.poolQueue.add(pool); + } } - int hash = getHash(value, needTypeCast); + } else { + use = "sequence"; + for (Row rowA : tableA.getRows()) { + Value value = rowA.getAsValue(joinPathA); + if (value.isNull()) { + continue; + } + int hash = getHash(value, needTypeCast); - if (rowsBHashMap.containsKey(hash)) { - for (Row rowB : rowsBHashMap.get(hash)) { - if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { - continue; - } else if (!equalOnSpecificPaths( - rowA, rowB, innerJoin.getPrefixA(), innerJoin.getPrefixB(), joinColumns)) { - continue; - } - Row joinedRow = - RowUtils.constructNewRow( - newHeader, - rowA, - rowB, - innerJoin.getPrefixA(), - innerJoin.getPrefixB(), - true, - joinColumns, - extraJoinPaths); - if (innerJoin.getFilter() != null) { - if (!FilterUtils.validate(innerJoin.getFilter(), joinedRow)) { + if (rowsBHashMap.containsKey(hash)) { + for (Row rowB : rowsBHashMap.get(hash)) { + if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { + continue; + } else if (!equalOnSpecificPaths( + rowA, rowB, innerJoin.getPrefixA(), innerJoin.getPrefixB(), joinColumns)) { continue; } + Row joinedRow = + RowUtils.constructNewRow( + newHeader, + rowA, + rowB, + innerJoin.getPrefixA(), + innerJoin.getPrefixB(), + true, + joinColumns, + extraJoinPaths); + if (innerJoin.getFilter() != null) { + if (!FilterUtils.validate(innerJoin.getFilter(), joinedRow)) { + continue; + } + } + transformedRows.add(joinedRow); } - transformedRows.add(joinedRow); } } } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "inner join use %s probe, row size: %s, cost time: %s", + use, tableA.getRowSize(), endTime - startTime)); return new Table(newHeader, transformedRows); } @@ -1391,6 +1514,7 @@ private RowStream executeHashOuterJoin(OuterJoin outerJoin, Table tableA, Table HashMap> rowsBHashMap = new HashMap<>(); HashMap> indexOfRowBHashMap = new HashMap<>(); + for (int indexB = 0; indexB < rowsB.size(); indexB++) { Value value = rowsB.get(indexB).getAsValue(joinPathB); if (value.isNull()) { @@ -1866,11 +1990,13 @@ private RowStream executeNestedLoopSingleJoin(SingleJoin singleJoin, Table table private RowStream executeHashSingleJoin(SingleJoin singleJoin, Table tableA, Table tableB) throws PhysicalException { // 检查左右两表需要进行额外连接的path - List extraJoinPaths = new ArrayList<>(); + List extraJoinPaths; if (!singleJoin.getExtraJoinPrefix().isEmpty()) { extraJoinPaths = getSamePathWithSpecificPrefix( tableA.getHeader(), tableB.getHeader(), singleJoin.getExtraJoinPrefix()); + } else { + extraJoinPaths = new ArrayList<>(); } // 计算建立和访问哈希表所用的path @@ -1891,47 +2017,128 @@ private RowStream executeHashSingleJoin(SingleJoin singleJoin, Table tableA, Tab checkNeedTypeCast(tableA.getRows(), tableB.getRows(), joinPathA, joinPathB); // 扫描右表建立哈希表 - HashMap> rowsBHashMap = + Map> rowsBHashMap = establishHashMap(tableB.getRows(), joinPathB, needTypeCast); Header newHeader = constructNewHead(tableA.getHeader(), tableB.getHeader(), true, extraJoinPaths); - List transformedRows = new ArrayList<>(); + List transformedRows = Collections.synchronizedList(new ArrayList<>()); int anotherRowSize = tableB.getHeader().getFieldSize(); - for (Row rowA : tableA.getRows()) { - Value value = rowA.getAsValue(joinPathA); - if (value.isNull()) { - continue; + + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() + && tableA.getRowSize() > config.getParallelGroupByRowsThreshold()) { + use = "parallel"; + CountDownLatch latch = new CountDownLatch(1); + ForkJoinPool pool = null; + try { + pool = RowUtils.poolQueue.take(); + pool.submit( + () -> { + Collections.synchronizedList(tableA.getRows()) + .parallelStream() + .forEach( + rowA -> { + Value value = rowA.getAsValue(joinPathA); + if (value.isNull()) { + return; + } + int hash = getHash(value, needTypeCast); + + boolean matched = false; + if (rowsBHashMap.containsKey(hash)) { + for (Row rowB : rowsBHashMap.get(hash)) { + try { + if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { + continue; + } + } catch (PhysicalException e) { + throw new RuntimeException(e); + } + Row joinedRow = + RowUtils.constructNewRow( + newHeader, rowA, rowB, true, extraJoinPaths); + if (singleJoin.getFilter() != null) { + try { + if (!FilterUtils.validate(singleJoin.getFilter(), joinedRow)) { + continue; + } + } catch (PhysicalException e) { + throw new RuntimeException(e); + } + } + if (matched) { + throw new RuntimeException( + "the return value of sub-query has more than one rows"); + } + matched = true; + transformedRows.add(joinedRow); + } + } + if (!matched) { + Row unmatchedRow = + RowUtils.constructUnmatchedRow( + newHeader, rowA, singleJoin.getPrefixA(), anotherRowSize, true); + transformedRows.add(unmatchedRow); + } + }); + latch.countDown(); + }); + try { + latch.await(); + } catch (InterruptedException e) { + throw new PhysicalException("Interrupt when latch await ", e); + } + } catch (InterruptedException e) { + throw new PhysicalException("Interrupt when parallel apply func", e); + } finally { + if (pool != null) { + RowUtils.poolQueue.add(pool); + } } - int hash = getHash(value, needTypeCast); + } else { + use = "sequence"; + for (Row rowA : tableA.getRows()) { + Value value = rowA.getAsValue(joinPathA); + if (value.isNull()) { + continue; + } + int hash = getHash(value, needTypeCast); - boolean matched = false; - if (rowsBHashMap.containsKey(hash)) { - for (Row rowB : rowsBHashMap.get(hash)) { - if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { - continue; - } - Row joinedRow = RowUtils.constructNewRow(newHeader, rowA, rowB, true, extraJoinPaths); - if (singleJoin.getFilter() != null) { - if (!FilterUtils.validate(singleJoin.getFilter(), joinedRow)) { + boolean matched = false; + if (rowsBHashMap.containsKey(hash)) { + for (Row rowB : rowsBHashMap.get(hash)) { + if (!equalOnSpecificPaths(rowA, rowB, extraJoinPaths)) { continue; } + Row joinedRow = RowUtils.constructNewRow(newHeader, rowA, rowB, true, extraJoinPaths); + if (singleJoin.getFilter() != null) { + if (!FilterUtils.validate(singleJoin.getFilter(), joinedRow)) { + continue; + } + } + if (matched) { + throw new PhysicalException("the return value of sub-query has more than one rows"); + } + matched = true; + transformedRows.add(joinedRow); } - if (matched) { - throw new PhysicalException("the return value of sub-query has more than one rows"); - } - matched = true; - transformedRows.add(joinedRow); } - } - if (!matched) { - Row unmatchedRow = - RowUtils.constructUnmatchedRow( - newHeader, rowA, singleJoin.getPrefixA(), anotherRowSize, true); - transformedRows.add(unmatchedRow); + if (!matched) { + Row unmatchedRow = + RowUtils.constructUnmatchedRow( + newHeader, rowA, singleJoin.getPrefixA(), anotherRowSize, true); + transformedRows.add(unmatchedRow); + } } } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "single join use %s probe, row size: %s, cost time: %s", + use, tableA.getRowSize(), endTime - startTime)); return new Table(newHeader, transformedRows); } @@ -2014,7 +2221,7 @@ private RowStream executeHashMarkJoin(MarkJoin markJoin, Table tableA, Table tab checkNeedTypeCast(tableA.getRows(), tableB.getRows(), joinPathA, joinPathB); // 扫描右表建立哈希表 - HashMap> rowsBHashMap = + Map> rowsBHashMap = establishHashMap(tableB.getRows(), joinPathB, needTypeCast); // 计算连接之后的header diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java index d5b0349589..0483af0bdb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java @@ -64,7 +64,7 @@ public class RowUtils { private static final Logger logger = LoggerFactory.getLogger(RowUtils.class); - private static final BlockingQueue poolQueue = new LinkedBlockingQueue<>(); + public static final BlockingQueue poolQueue = new LinkedBlockingQueue<>(); static { for (int i = 0; i < config.getParallelGroupByPoolNum(); i++) { @@ -170,9 +170,31 @@ public static boolean checkNeedTypeCast( return false; } - public static HashMap> establishHashMap( + public static Map> establishHashMap( + List rows, String joinPath, boolean needTypeCast) throws PhysicalException { + Map> result; + + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() + && rows.size() > config.getParallelGroupByRowsThreshold()) { + use = "parallel"; + result = parallelEstablishHashMap(rows, joinPath, needTypeCast); + } else { + use = "sequence"; + result = seqEstablishHashMap(rows, joinPath, needTypeCast); + } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "join use %s build, row size: %s, cost time: %s", + use, rows.size(), endTime - startTime)); + return result; + } + + private static Map> seqEstablishHashMap( List rows, String joinPath, boolean needTypeCast) { - HashMap> hashMap = new HashMap<>(); + Map> map = new HashMap<>(); for (Row row : rows) { Value value = row.getAsValue(joinPath); if (value == null) { @@ -180,10 +202,79 @@ public static HashMap> establishHashMap( } int hash = getHash(value, needTypeCast); - List l = hashMap.computeIfAbsent(hash, k -> new ArrayList<>()); + List l = map.computeIfAbsent(hash, k -> new ArrayList<>()); l.add(row); } - return hashMap; + return map; + } + + private static Map> parallelEstablishHashMap( + List rows, String joinPath, boolean needTypeCast) throws PhysicalException { + ForkJoinPool pool = null; + try { + pool = poolQueue.take(); + + int size = config.getParallelGroupByPoolSize(); + int range = rows.size() / size; + List>> listOfMaps = new ArrayList<>(); + + CountDownLatch latch = new CountDownLatch(size); + for (int i = 0; i < size; i++) { + int finalI = i; + listOfMaps.add(new HashMap<>()); + pool.submit( + () -> { + for (int j = finalI * range; j < Math.min((finalI + 1) * range, rows.size()); j++) { + Row row = rows.get(j); + Value value = row.getAsValue(joinPath); + if (value == null) { + continue; + } + int hash = getHash(value, needTypeCast); + + List l = listOfMaps.get(finalI).computeIfAbsent(hash, k -> new ArrayList<>()); + l.add(row); + } + latch.countDown(); + }); + } + latch.await(); + Map> mergedMap = + listOfMaps + .parallelStream() + .flatMap(map -> map.entrySet().stream()) // 将每个Map转换为流 + .collect( + Collectors.toConcurrentMap( + Map.Entry::getKey, // 键 + Map.Entry::getValue, // 值 + (list1, list2) -> { // 合并函数，用于处理重复键 + list1.addAll(list2); // 合并两个列表 + return list1; + })); + return mergedMap; + // Map> map = + // pool.submit( + // () -> + // Collections.synchronizedList(rows) + // .parallelStream() + // .collect( + // Collectors.groupingBy( + // row -> { + // Value value = row.getAsValue(joinPath); + // if (value == null) { + // return 0; // nullable value hashcode + // } + // return getHash(value, needTypeCast); + // }))) + // .get(); + // return map; + } catch (InterruptedException e) { + throw new PhysicalException("parallel build failed"); + } finally { + if (pool != null) { + poolQueue.add(pool); + } + } } /** @@ -568,11 +659,21 @@ public static List cacheGroupByResult(GroupBy groupBy, Table table) } Map> groups; - if (table.getRowSize() > config.getParallelGroupByRowsThreshold()) { + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() + && table.getRowSize() > config.getParallelGroupByRowsThreshold()) { + use = "parallel"; groups = parallelBuild(table, colIndex); } else { + use = "sequence"; groups = seqBuild(table, colIndex); } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "groupBy use %s build, row size: %s, cost time: %s", + use, table.getRowSize(), endTime - startTime)); return applyFunc(groupBy, fields, header, groups); } @@ -581,11 +682,21 @@ public static List applyFunc( GroupBy groupBy, List fields, Header header, Map> groups) throws PhysicalException { - if (groups.size() > config.getParallelApplyFuncGroupsThreshold()) { + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() + && groups.size() > config.getParallelApplyFuncGroupsThreshold()) { + use = "parallel"; parallelApplyFunc(groupBy, fields, header, groups); } else { + use = "sequence"; seqApplyFunc(groupBy, fields, header, groups); } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "groupBy use %s apply, row size: %s, cost time: %s", + use, groups.size(), endTime - startTime)); Header newHeader = new Header(fields); int fieldSize = newHeader.getFieldSize(); @@ -776,21 +887,26 @@ private static Map> parallelBuild(Table table, int[] colIn public static List cacheFilterResult(List rows, Filter filter) throws PhysicalException { - if (rows.size() > config.getParallelFilterThreshold()) { + List result; + String use; + long startTime = System.currentTimeMillis(); + if (config.isEnableParallelOperator() && rows.size() > config.getParallelFilterThreshold()) { + use = "parallel"; ForkJoinPool pool = null; try { pool = poolQueue.take(); - return rows.parallelStream() - .filter( - row -> { - try { - return FilterUtils.validate(filter, row); - } catch (PhysicalException e) { - logger.error("execute parallel filter error, cause by: ", e.getCause()); - return false; - } - }) - .collect(Collectors.toList()); + result = + rows.parallelStream() + .filter( + row -> { + try { + return FilterUtils.validate(filter, row); + } catch (PhysicalException e) { + logger.error("execute parallel filter error, cause by: ", e.getCause()); + return false; + } + }) + .collect(Collectors.toList()); } catch (InterruptedException e) { throw new PhysicalException("parallel filter failed"); } finally { @@ -799,18 +915,26 @@ public static List cacheFilterResult(List rows, Filter filter) } } } else { - return rows.stream() - .filter( - row -> { - try { - return FilterUtils.validate(filter, row); - } catch (PhysicalException e) { - logger.error("execute sequence filter error, cause by: ", e.getCause()); - return false; - } - }) - .collect(Collectors.toList()); + use = "sequence"; + result = + rows.stream() + .filter( + row -> { + try { + return FilterUtils.validate(filter, row); + } catch (PhysicalException e) { + logger.error("execute sequence filter error, cause by: ", e.getCause()); + return false; + } + }) + .collect(Collectors.toList()); } + long endTime = System.currentTimeMillis(); + logger.info( + String.format( + "select use %s filter, row size: %s, cost time: %s", + use, rows.size(), endTime - startTime)); + return result; } public static void sortRows(List rows, boolean asc, List sortByCols) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java index 3a65425d29..c04f1cda12 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java @@ -73,6 +73,8 @@ private PhysicalTask constructTask(Operator operator, RequestContext context) { } else { return new StoragePhysicalTask(operators, context); } + } else if (source.getType() == SourceType.IGinX) { + return new IGinXPhysicalTask(operator, context); } else { // 构建内存中的计划 OperatorSource operatorSource = (OperatorSource) source; Operator sourceOperator = operatorSource.getOperator(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index cc7c9e1cdf..8971fd377b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -87,8 +87,10 @@ public class StoragePhysicalTaskExecutor { private final int maxCachedPhysicalTaskPerStorage = ConfigDescriptor.getInstance().getConfig().getMaxCachedPhysicalTaskPerStorage(); + private final StorageUnitHook storageUnitHook; + private StoragePhysicalTaskExecutor() { - StorageUnitHook storageUnitHook = + storageUnitHook = (before, after) -> { if (before == null && after != null) { // 新增加 du，处理这种事件，其他事件暂时不处理 logger.info("new storage unit " + after.getId() + " come!"); @@ -390,15 +392,16 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { public void commit(List tasks) { for (StoragePhysicalTask task : tasks) { + String key; if (replicaDispatcher == null) { - storageTaskQueues - .get(task.getTargetFragment().getMasterStorageUnitId()) - .addTask(task); // 默认情况下，异步写备，查询只查主 + key = task.getTargetFragment().getMasterStorageUnitId(); // 默认情况下，异步写备，查询只查主 } else { - storageTaskQueues - .get(replicaDispatcher.chooseReplica(task)) - .addTask(task); // 在优化策略提供了选择器的情况下，利用选择器提供的结果 + key = replicaDispatcher.chooseReplica(task); // 在优化策略提供了选择器的情况下，利用选择器提供的结果 + } + if (!storageTaskQueues.containsKey(key)) { + storageUnitHook.onChange(null, task.getTargetFragment().getMasterStorageUnit()); } + storageTaskQueues.get(key).addTask(task); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/IGinXPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/IGinXPhysicalTask.java new file mode 100644 index 0000000000..093be2fb0a --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/IGinXPhysicalTask.java @@ -0,0 +1,24 @@ +package cn.edu.tsinghua.iginx.engine.physical.task; + +import cn.edu.tsinghua.iginx.engine.physical.task.visitor.TaskVisitor; +import cn.edu.tsinghua.iginx.engine.shared.RequestContext; +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import java.util.Collections; + +public class IGinXPhysicalTask extends UnaryMemoryPhysicalTask { + + public IGinXPhysicalTask(Operator operator, RequestContext context) { + super(Collections.singletonList(operator), null, context); + } + + public Operator getOperator() { + return getOperators().get(0); + } + + @Override + public void accept(TaskVisitor visitor) { + visitor.enter(); + visitor.visit(this); + visitor.leave(); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java index eee93e3026..5d7c2d0959 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java @@ -7,6 +7,7 @@ import cn.edu.tsinghua.iginx.engine.physical.task.visitor.TaskVisitor; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.operator.Load; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.UnaryOperator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -36,6 +37,9 @@ public PhysicalTask getParentTask() { @Override public TaskExecuteResult execute() { + if (getOperators().size() == 1 && getOperators().get(0).getType().equals(OperatorType.Load)) { + return executeLoad((Load) getOperators().get(0)); + } TaskExecuteResult parentResult = parentTask.getResult(); if (parentResult == null) { return new TaskExecuteResult( @@ -62,6 +66,18 @@ public TaskExecuteResult execute() { return new TaskExecuteResult(stream); } + private TaskExecuteResult executeLoad(Load load) { + OperatorMemoryExecutor executor = + OperatorMemoryExecutorFactory.getInstance().getMemoryExecutor(); + try { + RowStream stream = executor.executeUnaryOperator(load, null, getContext()); + return new TaskExecuteResult(stream); + } catch (PhysicalException e) { + logger.error("encounter error when execute load in memory: ", e); + return new TaskExecuteResult(e); + } + } + @Override public boolean notifyParentReady() { return parentReadyCount.incrementAndGet() == 1; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java index 276e26cd32..d6752467ea 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java @@ -1,9 +1,6 @@ package cn.edu.tsinghua.iginx.engine.physical.task.utils; -import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.MultipleMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.PhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.UnaryMemoryPhysicalTask; +import cn.edu.tsinghua.iginx.engine.physical.task.*; import java.util.List; public class TaskUtils { @@ -24,7 +21,11 @@ public static void getBottomTasks(List tasks, PhysicalTask root) { break; case UnaryMemory: UnaryMemoryPhysicalTask unaryMemoryPhysicalTask = (UnaryMemoryPhysicalTask) root; - getBottomTasks(tasks, unaryMemoryPhysicalTask.getParentTask()); + if (unaryMemoryPhysicalTask instanceof IGinXPhysicalTask) { + tasks.add(unaryMemoryPhysicalTask); + } else { + getBottomTasks(tasks, unaryMemoryPhysicalTask.getParentTask()); + } break; case MultipleMemory: MultipleMemoryPhysicalTask multipleMemoryPhysicalTask = (MultipleMemoryPhysicalTask) root; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java index 7208d8d8dc..7d76be8081 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java @@ -1,12 +1,6 @@ package cn.edu.tsinghua.iginx.engine.physical.task.visitor; -import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.GlobalPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.MultipleMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.PhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.TaskType; -import cn.edu.tsinghua.iginx.engine.physical.task.UnaryMemoryPhysicalTask; +import cn.edu.tsinghua.iginx.engine.physical.task.*; import java.util.ArrayList; import java.util.List; @@ -61,6 +55,11 @@ public void visit(GlobalPhysicalTask task) { collectTaskInfo(task); } + @Override + public void visit(IGinXPhysicalTask task) { + collectTaskInfo(task); + } + private void collectTaskInfo(PhysicalTask task) { TaskType type = task.getType(); StringBuilder builder = new StringBuilder(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java index 4a1eb3d259..3b3d318de0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java @@ -1,10 +1,6 @@ package cn.edu.tsinghua.iginx.engine.physical.task.visitor; -import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.GlobalPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.MultipleMemoryPhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.UnaryMemoryPhysicalTask; +import cn.edu.tsinghua.iginx.engine.physical.task.*; public interface TaskVisitor { @@ -26,4 +22,6 @@ default void leave() {} void visit(StoragePhysicalTask task); void visit(GlobalPhysicalTask task); + + void visit(IGinXPhysicalTask task); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java index 77c12b67da..ba6a547bc5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java @@ -43,6 +43,10 @@ public class RequestContext { private String warningMsg; + private String subPlanMsg; + + private long engineCostTime; + private void init() { this.id = SnowFlakeUtils.getInstance().nextId(); this.startTime = System.currentTimeMillis(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java index d1191f3ac4..86a0447ebb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java @@ -52,7 +52,7 @@ public class Result { private long jobId; private List jobIdList; - private Map configs; + private String configValue; private String exportByteStreamDir; @@ -169,7 +169,7 @@ public ExecuteSqlResp getExecuteSqlResp() { resp.setJobId(jobId); resp.setJobState(jobState); resp.setJobIdList(jobIdList); - resp.setConfigs(configs); + resp.setConfigValue(configValue); // INFILE AS CSV resp.setLoadCsvPath(loadCSVPath); resp.setSessionIDList(sessionIDs); @@ -265,6 +265,28 @@ public ExecuteStatementResp getExecuteStatementResp(int fetchSize) { return resp; } + public ExecuteSubPlanResp getExecuteSubPlanResp() { + ExecuteSubPlanResp resp = new ExecuteSubPlanResp(status); + if (status != RpcUtils.SUCCESS && status.code != StatusCode.PARTIAL_SUCCESS.getStatusCode()) { + return resp; + } + + resp.setPaths(paths); + resp.setTagsList(tagsList); + resp.setDataTypeList(dataTypes); + + if (valuesList != null) { + if (keys != null) { + ByteBuffer keyBuffer = ByteUtils.getByteBufferFromLongArray(keys); + resp.setKeys(keyBuffer); + resp.setQueryDataSet(new QueryDataSet(keyBuffer, valuesList, bitmapList)); + } else { + resp.setQueryDataSet(new QueryDataSet(ByteBuffer.allocate(0), valuesList, bitmapList)); + } + } + return resp; + } + public LoadCSVResp getLoadCSVResp() { LoadCSVResp resp = new LoadCSVResp(status); if (status != RpcUtils.SUCCESS && status.code != StatusCode.PARTIAL_SUCCESS.getStatusCode()) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java new file mode 100644 index 0000000000..1ac17622ee --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java @@ -0,0 +1,157 @@ +package cn.edu.tsinghua.iginx.engine.shared; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.exceptions.StatusCode; +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.thrift.Status; +import cn.edu.tsinghua.iginx.utils.Bitmap; +import cn.edu.tsinghua.iginx.utils.ByteUtils; +import cn.edu.tsinghua.iginx.utils.DataTypeUtils; +import cn.edu.tsinghua.iginx.utils.RpcUtils; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultUtils { + + private static final Logger logger = LoggerFactory.getLogger(ResultUtils.class); + + public static void setEmptyQueryResp( + RequestContext ctx, List paths, List types) { + Result result = new Result(RpcUtils.SUCCESS); + result.setKeys(new Long[0]); + result.setValuesList(new ArrayList<>()); + result.setBitmapList(new ArrayList<>()); + result.setPaths(paths); + result.setDataTypes(types); + ctx.setResult(result); + } + + public static void setResultFromRowStream(RequestContext ctx, RowStream stream) + throws PhysicalException { + Result result = null; + if (ctx.isUseStream()) { + Status status = RpcUtils.SUCCESS; + if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { + status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); + status.setMessage(ctx.getWarningMsg()); + } + result = new Result(status); + result.setResultStream(stream); + ctx.setResult(result); + return; + } + + if (stream == null) { + setEmptyQueryResp(ctx, new ArrayList<>(), new ArrayList<>()); + return; + } + + List paths = new ArrayList<>(); + List> tagsList = new ArrayList<>(); + List types = new ArrayList<>(); + stream + .getHeader() + .getFields() + .forEach( + field -> { + paths.add(field.getFullName()); + types.add(field.getType()); + if (field.getTags() == null) { + tagsList.add(new HashMap<>()); + } else { + tagsList.add(field.getTags()); + } + }); + + List timestampList = new ArrayList<>(); + List valuesList = new ArrayList<>(); + List bitmapList = new ArrayList<>(); + + boolean hasTimestamp = stream.getHeader().hasKey(); + while (stream.hasNext()) { + Row row = stream.next(); + + Object[] rowValues = row.getValues(); + valuesList.add(ByteUtils.getRowByteBuffer(rowValues, types)); + + Bitmap bitmap = new Bitmap(rowValues.length); + for (int i = 0; i < rowValues.length; i++) { + if (rowValues[i] != null) { + bitmap.mark(i); + } + } + bitmapList.add(ByteBuffer.wrap(bitmap.getBytes())); + + if (hasTimestamp) { + timestampList.add(row.getKey()); + } + } + + if (valuesList.isEmpty()) { // empty result + setEmptyQueryResp(ctx, paths, types); + return; + } + + Status status = RpcUtils.SUCCESS; + if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { + status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); + status.setMessage(ctx.getWarningMsg()); + } + result = new Result(status); + if (timestampList.size() != 0) { + Long[] timestamps = timestampList.toArray(new Long[timestampList.size()]); + result.setKeys(timestamps); + } + result.setValuesList(valuesList); + result.setBitmapList(bitmapList); + result.setPaths(paths); + result.setTagsList(tagsList); + result.setDataTypes(types); + ctx.setResult(result); + + stream.close(); + } + + public static void setShowTSRowStreamResult(RequestContext ctx, RowStream stream) + throws PhysicalException { + if (ctx.isUseStream()) { + Result result = new Result(RpcUtils.SUCCESS); + result.setResultStream(stream); + ctx.setResult(result); + return; + } + List paths = new ArrayList<>(); + // todo:need physical layer to support. + List> tagsList = new ArrayList<>(); + List types = new ArrayList<>(); + + while (stream.hasNext()) { + Row row = stream.next(); + Object[] rowValues = row.getValues(); + + if (rowValues.length == 2) { + paths.add(new String((byte[]) rowValues[0])); + DataType type = DataTypeUtils.getDataTypeFromString(new String((byte[]) rowValues[1])); + if (type == null) { + logger.warn("unknown data type [{}]", rowValues[1]); + } + types.add(type); + } else { + logger.warn("show columns result col size = {}", rowValues.length); + } + } + + Result result = new Result(RpcUtils.SUCCESS); + result.setPaths(paths); + result.setTagsList(tagsList); + result.setDataTypes(types); + ctx.setResult(result); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Load.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Load.java new file mode 100644 index 0000000000..2e48572197 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Load.java @@ -0,0 +1,36 @@ +package cn.edu.tsinghua.iginx.engine.shared.operator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; +import cn.edu.tsinghua.iginx.engine.shared.source.Source; + +public class Load extends AbstractUnaryOperator { + + private final Operator operator; + + private final int index; + + public Load(Source source, int index, Operator operator) { + super(OperatorType.Load, source); + this.index = index; + this.operator = operator; + } + + public Operator getOperator() { + return operator; + } + + @Override + public Operator copy() { + return new Load(getSource().copy(), index, operator.copy()); + } + + @Override + public String getInfo() { + return "Load" + index + ": " + getSource().toString(); + } + + @Override + public UnaryOperator copyWithSource(Source source) { + return new Load(source, index, operator.copy()); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java index 7cb4be2842..522a383323 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java @@ -65,7 +65,8 @@ public enum OperatorType { GroupBy, Distinct, ProjectWaitingForPath, - ValueToSelectedPath; + ValueToSelectedPath, + Load; private int value; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/FragmentsVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/FragmentsVisitor.java new file mode 100644 index 0000000000..2be132bf32 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/FragmentsVisitor.java @@ -0,0 +1,32 @@ +package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; + +import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; +import cn.edu.tsinghua.iginx.engine.shared.operator.MultipleOperator; +import cn.edu.tsinghua.iginx.engine.shared.operator.UnaryOperator; +import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; + +public class FragmentsVisitor implements OperatorVisitor { + + private int fragmentCount = 0; + + public int getFragmentCount() { + return fragmentCount; + } + + @Override + public void visit(UnaryOperator unaryOperator) { + if (unaryOperator.getSource().getType().equals(SourceType.Fragment)) { + fragmentCount++; + } + } + + @Override + public void visit(BinaryOperator binaryOperator) { + // do nothing + } + + @Override + public void visit(MultipleOperator multipleOperator) { + // do nothing + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/IGinXSource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/IGinXSource.java new file mode 100644 index 0000000000..9c4418aad4 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/IGinXSource.java @@ -0,0 +1,47 @@ +package cn.edu.tsinghua.iginx.engine.shared.source; + +import java.util.Objects; + +public class IGinXSource extends AbstractSource { + + private final String ip; + + private final int port; + + public IGinXSource(String ip, int port) { + super(SourceType.IGinX); + this.ip = ip; + this.port = port; + } + + public String getIp() { + return ip; + } + + public int getPort() { + return port; + } + + @Override + public Source copy() { + return new IGinXSource(ip, port); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IGinXSource that = (IGinXSource) o; + return port == that.port && Objects.equals(ip, that.ip); + } + + @Override + public int hashCode() { + return Objects.hash(ip, port); + } + + @Override + public String toString() { + return "IGinXSource[" + "ip='" + ip + '\'' + ", port=" + port + ']'; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java index c207473c74..5ac33f2e93 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java @@ -23,5 +23,6 @@ public enum SourceType { Fragment, Operator, + IGinX, Global, } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java index 8f1944891a..e0b4607e52 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java @@ -1005,7 +1005,9 @@ public boolean createInitialFragmentsAndStorageUnits( if (cache.hasFragment() && cache.hasStorageUnit()) { return false; } - checkInitialFragmentCompletion(initialFragments); + if (initialFragments != null) { + checkInitialFragmentCompletion(initialFragments); + } List newStorageUnits = new ArrayList<>(); try { storage.lockFragment(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java index ad16accb8c..baab7c67dd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java @@ -45,7 +45,6 @@ public void execute(RequestContext ctx) throws ExecutionException { } Result result = new Result(RpcUtils.SUCCESS); - result.setConfigs(configs); ctx.setResult(result); } catch (NoSuchFieldException e) { String errMsg = String.format("no such field, field=%s", configName); diff --git a/core/src/main/resources/log4j.properties b/core/src/main/resources/log4j.properties new file mode 100644 index 0000000000..ec4c999c6a --- /dev/null +++ b/core/src/main/resources/log4j.properties @@ -0,0 +1,17 @@ +log4j.rootLogger=INFO, console +log4j.additivity.org.apache=true +#console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=DEBUG +log4j.appender.console.ImmediateFlush=true +log4j.appender.console.Target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d [%t] %5p - [%C.%M:%L] %m%n +log4j.appender.dailyFile=org.apache.log4j.DailyRollingFileAppender +log4j.appender.dailyFile.Threshold=DEBUG +log4j.appender.dailyFile.ImmediateFlush=true +log4j.appender.dailyFile.Append=true +log4j.appender.dailyFile.File=logs/log.log4j +log4j.appender.dailyFile.DatePattern='.'yyyy-MM-dd +log4j.appender.dailyFile.layout=org.apache.log4j.PatternLayout +log4j.appender.dailyFile.layout.ConversionPattern=%d [%t] %5p - [%C.%M:%L] %m%n diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/MyCounter.java b/core/src/test/java/cn/edu/tsinghua/iginx/MyCounter.java new file mode 100644 index 0000000000..2954b4a942 --- /dev/null +++ b/core/src/test/java/cn/edu/tsinghua/iginx/MyCounter.java @@ -0,0 +1,38 @@ +package cn.edu.tsinghua.iginx; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +public class MyCounter { + + public static void main(String[] args) throws IOException { + String path = "/Users/cauchy-ny/Downloads/benchmark-3.log"; + FileInputStream inputStream = new FileInputStream(path); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + + int time = 300000; + int offset = time; + int count = 0; + + String str; + List list = new ArrayList<>(); + while ((str = reader.readLine()) != null) { + String[] arr = str.split(","); + if (arr.length != 3) { + continue; + } + int curTime = Integer.parseInt(arr[2]); + if (curTime < time) { + count++; + } else { + list.add(count); + count = 1; + time += offset; + } + } + list.add(count); + + list.forEach(System.out::println); + } +} diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/MyTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/MyTest.java new file mode 100644 index 0000000000..6e845f4a39 --- /dev/null +++ b/core/src/test/java/cn/edu/tsinghua/iginx/MyTest.java @@ -0,0 +1,119 @@ +package cn.edu.tsinghua.iginx; + +import static org.junit.Assert.assertEquals; + +import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +public class MyTest { + + private static Random random = new Random(); + + ForkJoinPool pool = new ForkJoinPool(5); + ExecutorService executor = Executors.newFixedThreadPool(5); + + @org.junit.Test + public void testA() throws ExecutionException, InterruptedException { + // Operator before = TreeBuilder.buildFilterFragmentTree(); + // String a = FastjsonSerializeUtils.serialize(before); + // Operator after = FastjsonSerializeUtils.deserialize(a, Operator.class); + int size = 1000_0000; + List rows = getRandomRows(size); + + long startTime, endTime; + + startTime = System.currentTimeMillis(); + Map> map1 = new HashMap<>(); + for (Row row : rows) { + int hash = row.index; + List l = map1.computeIfAbsent(hash, k -> new ArrayList<>()); + l.add(row); + } + endTime = System.currentTimeMillis(); + System.out.println("sequence cost time: " + (endTime - startTime)); + + startTime = System.currentTimeMillis(); + Map> map2 = + pool.submit( + () -> + rows.parallelStream() + .collect( + Collectors.groupingBy( + row -> { + return row.index; + }))) + .get(); + endTime = System.currentTimeMillis(); + System.out.println("parallel cost time: " + (endTime - startTime)); + + startTime = System.currentTimeMillis(); + int range = size / 5; + Map> map3 = new HashMap<>(); + List>> list = new ArrayList<>(); + + CountDownLatch latch = new CountDownLatch(5); + for (int i = 0; i < 5; i++) { + int finalI = i; + list.add(new HashMap<>()); + pool.submit( + () -> { + for (int j = finalI * range; j < (finalI + 1) * range; j++) { + Row row = rows.get(j); + int hash = row.index; + List l = list.get(finalI).computeIfAbsent(hash, k -> new ArrayList<>()); + l.add(row); + } + latch.countDown(); + }); + } + latch.await(); + // list.forEach(map -> + // map.forEach((key, value) -> + // map3.merge(key, value, (list1, list2) -> { + // list1.addAll(list2); // 合并两个列表 + // return list1; + // }) + // ) + // ); + Map> mergedMap = + pool.submit( + () -> + list.parallelStream() + .flatMap(map -> map.entrySet().stream()) // 将每个Map转换为流 + .collect( + Collectors.toConcurrentMap( + Map.Entry::getKey, // 键 + Map.Entry::getValue, // 值 + (list1, list2) -> { // 合并函数，用于处理重复键 + list1.addAll(list2); // 合并两个列表 + return list1; + }))) + .get(); + endTime = System.currentTimeMillis(); + System.out.println("new way cost time: " + (endTime - startTime)); + + assertEquals(map1.size(), map2.size()); + assertEquals(map2.size(), mergedMap.size()); + } + + private static List getRandomRows(int size) { + int range = size / 10; + List result = new ArrayList<>(); + for (int i = 0; i < size; i++) { + result.add(new Row(random.nextInt(range), random.nextInt(size))); + } + return result; + } + + static class Row { + int index; + int value; + + public Row(int index, int value) { + this.index = index; + this.value = value; + } + } +} diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 71ec5afd98..3617025c6a 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -24,6 +24,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; +import cn.edu.tsinghua.iginx.engine.physical.memory.execute.Table; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream.EmptyRowStream; import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; @@ -31,6 +32,7 @@ import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; @@ -343,7 +345,13 @@ private TaskExecuteResult executeProjectWithFilter( new ClearEmptyRowStreamWrapper( new IoTDBQueryRowStream( sessionPool.executeQueryStatement(statement), true, project, filter)); - return new TaskExecuteResult(rowStream); + + List newRows = new ArrayList<>(); + while (rowStream.hasNext()) { + newRows.add(rowStream.next()); + } + Table table = new Table(rowStream.getHeader(), newRows); + return new TaskExecuteResult(table); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { logger.error(e.getMessage()); return new TaskExecuteResult( diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java index 94ae728d8a..32d3b62c7a 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java @@ -18,7 +18,6 @@ */ package cn.edu.tsinghua.iginx.iotdb.query.entity; -import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.TEXT; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -206,9 +205,9 @@ private void cacheOneRow() throws SQLException, PhysicalException { } state = State.HAS_NEXT; cachedRow = new Row(header, timestamp, fields); - if (!validate(filter, cachedRow)) { - cacheOneRow(); - } + // if (!validate(filter, cachedRow)) { + // cacheOneRow(); + // } } else { state = State.NO_NEXT; cachedRow = null; diff --git a/pom.xml b/pom.xml index 0717dcbf28..58d747f614 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,8 @@ test zeppelin-interpreter tools-exportCsv + tools-tpch + tools-benchmark diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java index fda9f56de5..b19bae5eb3 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java @@ -926,6 +926,15 @@ public SessionExecuteSqlResult executeSql(String statement) return new SessionExecuteSqlResult(ref.resp); } + public SessionExecuteSubPlanResult executeSubPlan(String subPlanMsg) + throws SessionException, ExecutionException { + ExecuteSubPlanReq req = new ExecuteSubPlanReq(sessionId, subPlanMsg); + Reference ref = new Reference<>(); + executeWithCheck(() -> (ref.resp = client.executeSubPlan(req)).status); + + return new SessionExecuteSubPlanResult(ref.resp); + } + public SessionQueryDataSet queryLast( List paths, long startKey, TimePrecision timePrecision) throws SessionException, ExecutionException { diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java index a137aca080..2a911cc027 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java @@ -43,12 +43,14 @@ public class SessionExecuteSqlResult { private long jobId; private JobState jobState; private List jobIdList; - private Map configs; + private String configValue; private String loadCsvPath; private List sessionIDs; private Map rules; + private long totalCostTime; + // Only for mock test public SessionExecuteSqlResult() {} @@ -102,7 +104,7 @@ public SessionExecuteSqlResult(ExecuteSqlResp resp) { this.jobIdList = resp.getJobIdList(); break; case ShowConfig: - this.configs = resp.getConfigs(); + this.configValue = resp.getConfigValue(); break; case LoadCsv: this.loadCsvPath = resp.getLoadCsvPath(); @@ -131,6 +133,7 @@ private void constructQueryResult(ExecuteSqlResp resp) { } else { this.values = new ArrayList<>(); } + this.totalCostTime = resp.getCostTime(); } public List> getResultInList( @@ -182,8 +185,6 @@ public String getResultInString(boolean needFormatTime, String timePrecision) { return buildShowEligibleJobResult(); case ShowSessionID: return buildShowSessionIDResult(); - case ShowConfig: - return buildShowConfigResult(); case ShowRules: return buildShowRulesResult(); case GetReplicaNum: @@ -194,6 +195,8 @@ public String getResultInString(boolean needFormatTime, String timePrecision) { return "job id: " + jobId; case ShowJobStatus: return "Job status: " + jobState; + case ShowConfig: + return "config value: " + configValue + "\n"; default: return "No data to print." + "\n"; } @@ -208,9 +211,58 @@ private String buildQueryResult(boolean needFormatTime, String timePrecision) { builder.append(FormatUtils.formatResult(cache)); builder.append(FormatUtils.formatCount(cache.size() - 1)); + + if (paths != null && !paths.isEmpty() && paths.get(0).equals("Physical Tree")) { + builder.append(cacheMoreThings()); + } return builder.toString(); } + private String cacheMoreThings() { + Map map = new HashMap<>(); + for (List row : values) { + String timeStr = FormatUtils.valueToString(row.get(1)); + String name = FormatUtils.valueToString(row.get(3)).split(":")[0]; + int costTime = Integer.parseInt(timeStr.substring(0, timeStr.length() - 2)); + if (map.containsKey(name)) { + map.get(name).addUp(costTime); + } else { + map.put(name, new OpStats(name, costTime)); + } + } + + List list = new ArrayList<>(map.values()); + list.sort(Comparator.comparingInt(a -> a.costTime)); + + StringBuilder builder = new StringBuilder(); + list.forEach( + opStats -> + builder + .append( + String.format( + "[%s] cost time: %s, num: %s", opStats.name, opStats.costTime, opStats.num)) + .append("\n")); + builder.append("total cost: ").append(totalCostTime).append("ms\n"); + return builder.toString(); + } + + static class OpStats { + String name; + int num; + int costTime; + + public OpStats(String name, int costTime) { + this.num = 1; + this.name = name; + this.costTime = costTime; + } + + public void addUp(int costTime) { + this.num++; + this.costTime += costTime; + } + } + private List> cacheResult( boolean needFormatTime, String timeFormat, String timePrecision) { List> cache = new ArrayList<>(); @@ -381,21 +433,6 @@ private String buildShowSessionIDResult() { return builder.toString(); } - private String buildShowConfigResult() { - StringBuilder builder = new StringBuilder(); - if (configs != null) { - builder.append("Config Info:").append("\n"); - List> cache = new ArrayList<>(); - cache.add(new ArrayList<>(Arrays.asList("ConfigName", "ConfigValue"))); - configs.forEach( - (name, value) -> { - cache.add(new ArrayList<>(Arrays.asList(name, value))); - }); - builder.append(FormatUtils.formatResult(cache)); - } - return builder.toString(); - } - private String buildShowRulesResult() { StringBuilder builder = new StringBuilder(); if (rules != null) { @@ -570,8 +607,4 @@ public String getLoadCsvPath() { public List getSessionIDs() { return sessionIDs; } - - public Map getConfigs() { - return configs; - } } diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSubPlanResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSubPlanResult.java new file mode 100644 index 0000000000..e07377ffc3 --- /dev/null +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSubPlanResult.java @@ -0,0 +1,53 @@ +package cn.edu.tsinghua.iginx.session; + +import static cn.edu.tsinghua.iginx.utils.ByteUtils.getLongArrayFromByteBuffer; +import static cn.edu.tsinghua.iginx.utils.ByteUtils.getValuesFromBufferAndBitmaps; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.thrift.ExecuteSubPlanResp; +import java.util.ArrayList; +import java.util.List; + +public class SessionExecuteSubPlanResult { + + private final long[] keys; + private final List paths; + private final List> values; + private final List dataTypeList; + + public SessionExecuteSubPlanResult(ExecuteSubPlanResp resp) { + this.paths = resp.getPaths(); + this.dataTypeList = resp.getDataTypeList(); + + if (resp.getKeys() != null) { + this.keys = getLongArrayFromByteBuffer(resp.keys); + } else { + this.keys = null; + } + + // parse values + if (resp.getQueryDataSet() != null) { + this.values = + getValuesFromBufferAndBitmaps( + resp.dataTypeList, resp.queryDataSet.valuesList, resp.queryDataSet.bitmapList); + } else { + this.values = new ArrayList<>(); + } + } + + public long[] getKeys() { + return keys; + } + + public List getPaths() { + return paths; + } + + public List> getValues() { + return values; + } + + public List getDataTypeList() { + return dataTypeList; + } +} diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FastjsonSerializeUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FastjsonSerializeUtils.java new file mode 100644 index 0000000000..117d839223 --- /dev/null +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FastjsonSerializeUtils.java @@ -0,0 +1,19 @@ +package cn.edu.tsinghua.iginx.utils; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONReader; +import com.alibaba.fastjson2.JSONWriter; + +public class FastjsonSerializeUtils { + + public static String serialize(T obj) { + if (obj == null) { + throw new NullPointerException(); + } + return JSON.toJSONString(obj, JSONWriter.Feature.WriteClassName, JSONWriter.Feature.FieldBased); + } + + public static T deserialize(String json, Class clazz) { + return JSON.parseObject(json, clazz, JSONReader.Feature.SupportAutoType); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index f2c52e9295..62464c8d99 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -1,6 +1,5 @@ package cn.edu.tsinghua.iginx.integration.func.sql; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import cn.edu.tsinghua.iginx.exceptions.ExecutionException; @@ -17,10 +16,7 @@ import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.AfterClass; @@ -5961,125 +5957,6 @@ public void testConcurrentQuery() { executor.concurrentExecuteAndCompare(statementsAndExpectRes); } - @Test - public void testShowConfig() { - String statement = "show config \"memoryTaskThreadPoolSize\";"; - String expected = - "Config Info:\n" - + "+------------------------+-----------+\n" - + "| ConfigName|ConfigValue|\n" - + "+------------------------+-----------+\n" - + "|memoryTaskThreadPoolSize| 200|\n" - + "+------------------------+-----------+\n"; - executor.executeAndCompare(statement, expected); - - statement = "show config \"enableEnvParameter\";"; - expected = - "Config Info:\n" - + "+------------------+-----------+\n" - + "| ConfigName|ConfigValue|\n" - + "+------------------+-----------+\n" - + "|enableEnvParameter| false|\n" - + "+------------------+-----------+\n"; - executor.executeAndCompare(statement, expected); - - statement = "show config;"; - Map configs = executor.getSessionExecuteSqlResult(statement).getConfigs(); - Set expectedConfigNames = - new HashSet<>( - Arrays.asList( - "cachedTimeseriesProb", - "parallelFilterThreshold", - "memoryTaskThreadPoolSize", - "statisticsCollectorClassName", - "systemMemoryThreshold", - "streamParallelGroupByWorkerNum", - "password", - "constraintChecker", - "mqttHost", - "fragmentCompactionReadRatioThreshold", - "migrationPolicyClassName", - "metaStorage", - "tagPrefix", - "ip", - "enableRestService", - "parallelGroupByPoolNum", - "enablePushDown", - "loadBalanceCheckInterval", - "parallelApplyFuncGroupsThreshold", - "timePrecision", - "useStreamExecutor", - "parallelGroupByRowsThreshold", - "enableMonitor", - "databaseClassNames", - "mqttPort", - "clients", - "defaultUDFDir", - "ruleBasedOptimizer", - "instancesNumPerClient", - "physicalOptimizer", - "maxAsyncRetryTimes", - "enableInstantCompaction", - "maxCachedPhysicalTaskPerStorage", - "restIp", - "physicalTaskThreadPoolSizePerStorage", - "disorderMargin", - "enableEnvParameter", - "syncExecuteThreadPool", - "policyClassName", - "fragmentCacheThreshold", - "fragmentPerEngine", - "enableStorageGroupValueLimit", - "zookeeperConnectionString", - "minThriftWorkerThreadNum", - "udfList", - "migrationBatchSize", - "maxTimeseriesLoadBalanceThreshold", - "mqttMaxMessageSize", - "reshardFragmentTimeMargin", - "storageGroupValueLimit", - "replicaNum", - "enableMetaCacheControl", - "fragmentCompactionReadThreshold", - "storageEngineList", - "batchSizeImportCsv", - "restPort", - "asyncRestThreadPool", - "fragmentCompactionWriteThreshold", - "expectedStorageUnitNum", - "maxThriftWrokerThreadNum", - "isUTTestEnv", - "tagSuffix", - "port", - "asyncExecuteThreadPool", - "needInitBasicUDFFunctions", - "retryWait", - "reAllocatePeriod", - "pythonCMD", - "historicalPrefixList", - "transformMaxRetryTimes", - "transformTaskThreadPoolSize", - "enableFragmentCompaction", - "maxReshardFragmentsNum", - "statisticsLogInterval", - "enableMemoryControl", - "retryCount", - "tagNameAnnotation", - "etcdEndpoints", - "mqttPayloadFormatter", - "queryOptimizer", - "systemCpuThreshold", - "enableMQTT", - "mqttHandlerPoolSize", - "heapMemoryThreshold", - "systemResourceMetrics", - "maxTimeseriesLength", - "batchSize", - "parallelGroupByPoolSize", - "username")); - assertEquals(expectedConfigNames, configs.keySet()); - } - @Test public void testModifyRules() { String statement, expected; diff --git a/test/src/test/resources/fileReadAndWrite/byteStream/test.s1 b/test/src/test/resources/fileReadAndWrite/byteStream/test.s1 new file mode 100644 index 0000000000000000000000000000000000000000..7053bede70eb1c12ad87ab08ffb11036fae06b4d GIT binary patch literal 40 VcmZQzfB;4)%><>Hp)?DW1^@ua01E&B literal 0 HcmV?d00001 diff --git a/test/src/test/resources/fileReadAndWrite/byteStream/test.s2 b/test/src/test/resources/fileReadAndWrite/byteStream/test.s2 new file mode 100644 index 0000000000000000000000000000000000000000..0fa3234e1f0b5ca46306d5ad4604bd4b4cd253c9 GIT binary patch literal 40 acmZQz0D%YgV2a@flxA^&@OhxLkOKfs0s}b! literal 0 HcmV?d00001 diff --git a/test/src/test/resources/fileReadAndWrite/byteStream/test.s3 b/test/src/test/resources/fileReadAndWrite/byteStream/test.s3 new file mode 100644 index 0000000000000000000000000000000000000000..6ca6c0c8746ed7c9b3ac6017f1e998a9b065909f GIT binary patch literal 5 McmZQ%U}Rtf000gE1ONa4 literal 0 HcmV?d00001 diff --git a/test/src/test/resources/fileReadAndWrite/byteStream/test.s4 b/test/src/test/resources/fileReadAndWrite/byteStream/test.s4 new file mode 100644 index 0000000000..e45d59e162 --- /dev/null +++ b/test/src/test/resources/fileReadAndWrite/byteStream/test.s4 @@ -0,0 +1 @@ +aaabbbcccdddeee \ No newline at end of file diff --git a/test/src/test/resources/fileReadAndWrite/csv/test.csv b/test/src/test/resources/fileReadAndWrite/csv/test.csv new file mode 100644 index 0000000000..6d12ded171 --- /dev/null +++ b/test/src/test/resources/fileReadAndWrite/csv/test.csv @@ -0,0 +1,5 @@ +0,0,0.5,true,aaa +1,1,1.5,false,bbb +2,2,2.5,true,ccc +3,3,3.5,false,ddd +4,4,4.5,true,eee diff --git a/thrift/src/main/proto/rpc.thrift b/thrift/src/main/proto/rpc.thrift index 36209e4bb8..72194c76df 100644 --- a/thrift/src/main/proto/rpc.thrift +++ b/thrift/src/main/proto/rpc.thrift @@ -377,10 +377,25 @@ struct ExecuteSqlResp { 22: optional i64 jobId 23: optional JobState jobState 24: optional list jobIdList - 25: optional map configs + 25: optional string configValue 26: optional string loadCsvPath 27: optional list sessionIDList 28: optional map rules + 29: optional i64 costTime +} + +struct ExecuteSubPlanReq { + 1: required i64 sessionId + 2: required string subPlan +} + +struct ExecuteSubPlanResp { + 1: required Status status + 2: optional list paths + 3: optional list> tagsList + 4: optional list dataTypeList + 5: optional binary keys + 6: optional QueryDataSet queryDataSet } struct UpdateUserReq { @@ -764,4 +779,6 @@ service IService { ShowRulesResp showRules(1: ShowRulesReq req); Status setRules(1: SetRulesReq req); + + ExecuteSubPlanResp executeSubPlan(1: ExecuteSubPlanReq req); } diff --git a/tools-benchmark/pom.xml b/tools-benchmark/pom.xml new file mode 100644 index 0000000000..bec5cddfdf --- /dev/null +++ b/tools-benchmark/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + cn.edu.tsinghua + iginx + 0.6.0-SNAPSHOT + + + tools-benchmark + + + 8 + 8 + UTF-8 + + + + + commons-cli + commons-cli + 1.5.0 + + + cn.edu.tsinghua + iginx-session + ${project.version} + + + cn.edu.tsinghua + iginx-thrift + ${project.version} + + + org.apache.commons + commons-csv + 1.10.0 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + true + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + org.apache.maven.plugins + maven-assembly-plugin + 3.2.0 + + + src/assembly/tools.xml + + false + target + + + true + true + + + + + + tools-assembly + + single + + package + + + + + + diff --git a/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh b/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh new file mode 100644 index 0000000000..595946398f --- /dev/null +++ b/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# You can put your env variable here +# export JAVA_HOME=$JAVA_HOME + +if [[ -z "${IGINX_HOME}" ]]; then + export IGINX_HOME="$( + cd "$(dirname "$0")"/.. + pwd + )" +fi + +MAIN_CLASS=cn.edu.tsinghua.iginx.tools.benchmark.MyQueryBenchmark + +CLASSPATH="" +for f in ${IGINX_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}":"$f +done + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +#computing the memory size for the JVM options +calculate_heap_sizes() { + case "$(uname)" in + Linux) + system_memory_in_mb=$(free -m | sed -n '2p' | awk '{print $2}') + system_cpu_cores=$(egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo) + ;; + FreeBSD) + system_memory_in_bytes=$(sysctl hw.physmem | awk '{print $2}') + system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) + system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') + ;; + SunOS) + system_memory_in_mb=$(prtconf | awk '/Memory size:/ {print $3}') + system_cpu_cores=$(psrinfo | wc -l) + ;; + Darwin) + system_memory_in_bytes=$(sysctl hw.memsize | awk '{print $2}') + system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) + system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') + ;; + *) + # assume reasonable defaults for e.g. a modern desktop or + # cheap server + system_memory_in_mb="2048" + system_cpu_cores="2" + ;; + esac + + # some systems like the raspberry pi don't report cores, use at least 1 + if [ "$system_cpu_cores" -lt "1" ]; then + system_cpu_cores="1" + fi + + # set max heap size based on the following + # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB)) + # calculate 1/2 ram and cap to 1024MB + # calculate 1/4 ram and cap to 65536MB + # pick the max + half_system_memory_in_mb=$(expr $system_memory_in_mb / 2) + quarter_system_memory_in_mb=$(expr $half_system_memory_in_mb / 2) + if [ "$half_system_memory_in_mb" -gt "1024" ]; then + half_system_memory_in_mb="1024" + fi + if [ "$quarter_system_memory_in_mb" -gt "65536" ]; then + quarter_system_memory_in_mb="65536" + fi + if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]; then + max_heap_size_in_mb="$half_system_memory_in_mb" + else + max_heap_size_in_mb="$quarter_system_memory_in_mb" + fi + MAX_HEAP_SIZE="${max_heap_size_in_mb}M" +} + +calculate_heap_sizes +JMX_OPTS="" +JMX_OPTS="$JMX_OPTS -Xms${MAX_HEAP_SIZE}" +JMX_OPTS="$JMX_OPTS -Xmx${MAX_HEAP_SIZE}" + +# continue to other parameters +ICONF="$IGINX_HOME/conf/config.properties" +IDRIVER="$IGINX_HOME/driver/" + +export IGINX_CONF=$ICONF +export IGINX_DRIVER=$IDRIVER + +exec "$JAVA" -Duser.timezone=GMT+8 -cp "$CLASSPATH" "$MAIN_CLASS" "$@" + +exit $? diff --git a/tools-benchmark/src/assembly/tools.xml b/tools-benchmark/src/assembly/tools.xml new file mode 100644 index 0000000000..299015234a --- /dev/null +++ b/tools-benchmark/src/assembly/tools.xml @@ -0,0 +1,39 @@ + + + + tools + + dir + + false + + + lib + + + + + src/assembly/resources + ${file.separator} + + + \ No newline at end of file diff --git a/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java b/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java new file mode 100644 index 0000000000..a9edaa0146 --- /dev/null +++ b/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java @@ -0,0 +1,596 @@ +package cn.edu.tsinghua.iginx.tools.benchmark; + +import cn.edu.tsinghua.iginx.exceptions.ExecutionException; +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class MyQueryBenchmark { + + public static final String PREFIX = "explain physical "; + + public static final String Q1 = + "select\n" + + " lineitem.l_returnflag as l_returnflag,\n" + + " lineitem.l_linestatus as l_linestatus,\n" + + " sum(lineitem.l_quantity) as sum_qty,\n" + + " sum(lineitem.l_extendedprice) as sum_base_price,\n" + + " sum(tmp1) as sum_disc_price, \n" + + " sum(tmp2) as sum_charge, \n" + + " avg(lineitem.l_quantity) as avg_qty,\n" + + " avg(lineitem.l_extendedprice) as avg_price,\n" + + " avg(lineitem.l_discount) as avg_disc,\n" + + " count(*) as count_order\n" + + "from (\n" + + " select\n" + + " l_returnflag,\n" + + " l_linestatus,\n" + + " l_quantity,\n" + + " l_extendedprice,\n" + + " l_discount,\n" + + " l_extendedprice * (1 - l_discount) as tmp1,\n" + + " l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2\n" + + " from\n" + + " lineitem\n" + + " where\n" + + " lineitem.l_shipdate <= 9124416000\n" + + ")\n" + + "group by\n" + + " lineitem.l_returnflag,\n" + + " lineitem.l_linestatus\n" + + "order by\n" + + " lineitem.l_returnflag,\n" + + " lineitem.l_linestatus;"; + + public static final String Q2 = + "select\n" + + " supplier.s_acctbal,\n" + + " supplier.s_name,\n" + + " nation.n_name,\n" + + " part.p_partkey,\n" + + " part.p_mfgr,\n" + + " supplier.s_address,\n" + + " supplier.s_phone,\n" + + " supplier.s_comment\n" + + "from\n" + + " part\n" + + " join partsupp on part.p_partkey = partsupp.ps_partkey\n" + + " join supplier on supplier.s_suppkey = partsupp.ps_suppkey\n" + + " join nation on supplier.s_nationkey = nation.n_nationkey\n" + + " join region on nation.n_regionkey = region.r_regionkey\n" + + "where\n" + + " partsupp.ps_supplycost = (\n" + + " select\n" + + " min(partsupp.ps_supplycost)\n" + + " from\n" + + " partsupp\n" + + " join supplier on supplier.s_suppkey = partsupp.ps_suppkey\n" + + " join nation on supplier.s_nationkey = nation.n_nationkey\n" + + " join region on nation.n_regionkey = region.r_regionkey\n" + + " where\n" + + " part.p_partkey = partsupp.ps_partkey \n" + + " and region.r_name = 'AMERICA'\n" + + " )\n" + + "order by\n" + + " supplier.s_acctbal,\n" + + " nation.n_name,\n" + + " supplier.s_name,\n" + + " part.p_partkey;"; + + public static final String Q3 = + "select\n" + + " lineitem.l_orderkey,\n" + + " orders.o_orderdate,\n" + + " orders.o_shippriority\n" + + "from\n" + + " customer\n" + + " join orders on customer.c_custkey = orders.o_custkey\n" + + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" + + "where\n" + + " customer.c_mktsegment = 'BUILDING'\n" + + " and orders.o_orderdate < 9124416000\n" + + " and lineitem.l_shipdate > 0\n" + + "group by\n" + + " lineitem.l_orderkey,\n" + + " orders.o_orderdate,\n" + + " orders.o_shippriority\n" + + "order by\n" + + " orders.o_orderdate;"; + + public static final String Q4 = + "select\n" + + " orders.o_orderpriority,\n" + + " count(*) as order_count\n" + + "from\n" + + " orders\n" + + "where\n" + + " orders.o_orderdate >= 0\n" + + " and orders.o_orderdate < 9124416000\n" + + " and exists (\n" + + " select\n" + + " *\n" + + " from\n" + + " lineitem\n" + + " where\n" + + " lineitem.l_orderkey = orders.o_orderkey\n" + + " and lineitem.l_commitdate < lineitem.l_receiptdate\n" + + " )\n" + + "group by\n" + + " orders.o_orderpriority\n" + + "order by\n" + + " orders.o_orderpriority;"; + + public static final String Q5 = + "select \n" + + " nation.n_name,\n" + + " revenue\n" + + "from (\n" + + " select\n" + + " nation.n_name,\n" + + " sum(tmp) as revenue\n" + + " from (\n" + + " select\n" + + " nation.n_name,\n" + + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp\n" + + " from\n" + + " customer\n" + + " join orders on customer.c_custkey = orders.o_custkey\n" + + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" + + " join supplier on lineitem.l_suppkey = supplier.s_suppkey and customer.c_nationkey = supplier.s_nationkey\n" + + " join nation on supplier.s_nationkey = nation.n_nationkey\n" + + " join region on nation.n_regionkey = region.r_regionkey\n" + + " where\n" + + " region.r_name = \"EUROPE\"\n" + + " and orders.o_orderdate >= 0\n" + + " and orders.o_orderdate < 9124416000\n" + + ")\n" + + "group by\n" + + " nation.n_name\n" + + ")\n" + + "order by\n" + + " revenue desc;"; + + public static final String Q6 = + "select\n" + + " sum(tmp) as revenue\n" + + "from (\n" + + " select\n" + + " l_extendedprice * l_discount as tmp\n" + + " from\n" + + " lineitem\n" + + " where\n" + + " lineitem.l_shipdate >= 0\n" + + " and lineitem.l_shipdate < 9124416000\n" + + " and lineitem.l_discount >= 0.02 \n" + + " and lineitem.l_discount < 0.09\n" + + " and lineitem.l_quantity < 25\n" + + ");"; + + public static final String Q9 = + "select\n" + + " nation,\n" + + " date,\n" + + " sum(amount) as sum_profit\n" + + "from (\n" + + " select\n" + + " nation.n_name as nation,\n" + + " orders.o_orderdate as date,\n" + + " lineitem.l_extendedprice * (1 - lineitem.l_discount) - partsupp.ps_supplycost * lineitem.l_quantity as amount\n" + + " from\n" + + " part\n" + + " join lineitem on part.p_partkey = lineitem.l_partkey\n" + + " join supplier on supplier.s_suppkey = lineitem.l_suppkey\n" + + " join partsupp on partsupp.ps_suppkey = lineitem.l_suppkey and partsupp.ps_partkey = lineitem.l_partkey\n" + + " join orders on orders.o_orderkey = lineitem.l_orderkey\n" + + " join nation on supplier.s_nationkey = nation.n_nationkey\n" + + " where\n" + + " part.p_name like '.*yellow.*'\n" + + ")\n" + + "group by\n" + + " nation,\n" + + " date\n" + + "order by\n" + + " nation,\n" + + " date desc;"; + + public static final String Q10 = + "select\n" + + " customer.c_custkey,\n" + + " customer.c_name,\n" + + " revenue,\n" + + " customer.c_acctbal,\n" + + " nation.n_name,\n" + + " customer.c_address,\n" + + " customer.c_phone,\n" + + " customer.c_comment\n" + + "from (\n" + + " select\n" + + " customer.c_custkey,\n" + + " customer.c_name,\n" + + " sum(tmp) as revenue,\n" + + " customer.c_acctbal,\n" + + " nation.n_name,\n" + + " customer.c_address,\n" + + " customer.c_phone,\n" + + " customer.c_comment\n" + + " from (\n" + + " select\n" + + " customer.c_custkey,\n" + + " customer.c_name,\n" + + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp,\n" + + " customer.c_acctbal,\n" + + " nation.n_name,\n" + + " customer.c_address,\n" + + " customer.c_phone,\n" + + " customer.c_comment\n" + + " from\n" + + " customer\n" + + " join orders on customer.c_custkey = orders.o_custkey\n" + + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" + + " join nation on customer.c_nationkey = nation.n_nationkey\n" + + " where\n" + + " orders.o_orderdate >= 0\n" + + " and orders.o_orderdate < 9124416000\n" + + " and lineitem.l_returnflag = 'R'\n" + + " )\n" + + " group by\n" + + " customer.c_custkey,\n" + + " customer.c_name,\n" + + " customer.c_acctbal,\n" + + " customer.c_phone,\n" + + " nation.n_name,\n" + + " customer.c_address,\n" + + " customer.c_comment\n" + + ")\n" + + "order by\n" + + " revenue desc;"; + + public static final String Q13 = + "select\n" + + " c_count, \n" + + " custdist\n" + + "from (\n" + + " select\n" + + " c_count, \n" + + " count(*) as custdist\n" + + " from (\n" + + " select\n" + + " customer.c_custkey as c_custkey,\n" + + " count(orders.o_orderkey) as c_count\n" + + " from\n" + + " customer left outer join orders on customer.c_custkey = orders.o_custkey and !(orders.o_comment like '.*pending.*')\n" + + " group by\n" + + " customer.c_custkey\n" + + " )\n" + + " group by\n" + + " c_count\n" + + ")\n" + + "order by\n" + + " custdist,\n" + + " c_count desc;"; + + public static final String Q16 = + "select\n" + + " p_brand,\n" + + " p_type,\n" + + " p_size,\n" + + " supplier_cnt\n" + + "from (\n" + + " select\n" + + " part.p_brand as p_brand,\n" + + " part.p_type as p_type,\n" + + " part.p_size as p_size,\n" + + " count(distinct partsupp.ps_suppkey) as supplier_cnt\n" + + " from\n" + + " partsupp\n" + + " join part on part.p_partkey = partsupp.ps_partkey\n" + + " where\n" + + " part.p_brand != 'Brand#13'\n" + + " and !(part.p_type like '.*BURNISHED.*')\n" + + " and (\n" + + " part.p_size = 7\n" + + " or part.p_size = 8\n" + + " or part.p_size = 9\n" + + " or part.p_size = 21\n" + + " or part.p_size = 23\n" + + " or part.p_size = 26\n" + + " or part.p_size = 31\n" + + " or part.p_size = 49\n" + + " )\n" + + " and partsupp.ps_suppkey not in (\n" + + " select\n" + + " s_suppkey\n" + + " from\n" + + " supplier\n" + + " where\n" + + " supplier.s_comment like '.*Customer.*Complaints.*'\n" + + " )\n" + + " group by\n" + + " part.p_brand,\n" + + " part.p_type,\n" + + " part.p_size\n" + + ")\n" + + "order by\n" + + " supplier_cnt,\n" + + " p_brand,\n" + + " p_type,\n" + + " p_size;"; + + public static final String Q17 = + "select \n" + + " tmp2 / 7 as avg_yearly\n" + + "from (\n" + + " select\n" + + " sum(lineitem.l_extendedprice) as tmp2\n" + + " from\n" + + " lineitem\n" + + " join part on part.p_partkey = lineitem.l_partkey\n" + + " where\n" + + " part.p_brand = 'Brand#13'\n" + + " and part.p_container = 'JUMBO PKG'\n" + + " and lineitem.l_quantity < (\n" + + " select\n" + + " 0.2 * tmp\n" + + " from (\n" + + " select\n" + + " avg(l_quantity) as tmp\n" + + " from\n" + + " lineitem\n" + + " where\n" + + " lineitem.l_partkey = part.p_partkey\n" + + " )\n" + + " )\n" + + ");"; + + public static final String Q18 = + "select\n" + + " customer.c_name,\n" + + " customer.c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderdate,\n" + + " orders.o_totalprice,\n" + + " sum(lineitem.l_quantity)\n" + + "from\n" + + " customer\n" + + " join orders on customer.c_custkey = orders.o_custkey\n" + + " join lineitem on orders.o_orderkey = lineitem.l_orderkey\n" + + "where\n" + + " orders.o_orderkey in (\n" + + " select\n" + + " lineitem.l_orderkey\n" + + " from (\n" + + " select\n" + + " l_orderkey,\n" + + " sum(l_quantity)\n" + + " from\n" + + " lineitem\n" + + " group by\n" + + " l_orderkey \n" + + " having\n" + + " sum(lineitem.l_quantity) > 250\n" + + " )\n" + + " )\n" + + "group by\n" + + " customer.c_name,\n" + + " customer.c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderdate,\n" + + " orders.o_totalprice\n" + + "order by\n" + + " orders.o_totalprice,\n" + + " orders.o_orderdate;"; + + public static final String Q19 = + "select \n" + + " sum(tmp) as revenue\n" + + "from (\n" + + " select\n" + + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp\n" + + " from\n" + + " lineitem\n" + + " join part on part.p_partkey = lineitem.l_partkey\n" + + " where (\n" + + " part.p_brand = 'Brand#12'\n" + + " and (\n" + + " part.p_container = 'MED PKG'\n" + + " or part.p_container = 'JUMBO CASE'\n" + + " or part.p_container = 'MED BAG'\n" + + " or part.p_container = 'JUMBO CAN'\n" + + " )\n" + + " and lineitem.l_quantity >= 1 and lineitem.l_quantity <= 11\n" + + " and part.p_size >= 1 and part.p_size < 5\n" + + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" + + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" + + " )\n" + + " or (\n" + + " part.p_brand = 'Brand#22'\n" + + " and (\n" + + " part.p_container = 'MED PKG'\n" + + " or part.p_container = 'MED DRUM'\n" + + " or part.p_container = 'MED BAG'\n" + + " or part.p_container = 'JUMBO CAN'\n" + + " )\n" + + " and lineitem.l_quantity >= 17 and lineitem.l_quantity <= 27\n" + + " and part.p_size >= 1 and part.p_size < 10\n" + + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" + + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" + + " )\n" + + " or (\n" + + " part.p_brand = 'Brand#32'\n" + + " and (\n" + + " part.p_container = 'SM BAG'\n" + + " or part.p_container = 'SM CASE'\n" + + " or part.p_container = 'MED BOX'\n" + + " or part.p_container = 'LG PKG'\n" + + " )\n" + + " and lineitem.l_quantity >= 5 and lineitem.l_quantity <= 15\n" + + " and part.p_size >= 1 and part.p_size < 15\n" + + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" + + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" + + " )\n" + + ");"; + + public static final String Q20 = + "select\n" + + " supplier.s_name,\n" + + " supplier.s_address\n" + + "from\n" + + " supplier\n" + + " join nation on supplier.s_nationkey = nation.n_nationkey\n" + + "where\n" + + " supplier.s_suppkey in (\n" + + " select\n" + + " ps_suppkey\n" + + " from\n" + + " partsupp\n" + + " where\n" + + " partsupp.ps_partkey in (\n" + + " select\n" + + " p_partkey\n" + + " from\n" + + " part\n" + + " where\n" + + " part.p_name like '.*yellow.*'\n" + + " )\n" + + " and partsupp.ps_availqty > (\n" + + " select\n" + + " 0.5 * tmp\n" + + " from (\n" + + " select\n" + + " sum(l_quantity) as tmp\n" + + " from\n" + + " lineitem\n" + + " where\n" + + " lineitem.l_partkey = partsupp.ps_partkey\n" + + " and lineitem.l_suppkey = partsupp.ps_suppkey\n" + + " and lineitem.l_shipdate >= 0\n" + + " and lineitem.l_shipdate < 9124416000\n" + + " )\n" + + " )\n" + + " )\n" + + " and nation.n_name = '[NATION]'\n" + + "order by\n" + + " supplier.s_name;"; + + public static final String Q22 = + "select\n" + + " customer.c_phone,\n" + + " count(*) as numcust,\n" + + " sum(customer.c_acctbal) as totacctbal\n" + + "from (\n" + + " select\n" + + " c_phone,\n" + + " c_acctbal\n" + + " from\n" + + " customer\n" + + " where\n" + + " customer.c_acctbal > (\n" + + " select\n" + + " avg(c_acctbal)\n" + + " from\n" + + " customer\n" + + " where\n" + + " customer.c_acctbal > 0.00\n" + + " )\n" + + " and not exists (\n" + + " select\n" + + " *\n" + + " from\n" + + " orders\n" + + " where\n" + + " orders.o_custkey = customer.c_custkey\n" + + " )\n" + + " )\n" + + "group by\n" + + " customer.c_phone\n" + + "order by\n" + + " customer.c_phone;"; + + public static final Map QUERY_MAP = new HashMap<>(); + + static { + QUERY_MAP.put("q1", Q1); + QUERY_MAP.put("q2", Q2); + QUERY_MAP.put("q3", Q3); + QUERY_MAP.put("q4", Q4); + QUERY_MAP.put("q5", Q5); + QUERY_MAP.put("q6", Q6); + QUERY_MAP.put("q9", Q9); + QUERY_MAP.put("q10", Q10); + QUERY_MAP.put("q13", Q13); + QUERY_MAP.put("q16", Q16); + QUERY_MAP.put("q17", Q17); + QUERY_MAP.put("q18", Q18); + QUERY_MAP.put("q19", Q19); + QUERY_MAP.put("q20", Q20); + QUERY_MAP.put("q22", Q22); + } + + public static final ExecutorService pool = Executors.newCachedThreadPool(); + + public static final Random random = new Random(); + + public static void main(String[] args) + throws SessionException, ExecutionException, InterruptedException { + assert args.length >= 4; + String ip = args[0]; + int port = Integer.parseInt(args[1]); + int workerNum = Integer.parseInt(args[2]); + long timeout = Long.parseLong(args[3]); + + List queryList; + List queryNames; + if (args.length >= 5) { + queryList = new ArrayList<>(); + queryNames = new ArrayList<>(); + String[] queryCmds = args[4].split(","); + for (String cmd : queryCmds) { + cmd = cmd.trim().toLowerCase(); + if (QUERY_MAP.containsKey(cmd)) { + queryList.add(QUERY_MAP.get(cmd)); + queryNames.add(cmd); + } + } + } else { + queryList = + new ArrayList<>(Arrays.asList(Q1, Q2, Q3, Q5, Q6, Q9, Q10, Q13, Q16, Q17, Q18, Q19)); + queryNames = + new ArrayList<>( + Arrays.asList( + "q1", "q2", "q3", "q5", "q6", "q9", "q10", "q13", "q16", "q17", "q18", "q19")); + } + + long startTime = System.currentTimeMillis(); + long endTime = startTime + timeout; + CountDownLatch latch = new CountDownLatch(workerNum); + for (int i = 0; i < workerNum; i++) { + int finalI = i; + pool.submit( + () -> { + Session session = new Session(ip, port); + try { + session.openSession(); + long curTime = System.currentTimeMillis(); + while (curTime < endTime) { + int randomId = random.nextInt(queryList.size()); + String name = queryNames.get(randomId); + String query = queryList.get(randomId); + session.executeSql(PREFIX + query); + curTime = System.currentTimeMillis(); + System.out.printf("worker%s,q%s,%s\n", finalI, name, curTime - startTime); + } + session.closeSession(); + latch.countDown(); + System.out.printf("work%s quit.\n", finalI); + } catch (SessionException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } + latch.await(); + System.out.println("all workers finished tasks"); + } +} diff --git a/tools-tpch/pom.xml b/tools-tpch/pom.xml new file mode 100644 index 0000000000..b4f1524356 --- /dev/null +++ b/tools-tpch/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + cn.edu.tsinghua + iginx + 0.6.0-SNAPSHOT + + + tools-tpch + + + 8 + 8 + UTF-8 + + + + + commons-cli + commons-cli + 1.5.0 + + + cn.edu.tsinghua + iginx-session + ${project.version} + + + cn.edu.tsinghua + iginx-thrift + ${project.version} + + + org.apache.commons + commons-csv + 1.10.0 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + true + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + org.apache.maven.plugins + maven-assembly-plugin + 3.2.0 + + + src/assembly/tools.xml + + false + target + + + true + true + + + + + + tools-assembly + + single + + package + + + + + + diff --git a/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh b/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh new file mode 100644 index 0000000000..de4ebe4f3d --- /dev/null +++ b/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# You can put your env variable here +# export JAVA_HOME=$JAVA_HOME + +if [[ -z "${IGINX_HOME}" ]]; then + export IGINX_HOME="$( + cd "$(dirname "$0")"/.. + pwd + )" +fi + +MAIN_CLASS=cn.edu.tsinghua.iginx.tools.tpch.DataReader + +CLASSPATH="" +for f in ${IGINX_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}":"$f +done + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +#computing the memory size for the JVM options +calculate_heap_sizes() { + case "$(uname)" in + Linux) + system_memory_in_mb=$(free -m | sed -n '2p' | awk '{print $2}') + system_cpu_cores=$(egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo) + ;; + FreeBSD) + system_memory_in_bytes=$(sysctl hw.physmem | awk '{print $2}') + system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) + system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') + ;; + SunOS) + system_memory_in_mb=$(prtconf | awk '/Memory size:/ {print $3}') + system_cpu_cores=$(psrinfo | wc -l) + ;; + Darwin) + system_memory_in_bytes=$(sysctl hw.memsize | awk '{print $2}') + system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) + system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') + ;; + *) + # assume reasonable defaults for e.g. a modern desktop or + # cheap server + system_memory_in_mb="2048" + system_cpu_cores="2" + ;; + esac + + # some systems like the raspberry pi don't report cores, use at least 1 + if [ "$system_cpu_cores" -lt "1" ]; then + system_cpu_cores="1" + fi + + # set max heap size based on the following + # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB)) + # calculate 1/2 ram and cap to 1024MB + # calculate 1/4 ram and cap to 65536MB + # pick the max + half_system_memory_in_mb=$(expr $system_memory_in_mb / 2) + quarter_system_memory_in_mb=$(expr $half_system_memory_in_mb / 2) + if [ "$half_system_memory_in_mb" -gt "1024" ]; then + half_system_memory_in_mb="1024" + fi + if [ "$quarter_system_memory_in_mb" -gt "65536" ]; then + quarter_system_memory_in_mb="65536" + fi + if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]; then + max_heap_size_in_mb="$half_system_memory_in_mb" + else + max_heap_size_in_mb="$quarter_system_memory_in_mb" + fi + MAX_HEAP_SIZE="${max_heap_size_in_mb}M" +} + +calculate_heap_sizes +JMX_OPTS="" +JMX_OPTS="$JMX_OPTS -Xms${MAX_HEAP_SIZE}" +JMX_OPTS="$JMX_OPTS -Xmx${MAX_HEAP_SIZE}" + +# continue to other parameters +ICONF="$IGINX_HOME/conf/config.properties" +IDRIVER="$IGINX_HOME/driver/" + +export IGINX_CONF=$ICONF +export IGINX_DRIVER=$IDRIVER + +exec "$JAVA" -Duser.timezone=GMT+8 -cp "$CLASSPATH" "$MAIN_CLASS" "$@" + +exit $? diff --git a/tools-tpch/src/assembly/tools.xml b/tools-tpch/src/assembly/tools.xml new file mode 100644 index 0000000000..299015234a --- /dev/null +++ b/tools-tpch/src/assembly/tools.xml @@ -0,0 +1,39 @@ + + + + tools + + dir + + false + + + lib + + + + + src/assembly/resources + ${file.separator} + + + \ No newline at end of file diff --git a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java new file mode 100644 index 0000000000..51f3eda25a --- /dev/null +++ b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java @@ -0,0 +1,185 @@ +package cn.edu.tsinghua.iginx.tools.tpch; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DataReader { + + public static void main(String[] args) { + + assert args.length == 3; + + String path = args[0]; + + String host = args[1]; + int port = Integer.parseInt(args[2]); + + Map> tableColumns = new HashMap<>(); + tableColumns.put( + "supplier", + Arrays.asList( + "s_suppkey", + "s_name", + "s_address", + "s_nationkey", + "s_phone", + "s_acctbal", + "s_comment")); + tableColumns.put("nation", Arrays.asList("n_nationkey", "n_name", "n_regionkey", "n_comment")); + tableColumns.put("region", Arrays.asList("r_regionkey", "r_name", "r_comment")); + tableColumns.put( + "customer", + Arrays.asList( + "c_custkey", + "c_name", + "c_address", + "c_nationkey", + "c_phone", + "c_acctbal", + "c_mktsegment", + "c_comment")); + tableColumns.put( + "part", + Arrays.asList( + "p_partkey", + "p_name", + "p_mfgr", + "p_brand", + "p_type", + "p_size", + "p_container", + "p_retailprice", + "p_comment")); + tableColumns.put( + "partsupp", + Arrays.asList("ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment")); + tableColumns.put( + "orders", + Arrays.asList( + "o_orderkey", + "o_custkey", + "o_orderstatus", + "o_totalprice", + "o_orderdate", + "o_orderpriority", + "o_clerk", + "o_shippriority", + "o_comment")); + tableColumns.put( + "lineitem", + Arrays.asList( + "l_orderkey", + "l_partkey", + "l_suppkey", + "l_linenumber", + "l_quantity", + "l_extendedprice", + "l_discount", + "l_tax", + "l_returnflag", + "l_linestatus", + "l_shipdate", + "l_commitdate", + "l_receiptdate", + "l_shipinstruct", + "l_shipmode", + "l_comment")); + + Map> tableTypes = new HashMap<>(); + tableTypes.put( + "supplier", + Arrays.asList( + DataType.LONG, + DataType.BINARY, + DataType.BINARY, + DataType.LONG, + DataType.BINARY, + DataType.DOUBLE, + DataType.BINARY)); + tableTypes.put( + "nation", Arrays.asList(DataType.LONG, DataType.BINARY, DataType.LONG, DataType.BINARY)); + tableTypes.put("region", Arrays.asList(DataType.LONG, DataType.BINARY, DataType.BINARY)); + tableTypes.put( + "customer", + Arrays.asList( + DataType.LONG, + DataType.BINARY, + DataType.BINARY, + DataType.LONG, + DataType.BINARY, + DataType.DOUBLE, + DataType.BINARY, + DataType.BINARY)); + tableTypes.put( + "part", + Arrays.asList( + DataType.LONG, + DataType.BINARY, + DataType.BINARY, + DataType.BINARY, + DataType.BINARY, + DataType.LONG, + DataType.BINARY, + DataType.DOUBLE, + DataType.BINARY)); + tableTypes.put( + "partsupp", + Arrays.asList( + DataType.LONG, DataType.LONG, DataType.LONG, DataType.DOUBLE, DataType.BINARY)); + tableTypes.put( + "orders", + Arrays.asList( + DataType.LONG, + DataType.LONG, + DataType.BINARY, + DataType.DOUBLE, + DataType.LONG, + DataType.BINARY, + DataType.BINARY, + DataType.LONG, + DataType.BINARY)); + tableTypes.put( + "lineitem", + Arrays.asList( + DataType.LONG, + DataType.LONG, + DataType.LONG, + DataType.LONG, + DataType.LONG, + DataType.DOUBLE, + DataType.DOUBLE, + DataType.DOUBLE, + DataType.BINARY, + DataType.BINARY, + DataType.LONG, + DataType.LONG, + DataType.LONG, + DataType.BINARY, + DataType.BINARY, + DataType.BINARY)); + + for (String table : tableColumns.keySet()) { + TableReader reader = null; + List columns = tableColumns.get(table); + List types = tableTypes.get(table); + + try { + reader = new TableReader(path, table, columns, types, 100000, host, port); + + int index = 0; + System.out.println("loading " + table); + while (reader.hasNext()) { + System.out.println("loading " + index++ + " batch"); + reader.loadNextBatch(); + } + reader.close(); + } catch (Exception e) { + System.out.println(e.getMessage()); + e.printStackTrace(); + } + } + } +} diff --git a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java new file mode 100644 index 0000000000..03a4c513b6 --- /dev/null +++ b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java @@ -0,0 +1,134 @@ +package cn.edu.tsinghua.iginx.tools.tpch; + +import cn.edu.tsinghua.iginx.exceptions.ExecutionException; +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.thrift.TimePrecision; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class TableReader { + + private final String path; + + private final String tableName; + + private final List columns; + + private final List types; + + private final int batch; + + private final BufferedReader reader; + + private String str; + + private final Session session; + + private int indexKey = 0; + + private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + + public TableReader( + String path, + String tableName, + List columns, + List types, + int batch, + String host, + int port) + throws Exception { + this.path = path; + this.tableName = tableName; + this.types = types; + this.batch = batch; + + this.columns = new ArrayList<>(columns.size()); + for (String suffix : columns) { + this.columns.add(tableName + "." + suffix); + } + + FileInputStream inputStream = new FileInputStream(path + "/" + tableName + ".tbl"); + this.reader = new BufferedReader(new InputStreamReader(inputStream)); + this.str = reader.readLine(); + + this.session = new Session(host, port, "root", "root"); + this.session.openSession(); + } + + public boolean hasNext() { + return str != null; + } + + public void loadNextBatch() throws IOException { + int cnt = 0; + List lines = new ArrayList<>(); + while (str != null && cnt < batch) { + lines.add(str); + str = reader.readLine(); + cnt++; + } + + long[] keys = new long[lines.size()]; + Object[] values = new Object[lines.size()]; + + for (int i = 0; i < lines.size(); i++) { + String[] rowStrValues = lines.get(i).split("\\|"); + Object[] rowValues = new Object[columns.size()]; + assert rowValues.length == rowStrValues.length; + for (int j = 0; j < rowValues.length; j++) { + rowValues[j] = getValueByType(rowStrValues[j], columns.get(j), types.get(j)); + } + + keys[i] = indexKey; + values[i] = rowValues; + + str = reader.readLine(); + cnt++; + indexKey++; + } + + lines.clear(); + + try { + session.insertRowRecords(columns, keys, values, types, null, TimePrecision.NS); + } catch (SessionException | ExecutionException | NullPointerException e) { + throw new RuntimeException(e); + } + } + + private Object getValueByType(String strValue, String colName, DataType type) { + switch (type) { + case LONG: + if (colName.endsWith("date")) { + Date date = null; + try { + date = dateFormat.parse(strValue); + } catch (ParseException e) { + e.printStackTrace(); + return 0; + } + return date.getTime() / 1000; + } else { + return Long.parseLong(strValue); + } + case DOUBLE: + return Double.parseDouble(strValue); + default: + return strValue.getBytes(StandardCharsets.UTF_8); + } + } + + public void close() throws SessionException { + session.closeSession(); + } +} From d8d37ffb3079778eb5eca8b824b3d2f8a9b7bc69 Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Fri, 26 Apr 2024 16:03:10 +0800 Subject: [PATCH 2/6] dev parallel without stats --- .../cn/edu/tsinghua/iginx/IginxWorker.java | 2 +- .../tsinghua/iginx/conf/ConfigDescriptor.java | 2 +- .../iginx/engine/StatementExecutor.java | 91 +-- .../coordinator/ConnectManager.java | 2 +- .../naive/NaiveOperatorMemoryExecutor.java | 11 +- .../task/UnaryMemoryPhysicalTask.java | 2 +- .../iginx/engine/shared/ResultUtils.java | 2 +- pom.xml | 2 - .../edu/tsinghua/iginx/session/Session.java | 8 + tools-benchmark/pom.xml | 89 --- .../resources/sbin/iginx-benchmark.sh | 118 ---- tools-benchmark/src/assembly/tools.xml | 39 -- .../tools/benchmark/MyQueryBenchmark.java | 596 ------------------ tools-tpch/pom.xml | 89 --- .../src/assembly/resources/sbin/iginx-tpch.sh | 118 ---- tools-tpch/src/assembly/tools.xml | 39 -- .../tsinghua/iginx/tools/tpch/DataReader.java | 185 ------ .../iginx/tools/tpch/TableReader.java | 134 ---- 18 files changed, 20 insertions(+), 1509 deletions(-) delete mode 100644 tools-benchmark/pom.xml delete mode 100644 tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh delete mode 100644 tools-benchmark/src/assembly/tools.xml delete mode 100644 tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java delete mode 100644 tools-tpch/pom.xml delete mode 100644 tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh delete mode 100644 tools-tpch/src/assembly/tools.xml delete mode 100644 tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java delete mode 100644 tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java index 69ccc96e56..b3c3faf0e1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java @@ -564,7 +564,7 @@ public ExecuteSqlResp executeSql(ExecuteSqlReq req) { StatementExecutor executor = StatementExecutor.getInstance(); RequestContext ctx = contextBuilder.build(req); executor.execute(ctx); - logger.info("total cost time: " + (ctx.getEndTime() - ctx.getStartTime())); + LOGGER.info("total cost time: " + (ctx.getEndTime() - ctx.getStartTime())); ExecuteSqlResp resp = ctx.getResult().getExecuteSqlResp(); resp.setCostTime(ctx.getEndTime() - ctx.getStartTime()); return resp; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java index 9130c63ddb..b8a2db0f4a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java @@ -236,7 +236,7 @@ private void loadPropsFromFile() { properties.getProperty( "ruleBasedOptimizer", "NotFilterRemoveRule=on,FragmentPruningByFilterRule=on,ColumnPruningRule=on,FragmentPruningByPatternRule=on")); - properties.getProperty("ruleBasedOptimizer", "RemoveNotRule=on,FilterFragmentRule=on")); + properties.getProperty("ruleBasedOptimizer", "RemoveNotRule=on,FilterFragmentRule=on"); config.setDistributedQueryTriggerThreshold( Integer.parseInt(properties.getProperty("distributedQueryTriggerThreshold", "3"))); } catch (IOException e) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java index 40a1047b8d..049015a2a3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java @@ -81,7 +81,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -375,7 +374,7 @@ private void process(RequestContext ctx) throws StatementExecutionException, Phy } long endTime = System.currentTimeMillis(); long engineCostTime = endTime - startTime; - logger.info("engine cost time: " + engineCostTime); + LOGGER.info("engine cost time: " + engineCostTime); after(ctx, postPhysicalProcessors); ctx.setEngineCostTime(engineCostTime); @@ -395,7 +394,7 @@ private void process(RequestContext ctx) throws StatementExecutionException, Phy } private void processExplainLogicalPlan(RequestContext ctx, Plan plan) - throws PhysicalException, ExecutionException { + throws PhysicalException, StatementExecutionException { List fields = new ArrayList<>( Arrays.asList( @@ -825,92 +824,6 @@ private void setResult(RequestContext ctx, RowStream stream) } } - private void setResultFromRowStream(RequestContext ctx, RowStream stream) - throws PhysicalException { - Result result = null; - if (ctx.isUseStream()) { - Status status = RpcUtils.SUCCESS; - if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { - status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); - status.setMessage(ctx.getWarningMsg()); - } - result = new Result(status); - result.setResultStream(stream); - ctx.setResult(result); - return; - } - - if (stream == null) { - setEmptyQueryResp(ctx, new ArrayList<>()); - return; - } - - List paths = new ArrayList<>(); - List> tagsList = new ArrayList<>(); - List types = new ArrayList<>(); - stream - .getHeader() - .getFields() - .forEach( - field -> { - paths.add(field.getFullName()); - types.add(field.getType()); - if (field.getTags() == null) { - tagsList.add(new HashMap<>()); - } else { - tagsList.add(field.getTags()); - } - }); - - List timestampList = new ArrayList<>(); - List valuesList = new ArrayList<>(); - List bitmapList = new ArrayList<>(); - - boolean hasTimestamp = stream.getHeader().hasKey(); - while (stream.hasNext()) { - Row row = stream.next(); - - Object[] rowValues = row.getValues(); - valuesList.add(ByteUtils.getRowByteBuffer(rowValues, types)); - - Bitmap bitmap = new Bitmap(rowValues.length); - for (int i = 0; i < rowValues.length; i++) { - if (rowValues[i] != null) { - bitmap.mark(i); - } - } - bitmapList.add(ByteBuffer.wrap(bitmap.getBytes())); - - if (hasTimestamp) { - timestampList.add(row.getKey()); - } - } - - if (valuesList.isEmpty()) { // empty result - setEmptyQueryResp(ctx, paths); - return; - } - - Status status = RpcUtils.SUCCESS; - if (ctx.getWarningMsg() != null && !ctx.getWarningMsg().isEmpty()) { - status = new Status(StatusCode.PARTIAL_SUCCESS.getStatusCode()); - status.setMessage(ctx.getWarningMsg()); - } - result = new Result(status); - if (timestampList.size() != 0) { - Long[] timestamps = timestampList.toArray(new Long[timestampList.size()]); - result.setKeys(timestamps); - } - result.setValuesList(valuesList); - result.setBitmapList(bitmapList); - result.setPaths(paths); - result.setTagsList(tagsList); - result.setDataTypes(types); - ctx.setResult(result); - - stream.close(); - } - private void setShowTSRowStreamResult(RequestContext ctx, RowStream stream) throws PhysicalException { if (ctx.isUseStream()) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java index 3a443377fb..a3820111e5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/distributedquery/coordinator/ConnectManager.java @@ -1,7 +1,7 @@ package cn.edu.tsinghua.iginx.engine.distributedquery.coordinator; import cn.edu.tsinghua.iginx.engine.shared.source.IGinXSource; -import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.session.Session; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java index d7ce8735b5..1159c7634c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java @@ -62,8 +62,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.type.OuterJoinType; import cn.edu.tsinghua.iginx.engine.shared.source.EmptySource; import cn.edu.tsinghua.iginx.engine.shared.source.IGinXSource; -import cn.edu.tsinghua.iginx.exceptions.ExecutionException; -import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSubPlanResult; import cn.edu.tsinghua.iginx.thrift.DataType; @@ -588,8 +587,8 @@ private RowStream executeLoad(Load load) throws PhysicalException { try { SessionExecuteSubPlanResult result = session.executeSubPlan(subPlanMsg); return constructRowStream(result); - } catch (SessionException | ExecutionException e) { - logger.error("execute load fail, because: ", e); + } catch (SessionException e) { + LOGGER.error("execute load fail, because: ", e); throw new PhysicalException(e); } } @@ -1031,7 +1030,7 @@ private RowStream executeHashInnerJoin(InnerJoin innerJoin, Table tableA, Table } } long endTime = System.currentTimeMillis(); - logger.info( + LOGGER.info( String.format( "inner join use %s probe, row size: %s, cost time: %s", use, tableA.getRowSize(), endTime - startTime)); @@ -2075,7 +2074,7 @@ private RowStream executeHashSingleJoin(SingleJoin singleJoin, Table tableA, Tab } } long endTime = System.currentTimeMillis(); - logger.info( + LOGGER.info( String.format( "single join use %s probe, row size: %s, cost time: %s", use, tableA.getRowSize(), endTime - startTime)); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java index 4ab7607353..ab78a9cb00 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java @@ -73,7 +73,7 @@ private TaskExecuteResult executeLoad(Load load) { RowStream stream = executor.executeUnaryOperator(load, null, getContext()); return new TaskExecuteResult(stream); } catch (PhysicalException e) { - logger.error("encounter error when execute load in memory: ", e); + LOGGER.error("encounter error when execute load in memory: ", e); return new TaskExecuteResult(e); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java index 1ac17622ee..a3ca8ef8fc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/ResultUtils.java @@ -3,7 +3,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.exceptions.StatusCode; +import cn.edu.tsinghua.iginx.exception.StatusCode; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.Status; import cn.edu.tsinghua.iginx.utils.Bitmap; diff --git a/pom.xml b/pom.xml index 126a8a5b68..69888d1fd5 100644 --- a/pom.xml +++ b/pom.xml @@ -21,8 +21,6 @@ example test assembly - tools-tpch - tools-benchmark diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java index 5c7cb3d573..d8644895f1 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java @@ -949,6 +949,14 @@ public SessionExecuteSqlResult executePythonRegister(String statement) throws Se } } + public SessionExecuteSubPlanResult executeSubPlan(String subPlanMsg) throws SessionException { + ExecuteSubPlanReq req = new ExecuteSubPlanReq(sessionId, subPlanMsg); + Reference ref = new Reference<>(); + executeWithCheck(() -> (ref.resp = client.executeSubPlan(req)).status); + + return new SessionExecuteSubPlanResult(ref.resp); + } + public SessionQueryDataSet queryLast( List paths, long startKey, TimePrecision timePrecision) throws SessionException { return queryLast(paths, startKey, null, timePrecision); diff --git a/tools-benchmark/pom.xml b/tools-benchmark/pom.xml deleted file mode 100644 index bec5cddfdf..0000000000 --- a/tools-benchmark/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - 4.0.0 - - cn.edu.tsinghua - iginx - 0.6.0-SNAPSHOT - - - tools-benchmark - - - 8 - 8 - UTF-8 - - - - - commons-cli - commons-cli - 1.5.0 - - - cn.edu.tsinghua - iginx-session - ${project.version} - - - cn.edu.tsinghua - iginx-thrift - ${project.version} - - - org.apache.commons - commons-csv - 1.10.0 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 1.8 - 1.8 - UTF-8 - true - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.0 - - - org.apache.maven.plugins - maven-assembly-plugin - 3.2.0 - - - src/assembly/tools.xml - - false - target - - - true - true - - - - - - tools-assembly - - single - - package - - - - - - diff --git a/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh b/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh deleted file mode 100644 index 595946398f..0000000000 --- a/tools-benchmark/src/assembly/resources/sbin/iginx-benchmark.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# You can put your env variable here -# export JAVA_HOME=$JAVA_HOME - -if [[ -z "${IGINX_HOME}" ]]; then - export IGINX_HOME="$( - cd "$(dirname "$0")"/.. - pwd - )" -fi - -MAIN_CLASS=cn.edu.tsinghua.iginx.tools.benchmark.MyQueryBenchmark - -CLASSPATH="" -for f in ${IGINX_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done - -if [ -n "$JAVA_HOME" ]; then - for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do - if [ -x "$java" ]; then - JAVA="$java" - break - fi - done -else - JAVA=java -fi - -#computing the memory size for the JVM options -calculate_heap_sizes() { - case "$(uname)" in - Linux) - system_memory_in_mb=$(free -m | sed -n '2p' | awk '{print $2}') - system_cpu_cores=$(egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo) - ;; - FreeBSD) - system_memory_in_bytes=$(sysctl hw.physmem | awk '{print $2}') - system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) - system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') - ;; - SunOS) - system_memory_in_mb=$(prtconf | awk '/Memory size:/ {print $3}') - system_cpu_cores=$(psrinfo | wc -l) - ;; - Darwin) - system_memory_in_bytes=$(sysctl hw.memsize | awk '{print $2}') - system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) - system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') - ;; - *) - # assume reasonable defaults for e.g. a modern desktop or - # cheap server - system_memory_in_mb="2048" - system_cpu_cores="2" - ;; - esac - - # some systems like the raspberry pi don't report cores, use at least 1 - if [ "$system_cpu_cores" -lt "1" ]; then - system_cpu_cores="1" - fi - - # set max heap size based on the following - # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB)) - # calculate 1/2 ram and cap to 1024MB - # calculate 1/4 ram and cap to 65536MB - # pick the max - half_system_memory_in_mb=$(expr $system_memory_in_mb / 2) - quarter_system_memory_in_mb=$(expr $half_system_memory_in_mb / 2) - if [ "$half_system_memory_in_mb" -gt "1024" ]; then - half_system_memory_in_mb="1024" - fi - if [ "$quarter_system_memory_in_mb" -gt "65536" ]; then - quarter_system_memory_in_mb="65536" - fi - if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]; then - max_heap_size_in_mb="$half_system_memory_in_mb" - else - max_heap_size_in_mb="$quarter_system_memory_in_mb" - fi - MAX_HEAP_SIZE="${max_heap_size_in_mb}M" -} - -calculate_heap_sizes -JMX_OPTS="" -JMX_OPTS="$JMX_OPTS -Xms${MAX_HEAP_SIZE}" -JMX_OPTS="$JMX_OPTS -Xmx${MAX_HEAP_SIZE}" - -# continue to other parameters -ICONF="$IGINX_HOME/conf/config.properties" -IDRIVER="$IGINX_HOME/driver/" - -export IGINX_CONF=$ICONF -export IGINX_DRIVER=$IDRIVER - -exec "$JAVA" -Duser.timezone=GMT+8 -cp "$CLASSPATH" "$MAIN_CLASS" "$@" - -exit $? diff --git a/tools-benchmark/src/assembly/tools.xml b/tools-benchmark/src/assembly/tools.xml deleted file mode 100644 index 299015234a..0000000000 --- a/tools-benchmark/src/assembly/tools.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - tools - - dir - - false - - - lib - - - - - src/assembly/resources - ${file.separator} - - - \ No newline at end of file diff --git a/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java b/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java deleted file mode 100644 index a9edaa0146..0000000000 --- a/tools-benchmark/src/main/java/cn/edu/tsinghua/iginx/tools/benchmark/MyQueryBenchmark.java +++ /dev/null @@ -1,596 +0,0 @@ -package cn.edu.tsinghua.iginx.tools.benchmark; - -import cn.edu.tsinghua.iginx.exceptions.ExecutionException; -import cn.edu.tsinghua.iginx.exceptions.SessionException; -import cn.edu.tsinghua.iginx.session.Session; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class MyQueryBenchmark { - - public static final String PREFIX = "explain physical "; - - public static final String Q1 = - "select\n" - + " lineitem.l_returnflag as l_returnflag,\n" - + " lineitem.l_linestatus as l_linestatus,\n" - + " sum(lineitem.l_quantity) as sum_qty,\n" - + " sum(lineitem.l_extendedprice) as sum_base_price,\n" - + " sum(tmp1) as sum_disc_price, \n" - + " sum(tmp2) as sum_charge, \n" - + " avg(lineitem.l_quantity) as avg_qty,\n" - + " avg(lineitem.l_extendedprice) as avg_price,\n" - + " avg(lineitem.l_discount) as avg_disc,\n" - + " count(*) as count_order\n" - + "from (\n" - + " select\n" - + " l_returnflag,\n" - + " l_linestatus,\n" - + " l_quantity,\n" - + " l_extendedprice,\n" - + " l_discount,\n" - + " l_extendedprice * (1 - l_discount) as tmp1,\n" - + " l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2\n" - + " from\n" - + " lineitem\n" - + " where\n" - + " lineitem.l_shipdate <= 9124416000\n" - + ")\n" - + "group by\n" - + " lineitem.l_returnflag,\n" - + " lineitem.l_linestatus\n" - + "order by\n" - + " lineitem.l_returnflag,\n" - + " lineitem.l_linestatus;"; - - public static final String Q2 = - "select\n" - + " supplier.s_acctbal,\n" - + " supplier.s_name,\n" - + " nation.n_name,\n" - + " part.p_partkey,\n" - + " part.p_mfgr,\n" - + " supplier.s_address,\n" - + " supplier.s_phone,\n" - + " supplier.s_comment\n" - + "from\n" - + " part\n" - + " join partsupp on part.p_partkey = partsupp.ps_partkey\n" - + " join supplier on supplier.s_suppkey = partsupp.ps_suppkey\n" - + " join nation on supplier.s_nationkey = nation.n_nationkey\n" - + " join region on nation.n_regionkey = region.r_regionkey\n" - + "where\n" - + " partsupp.ps_supplycost = (\n" - + " select\n" - + " min(partsupp.ps_supplycost)\n" - + " from\n" - + " partsupp\n" - + " join supplier on supplier.s_suppkey = partsupp.ps_suppkey\n" - + " join nation on supplier.s_nationkey = nation.n_nationkey\n" - + " join region on nation.n_regionkey = region.r_regionkey\n" - + " where\n" - + " part.p_partkey = partsupp.ps_partkey \n" - + " and region.r_name = 'AMERICA'\n" - + " )\n" - + "order by\n" - + " supplier.s_acctbal,\n" - + " nation.n_name,\n" - + " supplier.s_name,\n" - + " part.p_partkey;"; - - public static final String Q3 = - "select\n" - + " lineitem.l_orderkey,\n" - + " orders.o_orderdate,\n" - + " orders.o_shippriority\n" - + "from\n" - + " customer\n" - + " join orders on customer.c_custkey = orders.o_custkey\n" - + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" - + "where\n" - + " customer.c_mktsegment = 'BUILDING'\n" - + " and orders.o_orderdate < 9124416000\n" - + " and lineitem.l_shipdate > 0\n" - + "group by\n" - + " lineitem.l_orderkey,\n" - + " orders.o_orderdate,\n" - + " orders.o_shippriority\n" - + "order by\n" - + " orders.o_orderdate;"; - - public static final String Q4 = - "select\n" - + " orders.o_orderpriority,\n" - + " count(*) as order_count\n" - + "from\n" - + " orders\n" - + "where\n" - + " orders.o_orderdate >= 0\n" - + " and orders.o_orderdate < 9124416000\n" - + " and exists (\n" - + " select\n" - + " *\n" - + " from\n" - + " lineitem\n" - + " where\n" - + " lineitem.l_orderkey = orders.o_orderkey\n" - + " and lineitem.l_commitdate < lineitem.l_receiptdate\n" - + " )\n" - + "group by\n" - + " orders.o_orderpriority\n" - + "order by\n" - + " orders.o_orderpriority;"; - - public static final String Q5 = - "select \n" - + " nation.n_name,\n" - + " revenue\n" - + "from (\n" - + " select\n" - + " nation.n_name,\n" - + " sum(tmp) as revenue\n" - + " from (\n" - + " select\n" - + " nation.n_name,\n" - + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp\n" - + " from\n" - + " customer\n" - + " join orders on customer.c_custkey = orders.o_custkey\n" - + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" - + " join supplier on lineitem.l_suppkey = supplier.s_suppkey and customer.c_nationkey = supplier.s_nationkey\n" - + " join nation on supplier.s_nationkey = nation.n_nationkey\n" - + " join region on nation.n_regionkey = region.r_regionkey\n" - + " where\n" - + " region.r_name = \"EUROPE\"\n" - + " and orders.o_orderdate >= 0\n" - + " and orders.o_orderdate < 9124416000\n" - + ")\n" - + "group by\n" - + " nation.n_name\n" - + ")\n" - + "order by\n" - + " revenue desc;"; - - public static final String Q6 = - "select\n" - + " sum(tmp) as revenue\n" - + "from (\n" - + " select\n" - + " l_extendedprice * l_discount as tmp\n" - + " from\n" - + " lineitem\n" - + " where\n" - + " lineitem.l_shipdate >= 0\n" - + " and lineitem.l_shipdate < 9124416000\n" - + " and lineitem.l_discount >= 0.02 \n" - + " and lineitem.l_discount < 0.09\n" - + " and lineitem.l_quantity < 25\n" - + ");"; - - public static final String Q9 = - "select\n" - + " nation,\n" - + " date,\n" - + " sum(amount) as sum_profit\n" - + "from (\n" - + " select\n" - + " nation.n_name as nation,\n" - + " orders.o_orderdate as date,\n" - + " lineitem.l_extendedprice * (1 - lineitem.l_discount) - partsupp.ps_supplycost * lineitem.l_quantity as amount\n" - + " from\n" - + " part\n" - + " join lineitem on part.p_partkey = lineitem.l_partkey\n" - + " join supplier on supplier.s_suppkey = lineitem.l_suppkey\n" - + " join partsupp on partsupp.ps_suppkey = lineitem.l_suppkey and partsupp.ps_partkey = lineitem.l_partkey\n" - + " join orders on orders.o_orderkey = lineitem.l_orderkey\n" - + " join nation on supplier.s_nationkey = nation.n_nationkey\n" - + " where\n" - + " part.p_name like '.*yellow.*'\n" - + ")\n" - + "group by\n" - + " nation,\n" - + " date\n" - + "order by\n" - + " nation,\n" - + " date desc;"; - - public static final String Q10 = - "select\n" - + " customer.c_custkey,\n" - + " customer.c_name,\n" - + " revenue,\n" - + " customer.c_acctbal,\n" - + " nation.n_name,\n" - + " customer.c_address,\n" - + " customer.c_phone,\n" - + " customer.c_comment\n" - + "from (\n" - + " select\n" - + " customer.c_custkey,\n" - + " customer.c_name,\n" - + " sum(tmp) as revenue,\n" - + " customer.c_acctbal,\n" - + " nation.n_name,\n" - + " customer.c_address,\n" - + " customer.c_phone,\n" - + " customer.c_comment\n" - + " from (\n" - + " select\n" - + " customer.c_custkey,\n" - + " customer.c_name,\n" - + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp,\n" - + " customer.c_acctbal,\n" - + " nation.n_name,\n" - + " customer.c_address,\n" - + " customer.c_phone,\n" - + " customer.c_comment\n" - + " from\n" - + " customer\n" - + " join orders on customer.c_custkey = orders.o_custkey\n" - + " join lineitem on lineitem.l_orderkey = orders.o_orderkey\n" - + " join nation on customer.c_nationkey = nation.n_nationkey\n" - + " where\n" - + " orders.o_orderdate >= 0\n" - + " and orders.o_orderdate < 9124416000\n" - + " and lineitem.l_returnflag = 'R'\n" - + " )\n" - + " group by\n" - + " customer.c_custkey,\n" - + " customer.c_name,\n" - + " customer.c_acctbal,\n" - + " customer.c_phone,\n" - + " nation.n_name,\n" - + " customer.c_address,\n" - + " customer.c_comment\n" - + ")\n" - + "order by\n" - + " revenue desc;"; - - public static final String Q13 = - "select\n" - + " c_count, \n" - + " custdist\n" - + "from (\n" - + " select\n" - + " c_count, \n" - + " count(*) as custdist\n" - + " from (\n" - + " select\n" - + " customer.c_custkey as c_custkey,\n" - + " count(orders.o_orderkey) as c_count\n" - + " from\n" - + " customer left outer join orders on customer.c_custkey = orders.o_custkey and !(orders.o_comment like '.*pending.*')\n" - + " group by\n" - + " customer.c_custkey\n" - + " )\n" - + " group by\n" - + " c_count\n" - + ")\n" - + "order by\n" - + " custdist,\n" - + " c_count desc;"; - - public static final String Q16 = - "select\n" - + " p_brand,\n" - + " p_type,\n" - + " p_size,\n" - + " supplier_cnt\n" - + "from (\n" - + " select\n" - + " part.p_brand as p_brand,\n" - + " part.p_type as p_type,\n" - + " part.p_size as p_size,\n" - + " count(distinct partsupp.ps_suppkey) as supplier_cnt\n" - + " from\n" - + " partsupp\n" - + " join part on part.p_partkey = partsupp.ps_partkey\n" - + " where\n" - + " part.p_brand != 'Brand#13'\n" - + " and !(part.p_type like '.*BURNISHED.*')\n" - + " and (\n" - + " part.p_size = 7\n" - + " or part.p_size = 8\n" - + " or part.p_size = 9\n" - + " or part.p_size = 21\n" - + " or part.p_size = 23\n" - + " or part.p_size = 26\n" - + " or part.p_size = 31\n" - + " or part.p_size = 49\n" - + " )\n" - + " and partsupp.ps_suppkey not in (\n" - + " select\n" - + " s_suppkey\n" - + " from\n" - + " supplier\n" - + " where\n" - + " supplier.s_comment like '.*Customer.*Complaints.*'\n" - + " )\n" - + " group by\n" - + " part.p_brand,\n" - + " part.p_type,\n" - + " part.p_size\n" - + ")\n" - + "order by\n" - + " supplier_cnt,\n" - + " p_brand,\n" - + " p_type,\n" - + " p_size;"; - - public static final String Q17 = - "select \n" - + " tmp2 / 7 as avg_yearly\n" - + "from (\n" - + " select\n" - + " sum(lineitem.l_extendedprice) as tmp2\n" - + " from\n" - + " lineitem\n" - + " join part on part.p_partkey = lineitem.l_partkey\n" - + " where\n" - + " part.p_brand = 'Brand#13'\n" - + " and part.p_container = 'JUMBO PKG'\n" - + " and lineitem.l_quantity < (\n" - + " select\n" - + " 0.2 * tmp\n" - + " from (\n" - + " select\n" - + " avg(l_quantity) as tmp\n" - + " from\n" - + " lineitem\n" - + " where\n" - + " lineitem.l_partkey = part.p_partkey\n" - + " )\n" - + " )\n" - + ");"; - - public static final String Q18 = - "select\n" - + " customer.c_name,\n" - + " customer.c_custkey,\n" - + " orders.o_orderkey,\n" - + " orders.o_orderdate,\n" - + " orders.o_totalprice,\n" - + " sum(lineitem.l_quantity)\n" - + "from\n" - + " customer\n" - + " join orders on customer.c_custkey = orders.o_custkey\n" - + " join lineitem on orders.o_orderkey = lineitem.l_orderkey\n" - + "where\n" - + " orders.o_orderkey in (\n" - + " select\n" - + " lineitem.l_orderkey\n" - + " from (\n" - + " select\n" - + " l_orderkey,\n" - + " sum(l_quantity)\n" - + " from\n" - + " lineitem\n" - + " group by\n" - + " l_orderkey \n" - + " having\n" - + " sum(lineitem.l_quantity) > 250\n" - + " )\n" - + " )\n" - + "group by\n" - + " customer.c_name,\n" - + " customer.c_custkey,\n" - + " orders.o_orderkey,\n" - + " orders.o_orderdate,\n" - + " orders.o_totalprice\n" - + "order by\n" - + " orders.o_totalprice,\n" - + " orders.o_orderdate;"; - - public static final String Q19 = - "select \n" - + " sum(tmp) as revenue\n" - + "from (\n" - + " select\n" - + " lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp\n" - + " from\n" - + " lineitem\n" - + " join part on part.p_partkey = lineitem.l_partkey\n" - + " where (\n" - + " part.p_brand = 'Brand#12'\n" - + " and (\n" - + " part.p_container = 'MED PKG'\n" - + " or part.p_container = 'JUMBO CASE'\n" - + " or part.p_container = 'MED BAG'\n" - + " or part.p_container = 'JUMBO CAN'\n" - + " )\n" - + " and lineitem.l_quantity >= 1 and lineitem.l_quantity <= 11\n" - + " and part.p_size >= 1 and part.p_size < 5\n" - + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" - + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" - + " )\n" - + " or (\n" - + " part.p_brand = 'Brand#22'\n" - + " and (\n" - + " part.p_container = 'MED PKG'\n" - + " or part.p_container = 'MED DRUM'\n" - + " or part.p_container = 'MED BAG'\n" - + " or part.p_container = 'JUMBO CAN'\n" - + " )\n" - + " and lineitem.l_quantity >= 17 and lineitem.l_quantity <= 27\n" - + " and part.p_size >= 1 and part.p_size < 10\n" - + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" - + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" - + " )\n" - + " or (\n" - + " part.p_brand = 'Brand#32'\n" - + " and (\n" - + " part.p_container = 'SM BAG'\n" - + " or part.p_container = 'SM CASE'\n" - + " or part.p_container = 'MED BOX'\n" - + " or part.p_container = 'LG PKG'\n" - + " )\n" - + " and lineitem.l_quantity >= 5 and lineitem.l_quantity <= 15\n" - + " and part.p_size >= 1 and part.p_size < 15\n" - + " and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG')\n" - + " and lineitem.l_shipinstruct = 'DELIVER IN PERSON'\n" - + " )\n" - + ");"; - - public static final String Q20 = - "select\n" - + " supplier.s_name,\n" - + " supplier.s_address\n" - + "from\n" - + " supplier\n" - + " join nation on supplier.s_nationkey = nation.n_nationkey\n" - + "where\n" - + " supplier.s_suppkey in (\n" - + " select\n" - + " ps_suppkey\n" - + " from\n" - + " partsupp\n" - + " where\n" - + " partsupp.ps_partkey in (\n" - + " select\n" - + " p_partkey\n" - + " from\n" - + " part\n" - + " where\n" - + " part.p_name like '.*yellow.*'\n" - + " )\n" - + " and partsupp.ps_availqty > (\n" - + " select\n" - + " 0.5 * tmp\n" - + " from (\n" - + " select\n" - + " sum(l_quantity) as tmp\n" - + " from\n" - + " lineitem\n" - + " where\n" - + " lineitem.l_partkey = partsupp.ps_partkey\n" - + " and lineitem.l_suppkey = partsupp.ps_suppkey\n" - + " and lineitem.l_shipdate >= 0\n" - + " and lineitem.l_shipdate < 9124416000\n" - + " )\n" - + " )\n" - + " )\n" - + " and nation.n_name = '[NATION]'\n" - + "order by\n" - + " supplier.s_name;"; - - public static final String Q22 = - "select\n" - + " customer.c_phone,\n" - + " count(*) as numcust,\n" - + " sum(customer.c_acctbal) as totacctbal\n" - + "from (\n" - + " select\n" - + " c_phone,\n" - + " c_acctbal\n" - + " from\n" - + " customer\n" - + " where\n" - + " customer.c_acctbal > (\n" - + " select\n" - + " avg(c_acctbal)\n" - + " from\n" - + " customer\n" - + " where\n" - + " customer.c_acctbal > 0.00\n" - + " )\n" - + " and not exists (\n" - + " select\n" - + " *\n" - + " from\n" - + " orders\n" - + " where\n" - + " orders.o_custkey = customer.c_custkey\n" - + " )\n" - + " )\n" - + "group by\n" - + " customer.c_phone\n" - + "order by\n" - + " customer.c_phone;"; - - public static final Map QUERY_MAP = new HashMap<>(); - - static { - QUERY_MAP.put("q1", Q1); - QUERY_MAP.put("q2", Q2); - QUERY_MAP.put("q3", Q3); - QUERY_MAP.put("q4", Q4); - QUERY_MAP.put("q5", Q5); - QUERY_MAP.put("q6", Q6); - QUERY_MAP.put("q9", Q9); - QUERY_MAP.put("q10", Q10); - QUERY_MAP.put("q13", Q13); - QUERY_MAP.put("q16", Q16); - QUERY_MAP.put("q17", Q17); - QUERY_MAP.put("q18", Q18); - QUERY_MAP.put("q19", Q19); - QUERY_MAP.put("q20", Q20); - QUERY_MAP.put("q22", Q22); - } - - public static final ExecutorService pool = Executors.newCachedThreadPool(); - - public static final Random random = new Random(); - - public static void main(String[] args) - throws SessionException, ExecutionException, InterruptedException { - assert args.length >= 4; - String ip = args[0]; - int port = Integer.parseInt(args[1]); - int workerNum = Integer.parseInt(args[2]); - long timeout = Long.parseLong(args[3]); - - List queryList; - List queryNames; - if (args.length >= 5) { - queryList = new ArrayList<>(); - queryNames = new ArrayList<>(); - String[] queryCmds = args[4].split(","); - for (String cmd : queryCmds) { - cmd = cmd.trim().toLowerCase(); - if (QUERY_MAP.containsKey(cmd)) { - queryList.add(QUERY_MAP.get(cmd)); - queryNames.add(cmd); - } - } - } else { - queryList = - new ArrayList<>(Arrays.asList(Q1, Q2, Q3, Q5, Q6, Q9, Q10, Q13, Q16, Q17, Q18, Q19)); - queryNames = - new ArrayList<>( - Arrays.asList( - "q1", "q2", "q3", "q5", "q6", "q9", "q10", "q13", "q16", "q17", "q18", "q19")); - } - - long startTime = System.currentTimeMillis(); - long endTime = startTime + timeout; - CountDownLatch latch = new CountDownLatch(workerNum); - for (int i = 0; i < workerNum; i++) { - int finalI = i; - pool.submit( - () -> { - Session session = new Session(ip, port); - try { - session.openSession(); - long curTime = System.currentTimeMillis(); - while (curTime < endTime) { - int randomId = random.nextInt(queryList.size()); - String name = queryNames.get(randomId); - String query = queryList.get(randomId); - session.executeSql(PREFIX + query); - curTime = System.currentTimeMillis(); - System.out.printf("worker%s,q%s,%s\n", finalI, name, curTime - startTime); - } - session.closeSession(); - latch.countDown(); - System.out.printf("work%s quit.\n", finalI); - } catch (SessionException | ExecutionException e) { - throw new RuntimeException(e); - } - }); - } - latch.await(); - System.out.println("all workers finished tasks"); - } -} diff --git a/tools-tpch/pom.xml b/tools-tpch/pom.xml deleted file mode 100644 index b4f1524356..0000000000 --- a/tools-tpch/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - 4.0.0 - - cn.edu.tsinghua - iginx - 0.6.0-SNAPSHOT - - - tools-tpch - - - 8 - 8 - UTF-8 - - - - - commons-cli - commons-cli - 1.5.0 - - - cn.edu.tsinghua - iginx-session - ${project.version} - - - cn.edu.tsinghua - iginx-thrift - ${project.version} - - - org.apache.commons - commons-csv - 1.10.0 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 1.8 - 1.8 - UTF-8 - true - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.0 - - - org.apache.maven.plugins - maven-assembly-plugin - 3.2.0 - - - src/assembly/tools.xml - - false - target - - - true - true - - - - - - tools-assembly - - single - - package - - - - - - diff --git a/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh b/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh deleted file mode 100644 index de4ebe4f3d..0000000000 --- a/tools-tpch/src/assembly/resources/sbin/iginx-tpch.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# You can put your env variable here -# export JAVA_HOME=$JAVA_HOME - -if [[ -z "${IGINX_HOME}" ]]; then - export IGINX_HOME="$( - cd "$(dirname "$0")"/.. - pwd - )" -fi - -MAIN_CLASS=cn.edu.tsinghua.iginx.tools.tpch.DataReader - -CLASSPATH="" -for f in ${IGINX_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done - -if [ -n "$JAVA_HOME" ]; then - for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do - if [ -x "$java" ]; then - JAVA="$java" - break - fi - done -else - JAVA=java -fi - -#computing the memory size for the JVM options -calculate_heap_sizes() { - case "$(uname)" in - Linux) - system_memory_in_mb=$(free -m | sed -n '2p' | awk '{print $2}') - system_cpu_cores=$(egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo) - ;; - FreeBSD) - system_memory_in_bytes=$(sysctl hw.physmem | awk '{print $2}') - system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) - system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') - ;; - SunOS) - system_memory_in_mb=$(prtconf | awk '/Memory size:/ {print $3}') - system_cpu_cores=$(psrinfo | wc -l) - ;; - Darwin) - system_memory_in_bytes=$(sysctl hw.memsize | awk '{print $2}') - system_memory_in_mb=$(expr $system_memory_in_bytes / 1024 / 1024) - system_cpu_cores=$(sysctl hw.ncpu | awk '{print $2}') - ;; - *) - # assume reasonable defaults for e.g. a modern desktop or - # cheap server - system_memory_in_mb="2048" - system_cpu_cores="2" - ;; - esac - - # some systems like the raspberry pi don't report cores, use at least 1 - if [ "$system_cpu_cores" -lt "1" ]; then - system_cpu_cores="1" - fi - - # set max heap size based on the following - # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB)) - # calculate 1/2 ram and cap to 1024MB - # calculate 1/4 ram and cap to 65536MB - # pick the max - half_system_memory_in_mb=$(expr $system_memory_in_mb / 2) - quarter_system_memory_in_mb=$(expr $half_system_memory_in_mb / 2) - if [ "$half_system_memory_in_mb" -gt "1024" ]; then - half_system_memory_in_mb="1024" - fi - if [ "$quarter_system_memory_in_mb" -gt "65536" ]; then - quarter_system_memory_in_mb="65536" - fi - if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]; then - max_heap_size_in_mb="$half_system_memory_in_mb" - else - max_heap_size_in_mb="$quarter_system_memory_in_mb" - fi - MAX_HEAP_SIZE="${max_heap_size_in_mb}M" -} - -calculate_heap_sizes -JMX_OPTS="" -JMX_OPTS="$JMX_OPTS -Xms${MAX_HEAP_SIZE}" -JMX_OPTS="$JMX_OPTS -Xmx${MAX_HEAP_SIZE}" - -# continue to other parameters -ICONF="$IGINX_HOME/conf/config.properties" -IDRIVER="$IGINX_HOME/driver/" - -export IGINX_CONF=$ICONF -export IGINX_DRIVER=$IDRIVER - -exec "$JAVA" -Duser.timezone=GMT+8 -cp "$CLASSPATH" "$MAIN_CLASS" "$@" - -exit $? diff --git a/tools-tpch/src/assembly/tools.xml b/tools-tpch/src/assembly/tools.xml deleted file mode 100644 index 299015234a..0000000000 --- a/tools-tpch/src/assembly/tools.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - tools - - dir - - false - - - lib - - - - - src/assembly/resources - ${file.separator} - - - \ No newline at end of file diff --git a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java deleted file mode 100644 index 51f3eda25a..0000000000 --- a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/DataReader.java +++ /dev/null @@ -1,185 +0,0 @@ -package cn.edu.tsinghua.iginx.tools.tpch; - -import cn.edu.tsinghua.iginx.thrift.DataType; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class DataReader { - - public static void main(String[] args) { - - assert args.length == 3; - - String path = args[0]; - - String host = args[1]; - int port = Integer.parseInt(args[2]); - - Map> tableColumns = new HashMap<>(); - tableColumns.put( - "supplier", - Arrays.asList( - "s_suppkey", - "s_name", - "s_address", - "s_nationkey", - "s_phone", - "s_acctbal", - "s_comment")); - tableColumns.put("nation", Arrays.asList("n_nationkey", "n_name", "n_regionkey", "n_comment")); - tableColumns.put("region", Arrays.asList("r_regionkey", "r_name", "r_comment")); - tableColumns.put( - "customer", - Arrays.asList( - "c_custkey", - "c_name", - "c_address", - "c_nationkey", - "c_phone", - "c_acctbal", - "c_mktsegment", - "c_comment")); - tableColumns.put( - "part", - Arrays.asList( - "p_partkey", - "p_name", - "p_mfgr", - "p_brand", - "p_type", - "p_size", - "p_container", - "p_retailprice", - "p_comment")); - tableColumns.put( - "partsupp", - Arrays.asList("ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment")); - tableColumns.put( - "orders", - Arrays.asList( - "o_orderkey", - "o_custkey", - "o_orderstatus", - "o_totalprice", - "o_orderdate", - "o_orderpriority", - "o_clerk", - "o_shippriority", - "o_comment")); - tableColumns.put( - "lineitem", - Arrays.asList( - "l_orderkey", - "l_partkey", - "l_suppkey", - "l_linenumber", - "l_quantity", - "l_extendedprice", - "l_discount", - "l_tax", - "l_returnflag", - "l_linestatus", - "l_shipdate", - "l_commitdate", - "l_receiptdate", - "l_shipinstruct", - "l_shipmode", - "l_comment")); - - Map> tableTypes = new HashMap<>(); - tableTypes.put( - "supplier", - Arrays.asList( - DataType.LONG, - DataType.BINARY, - DataType.BINARY, - DataType.LONG, - DataType.BINARY, - DataType.DOUBLE, - DataType.BINARY)); - tableTypes.put( - "nation", Arrays.asList(DataType.LONG, DataType.BINARY, DataType.LONG, DataType.BINARY)); - tableTypes.put("region", Arrays.asList(DataType.LONG, DataType.BINARY, DataType.BINARY)); - tableTypes.put( - "customer", - Arrays.asList( - DataType.LONG, - DataType.BINARY, - DataType.BINARY, - DataType.LONG, - DataType.BINARY, - DataType.DOUBLE, - DataType.BINARY, - DataType.BINARY)); - tableTypes.put( - "part", - Arrays.asList( - DataType.LONG, - DataType.BINARY, - DataType.BINARY, - DataType.BINARY, - DataType.BINARY, - DataType.LONG, - DataType.BINARY, - DataType.DOUBLE, - DataType.BINARY)); - tableTypes.put( - "partsupp", - Arrays.asList( - DataType.LONG, DataType.LONG, DataType.LONG, DataType.DOUBLE, DataType.BINARY)); - tableTypes.put( - "orders", - Arrays.asList( - DataType.LONG, - DataType.LONG, - DataType.BINARY, - DataType.DOUBLE, - DataType.LONG, - DataType.BINARY, - DataType.BINARY, - DataType.LONG, - DataType.BINARY)); - tableTypes.put( - "lineitem", - Arrays.asList( - DataType.LONG, - DataType.LONG, - DataType.LONG, - DataType.LONG, - DataType.LONG, - DataType.DOUBLE, - DataType.DOUBLE, - DataType.DOUBLE, - DataType.BINARY, - DataType.BINARY, - DataType.LONG, - DataType.LONG, - DataType.LONG, - DataType.BINARY, - DataType.BINARY, - DataType.BINARY)); - - for (String table : tableColumns.keySet()) { - TableReader reader = null; - List columns = tableColumns.get(table); - List types = tableTypes.get(table); - - try { - reader = new TableReader(path, table, columns, types, 100000, host, port); - - int index = 0; - System.out.println("loading " + table); - while (reader.hasNext()) { - System.out.println("loading " + index++ + " batch"); - reader.loadNextBatch(); - } - reader.close(); - } catch (Exception e) { - System.out.println(e.getMessage()); - e.printStackTrace(); - } - } - } -} diff --git a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java b/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java deleted file mode 100644 index 03a4c513b6..0000000000 --- a/tools-tpch/src/main/java/cn/edu/tsinghua/iginx/tools/tpch/TableReader.java +++ /dev/null @@ -1,134 +0,0 @@ -package cn.edu.tsinghua.iginx.tools.tpch; - -import cn.edu.tsinghua.iginx.exceptions.ExecutionException; -import cn.edu.tsinghua.iginx.exceptions.SessionException; -import cn.edu.tsinghua.iginx.session.Session; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.thrift.TimePrecision; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class TableReader { - - private final String path; - - private final String tableName; - - private final List columns; - - private final List types; - - private final int batch; - - private final BufferedReader reader; - - private String str; - - private final Session session; - - private int indexKey = 0; - - private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - - public TableReader( - String path, - String tableName, - List columns, - List types, - int batch, - String host, - int port) - throws Exception { - this.path = path; - this.tableName = tableName; - this.types = types; - this.batch = batch; - - this.columns = new ArrayList<>(columns.size()); - for (String suffix : columns) { - this.columns.add(tableName + "." + suffix); - } - - FileInputStream inputStream = new FileInputStream(path + "/" + tableName + ".tbl"); - this.reader = new BufferedReader(new InputStreamReader(inputStream)); - this.str = reader.readLine(); - - this.session = new Session(host, port, "root", "root"); - this.session.openSession(); - } - - public boolean hasNext() { - return str != null; - } - - public void loadNextBatch() throws IOException { - int cnt = 0; - List lines = new ArrayList<>(); - while (str != null && cnt < batch) { - lines.add(str); - str = reader.readLine(); - cnt++; - } - - long[] keys = new long[lines.size()]; - Object[] values = new Object[lines.size()]; - - for (int i = 0; i < lines.size(); i++) { - String[] rowStrValues = lines.get(i).split("\\|"); - Object[] rowValues = new Object[columns.size()]; - assert rowValues.length == rowStrValues.length; - for (int j = 0; j < rowValues.length; j++) { - rowValues[j] = getValueByType(rowStrValues[j], columns.get(j), types.get(j)); - } - - keys[i] = indexKey; - values[i] = rowValues; - - str = reader.readLine(); - cnt++; - indexKey++; - } - - lines.clear(); - - try { - session.insertRowRecords(columns, keys, values, types, null, TimePrecision.NS); - } catch (SessionException | ExecutionException | NullPointerException e) { - throw new RuntimeException(e); - } - } - - private Object getValueByType(String strValue, String colName, DataType type) { - switch (type) { - case LONG: - if (colName.endsWith("date")) { - Date date = null; - try { - date = dateFormat.parse(strValue); - } catch (ParseException e) { - e.printStackTrace(); - return 0; - } - return date.getTime() / 1000; - } else { - return Long.parseLong(strValue); - } - case DOUBLE: - return Double.parseDouble(strValue); - default: - return strValue.getBytes(StandardCharsets.UTF_8); - } - } - - public void close() throws SessionException { - session.closeSession(); - } -} From 249d9293c154fada034bc6f7c6c38c542938389e Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Wed, 8 May 2024 10:41:51 +0800 Subject: [PATCH 3/6] fix problems when resolving conflicts --- conf/config.properties | 2 +- .../tsinghua/iginx/engine/shared/Result.java | 4 ++-- .../sql/statement/ShowConfigStatement.java | 1 + .../session/SessionExecuteSqlResult.java | 23 +++++++++++++++---- thrift/src/main/proto/rpc.thrift | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index dbed3da2ea..2d6fe6420c 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -188,7 +188,7 @@ restIp=0.0.0.0 restPort=6666 # 是否启用 rest 服务 -enableRestService=false +enableRestService=true # 乱序数据 margin, 单位是秒 disorderMargin=10 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java index 6631c67db4..553297f2f9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java @@ -52,7 +52,7 @@ public class Result { private long jobId; private List jobIdList; - private String configValue; + private Map configs; private String exportByteStreamDir; @@ -169,7 +169,7 @@ public ExecuteSqlResp getExecuteSqlResp() { resp.setJobId(jobId); resp.setJobState(jobState); resp.setJobIdList(jobIdList); - resp.setConfigValue(configValue); + resp.setConfigs(configs); // INFILE AS CSV resp.setLoadCsvPath(loadCSVPath); resp.setSessionIDList(sessionIDs); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java index c72b8c456a..a5221df0eb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java @@ -45,6 +45,7 @@ public void execute(RequestContext ctx) throws StatementExecutionException { } Result result = new Result(RpcUtils.SUCCESS); + result.setConfigs(configs); ctx.setResult(result); } catch (NoSuchFieldException e) { String errMsg = String.format("no such field, field=%s", configName); diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java index a680efd94d..3003f2a189 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java @@ -43,7 +43,7 @@ public class SessionExecuteSqlResult { private long jobId; private JobState jobState; private List jobIdList; - private String configValue; + private Map configs; private String loadCsvPath; private List sessionIDs; @@ -104,7 +104,7 @@ public SessionExecuteSqlResult(ExecuteSqlResp resp) { this.jobIdList = resp.getJobIdList(); break; case ShowConfig: - this.configValue = resp.getConfigValue(); + this.configs = resp.getConfigs(); break; case LoadCsv: this.loadCsvPath = resp.getLoadCsvPath(); @@ -185,6 +185,8 @@ public String getResultInString(boolean needFormatTime, String timePrecision) { return buildShowEligibleJobResult(); case ShowSessionID: return buildShowSessionIDResult(); + case ShowConfig: + return buildShowConfigResult(); case ShowRules: return buildShowRulesResult(); case GetReplicaNum: @@ -195,8 +197,6 @@ public String getResultInString(boolean needFormatTime, String timePrecision) { return "job id: " + jobId; case ShowJobStatus: return "Job status: " + jobState; - case ShowConfig: - return "config value: " + configValue + "\n"; default: return "No data to print." + "\n"; } @@ -433,6 +433,21 @@ private String buildShowSessionIDResult() { return builder.toString(); } + private String buildShowConfigResult() { + StringBuilder builder = new StringBuilder(); + if (configs != null) { + builder.append("Config Info:").append("\n"); + List> cache = new ArrayList<>(); + cache.add(new ArrayList<>(Arrays.asList("ConfigName", "ConfigValue"))); + configs.forEach( + (name, value) -> { + cache.add(new ArrayList<>(Arrays.asList(name, value))); + }); + builder.append(FormatUtils.formatResult(cache)); + } + return builder.toString(); + } + private String buildShowRulesResult() { StringBuilder builder = new StringBuilder(); if (rules != null) { diff --git a/thrift/src/main/proto/rpc.thrift b/thrift/src/main/proto/rpc.thrift index 2c5c56278e..f5bd1aaf0e 100644 --- a/thrift/src/main/proto/rpc.thrift +++ b/thrift/src/main/proto/rpc.thrift @@ -382,7 +382,7 @@ struct ExecuteSqlResp { 22: optional i64 jobId 23: optional JobState jobState 24: optional list jobIdList - 25: optional string configValue + 25: optional map configs 26: optional string loadCsvPath 27: optional list sessionIDList 28: optional map rules From 3eefa63d5e7e0fba7901ee1c3d6f64e00218b92e Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Wed, 8 May 2024 17:19:16 +0800 Subject: [PATCH 4/6] fix problems when resolving conflicts --- conf/config.properties | 2 +- .../iginx/iotdb/query/entity/IoTDBQueryRowStream.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 2d6fe6420c..f1c602bf68 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -94,7 +94,7 @@ minThriftWorkerThreadNum = 20 maxThriftWrokerThreadNum = 2147483647 # 分布式查询的触发分片数 -distributedQueryTriggerThreshold = 3 +distributedQueryTriggerThreshold = 100 #################### ### Migration 相关配置 diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java index e596025c92..ead62c28f9 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java @@ -22,6 +22,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; +import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; @@ -204,9 +205,9 @@ private void cacheOneRow() throws SQLException, PhysicalException { } state = State.HAS_NEXT; cachedRow = new Row(header, timestamp, fields); - // if (!validate(filter, cachedRow)) { - // cacheOneRow(); - // } + if (!FilterUtils.validate(filter, cachedRow)) { + cacheOneRow(); + } } else { state = State.NO_NEXT; cachedRow = null; From 7bc4edbb9114ab47f6f57f2fed7f00e5db3ab29f Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Tue, 14 May 2024 10:34:22 +0800 Subject: [PATCH 5/6] fix problems when resolving conflicts --- antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 | 2 +- .../src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 index 50bc1c23f8..8155a47c3d 100644 --- a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 +++ b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 @@ -25,7 +25,7 @@ statement | SHOW jobStatus TRANSFORM JOB # showEligibleJobStatement | REMOVE HISTORYDATASOURCE removedStorageEngine (COMMA removedStorageEngine)* # removeHistoryDataSourceStatement | SET CONFIG configName = stringLiteral configValue = stringLiteral # setConfigStatement - | SHOW CONFIG configName = stringLiteral # showConfigStatement + | SHOW CONFIG (configName = stringLiteral)? # showConfigStatement | SHOW SESSIONID # showSessionIDStatement | COMPACT # compactStatement | SHOW RULES # showRulesStatement diff --git a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java index e8b050f203..8584c8569b 100644 --- a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java +++ b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java @@ -389,7 +389,6 @@ private static void processSql(String sql) { } catch (SessionException e) { System.out.println(e.getMessage()); } catch (Exception e) { - e.printStackTrace(); System.out.println( "Execute Error: encounter error(s) when executing sql statement, " + "see server log for more details."); @@ -447,7 +446,6 @@ private static void processSqlWithStream(String sql) { } catch (SessionException e) { System.out.println(e.getMessage()); } catch (Exception e) { - e.printStackTrace(); System.out.println( "Execute Error: encounter error(s) when executing sql statement, " + "see server log for more details."); From fd7f46b9fa7418ec3cd7a8cd25754a720af4bb7b Mon Sep 17 00:00:00 2001 From: Jensen Wang <1092792221@qq.com> Date: Fri, 7 Jun 2024 16:53:49 +0800 Subject: [PATCH 6/6] fix conflicts --- .../edu/tsinghua/iginx/session/SessionExecuteSqlResult.java | 4 ++++ .../edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java | 1 + 2 files changed, 5 insertions(+) diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java index 85cdc7043d..5d08a7a149 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java @@ -629,4 +629,8 @@ public String getUDFModulePath() { public List getSessionIDs() { return sessionIDs; } + + public Map getConfigs() { + return configs; + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index a47d271370..1c1f7ec600 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -19,6 +19,7 @@ import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set;