Merge branch 'main' into test/add-tests

IGinX-THU · May 11, 2024 · 9dce29a · 9dce29a
2 parents 6a33e29 + c008d6d
commit 9dce29a
Show file tree

Hide file tree

Showing 54 changed files with 1,850 additions and 610 deletions.
diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml
@@ -167,4 +167,22 @@ runs:
               paths: conf/config.properties
               statements: |
                   s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g
-                  s/#storageEngineList=127.0.0.1#5432/storageEngineList=127.0.0.1#5432/g
+                  s/#storageEngineList=127.0.0.1#5432#relational#engine=postgresql/storageEngineList=127.0.0.1#5432#relational#engine=postgresql/g
+
+        - if: inputs.DB-name == 'MySQL'
+          name: chan
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql.sh" 3306 3307 3308
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql_windows.sh" 3306 3307 3308
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mysql_macos.sh" 3306 3307 3308
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml
@@ -22,6 +22,9 @@ inputs:
 runs:
     using: "composite"
     steps:
+        - name: Set up Docker Buildx
+          uses: crazy-max/ghaction-setup-docker@v3
+
         # need to set up timezone and enlarge JVM heap size manually on windows
         - if: runner.os == 'Windows'
           name: Set Dynamic Timezone

diff --git a/.github/scripts/dataSources/mysql.sh b/.github/scripts/dataSources/mysql.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties
+
+sed -i "s^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path^storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=/home/runner/work/IGinX/IGinX/dataSources/relational/src/main/resources/mysql-meta-template.properties^g" conf/config.properties
+
+for port in "$@"
+do
+    docker run --name mysql${port} -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p ${port}:3306 --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 -d mysql:8.0.26
+done
diff --git a/.github/scripts/dataSources/mysql_macos.sh b/.github/scripts/dataSources/mysql_macos.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties
+
+sed -i  "" "s^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path^storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=/Users/runner/work/IGinX/IGinX/dataSources/relational/src/main/resources/mysql-meta-template.properties^g" conf/config.properties
+
+brew install [email protected] | tee brew_install_output.txt
+
+awk -F'/' '/\/usr\/local\/Cellar\/[email protected]/ {print $0}' brew_install_output.txt
+
+MYSQL_VERSION=$(awk -F'/' '/\/usr\/local\/Cellar\/[email protected]/ && !done {split($0, version, "/"); print version[6]; done=1}' brew_install_output.txt)
+echo "MYSQL_VERSION: $MYSQL_VERSION"
+
+for port in "$@"
+do
+    mkdir -p ./data${port}
+    sudo touch /usr/local/etc/my${port}.cnf
+    sudo sh -c "echo '[mysqld]' > /usr/local/etc/my${port}.cnf"
+    sudo sh -c "echo 'port=${port}' >> /usr/local/etc/my${port}.cnf"
+    sudo sh -c "echo 'mysqlx_port=$((33060+${port}-3306))' >> /usr/local/etc/my${port}.cnf"
+    sudo sh -c "echo 'user=root' >> /usr/local/etc/my${port}.cnf"
+    sudo sh -c "echo 'socket=/tmp/mysql${port}.soc' >> /usr/local/etc/my${port}.cnf"
+
+    /usr/local/Cellar/[email protected]/${MYSQL_VERSION}/bin/mysqld --defaults-file=/usr/local/etc/my${port}.cnf --mysqlx=0 --user=root --initialize-insecure --datadir=./data${port} --port=${port}
+
+    /usr/local/Cellar/[email protected]/${MYSQL_VERSION}/bin/mysqld --defaults-file=/usr/local/etc/my${port}.cnf --mysqlx=0 --datadir=./data${port} --port=${port} &
+done
diff --git a/.github/scripts/dataSources/mysql_windows.sh b/.github/scripts/dataSources/mysql_windows.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties
+
+sed -i "s^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path^storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=D:/a/IGinX/IGinX/dataSources/relational/src/main/resources/mysql-meta-template.properties^g" conf/config.properties
+
+choco install mysql --version 8.0.26 --params "/InstallLocation:./"
+
+echo "ALTER USER 'root'@'localhost' IDENTIFIED BY 'mysql';" > init.txt
+
+for port in "$@"
+do
+    mkdir ./mysql/data${port}
+    ./mysql/mysql-8.0.26-winx64/bin/mysqld --initialize-insecure --datadir=./mysql/data$port --port=$port --console
+    ./mysql/mysql-8.0.26-winx64/bin/mysqld --console --datadir=./mysql/data$port --port=$port &
+done
+
+for port in "$@"
+do
+    ./mysql/mysql-8.0.26-winx64/bin/mysql -v -u root --port=$port
+done
diff --git a/.github/scripts/iginx/iginx_udf_path.sh b/.github/scripts/iginx/iginx_udf_path.sh
@@ -19,5 +19,5 @@ else
 
     sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh"
 
-    sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &"
+    sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx-u.log 2>&1 &"
 fi
diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml
@@ -30,6 +30,7 @@ jobs:
                         "Redis",
                         "MongoDB",
                         "Parquet",
+                        "MySQL",
                     ]
                 metadata: [zookeeper]
         runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml
@@ -21,13 +21,14 @@ jobs:
                 os: [ubuntu-latest, macos-13, windows-latest]
                 DB-name:
                     [
+                        "FileSystem",
                         "IoTDB12",
                         "InfluxDB",
-                        "Redis",
-                        "Parquet",
                         "PostgreSQL",
-                        "FileSystem",
+                        "Redis",
                         "MongoDB",
+                        "Parquet",
+                        "MySQL",
                     ]
                 metadata: [zookeeper]
         runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml
@@ -21,13 +21,14 @@ jobs:
                 os: [ubuntu-latest, macos-13, windows-latest]
                 DB-name:
                     [
+                        "FileSystem",
                         "IoTDB12",
                         "InfluxDB",
-                        "Parquet",
                         "PostgreSQL",
-                        "FileSystem",
                         "Redis",
                         "MongoDB",
+                        "Parquet",
+                        "MySQL",
                     ]
                 metadata: [zookeeper]
         runs-on: ${{ matrix.os }}

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -60,7 +60,7 @@
         </dependency>
         <dependency>
             <groupId>cn.edu.tsinghua</groupId>
-            <artifactId>postgresql</artifactId>
+            <artifactId>relational</artifactId>
         </dependency>
         <dependency>
             <groupId>cn.edu.tsinghua</groupId>

diff --git a/conf/config.properties b/conf/config.properties
@@ -18,7 +18,8 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool
 #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false
 #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1
 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres
-#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false
+#storageEngineList=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false
+#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path
 #storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY
 #storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0
 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000
@@ -35,7 +36,7 @@ defaultUDFDir=udf_funcs
 replicaNum=0
 
 # 底层数据库类名
-databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage
+databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage
 #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage
 
 # 内存任务执行线程池

diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java
@@ -53,7 +53,7 @@ public class Config {
   private TimePrecision timePrecision = TimePrecision.NS;
 
   private String databaseClassNames =
-      "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage";
+      "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage";
   // ,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage
 
   private String policyClassName = "cn.edu.tsinghua.iginx.policy.naive.NaivePolicy";

diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java
@@ -86,7 +86,7 @@ private void loadPropsFromFile() {
       config.setDatabaseClassNames(
           properties.getProperty(
               "databaseClassNames",
-              "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage"));
+              "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage"));
       // ,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage
 
       config.setPolicyClassName(

diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java
@@ -11,7 +11,7 @@ public class RestUtils {
   public static final long TITLE_KEY = 9223372036854775806L;
   public static final long MAX_KEY = 9223372036854775807L;
   public static final long ANNOTATION_START_KEY = 10L;
-  public static final String ANNOTATION_SEQUENCE = "TITLE.DESCRIPTION";
+  public static final String ANNOTATION_SEQUENCE = "title.description";
 
   public static DataType checkType(SessionQueryDataSet sessionQueryDataSet) {
     int n = sessionQueryDataSet.getKeys().length;

diff --git a/dataSources/pom.xml b/dataSources/pom.xml
@@ -34,8 +34,8 @@
         <module>mongodb</module>
         <module>opentsdb</module>
         <module>parquet</module>
-        <module>postgresql</module>
         <module>redis</module>
+        <module>relational</module>
     </modules>
 
     <properties>

diff --git a/...tgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java b/...tgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java
diff --git a/...ava/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java b/...ava/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java
diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java
diff --git a/dataSources/relational/.gitignore b/dataSources/relational/.gitignore
@@ -0,0 +1,38 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store