From 9e049a43f017b2ac305aaf1276111cf6276598a1 Mon Sep 17 00:00:00 2001 From: RemHero <80055724+RemHero@users.noreply.github.com> Date: Tue, 7 May 2024 09:59:34 +0800 Subject: [PATCH] refactor(test): enhance test workflow (#310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修改测试使本地可运行 - 将macos-latest测试环境改为macos-13 --- .github/workflows/DB-CE.yml | 2 +- .github/workflows/case-regression.yml | 2 +- .github/workflows/codeql.yml | 2 +- .../workflows/standalone-test-pushdown.yml | 2 +- .github/workflows/standalone-test.yml | 2 +- .github/workflows/unit-mds.yml | 4 +- .github/workflows/unit-test.yml | 2 +- .../iginx/integration/tool/ConfLoader.java | 56 +++++++++++-------- test/src/test/resources/testConfig.properties | 8 +++ 9 files changed, 50 insertions(+), 30 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index af53760bee..22511ba399 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -20,7 +20,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] DB-name: [ "FileSystem", diff --git a/.github/workflows/case-regression.yml b/.github/workflows/case-regression.yml index b5414947ef..474a112472 100644 --- a/.github/workflows/case-regression.yml +++ b/.github/workflows/case-regression.yml @@ -20,7 +20,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] metadata: [zookeeper] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e2022d7496..4861a044d6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,7 @@ jobs: # - https://gh.io/supported-runners-and-hardware-resources # - https://gh.io/using-larger-runners # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + runs-on: ${{ (matrix.language == 'swift' && 'macos-13') || 'ubuntu-latest' }} timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: # required for all workflows diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml index 1b0b9dfd98..148995a895 100644 --- a/.github/workflows/standalone-test-pushdown.yml +++ b/.github/workflows/standalone-test-pushdown.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] DB-name: [ "IoTDB12", diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 9f920defed..b14d6329eb 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] DB-name: [ "IoTDB12", diff --git a/.github/workflows/unit-mds.yml b/.github/workflows/unit-mds.yml index 3af707a51b..521f20afdb 100644 --- a/.github/workflows/unit-mds.yml +++ b/.github/workflows/unit-mds.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: java: [8] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-13] runs-on: ${{ matrix.os}} env: STORAGE: zookeeper @@ -54,7 +54,7 @@ jobs: max-parallel: 20 matrix: java: [8] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-13] runs-on: ${{ matrix.os}} env: STORAGE: etcd diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index c3788bd0e2..5d3cc6d68e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -19,7 +19,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java index 218d2ecaf7..cecfb35d66 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java @@ -3,6 +3,7 @@ import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.FileReader; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -35,12 +36,20 @@ public class ConfLoader { private static final String DBCE_TEST_WAY = "./src/test/resources/dbce-test-way.txt"; + private static final String DEFAULT_DB = "stand_alone_DB"; + + private static final String DEFAULT_IS_SCALING = "is_scaling"; + + private static final String DEFAULT_DBCE_TEST_WAY = "DBCE_test_way"; + private static List storageEngines = new ArrayList<>(); private Map> taskMap = new HashMap<>(); private static String confPath; + private static Properties properties = null; + private boolean DEBUG = false; private void logInfo(String info, Object... args) { @@ -49,32 +58,45 @@ private void logInfo(String info, Object... args) { } } + private String getTestProperty(String path, String defaultProperty) { + String result = null; + File file = new File(path); + if (file.exists()) { + result = FileReader.convertToString(path); + } else { + logInfo(path + "does not exist and use default storage type"); + result = properties.getProperty(defaultProperty); + logInfo("default property: {}", result); + } + return result; + } + public String getStorageType() { - String storageType = FileReader.convertToString(RUNNING_STORAGE); - logInfo("run the test on {}", storageType); - return storageType; + return getTestProperty(RUNNING_STORAGE, DEFAULT_DB); } public boolean isScaling() { - String isScaling = FileReader.convertToString(IS_SCALING); - logInfo("isScaling: {}", isScaling); - return Boolean.parseBoolean(isScaling == null ? "false" : isScaling); + return Boolean.parseBoolean(getTestProperty(IS_SCALING, DEFAULT_IS_SCALING)); } public String getDBCETestWay() { - String dbceTestWay = FileReader.convertToString(DBCE_TEST_WAY); - logInfo("dbceTestWay: {}", dbceTestWay); - return dbceTestWay; + return getTestProperty(DBCE_TEST_WAY, DEFAULT_DBCE_TEST_WAY); } public ConfLoader(String confPath) { this.confPath = confPath; + try { + if (properties == null) { + InputStream in = Files.newInputStream(Paths.get(confPath)); + properties = new Properties(); + properties.load(in); + } + } catch (IOException e) { + LOGGER.error("load conf failure: ", e); + } } public void loadTestConf() throws IOException { - InputStream in = Files.newInputStream(Paths.get(confPath)); - Properties properties = new Properties(); - properties.load(in); logInfo("loading the test conf..."); String property = properties.getProperty(STORAGE_ENGINE_LIST); if (property == null || property.isEmpty()) { @@ -105,16 +127,6 @@ public void loadTestConf() throws IOException { public DBConf loadDBConf(String storageEngine) { DBConf dbConf = new DBConf(); - Properties properties; - try { - InputStream in = Files.newInputStream(Paths.get(confPath)); - properties = new Properties(); - properties.load(in); - } catch (IOException e) { - LOGGER.error("load conf failure: ", e); - return dbConf; - } - logInfo("loading the DB conf..."); String property = properties.getProperty(STORAGE_ENGINE_LIST); if (property == null || property.isEmpty()) { diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index 961a8bffe3..e6b4c1a1d6 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -58,3 +58,11 @@ PostgreSQL_port=5432,5433,5434 MongoDB_port=27017,27018,27019 Redis_port=6379,6380,6381 FileSystem_port=6667,6668,6669 + +# Local test stand-alone +stand_alone_DB=IoTDB12 + +# Local test DB-CE +is_scaling=false +DBCE_test_way=oriHasDataExpHasData +