Merge branch 'main' of github.com:Altinity/clickhouse-regression

Altinity · Feb 26, 2024 · 4c4000a · 4c4000a
2 parents 2069882 + ce18ff5
commit 4c4000a
Show file tree

Hide file tree

Showing 58 changed files with 6,106 additions and 2,620 deletions.
diff --git a/.github/workflows/run-arm-regression.yml b/.github/workflows/run-arm-regression.yml
@@ -40,6 +40,7 @@ on:
           - key_value
           - ldap
           - lightweight_delete
+          - memory
           - parquet_all
           - parquet
           - parquet_minio
@@ -59,6 +60,8 @@ on:
           - tiered_storage_local
           - tiered_storage_minio
           - vfs_all
+          - vfs_aws
+          - vfs_gcs
           - vfs_minio
           - window_functions
       artifacts:
@@ -189,7 +192,7 @@ jobs:
           path: ${{ env.artifact_paths}}
 
   atomic_insert:
-    runs-on: [self-hosted, type-cax41]
+    runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce]
     timeout-minutes: 180
     env:
       SUITE: atomic_insert
@@ -968,6 +971,38 @@ jobs:
           name: ${{ env.SUITE }}-artifacts
           path: ${{ env.artifact_paths}}
 
+  memory:
+    runs-on: [self-hosted, type-cax31, image-arm-app-docker-ce]
+    timeout-minutes: 1800
+    env:
+      SUITE: memory
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-artifacts
+          path: ${{ env.artifact_paths}}
+
   parquet:
     runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce]
     timeout-minutes: 300
@@ -1484,9 +1519,83 @@ jobs:
           name: ${{ env.SUITE }}-minio-artifacts
           path: ${{ env.artifact_paths}}
 
+  vfs_aws:
+    runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce]
+    timeout-minutes: 300
+    env:
+      SUITE: vfs
+      STORAGE: /aws
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'vfs_aws' || inputs.suite == 'vfs_all' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --storage aws_s3
+              --aws-s3-access-key ${{ secrets.AWS_ACCESS_KEY }}
+              --aws-s3-key-id ${{ secrets.AWS_KEY_ID }}
+              --aws-s3-uri https://s3.${{ secrets.AWS_REGION}}.amazonaws.com/${{ secrets.AWS_BUCKET }}/data/
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-aws-artifacts
+          path: ${{ env.artifact_paths}}
+
+  vfs_gcs:
+    runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce]
+    timeout-minutes: 300
+    env:
+      SUITE: vfs
+      STORAGE: /gcs
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'vfs_gcs' || inputs.suite == 'vfs_all' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --storage gcs
+              --gcs-uri ${{ secrets.GCS_URI }}
+              --gcs-key-id ${{ secrets.GCS_KEY_ID }}
+              --gcs-key-secret ${{ secrets.GCS_KEY_SECRET }}
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-gcs-artifacts
+          path: ${{ env.artifact_paths}}
+
   vfs_minio:
     runs-on: [self-hosted, type-cax41, image-arm-app-docker-ce]
-    timeout-minutes: 180
+    timeout-minutes: 300
     env:
       SUITE: vfs
       STORAGE: /minio
@@ -1506,7 +1615,6 @@ jobs:
               --storage minio
               --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
               --clickhouse-version ${{ env.version }}
-              --allow-vfs
               ${{ env.args }}
 
       - name: Create and upload logs

diff --git a/.github/workflows/run-regression.yml b/.github/workflows/run-regression.yml
@@ -40,6 +40,7 @@ on:
           - key_value
           - ldap
           - lightweight_delete
+          - memory
           - parquet_all
           - parquet
           - parquet_minio
@@ -59,6 +60,8 @@ on:
           - tiered_storage_local
           - tiered_storage_minio
           - vfs_all
+          - vfs_aws
+          - vfs_gcs
           - vfs_minio
           - window_functions
       artifacts:
@@ -968,6 +971,38 @@ jobs:
           name: ${{ env.SUITE }}-artifacts
           path: ${{ env.artifact_paths}}
 
+  memory:
+    runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce]
+    timeout-minutes: 1800
+    env:
+      SUITE: memory
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'memory' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-artifacts
+          path: ${{ env.artifact_paths}}
+
   parquet:
     runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce]
     timeout-minutes: 300
@@ -1484,6 +1519,81 @@ jobs:
           name: ${{ env.SUITE }}-minio-artifacts
           path: ${{ env.artifact_paths}}
 
+  vfs_aws:
+    runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce]
+    timeout-minutes: 300
+    env:
+      SUITE: vfs
+      STORAGE: /aws
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'vfs_aws' || inputs.suite == 'vfs_all' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --storage aws_s3
+              --aws-s3-bucket ${{ secrets.AWS_BUCKET }}
+              --aws-s3-region ${{ secrets.AWS_REGION }}
+              --aws-s3-key-id ${{ secrets.AWS_KEY_ID }}
+              --aws-s3-access-key ${{ secrets.AWS_ACCESS_KEY }}
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-aws-artifacts
+          path: ${{ env.artifact_paths}}
+
+  vfs_gcs:
+    runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce]
+    timeout-minutes: 300
+    env:
+      SUITE: vfs
+      STORAGE: /gcs
+    if: ${{ inputs.suite == 'all' || inputs.suite == 'vfs_gcs' || inputs.suite == 'vfs_all' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup
+        run: .github/setup.sh
+
+      - name: Run ${{ env.SUITE }} suite
+        run: python3
+              -u ${{ env.SUITE }}/regression.py
+              --storage gcs
+              --gcs-uri ${{ secrets.GCS_URI }}
+              --gcs-key-id ${{ secrets.GCS_KEY_ID }}
+              --gcs-key-secret ${{ secrets.GCS_KEY_SECRET }}
+              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
+              --clickhouse-version ${{ env.version }}
+              ${{ env.args }}
+
+      - name: Create and upload logs
+        if: always()
+        run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ env.SUITE }}-gcs-artifacts
+          path: ${{ env.artifact_paths}}
+
   vfs_minio:
     runs-on: [self-hosted, type-cpx51, image-x86-app-docker-ce]
     timeout-minutes: 72000
@@ -1506,7 +1616,6 @@ jobs:
               --storage minio
               --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
               --clickhouse-version ${{ env.version }}
-              --allow-vfs
               ${{ env.args }}
 
       - name: Create and upload logs