Address @xushiyan PR comments

apache · Sep 10, 2024 · 5a747fd · 5a747fd
1 parent f43852e
commit 5a747fd
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 65 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -79,10 +79,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Spin up minio
-        run: docker compose -f docker/docker-compose.yaml up -d
+      - name: Spin up minio for s3 tests
+        run: docker compose -f docker/docker-compose.yaml --profile s3 up -d
       - name: Run Rust integration tests
-        run: cd integration_test && cargo test --config env.toml
+        run: docker compose -f docker/docker-compose.yaml run integration_test_rust
 
   python-tests:
     strategy:

diff --git a/docker/Dockerfile → docker/Dockerfile.minio b/docker/Dockerfile → docker/Dockerfile.minio
diff --git a/docker/copy_tables.sh b/docker/copy_tables.sh
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
@@ -21,10 +21,14 @@ services:
   minio:
     image: quay.io/minio/minio
     container_name: minio
+    profiles:
+      - s3
     ports:
       - "9000:9000"
+    networks:
+      - hudi_network
     env_file:
-      - s3.env
+      - hudi.env
     command: server /data
     healthcheck:
       test: mc ready local
@@ -36,16 +40,53 @@ services:
 
   mc:
     build:
-      context: .
-      dockerfile: Dockerfile
+      context: ./
+      dockerfile: ./Dockerfile.minio
     container_name: mc
+    networks:
+      - hudi_network
+    profiles:
+      - s3
     depends_on:
       minio:
         condition: service_healthy
     env_file:
-      - s3.env
+      - hudi.env
     volumes:
       - ../crates/tests/data/tables:/opt/data:ro
-      - ./copy_tables.sh:/opt/copy_tables.sh
-    entrypoint: /opt/copy_tables.sh
+    entrypoint: >
+      /bin/bash -c '
+        set -eux
+        mc config host add store http://minio:9000 $$MINIO_ACCESS_KEY $$MINIO_SECRET_KEY
+        mc mb store/$$MINIO_TEST_BUCKET
+        find /opt/data -name "*.zip" -exec unzip {} -d /tmp/data/ \;
+        find /tmp/data -type d -mindepth 1 -maxdepth 1 -exec mc cp --recursive {} store/$$MINIO_TEST_BUCKET \;
+        mc ls store/$$MINIO_TEST_BUCKET
+      '
+  
+  integration_test_rust:
+    image: rust:1.75
+    container_name: integration_test_rust
+    networks:
+      - hudi_network
+    profiles:
+      - integration_test
+    env_file:
+      - hudi.env
+    volumes:
+      - ../crates:/app/crates:ro
+      - ../python:/app/python:ro
+      - ../integration_test:/app/integration_test
+      - ../Cargo.toml:/app/Cargo.toml:ro
+    entrypoint: >
+      /bin/sh -c "
+        set -eux
+        apt update && apt install -y --no-install-recommends libpython3.11-dev
+        cd /app/integration_test
+        cargo test
+      "
+
 
+networks:
+  hudi_network:
+    driver: bridge
diff --git a/docker/s3.env → docker/hudi.env b/docker/s3.env → docker/hudi.env
@@ -15,8 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# setup minio
 MINIO_ROOT_USER=minio
 MINIO_ROOT_PASSWORD=minio123
 MINIO_ACCESS_KEY=minio
 MINIO_SECRET_KEY=minio123
 MINIO_TEST_BUCKET=test-bucket
+
+# s3 integration test
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+AWS_ENDPOINT=http://minio:9000
+INTEGRATION_TEST_S3_BUCKET=test-bucket
diff --git a/integration_test/env.toml b/integration_test/env.toml
diff --git a/integration_test/src/lib.rs → integration_test/src/main.rs b/integration_test/src/lib.rs → integration_test/src/main.rs
@@ -16,3 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+fn main() {
+    println!("hudi-rs integration tests")
+}