Use parquet CLI to combine

.github/workflows/run.yml

@@ -127,15 +127,17 @@ jobs:
         with:
           name: groups
 
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - run: cargo install parquet -F cli
+
       - name: Download links
         run: cat ${{ matrix.index }} | jq -rc '.[]'
 
-      - name: Setup DuckDB
-        run: |
-          wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/duckdb_cli-linux-amd64.zip -O /tmp/duckdb.zip
-          unzip /tmp/duckdb.zip -d ${{ github.workspace }}
-          chmod +x ${{ github.workspace }}/duckdb
-
       - name: Debug
         run: |
           echo "Links for ${{ matrix.index }}" 
@@ -149,9 +151,22 @@ jobs:
       - run: ls -la ${{ github.workspace }}/input/
 
       - name: Combine
-        run: ${{ github.workspace }}/duckdb -echo foo.db < ${{ github.workspace }}/sql/combine.sql
+        run: parquet-concat ${{ github.workspace }}/input/*.parquet ${{ github.workspace }}/merged.parquet
+
+      - name: Merged size
+        run: du -hs ${{ github.workspace }}/merged.parquet
 
-      - run: ls -la ${{ github.workspace }}/*.parquet
+      - name: Rewrite
+        run: |
+          parquet-rewrite --compression=zstd \
+            --input=${{ github.workspace }}/merged.parquet \
+            --output=${{ github.workspace }}/output.parquet \
+            --writer-version=2.0 \
+            --statistics-enabled=page \
+            --bloom-filter-enabled=true
+
+      - name: Output size
+        run: du -hs ${{ github.workspace }}/merged.parquet
 
       - name: Upload Assets
         uses: shogo82148/actions-upload-release-asset@v1