Combine?

pypi-data · Jul 29, 2023 · 629ca3e · 629ca3e
1 parent bd8af9b
commit 629ca3e
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/.github/workflows/unique_python_files.yml b/.github/workflows/unique_python_files.yml
@@ -59,14 +59,14 @@ jobs:
         run: |
           sudo apt-get install parallel
 
-      - name: Combine
+      - name: Ingest
         run: |
           mkdir combined/
           find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n2 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 combined/{#}.parquet {}
 
-      - name: Complete
+      - name: Combine
         run: |
-          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 unique-python-files.parquet combined/*.parquet
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --output=parquet --threads=2 unique-python-files.parquet combined/*.parquet
 
       - name: List
         run: ls combined/

diff --git a/sql/unique_python_files_combine.prql b/sql/unique_python_files_combine.prql
@@ -0,0 +1,10 @@
+prql target:sql.duckdb
+
+let any_value = column -> s"any_value({column})"
+
+from (read_parquet $1)
+select {hash, path}
+group {hash} (
+aggregate {
+  path = any_value(path)
+})