Snappy?

pypi-data · Aug 2, 2023 · 05f8835 · 05f8835
1 parent 22d3c5a
commit 05f8835
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/.github/workflows/unique_python_files.yml b/.github/workflows/unique_python_files.yml
@@ -62,7 +62,7 @@ jobs:
       - name: Ingest
         run: |
           mkdir combined/
-          find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n4 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 combined/{#}.parquet {}
+          find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n3 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 combined/{#}.parquet {}
           
       - name: List
         run: ls combined/

diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py
@@ -226,7 +226,7 @@ def print_thread():
             print(f'\n\nper_thread_output {output_sql}\n\n\n')
             conn.execute(output_sql)
         else:
-            sql_obj.to_parquet(str(output_file), compression="zstd")
+            sql_obj.to_parquet(str(output_file), compression="snappy")
     else:
         sql_obj.to_table("temp_table")
         conn.execute(f'COPY temp_table TO \'{output_file}\' (FORMAT JSON, array TRUE)')