3gb?

pypi-data · Jul 29, 2023 · fef63cd · fef63cd
1 parent 4a9ef27
commit fef63cd
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 2 deletions.
diff --git a/.github/workflows/unique_python_files.yml b/.github/workflows/unique_python_files.yml
@@ -66,7 +66,7 @@ jobs:
 
       - name: Combine
         run: |
-          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --output=parquet --threads=2 unique-python-files.parquet combined/*.parquet
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --output=parquet --memory=3 --threads=2 unique-python-files.parquet combined/*.parquet
 
       - name: List
         run: ls combined/

diff --git a/sql/unique_python_files.prql b/sql/unique_python_files.prql
@@ -10,3 +10,4 @@ group {hash} (
 aggregate {
   path = any_value(path)
 })
+sort {hash}
diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py
@@ -106,6 +106,7 @@ def run_sql(
         parameter: Annotated[Optional[List[str]], typer.Argument()] = None,
         output: Annotated[OutputFormat, typer.Option()] = OutputFormat.PARQUET,
         threads: Annotated[int, typer.Option()] = 2,
+        memory: Annotated[int, typer.Option()] = 6,
         no_limits: Annotated[bool, typer.Option()] = False,
         profile: Annotated[bool, typer.Option()] = False,
         db: Annotated[Optional[str], typer.Option()] = None,
@@ -149,7 +150,7 @@ def run_sql(
             # sql = f'EXPLAIN ANALYZE ({sql})'
 
         if not no_limits:
-            limits = f"PRAGMA threads={threads}; PRAGMA memory_limit='6GB';"
+            limits = f"PRAGMA threads={threads}; PRAGMA memory_limit='{memory}GB';"
             conn.executemany(limits)
 
     print(sql)