Revert "try this"

This reverts commit 17c7fb9
pypi-data · Jul 29, 2023 · d78f3f6 · d78f3f6
1 parent 6ecdef9
commit d78f3f6
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/sql/combine.sql b/sql/combine.sql
@@ -1 +1,4 @@
-SELECT * FROM read_parquet('input/*.parquet', union_by_name=True);
+PRAGMA memory_limit='2GB';
+PRAGMA threads=4;
+CREATE TABLE temp_table AS SELECT * FROM read_parquet('input/*.parquet', union_by_name=True);
+COPY temp_table TO 'output.parquet' (FORMAT PARQUET, COMPRESSION zstd);
diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py
@@ -93,11 +93,12 @@ def run_sql(
     if prql_file.name.endswith(".sql"):
         sql = prql_file.read_text()
         # Can't get it to work without doing this. So dumb.
-        compiled_sql = sql.replace('$1', json.dumps(parameter))
+        sql = sql.replace('$1', json.dumps(parameter))
+        sql = f"{sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd);"
+        parameter = []
     else:
-        # to-do: make this not shit....
-        compiled_sql = prql.compile(prql_file.read_text(), options=options).replace('$1', json.dumps(parameter))
-    sql = f"CREATE TABLE temp_table AS {compiled_sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd)"
+        compiled_sql = prql.compile(prql_file.read_text(), options=options)
+        sql = f"CREATE TABLE temp_table AS {compiled_sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd)"
     print(sql)
 
     print("\n\n\n")