From 6ecdef9d708746843a0888deb7620bf803a007fc Mon Sep 17 00:00:00 2001 From: Tom Forbes Date: Sat, 29 Jul 2023 03:04:27 +0100 Subject: [PATCH] Try this --- sql/stats.prql | 5 +---- src/pypi_data/cli.py | 8 +++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/sql/stats.prql b/sql/stats.prql index af82ede3..3abe63cb 100644 --- a/sql/stats.prql +++ b/sql/stats.prql @@ -91,15 +91,12 @@ let stats_over_time = ( total_size = sum(size) | as bigint, total_lines = sum(lines) | as bigint, total_hours = s"date_diff('hours', min({uploaded_on}), max({uploaded_on}))", - lines_per_hour = total_lines / total_hours | as bigint, - bytes_per_hour = total_size / total_hours | as bigint, - files_per_hour = total_files / total_hours | as bigint, - releases_per_hour = total_uploads / total_hours | as bigint, } ) ) + relation_to_json(total_stats) append (relation_to_json extension_stats) append (relation_to_json binary_extension_stats) diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py index cdc57926..d2404499 100644 --- a/src/pypi_data/cli.py +++ b/src/pypi_data/cli.py @@ -94,10 +94,9 @@ def run_sql( sql = prql_file.read_text() # Can't get it to work without doing this. So dumb. compiled_sql = sql.replace('$1', json.dumps(parameter)) - # sql = f"{sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd);" - parameter = [] else: - compiled_sql = prql.compile(prql_file.read_text(), options=options) + # to-do: make this not shit.... + compiled_sql = prql.compile(prql_file.read_text(), options=options).replace('$1', json.dumps(parameter)) sql = f"CREATE TABLE temp_table AS {compiled_sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd)" print(sql) @@ -124,8 +123,7 @@ def print_thread(): con.executemany(f"PRAGMA threads=4; " f"PRAGMA memory_limit='2GB'; " # f"PRAGMA enable_profiling;" - f"{sql}", - parameters=[[parameter]] if parameter else []) + f"{sql}") try: for name, plan in con.fetchall(): print(name)