Skip to content

Commit

Permalink
Try this
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Jul 29, 2023
1 parent e28e68c commit 6ecdef9
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 9 deletions.
5 changes: 1 addition & 4 deletions sql/stats.prql
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,12 @@ let stats_over_time = (
total_size = sum(size) | as bigint,
total_lines = sum(lines) | as bigint,
total_hours = s"date_diff('hours', min({uploaded_on}), max({uploaded_on}))",
lines_per_hour = total_lines / total_hours | as bigint,
bytes_per_hour = total_size / total_hours | as bigint,
files_per_hour = total_files / total_hours | as bigint,
releases_per_hour = total_uploads / total_hours | as bigint,
}
)
)



relation_to_json(total_stats)
append (relation_to_json extension_stats)
append (relation_to_json binary_extension_stats)
Expand Down
8 changes: 3 additions & 5 deletions src/pypi_data/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,9 @@ def run_sql(
sql = prql_file.read_text()
# Can't get it to work without doing this. So dumb.
compiled_sql = sql.replace('$1', json.dumps(parameter))
# sql = f"{sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd);"
parameter = []
else:
compiled_sql = prql.compile(prql_file.read_text(), options=options)
# to-do: make this not shit....
compiled_sql = prql.compile(prql_file.read_text(), options=options).replace('$1', json.dumps(parameter))
sql = f"CREATE TABLE temp_table AS {compiled_sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd)"
print(sql)

Expand All @@ -124,8 +123,7 @@ def print_thread():
con.executemany(f"PRAGMA threads=4; "
f"PRAGMA memory_limit='2GB'; "
# f"PRAGMA enable_profiling;"
f"{sql}",
parameters=[[parameter]] if parameter else [])
f"{sql}")
try:
for name, plan in con.fetchall():
print(name)
Expand Down

0 comments on commit 6ecdef9

Please sign in to comment.