Skip to content

Commit

Permalink
Maybe?
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Jul 29, 2023
1 parent 4985c6d commit e33103a
Showing 1 changed file with 8 additions and 6 deletions.
14 changes: 8 additions & 6 deletions src/pypi_data/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,18 +96,20 @@ def run_sql(
sql = sql.replace('$1', json.dumps(parameter))
sql = f"{sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd);"
parameter = []
conn = duckdb
else:
compiled_sql = prql.compile(prql_file.read_text(), options=options)
sql = f"CREATE TABLE temp_table AS {compiled_sql}; COPY temp_table TO '{output_file}' (FORMAT PARQUET, COMPRESSION zstd)"
conn = duckdb.connect("file.db")
print(sql)

print("\n\n\n")
# x = duckdb.execute(sql, parameters=[parameter] if parameter else [])
# import pprint
# pprint.pprint(x.fetchall())
con = duckdb.connect('file.db')
con.install_extension("httpfs")
con.load_extension("httpfs")
# pprint.pprint(x.fetchall()
# )
conn.install_extension("httpfs")
conn.load_extension("httpfs")

def print_thread():
psutil.cpu_percent()
Expand All @@ -121,12 +123,12 @@ def print_thread():
t = threading.Thread(target=print_thread, daemon=True)
t.start()
# duckdb.execute("PRAGMA EXPLAIN_OUTPUT='ALL';")
con.executemany(f"PRAGMA threads=4; "
conn.executemany(f"PRAGMA threads=4; "
f"PRAGMA memory_limit='2GB'; "
# f"PRAGMA enable_profiling;"
f"{sql}")
try:
for name, plan in con.fetchall():
for name, plan in conn.fetchall():
print(name)
print(plan)
except duckdb.InvalidInputException:
Expand Down

0 comments on commit e33103a

Please sign in to comment.