try fixing stats

pypi-data · Jul 29, 2023 · 69c1eaa · 69c1eaa
1 parent c3db870
commit 69c1eaa
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml
@@ -53,7 +53,7 @@ jobs:
       - name: Generate stats
         run: |
           cat links/dataset.txt
-          poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats/totals.json --output=json --no-limits $(cat links/dataset.txt)
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats/totals.json --output=json --threads=4 $(cat links/dataset.txt)
 
       - run: |
           cp stats/totals.json t

diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py
@@ -189,10 +189,11 @@ def print_thread():
     elif output == OutputFormat.PARQUET:
         sql.to_parquet(str(output_file), compression="zstd")
     else:
-        df: pd.DataFrame = sql.to_df()
-        df.set_index("name", inplace=True)
-        df["stat"] = df["stat"].apply(lambda x: json.loads(x))
-        df.to_json(output_file, orient="index", lines=False, indent=2)
+        sql.to_table("temp_table")
+        # df: pd.DataFrame = sql.to_df()
+        # df.set_index("name", inplace=True)
+        # df["stat"] = df["stat"].apply(lambda x: json.loads(x))
+        # df.to_json(output_file, orient="index", lines=False, indent=2)
 
 
 if __name__ == "__main__":