From 99628f88ad34655af28b1f04f9c120970abc6a29 Mon Sep 17 00:00:00 2001
From: Tom Forbes <tom@tomforb.es>
Date: Thu, 27 Jul 2023 00:46:36 +0100
Subject: [PATCH] ok

---
 .github/workflows/run.yml | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index a409bdb4..9e59c64b 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -138,14 +138,14 @@ jobs:
       - name: Download links
         run: cat ${{ matrix.index }} | jq -rc '.[]'
 
-#      - name: Download
-#        run: |
-#          mkdir input/
-#          cat ${{ matrix.index }} | jq -rc '.[]' | parallel wget -nv -O input/{#}.parquet {}
+      - name: Download
+        run: |
+          mkdir input/
+          cat ${{ matrix.index }} | jq -rc '.[]' | parallel wget -nv -O input/{#}.parquet {}
 
       - name: Combine
         run: |
-          poetry run pypi-data run-sql ${{ github.workspace }}/sql/combine.sql output.parquet $(cat ${{ matrix.index }} | jq -rc '.[]')
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/combine.sql output.parquet input/*
 
       - name: Upload Assets
         uses: shogo82148/actions-upload-release-asset@v1
@@ -239,10 +239,22 @@ jobs:
 
       - name: Download links
         run: cat download_links.txt
+
+      - name: Setup wget2
+        run: sudo apt-get update && sudo apt-get install wget2
+
+      - name: Download releases
+        run: |
+          mkdir data/
+          wget2 $(cat download_links.txt) --compression -P data/
           
       - name: Combine
         run: |
-          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql unique-python-files.parquet $(cat download_links.txt)
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql unique-python-files.parquet data/*
+          
+      - name: Stats
+        run: |
+          poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats.parquet data/*
 
       - name: Upload Assets
         id: upload