run.yml

# On every push this script is executed
on:
  workflow_dispatch:
  push:
    branches:
      - main

#concurrency: data

name: Build data
jobs:
  generate-matrix:
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install root
        run: poetry install --only-root

      - name: Generate token
        id: generate_token
        uses: pypi-data/github-app-token@v1
        with:
          app_id: ${{ secrets.APP_ID }}
          private_key: ${{ secrets.APP_PRIVATE_KEY }}

      - env:
          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
        id: groups
        run: |
          mkdir output/
          poetry run pypi-data group-index-urls output/
          echo "matrix=$(cat output/groups.json)" >> "$GITHUB_OUTPUT"

      - uses: actions/upload-artifact@v3
        with:
          name: groups
          path: output/groups/
          retention-days: 1

      - name: "Set current date as env variable"
        run: |
          echo "tag_name=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_OUTPUT
        id: version

      - name: Create Release
        id: create-release
        uses: shogo82148/actions-create-release@v1
        with:
          draft: true
          release_name: ${{ steps.version.outputs.tag_name }}
          tag_name: ${{ steps.version.outputs.tag_name }}
          commitish: ${{ github.sha }}

    outputs:
      matrix: ${{ steps.groups.outputs.matrix }}
      upload_url: ${{ steps.create-release.outputs.upload_url }}
      release_id: ${{ steps.create-release.outputs.id }}

  combine:
    needs: [generate-matrix]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        index: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'
          remove-docker-images: 'true'

      - name: checkout
        uses: actions/checkout@v3

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install root
        run: poetry install --only-root

      - uses: actions/download-artifact@v3
        with:
          name: groups

      - name: Download links
        run: cat ${{ matrix.index }} | jq -rc '.[]'

#      - name: Download
#        run: |
#          mkdir input/
#          cat ${{ matrix.index }} | jq -rc '.[]' | parallel wget -nv -O input/{#}.parquet {}

      - name: Combine
        run: |
          poetry run pypi-data run-sql ${{ github.workspace }}/sql/combine.sql output.parquet $(cat ${{ matrix.index }} | jq -rc '.[]')

      - name: Upload Assets
        uses: shogo82148/actions-upload-release-asset@v1
        with:
          upload_url: ${{ needs.generate-matrix.outputs.upload_url }}
          asset_path: output.parquet
          asset_name: index-${{ matrix.index }}.parquet

  makepublic:
    needs: [generate-matrix, combine]
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Publish release
        uses: StuYarrow/publish-release@v1.1.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          id: ${{ needs.generate-matrix.outputs.release_id }}

      - name: Get download links
        id: get_download_links
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const response = await github.rest.repos.listReleaseAssets({
              owner: context.repo.owner,
              repo: context.repo.repo,
              release_id: ${{ needs.generate-matrix.outputs.release_id }},
            });
            const assets = response.data;
            // Save the download links to a file
            require('fs').writeFileSync('download_links.txt', assets.map(asset => asset.browser_download_url).join('\n'));

      - uses: EndBug/add-and-commit@v9
        with:
          add: 'download_links.txt'
          author_email: "41898282+github-actions[bot]@users.noreply.github.com"
          author_name: "commit-bot"
          message: "Add download links for asset ${{ needs.generate-matrix.outputs.release_id }}"
          push: true
          fetch: true
          pull: '--rebase --autostash'

  unique_python_files:
    needs: [ makepublic, generate-matrix ]
    runs-on: ubuntu-latest
    steps:
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install root
        run: poetry install --only-root

      - name: Download links
        run: cat download_links.txt
          
      - name: Combine
        run: |
          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql unique-python-files.parquet $(cat download_links.txt)

      - name: Upload Assets
        id: upload
        uses: shogo82148/actions-upload-release-asset@v1
        with:
          upload_url: ${{ needs.generate-matrix.outputs.upload_url }}
          asset_name: unique-python-files.parquet

      - name: Create download links
        run: |
          echo "${{ steps.upload.outputs.browser_download_url }}" > only_python_download_links.txt

      - uses: EndBug/add-and-commit@v9
        with:
          add: 'only_python_download_links.txt'
          author_email: "41898282+github-actions[bot]@users.noreply.github.com"
          author_name: "commit-bot"
          message: "Add only python links for asset ${{ needs.generate-matrix.outputs.release_id }}"
          push: true
          fetch: true
          pull: '--rebase --autostash'

  generate_stats:
    needs: [ makepublic, generate-matrix ]
    runs-on: ubuntu-latest
    steps:
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install root
        run: poetry install --only-root

      - name: Combine
        run: |
          poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats.parquet $(cat download_links.txt)
#
#      - name: Upload Assets
#        id: upload
#        uses: shogo82148/actions-upload-release-asset@v1
#        with:
#          upload_url: ${{ needs.generate-matrix.outputs.upload_url }}
#          asset_name: unique-python-files.parquet
#
#      - name: Create download links
#        run: |
#          echo "${{ steps.upload.outputs.browser_download_url }}" > only_python_download_links.txt
#
#      - uses: EndBug/add-and-commit@v9
#        with:
#          add: 'only_python_download_links.txt'
#          author_email: "41898282+github-actions[bot]@users.noreply.github.com"
#          author_name: "commit-bot"
#          message: "Add only python links for asset ${{ needs.generate-matrix.outputs.release_id }}"
#          push: true
#          fetch: true
#          pull: '--rebase --autostash'