unique_python_files.yml

# On every push this script is executed
on:
  workflow_dispatch:

#concurrency: data

name: Unique Python files
jobs:
  unique_python_files:
    timeout-minutes: 120
    runs-on: ubuntu-latest
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'
          remove-docker-images: 'true'

      - name: checkout
        uses: actions/checkout@v3

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install root
        run: poetry install --only-root

      - name: Download links
        run: |
          mkdir dataset/
          cat links/dataset.txt | xargs -P 5 -n 4 wget --no-verbose -P dataset/

      - name: Install parallel
        run: |
          sudo apt-get install parallel

      - name: Ingest
        run: |
          mkdir combined/
          find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n4 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 combined/{#}.parquet {}
          
      - name: List
        run: ls combined/

      - name: Combine
        run: |
          poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --output=parquet --memory=3 --threads=2 unique-python-files.parquet combined/*.parquet

      - name: Gets latest created release info
        id: latest_release_info
        uses: jossef/action-latest-release-info@v1.2.1
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Upload Assets
        id: upload
        uses: shogo82148/actions-upload-release-asset@v1
        with:
          upload_url: ${{ steps.latest_release_info.outputs.upload_url }}
          asset_name: unique-python-files.parquet
          asset_path: unique-python-files.parquet
          overwrite: true

      - name: Create download links
        run: |
          echo "${{ steps.upload.outputs.browser_download_url }}" > links/unique_python_files.txt

      - uses: EndBug/add-and-commit@v9
        with:
          add: 'links/unique_python_files.txt'
          author_email: "41898282+github-actions[bot]@users.noreply.github.com"
          author_name: "commit-bot"
          message: "Add only python links for asset ${{ needs.generate-matrix.outputs.release_id }}"
          push: true
          fetch: true
          pull: '--rebase --autostash'