Build data #826
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # On every push this script is executed | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 3 * * *" | |
| concurrency: data | |
| name: Build data | |
| jobs: | |
| build-data: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| packages: write | |
| steps: | |
| - name: Max build space | |
| run: | | |
| rm -rf /usr/share/dotnet/ & | |
| rm -rf /usr/local/lib/android/ & | |
| rm -rf /opt/ghc/ & | |
| rm -rf /opt/hostedtoolcache/CodeQL/ & | |
| sudo docker image prune --all --force & | |
| df -h | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| enable-cache: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: "pyproject.toml" | |
| - name: Install deps | |
| run: uv sync --frozen | |
| - name: "Set current date as env variable" | |
| run: | | |
| echo "tag_name=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_OUTPUT | |
| id: version | |
| - name: Create Release | |
| id: create-release | |
| uses: shogo82148/actions-create-release@v1 | |
| with: | |
| overwrite: true | |
| draft: true | |
| release_name: ${{ steps.version.outputs.tag_name }} | |
| tag_name: ${{ steps.version.outputs.tag_name }} | |
| commitish: ${{ github.sha }} | |
| - name: Generate token | |
| id: generate_token | |
| uses: pypi-data/github-app-token@v2 | |
| with: | |
| app_id: ${{ secrets.APP_ID }} | |
| private_key: ${{ secrets.APP_PRIVATE_KEY }} | |
| revoke: false | |
| - name: Download hishel cache | |
| uses: actions/cache/restore@v4 | |
| id: hisel-cache | |
| with: | |
| path: .hishel.sqlite | |
| key: ${{ runner.os }}-hishel-${{ hashFiles('links/*.json') }} | |
| - name: Generate Repo Metadata | |
| env: | |
| GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} | |
| run: uv run pypi-data load-repos repos-with-packages.jsonl.gz links/ | |
| - name: Upload hishel cache | |
| uses: actions/cache/save@v4 | |
| if: steps.hisel-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: .hishel.sqlite | |
| key: ${{ steps.hisel-cache.outputs.cache-primary-key }} | |
| - name: Upload Repos with packages | |
| uses: shogo82148/actions-upload-release-asset@v1 | |
| with: | |
| upload_url: ${{ steps.create-release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/repos-with-packages.jsonl.gz | |
| - name: Remove repos-with-packages.jsonl.gz | |
| run: rm repos-with-packages.jsonl.gz | |
| - name: Create dataset | |
| run: uv run pypi-data merge-datasets links/repositories.json dataset/ | |
| - name: Debug | |
| run: ls -la dataset/ | |
| - name: Upload Dataset | |
| uses: shogo82148/actions-upload-release-asset@v1 | |
| id: upload-dataset | |
| with: | |
| upload_url: ${{ steps.create-release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/dataset/dataset-*.parquet | |
| asset_content_type: application/vnd.apache.parquet | |
| - name: Publish Draft Release | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const releaseId = "${{ steps.create-release.outputs.id }}"; | |
| const response = await github.rest.repos.updateRelease({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| release_id: releaseId, | |
| draft: false | |
| }); | |
| if (response.status === 200) { | |
| core.info(`Release ${releaseId} successfully published.`); | |
| } else { | |
| core.setFailed(`Failed to publish release ${releaseId}.`); | |
| } | |
| - name: Get download links | |
| id: get_download_links | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const response = await github.rest.repos.listReleaseAssets({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| release_id: ${{ steps.create-release.outputs.id }}, | |
| }); | |
| const assets = response.data; | |
| const urls = assets.filter(asset => asset.name.endsWith(".parquet")).map(asset => asset.browser_download_url) | |
| // Save the download links to a file | |
| require('fs').writeFileSync('links/dataset.txt', urls.join('\n')); | |
| const jsonUrls = assets.filter(asset => asset.name == "repos-with-packages.jsonl.gz").map(asset => asset.browser_download_url) | |
| require('fs').writeFileSync('links/repositories-with-packages.txt', jsonUrls.join('\n')); | |
| - uses: EndBug/add-and-commit@v9 | |
| with: | |
| add: links/* | |
| message: "Add repository URLs" | |
| push: true | |
| fetch: true | |
| pull: '--rebase --autostash' |