Skip to content

Commit

Permalink
Try this
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Jul 28, 2023
1 parent dcb22ea commit 2c5d5bf
Showing 1 changed file with 158 additions and 0 deletions.
158 changes: 158 additions & 0 deletions .github/workflows/stats.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
# On every push this script is executed
on:
workflow_dispatch:

#concurrency: data

name: Build data
jobs:
unique_python_files:
runs-on: ubuntu-latest
steps:
- name: Maximize build space
uses: easimon/maximize-build-space@master
with:
remove-dotnet: 'true'
remove-android: 'true'
remove-haskell: 'true'
remove-codeql: 'true'
remove-docker-images: 'true'

- name: checkout
uses: actions/checkout@v3

- name: Set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: 'pip'

- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root

- name: Install root
run: poetry install --only-root

- name: Download links
run: cat links/dataset.txt

- name: Setup wget2
run: sudo apt-get update && sudo apt-get install wget2

- name: Download releases
run: |
mkdir data/
wget2 $(cat links/dataset.txt) --compression -P data/
- name: Combine
run: |
poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql unique-python-files.parquet data/*
- name: Stats
run: |
poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats.parquet data/*
- name: Upload Assets
id: upload
uses: shogo82148/actions-upload-release-asset@v1
with:
upload_url: ${{ needs.generate-matrix.outputs.upload_url }}
asset_name: unique-python-files.parquet

- name: Create download links
run: |
echo "${{ steps.upload.outputs.browser_download_url }}" > data/python_files_dataset.txt
- uses: EndBug/add-and-commit@v9
with:
add: 'data/python_files_dataset.txt'
author_email: "41898282+github-actions[bot]@users.noreply.github.com"
author_name: "commit-bot"
message: "Add only python links for asset ${{ needs.generate-matrix.outputs.release_id }}"
push: true
fetch: true
pull: '--rebase --autostash'

generate_stats:
runs-on: ubuntu-latest
steps:
- name: Maximize build space
uses: easimon/maximize-build-space@master
with:
remove-dotnet: 'true'
remove-android: 'true'
remove-haskell: 'true'
remove-codeql: 'true'
remove-docker-images: 'true'

- name: checkout
uses: actions/checkout@v3

- name: Set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: 'pip'

- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root

- name: Install root
run: poetry install --only-root

- name: Combine
run: |
poetry run pypi-data run-sql ${{ github.workspace }}/sql/stats.prql stats.parquet $(cat links/dataset.txt)
# - name: Upload Assets
# id: upload
# uses: shogo82148/actions-upload-release-asset@v1
# with:
# upload_url: ${{ needs.generate-matrix.outputs.upload_url }}
# asset_name: unique-python-files.parquet
#
# - name: Create download links
# run: |
# echo "${{ steps.upload.outputs.browser_download_url }}" > only_python_download_links.txt
#
# - uses: EndBug/add-and-commit@v9
# with:
# add: 'only_python_download_links.txt'
# author_email: "41898282+github-actions[bot]@users.noreply.github.com"
# author_name: "commit-bot"
# message: "Add only python links for asset ${{ needs.generate-matrix.outputs.release_id }}"
# push: true
# fetch: true
# pull: '--rebase --autostash'

0 comments on commit 2c5d5bf

Please sign in to comment.