Rerun demix #237
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rerun demix | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| rerun_demix: | |
| permissions: | |
| contents: write | |
| id-token: write | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup nextflow | |
| uses: nf-core/setup-nextflow@v1 | |
| - name: Setup Python | |
| run: | | |
| echo ${{secrets.DPILZ_USR_PWD}} | sudo -S dnf install python3 -y | |
| echo ${{secrets.DPILZ_USR_PWD}} | sudo -S dnf install python3-pip -y | |
| pip3 install pandas numpy pyyaml ffq epiweeks pyarrow git+https://github.com/outbreak-info/python-outbreak-info.git@new_docs | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| with: | |
| version: '>= 363.0.0' | |
| - id: 'auth' | |
| name: 'Authenticate with gcloud' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| workload_identity_provider: 'projects/12767718289/locations/global/workloadIdentityPools/github/providers/freyja-sra' | |
| service_account: '[email protected]' | |
| - name: 'Get samples to re-run' | |
| run: | | |
| mkdir demix_rerun | |
| python scripts/get_samples_for_demix_rerun.py | |
| - name: 'Get variants output' | |
| run: | | |
| bash scripts/get_variants_output.sh | |
| - name: 'Rerun demix' | |
| run: | | |
| export NXF_ENABLE_VIRTUAL_THREADS=false | |
| nextflow run main.nf \ | |
| -entry rerun_demix \ | |
| -profile docker \ | |
| --accession_list data/samples_to_rerun.csv \ | |
| --variants_dir demix_rerun \ | |
| - name: 'Aggregate demix results' | |
| run: | | |
| python scripts/aggregate_demix.py | |
| - name: 'Replace old demix results' | |
| run: | | |
| gcloud storage cp gs://outbreak-ww-data/aggregate/aggregate_demix.json outputs/aggregate/aggregate_demix.json | |
| gcloud storage cp gs://outbreak-ww-data/aggregate/aggregate_metadata.json outputs/aggregate/aggregate_metadata.json | |
| python scripts/update_aggregate_demix.py | |
| rm -rf outputs/aggregate/aggregate_demix_new.json | |
| - name: 'Aggregate demix by week' | |
| run: | | |
| python scripts/aggregate_demix_by_week.py | |
| - id: 'upload-demix' | |
| name: 'Upload Demix to Cloud Storage' | |
| uses: 'google-github-actions/upload-cloud-storage@v2' | |
| with: | |
| path: 'outputs/demix' | |
| destination: 'outbreak-ww-data/demix/' | |
| parent: false | |
| project_id: 'andersen-lab-primary' | |
| - id: 'upload-aggregate' | |
| name: 'Upload Aggregate to Cloud Storage' | |
| uses: 'google-github-actions/upload-cloud-storage@v2' | |
| with: | |
| path: 'outputs/aggregate' | |
| destination: 'outbreak-ww-data/aggregate/' | |
| parent: false | |
| project_id: 'andersen-lab-primary' | |