Hourly Job #8
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hourly Job | |
| on: | |
| schedule: | |
| # Runs at the start of every hour | |
| - cron: '0 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| run-script: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v4 | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r src/data_collection/requirements.txt | |
| - name: Run the reddit crawler | |
| env: | |
| REDDIT_APP_ID: ${{ secrets.REDDIT_APP_ID }} | |
| REDDIT_APP_KEY: ${{ secrets.REDDIT_APP_KEY }} | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| run: python src/data_collection/reddit_crawler.py --post_limit 15 | |
| - name: Run the watchdb populator to | |
| env: | |
| REDDIT_APP_ID: ${{ secrets.REDDIT_APP_ID }} | |
| REDDIT_APP_KEY: ${{ secrets.REDDIT_APP_KEY }} | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| run: python src/data_collection/watchdb_populator.py --num_requests 25 |