Hourly Job #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hourly Job | |
| on: | |
| schedule: | |
| # Runs at the start of every hour | |
| - cron: '0 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| run-script: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v4 | |
| - name: Verify environment variables | |
| run: | | |
| echo "REDDIT_APP_ID is set: ${{ secrets.REDDIT_APP_ID != '' }}" | |
| echo "REDDIT_APP_KEY is set: ${{ secrets.REDDIT_APP_KEY != '' }}" | |
| echo "SUPABASE_WATCHDB_URL is set: ${{ secrets.SUPABASE_WATCHDB_URL != '' }}" | |
| echo "SUPABASE_WATCHDB_SERVICE_ROLE_KEY is set: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY != '' }}" | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r src/data_collection/requirements.txt | |
| - name: Run the reddit crawler | |
| env: | |
| REDDIT_APP_ID: ${{ secrets.REDDIT_APP_ID }} | |
| REDDIT_APP_KEY: ${{ secrets.REDDIT_APP_KEY }} | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| run: python src/data_collection/reddit_crawler.py --post_limit 15 | |
| - name: Run the watchdb populator to | |
| env: | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| OPENAI_API_CHRONO_KEY: ${{ secrets.OPENAI_API_CHRONO_KEY }} | |
| run: python src/data_collection/watchdb_populator.py --num_requests 25 |