Hourly Job #496
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hourly Job | |
| on: | |
| schedule: | |
| # Runs at the start of every hour | |
| - cron: '0 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| run-script: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v4 | |
| - name: Verify environment variables | |
| run: | | |
| echo "REDDIT_APP_ID is set: ${{ secrets.REDDIT_APP_ID != '' }}" | |
| echo "REDDIT_APP_KEY is set: ${{ secrets.REDDIT_APP_KEY != '' }}" | |
| echo "OPENAI_API_CHRONO_KEY is set: ${{ secrets.OPENAI_API_CHRONO_KEY != '' }}" | |
| echo "DB_USER is set: ${{ secrets.DB_USER != '' }}" | |
| echo "DB_PASSWORD is set: ${{ secrets.DB_PASSWORD != '' }}" | |
| echo "DB_HOST_URL is set: ${{ secrets.DB_HOST_URL != '' }}" | |
| echo "DB_PORT is set: ${{ secrets.DB_PORT != '' }}" | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r src/data_collection/requirements.txt | |
| - name: Run the reddit crawler | |
| env: | |
| REDDIT_APP_ID: ${{ secrets.REDDIT_APP_ID }} | |
| REDDIT_APP_KEY: ${{ secrets.REDDIT_APP_KEY }} | |
| DB_USER: ${{ secrets.DB_USER }} | |
| DB_PASSWORD: ${{ secrets.DB_PASSWORD }} | |
| DB_HOST_URL: ${{ secrets.DB_HOST_URL }} | |
| DB_PORT: ${{ secrets.DB_PORT }} | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| run: python src/data_collection/reddit_crawler.py --post_limit 15 --comments_limit 25 | |
| - name: Run the watchdb populator to | |
| env: | |
| OPENAI_API_CHRONO_KEY: ${{ secrets.OPENAI_API_CHRONO_KEY }} | |
| DB_USER: ${{ secrets.DB_USER }} | |
| DB_PASSWORD: ${{ secrets.DB_PASSWORD }} | |
| DB_HOST_URL: ${{ secrets.DB_HOST_URL }} | |
| DB_PORT: ${{ secrets.DB_PORT }} | |
| SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }} | |
| SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }} | |
| run: python src/data_collection/watchdb_populator.py --num_requests 25 |