main.yml

name: Hourly Job

on:
  schedule:
    # Runs at the start of every hour
    - cron: '0 * * * *'
  workflow_dispatch:

jobs:
  run-script:
    runs-on: ubuntu-latest

    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
  
    - name: Verify environment variables
      run: |
        echo "REDDIT_APP_ID is set: ${{ secrets.REDDIT_APP_ID != '' }}"
        echo "REDDIT_APP_KEY is set: ${{ secrets.REDDIT_APP_KEY != '' }}"
        echo "OPENAI_API_CHRONO_KEY is set: ${{ secrets.OPENAI_API_CHRONO_KEY != '' }}"
        echo "DB_USER is set: ${{ secrets.DB_USER != '' }}"
        echo "DB_PASSWORD is set: ${{ secrets.DB_PASSWORD != '' }}"
        echo "DB_HOST_URL is set: ${{ secrets.DB_HOST_URL != '' }}"
        echo "DB_PORT is set: ${{ secrets.DB_PORT != '' }}"
        
    - name: setup python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    
    - name: install python packages
      run: |
        python -m pip install --upgrade pip
        pip install -r src/data_collection/requirements.txt

    - name: Run the reddit crawler
      env:
        REDDIT_APP_ID: ${{ secrets.REDDIT_APP_ID }}
        REDDIT_APP_KEY: ${{ secrets.REDDIT_APP_KEY }}
        DB_USER: ${{ secrets.DB_USER }}
        DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
        DB_HOST_URL: ${{ secrets.DB_HOST_URL }}
        DB_PORT: ${{ secrets.DB_PORT }}
        SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }}
        SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }}
      run: python src/data_collection/reddit_crawler.py --post_limit 15
    
    - name: Run the watchdb populator to
      env:
          OPENAI_API_CHRONO_KEY: ${{ secrets.OPENAI_API_CHRONO_KEY }}
          DB_USER: ${{ secrets.DB_USER }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
          DB_HOST_URL: ${{ secrets.DB_HOST_URL }}
          DB_PORT: ${{ secrets.DB_PORT }}
          SUPABASE_WATCHDB_URL: ${{ secrets.SUPABASE_WATCHDB_URL }}
          SUPABASE_WATCHDB_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_WATCHDB_SERVICE_ROLE_KEY }}
      run: python src/data_collection/watchdb_populator.py --num_requests 25