test-notebooks.yml

name: Test Notebooks
on:
  pull_request:
    types: [closed]
    branches:
      - main
    paths:
      - "agentops/**"
      - "examples/**"
      - "tests/**"
      - ".github/workflows/test-notebooks.yml"
  workflow_dispatch:
jobs:
  test-notebooks:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.11']
      fail-fast: false
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -U jupyter
    - name: Create .env file
      run: |
        echo "AGENTOPS_API_KEY=${{ secrets.AGENTOPS_API_KEY }}" >> .env
        echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env
        echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> .env
        echo "CO_API_KEY=${{ secrets.CO_API_KEY }}" >> .env
        echo "GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}" >> .env
        echo "MULTION_API_KEY=${{ secrets.MULTION_API_KEY }}" >> .env
        echo "SERPER_API_KEY=${{ secrets.SERPER_API_KEY }}" >> .env
    - name: Install AgentOps from main branch and remove agentops install from notebooks
      run: |
        pip install git+https://github.com/AgentOps-AI/agentops.git@main
        find . -name '*.ipynb' -exec sed -i '/^%pip install.*agentops/d' {} +
    - name: Run notebooks and check for errors
      run: |
        mkdir -p logs
        exit_code=0

        exclude_notebooks=(
          "./examples/crewai_examples/job_posting.ipynb"
          "./examples/demos/agentchat_agentops.ipynb"
        )

        for notebook in $(find . -name '*.ipynb'); do

          skip=false
          for excluded in "${exclude_notebooks[@]}"; do
            if [[ "$notebook" == "$excluded" ]]; then
              skip=true
              echo "Skipping excluded notebook: $notebook"
              break
            fi
          done

          $skip && continue

          notebook_name=$(basename "$notebook" .ipynb)
          notebook_path=$(realpath "$notebook")
          notebook_dir=$(dirname "$notebook_path")
          
          # Run the notebook
          jupyter execute "$notebook_path" || true
          
          # Check if agentops.log was created
          if [ -f "${notebook_dir}/agentops.log" ]; then
            dest_log="logs/agentops-${notebook_name}.log"
            mv "${notebook_dir}/agentops.log" "$dest_log"
            
            # Check agentops log for errors or warnings
            if grep -E "ERROR|WARNING" "$dest_log"; then
              echo "Errors or warnings found in $dest_log for Python ${{ matrix.python-version }}"
              exit_code=1
            else
              echo "No errors or warnings found in $dest_log for Python ${{ matrix.python-version }}"
            fi
          else
            echo "No agentops.log generated for $notebook_name"
          fi
        done

        # Check if any logs were found
        if [ $(find logs -name 'agentops-*.log' | wc -l) -eq 0 ]; then
          echo "No agentops.log files were generated for any notebook"
        fi

        exit $exit_code

    - name: Upload logs as artifacts
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: notebook-logs-${{ matrix.python-version }}
        path: logs/agentops-*.log
        if-no-files-found: warn