populate_labelstudio.yml

name: Populate LabelStudio

on:
  workflow_dispatch:
    inputs:
      crawl_id:
        description: 'Common Crawl Corpus'
        required: true
        default: 'CC-MAIN-2024-10'
      url:
        description: 'URL type'
        required: true
        default: '*.gov'
      keyword:
        description: 'keyword'
        required: true
        default: 'police'
      pages:
        description: 'num pages'
        required: true
        default: '2'
      record_type:
        description: 'record type'
        required: false
        

jobs:
  run-script:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r annotation_pipeline/requirements.txt

    - name: Run main script
      env:
        HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
        LABEL_STUDIO_ACCESS_TOKEN: ${{ secrets.LABEL_STUDIO_ACCESS_TOKEN }}
        LABEL_STUDIO_PROJECT_ID: ${{ secrets.LABEL_STUDIO_PROJECT_ID }}
        LABEL_STUDIO_ORGANIZATION: ${{ secrets.LABEL_STUDIO_ORGANIZATION }}
      run: |
        if [ -n "${{ github.event.inputs.record_type }}" ]; then
          python annotation_pipeline/populate_labelstudio.py ${{ github.event.inputs.crawl_id }} "${{ github.event.inputs.url }}" ${{ github.event.inputs.keyword }} --pages ${{ github.event.inputs.pages }} --record_type "${{ github.event.inputs.record_type }}"
        else
          python annotation_pipeline/populate_labelstudio.py ${{ github.event.inputs.crawl_id }} "${{ github.event.inputs.url }}" ${{ github.event.inputs.keyword }} --pages ${{ github.event.inputs.pages }}
        fi

    - name: Commit and push outputs
      run: |
        git config --global user.name "github-actions[bot]"
        git config --global user.email "action@github.com"
        git add annotation_pipeline/data/batch_info.csv
        git add annotation_pipeline/data/cache.json
        git add annotation_pipeline/data/tag_collector/*
        git commit -m "Update batch info and collected urls & tags"
        git push