unit-test.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: unittest

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

permissions:
  contents: read

jobs:
  unittest-single:
    runs-on: [self-hosted, linux]
    environment: Testing
    steps:
    - uses: actions/checkout@v3
      with:
        path: dj-${{ github.run_id }}

    - name: Setup docker compose
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      run: |
        docker compose up -d

    - name: Install data-juicer
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      run: |
        docker compose exec ray-head pip install -e .\[all\]
        docker compose exec ray-worker pip install -e .\[all\]

    - name: Clean dataset cache
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      run: |
        docker compose exec ray-head rm -rf /data/huggingface/dataset

    - name: Run unittest standalone
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      run: |
        docker compose exec ray-head python tests/run.py --tag standalone

    - name: Run unittest ray
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      run: |
        docker compose exec ray-head python tests/run.py --tag ray

    - name: Remove docker compose
      working-directory: dj-${{ github.run_id }}/.github/workflows/docker
      if: always()
      run: |
        docker compose down --remove-orphans

    - name: Cleanup workspace
      if: always()
      run: |
        rm -rf  dj-${{ github.run_id }}