Add more unittest #974

	# This workflow will install Python dependencies, run tests and lint with a single version of Python
	# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

	name: unittest

	on:
	workflow_dispatch:
	pull_request:
	push:
	branches:
	- main

	permissions:
	contents: read

	jobs:
	unittest-single:
	runs-on: [self-hosted, linux]
	environment: Testing
	steps:
	- uses: actions/checkout@v3
	with:
	path: dj-${{ github.run_id }}

	- name: Setup docker compose
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	run: \|
	docker compose up -d

	- name: Install data-juicer
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	run: \|
	docker compose exec ray-head pip install -e .\[all\]
	docker compose exec ray-worker pip install -e .\[all\]

	- name: Clean dataset cache
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	run: \|
	docker compose exec ray-head rm -rf /data/huggingface/dataset

	- name: Run unittest standalone
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	run: \|
	docker compose exec ray-head python tests/run.py --tag standalone

	- name: Run unittest ray
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	run: \|
	docker compose exec ray-head python tests/run.py --tag ray

	- name: Remove docker compose
	working-directory: dj-${{ github.run_id }}/.github/workflows/docker
	if: always()
	run: \|
	docker compose down --remove-orphans

	- name: Cleanup workspace
	if: always()
	run: \|
	rm -rf dj-${{ github.run_id }}

Provide feedback