python.yaml

name: Python CI

on:
  push:
    branches:
      - '*'
  pull_request:
    types: ['opened', 'reopened', 'synchronize']

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ "ubuntu-latest" ]
        python-version: [ "3.10" ]

    steps:
      - name: Check out code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Set up Python environment
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: flake8 Lint
        uses: py-actions/flake8@v2
        with:
          path: "python/xorbits"
          args: "--config python/setup.cfg"
      - name: black
        uses: psf/black@stable
        with:
          src: "python/xorbits"
          options: "--check"
          version: "23.12.0"
      - uses: isort/isort-action@master
        with:
          sortPaths: "python/xorbits"
          configuration: "--check-only --diff --sp python/setup.cfg"
      - name: mypy
        run: pip install mypy && cd python && mypy xorbits
      - name: codespell
        run: pip install codespell==2.2.6 && cd python && codespell xorbits
      - name: Set up Node.js
        uses: actions/setup-node@v1
        with:
          node-version: 16
      # ESLint and Prettier must be in `package.json`
      - name: Install Node.js dependencies
        run: cd python/xorbits/web/ui && npm ci
      - name: ESLint Check
        run: cd python/xorbits/web/ui && npx eslint .
      - name: Prettier Check
        run: cd python/xorbits/web/ui && ./node_modules/.bin/prettier --check .

  build_test_job:
    if: github.repository == 'xorbitsai/xorbits'
    runs-on: ${{ matrix.os }}
    needs: lint
    env:
      CONDA_ENV: xorbits-test
      SELF_HOST_PYTHON: /root/miniconda3/bin/python
      SELF_HOST_CONDA: /root/miniconda3/bin/conda
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-13", "windows-latest"]
        python-version: ["3.9", "3.10", "3.11"]
        module: ["xorbits", "xorbits/numpy", "xorbits/pandas", "kubernetes"]
        exclude:
          - { os: macos-13, python-version: 3.10}
          - { os: macos-13, python-version: 3.9}
          - { os: windows-latest, python-version: 3.10}
          - { os: windows-latest, python-version: 3.9}
          - { os: windows-latest, module: kubernetes}
          - { os: macos-13, module: kubernetes}
        include:
          - { os: ubuntu-latest, module: _mars/dataframe, python-version: 3.9 }
          - { os: ubuntu-latest, module: learn, python-version: 3.9 }
          - { os: ubuntu-latest, module: mars-core, python-version: 3.9 }
          - { os: ubuntu-20.04, module: hadoop, python-version: 3.9 }
          - { os: ubuntu-latest, module: vineyard, python-version: 3.9 }
          - { os: ubuntu-latest, module: external-storage, python-version: 3.9 }
          - { os: ubuntu-latest, module: compatibility, python-version: 3.9 }
          - { os: ubuntu-latest, module: doc-build, python-version: 3.9 }
          - { os: [self-hosted, gpu], module: gpu, python-version: 3.11}
          - { os: ubuntu-latest, module: jax, python-version: 3.9 }
          - { os: juicefs-ci, module: kubernetes-juicefs, python-version: 3.9 }
          - { os: ubuntu-latest, module: slurm, python-version: 3.9 }
          - { os: ubuntu-latest, module: datasets, python-version: 3.9 }
    steps:
    - name: Check out code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        submodules: recursive
        
    - name: Add msbuild to PATH
      if: ${{ matrix.os == 'windows-latest'}}
      uses: microsoft/setup-msbuild@v1.1
    
    - name: Set up conda ${{ matrix.python-version }}
      uses: conda-incubator/setup-miniconda@v3
      if: ${{ matrix.module != 'gpu' && matrix.module != 'kubernetes-juicefs' }}
      with:
        python-version: ${{ matrix.python-version }}
        activate-environment: ${{ env.CONDA_ENV }}

    - name: Start minikube
      uses: medyagh/setup-minikube@master
      if: ${{ matrix.module == 'kubernetes' }}
      with:
        driver: none
        kubernetes-version: v1.23.12
        minikube-version: 1.31.2

    - name: Install ucx dependencies
      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}}
      run: |
        conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
    - name: Install libomp (macOS)
      if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-13' }}
      run: brew install libomp
    - name: Install dependencies
      env:
        MODULE: ${{ matrix.module }}
        PYTHON: ${{ matrix.python-version }}
      if: ${{ matrix.module != 'gpu' }}
      run: |
        pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
        pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr

        if [[ "$MODULE" == "xorbits/pandas" ]]; then
          pip install openpyxl
        fi
        if [[ "$MODULE" == "mars-core" ]]; then
          pip install oss2
        fi
        if [[ "$MODULE" == "kubernetes" ]]; then
          pip install kubernetes
        fi
        if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
          pip install kubernetes
          if helm version --short >/dev/null 2>&1; then
            echo "Helm installed. Skipping installation."
          else
            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          fi
          if helm repo list | grep "juicefs" >/dev/null 2>&1; then
            echo "Repo juicefs existed. Skipping adding to helm repos."
          else
            helm repo add juicefs https://juicedata.github.io/charts/
            helm repo update
            helm fetch --untar juicefs/juicefs-csi-driver
          fi
          if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
            echo "Juicefs-csi-driver installed. Skipping installation."
          else
            helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
          fi
          sleep 60
          kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
          sleep 60
        fi
        if [[ "$MODULE" == "hadoop" ]]; then
          ../CI/install-hadoop.sh
          echo "import coverage; coverage.process_startup()" > \
            $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
          conda install --quiet --yes -c conda-forge skein libffi conda-pack grpcio=1.42.0
        fi
        if [[ "$MODULE" == "vineyard" ]]; then
          pip install "vineyard<0.16.1" -i https://pypi.org/simple
          mkdir -p /tmp/etcd-download-test
          export ETCD_VER=v3.4.13
          export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
          curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
          tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
          sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
          sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
          rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
        fi
        if [[ "$MODULE" == "external-storage" ]]; then
          export CUR_DIR=$(pwd)
          mkdir -p /tmp/alluxio-download-test
          cd /tmp/alluxio-download-test
          wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
          tar -xzf alluxio-2.9.3-bin.tar.gz
          cd alluxio-2.9.3
          cp conf/alluxio-env.sh.template conf/alluxio-env.sh
          cp conf/alluxio-site.properties.template conf/alluxio-site.properties
          echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
          cd $CUR_DIR
          sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
          rm -R /tmp/alluxio-download-test
          unset CUR_DIR
          export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
          ${ALLUXIO_HOME}/bin/alluxio format
          ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
          curl -sSL https://d.juicefs.com/install | sh -
          curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
          sudo apt-get update
          sudo apt-get install redis
          export JUICEFS_HOME="/usr/local/bin/juicefs"
        fi
        if [[ "$PYTHON" == '3.9' ]]; then
          pip install h5py zarr matplotlib fastparquet
          conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
            "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
        fi
        if [[ "$MODULE" == "_mars/learn" ]]; then
          pip install scikit-learn
          pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
            statsmodels tsfresh dask[complete] mimesis\<9.0.0
        fi
        if [[ "$MODULE" == "learn" ]]; then
          pip install xgboost lightgbm
        fi
        if [[ "$MODULE" == "compatibility" ]]; then
          # test if compatible with older versions
          pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
        fi
        if [[ "$MODULE" == "jax" ]]; then
          # test jax
          pip install "jax>=0.4.0" "jaxlib>=0.4.0"
        fi
        if [[ "$MODULE" == "datasets" ]]; then
          # test huggingface datasets
          pip install datasets torch
        fi
        pip install -e ".[dev,extra,aws]"
      working-directory: ./python

    - name: Install on JuiceFsCI
      if: ${{ matrix.module == 'kubernetes-juicefs' }}
      run: |
        pip install -U xoscar
        python setup.py build_ext -i
      working-directory: ./python

    - name: Slurm Setup Job queuing system
      if: ${{ matrix.module == 'slurm' }}
      run: |
        source CI/slurm/${{ matrix.module }}.sh
        jobqueue_before_install

    - name: Slurm Install xorbits
      if: ${{ matrix.module == 'slurm' }}
      run: |
        source CI/slurm/${{ matrix.module }}.sh
        jobqueue_install
      
    - name: Install on GPU
      if: ${{ matrix.module == 'gpu' }}
      run: |
        pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12==24.8.*
        pip install ucx-py-cu12 cython "numpy>=1.14.0,<2.0.0" cloudpickle scikit-learn \
          pyyaml psutil tornado sqlalchemy defusedxml tqdm uvloop coverage \
          pytest pytest-cov pytest-timeout pytest-forked pytest-asyncio pytest-mock
        pip install -U xoscar
        python setup.py build_ext -i
      working-directory: ./python

    - name: Build doc
      if: ${{ matrix.module == 'doc-build' }}
      run: |
        make html
        make html_zh_cn
      working-directory: ./doc

    - name: Test with pytest
      if: ${{ matrix.module != 'doc-build' && matrix.module != 'gpu' }}
      env:
        MODULE: ${{ matrix.module }}
      run: |
        if [[ "$MODULE" == "xorbits" ]]; then
          pytest --ignore xorbits/_mars/ --ignore xorbits/pandas --ignore xorbits/numpy \
            --ignore xorbits/xgboost --ignore xorbits/lightgbm --ignore xorbits/sklearn \
            --ignore xorbits/datasets \
            --ignore xorbits/core/tests/test_execution_exit.py \
            --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
            -k "not test_execution_with_process_exit_message" \
            xorbits
          # workaround: this case will hang, run it separately.
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
            -k "test_execution_with_process_exit_message" \
            xorbits/core/tests/test_execution.py
        elif [[ "$MODULE" == "xorbits/pandas" ]]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml \
            --cov=xorbits xorbits/pandas
        elif [[ "$MODULE" == "xorbits/numpy" ]]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
            -k "not test_numpy_fallback" \
            xorbits/numpy
          # workaround: this case will hang, run it separately.
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml \
            --cov=xorbits \
            -k "test_numpy_fallback" \
            xorbits/numpy/numpy_adapters/tests/test_numpy_adapters.py
        elif [[ "$MODULE" == "mars-core" ]]; then
          pytest --forked --log-level=DEBUG --ignore xorbits/_mars/dataframe --ignore xorbits/_mars/tensor \
            --ignore xorbits/_mars/learn --ignore xorbits/_mars/remote \
            --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars
        elif [[ "$MODULE" == "kubernetes" ]]; then
          pytest --ignore xorbits/_mars/ --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/deploy/kubernetes
        elif [[ "$MODULE" == "kubernetes-juicefs" ]]; then
          pytest --ignore xorbits/_mars/ --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/deploy/kubernetes/external_storage/juicefs
        elif [[ "$MODULE" == "slurm" ]]; then
          docker exec c1 /bin/bash -c "pip install xorbits"
          docker exec c2 /bin/bash -c "pip install xorbits"
          docker exec slurmctld /bin/bash -c \
          "pytest /xorbits/python/xorbits/deploy/slurm/tests/test_slurm.py "
        elif [[ "$MODULE" == "hadoop" ]]; then
          export WITH_HADOOP="1"
          export HADOOP_HOME="/usr/local/hadoop"
          export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
          export HADOOP_INSTALL=$HADOOP_HOME
          export HADOOP_MAPRED_HOME=$HADOOP_HOME
          export HADOOP_COMMON_HOME=$HADOOP_HOME
          export HADOOP_HDFS_HOME=$HADOOP_HOME
          export YARN_HOME=$HADOOP_HOME
          export HADOOP_COMMON_LIB_NATIVE_DIR="$HADOOP_HOME/lib/native"
          export PATH="$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin"
          pytest --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop
        elif [[ "$MODULE" == "vineyard" ]]; then
          pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py
          pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/deploy/oscar/tests/test_local.py
          pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits -k "vineyard" \
            xorbits/_mars/tensor/datastore/tests/test_datastore_execution.py \
            xorbits/_mars/dataframe/datastore/tests/test_datastore_execution.py
        elif [[ "$MODULE" == "external-storage" ]]; then
          export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
          export JUICEFS_HOME="/usr/local/bin/juicefs"
          pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py
        elif [[ "$MODULE" == "_mars/learn" ]]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
            xorbits/$MODULE xorbits/_mars/contrib/dask/tests/test_dask.py
        elif [[ "$MODULE" == "learn" ]]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
            xorbits/xgboost xorbits/lightgbm
        elif [ "$MODULE" == "jax" ]; then
          pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/tensor/fuse/tests/test_runtime_fusion.py
          pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/tensor/
        elif [ "$MODULE" == "datasets" ]; then
          pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/datasets/backends
          pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/datasets/tests
        elif [ "$MODULE" == "compatibility" ]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/_mars/dataframe
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/_mars/services/storage
        else
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/$MODULE
        fi
      working-directory: ./python
    
    - name: Test with pytest GPU
      if: ${{ matrix.module == 'gpu' }}
      run: |
        pytest -m cuda --gpu --ignore xorbits/datasets \
          --ignore xorbits/sklearn --ignore xorbits/_mars/learn \
          --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
          xorbits
      working-directory: ./python


    - name: Cleanup on slurm
      if: ${{ matrix.module == 'slurm' }}
      run: |
        source CI/slurm/${{ matrix.module }}.sh
        jobqueue_after_script

    - name: Report coverage data
      uses: codecov/codecov-action@v4
      with:
        working-directory: ./python
        flags: unittests
        token: ${{ secrets.CODECOV_TOKEN }}