DOC: update CI environment dependencies for doc build (#814)

Signed-off-by: Dawnfz <[email protected]>
xorbitsai · Sep 19, 2024 · dd46556 · dd46556
1 parent c765e57
commit dd46556
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 104 deletions.
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -129,7 +129,7 @@ jobs:
  minikube-version: 1.31.2
 
  - name: Install ucx dependencies
- if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}}
+ if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.module != 'doc-build') }}
  run: |
  conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
  - name: Install libomp (macOS)
@@ -141,108 +141,114 @@ jobs:
  PYTHON: ${{ matrix.python-version }}
  if: ${{ matrix.module != 'gpu' }}
  run: |
- pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
- pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
+ if [[ "$MODULE" == "doc-build" ]]; then
+ pip install --upgrade --no-cache-dir pip setuptools
+ pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext
+ pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]"
+ else
+ pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
+ pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
 
- if [[ "$MODULE" == "xorbits/pandas" ]]; then
- pip install openpyxl
- fi
- if [[ "$MODULE" == "mars-core" ]]; then
- pip install oss2
- fi
- if [[ "$MODULE" == "kubernetes" ]]; then
- pip install kubernetes
- fi
- if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
- pip install kubernetes
- if helm version --short >/dev/null 2>&1; then
- echo "Helm installed. Skipping installation."
- else
- curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+ if [[ "$MODULE" == "xorbits/pandas" ]]; then
+ pip install openpyxl
  fi
- if helm repo list | grep "juicefs" >/dev/null 2>&1; then
- echo "Repo juicefs existed. Skipping adding to helm repos."
- else
- helm repo add juicefs https://juicedata.github.io/charts/
- helm repo update
- helm fetch --untar juicefs/juicefs-csi-driver
+ if [[ "$MODULE" == "mars-core" ]]; then
+ pip install oss2
  fi
- if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
- echo "Juicefs-csi-driver installed. Skipping installation."
- else
- helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
+ if [[ "$MODULE" == "kubernetes" ]]; then
+ pip install kubernetes
  fi
- sleep 60
- kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
- sleep 60
- fi
- if [[ "$MODULE" == "hadoop" ]]; then
- ../CI/install-hadoop.sh
- echo "import coverage; coverage.process_startup()" > \
- $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
- conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0
- fi
- if [[ "$MODULE" == "vineyard" ]]; then
- pip install vineyard
- mkdir -p /tmp/etcd-download-test
- export ETCD_VER=v3.4.13
- export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
- curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
- tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
- sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
- sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
- rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
- fi
- if [[ "$MODULE" == "external-storage" ]]; then
- export CUR_DIR=$(pwd)
- mkdir -p /tmp/alluxio-download-test
- cd /tmp/alluxio-download-test
- wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
- tar -xzf alluxio-2.9.3-bin.tar.gz
- cd alluxio-2.9.3
- cp conf/alluxio-env.sh.template conf/alluxio-env.sh
- cp conf/alluxio-site.properties.template conf/alluxio-site.properties
- echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
- cd $CUR_DIR
- sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
- rm -R /tmp/alluxio-download-test
- unset CUR_DIR
- export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
- ${ALLUXIO_HOME}/bin/alluxio format
- ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
- curl -sSL https://d.juicefs.com/install | sh -
- curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
- echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
- sudo apt-get update
- sudo apt-get install redis
- export JUICEFS_HOME="/usr/local/bin/juicefs"
- fi
- if [[ "$PYTHON" == '3.9' ]]; then
- pip install h5py zarr matplotlib fastparquet
- conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
- "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
- fi
- if [[ "$MODULE" == "_mars/learn" ]]; then
- pip install scikit-learn
- pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
- statsmodels tsfresh dask[complete] mimesis\<9.0.0
- fi
- if [[ "$MODULE" == "learn" ]]; then
- pip install xgboost lightgbm
- fi
- if [[ "$MODULE" == "compatibility" ]]; then
- # test if compatible with older versions
- pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
- fi
- if [[ "$MODULE" == "jax" ]]; then
- # test jax
- pip install "jax>=0.4.0" "jaxlib>=0.4.0"
- fi
- if [[ "$MODULE" == "datasets" ]]; then
- # test huggingface datasets
- pip install datasets torch
+ if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
+ pip install kubernetes
+ if helm version --short >/dev/null 2>&1; then
+ echo "Helm installed. Skipping installation."
+ else
+ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+ fi
+ if helm repo list | grep "juicefs" >/dev/null 2>&1; then
+ echo "Repo juicefs existed. Skipping adding to helm repos."
+ else
+ helm repo add juicefs https://juicedata.github.io/charts/
+ helm repo update
+ helm fetch --untar juicefs/juicefs-csi-driver
+ fi
+ if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
+ echo "Juicefs-csi-driver installed. Skipping installation."
+ else
+ helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
+ fi
+ sleep 60
+ kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
+ sleep 60
+ fi
+ if [[ "$MODULE" == "hadoop" ]]; then
+ ../CI/install-hadoop.sh
+ echo "import coverage; coverage.process_startup()" > \
+ $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
+ conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0
+ fi
+ if [[ "$MODULE" == "vineyard" ]]; then
+ pip install vineyard
+ mkdir -p /tmp/etcd-download-test
+ export ETCD_VER=v3.4.13
+ export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+ curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
+ tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
+ sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
+ sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
+ rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
+ fi
+ if [[ "$MODULE" == "external-storage" ]]; then
+ export CUR_DIR=$(pwd)
+ mkdir -p /tmp/alluxio-download-test
+ cd /tmp/alluxio-download-test
+ wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
+ tar -xzf alluxio-2.9.3-bin.tar.gz
+ cd alluxio-2.9.3
+ cp conf/alluxio-env.sh.template conf/alluxio-env.sh
+ cp conf/alluxio-site.properties.template conf/alluxio-site.properties
+ echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
+ cd $CUR_DIR
+ sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
+ rm -R /tmp/alluxio-download-test
+ unset CUR_DIR
+ export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
+ ${ALLUXIO_HOME}/bin/alluxio format
+ ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
+ curl -sSL https://d.juicefs.com/install | sh -
+ curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+ echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+ sudo apt-get update
+ sudo apt-get install redis
+ export JUICEFS_HOME="/usr/local/bin/juicefs"
+ fi
+ if [[ "$PYTHON" == '3.9' ]]; then
+ pip install h5py zarr matplotlib fastparquet
+ conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
+ "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
+ fi
+ if [[ "$MODULE" == "_mars/learn" ]]; then
+ pip install scikit-learn
+ pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
+ statsmodels tsfresh dask[complete] mimesis\<9.0.0
+ fi
+ if [[ "$MODULE" == "learn" ]]; then
+ pip install xgboost lightgbm
+ fi
+ if [[ "$MODULE" == "compatibility" ]]; then
+ # test if compatible with older versions
+ pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
+ fi
+ if [[ "$MODULE" == "jax" ]]; then
+ # test jax
+ pip install "jax>=0.4.0" "jaxlib>=0.4.0"
+ fi
+ if [[ "$MODULE" == "datasets" ]]; then
+ # test huggingface datasets
+ pip install datasets torch
+ fi
+ pip install -e ".[dev,extra,aws]"
  fi
- pip install -e ".[dev,extra,aws]"
  working-directory: ./python
 
  - name: Install on JuiceFsCI

diff --git a/python/xorbits/lightgbm/__init__.py b/python/xorbits/lightgbm/__init__.py
@@ -22,7 +22,8 @@ def __dir__():
  try:
  import lightgbm
  except ImportError: # pragma: no cover
- raise AttributeError("lightgbm is required but not installed.")
+ # raise AttributeError("lightgbm is required but not installed.")
+ lightgbm = None
  from .mars_adapters import MARS_LIGHTGBM_CALLABLES
 
  return list(MARS_LIGHTGBM_CALLABLES.keys())
@@ -34,7 +35,8 @@ def __getattr__(name: str):
  try:
  import lightgbm
  except ImportError: # pragma: no cover
- raise AttributeError("lightgbm is required but not installed.")
+ # raise AttributeError("lightgbm is required but not installed.")
+ lightgbm = None
  from .mars_adapters import MARS_LIGHTGBM_CALLABLES
 
  if name in MARS_LIGHTGBM_CALLABLES:

diff --git a/python/xorbits/lightgbm/mars_adapters/core.py b/python/xorbits/lightgbm/mars_adapters/core.py
@@ -16,7 +16,10 @@
 import inspect
 from typing import Callable, Dict, List, Optional
 
-import lightgbm
+try:
+ import lightgbm
+except ImportError: # pragma: no cover
+ lightgbm = None
 
 from ..._mars.learn.contrib import lightgbm as mars_lightgbm
 from ..._mars.learn.contrib.lightgbm.classifier import (
@@ -97,6 +100,8 @@ def _install_cls_members(module_callables: Dict[str, Callable]):
 
 MARS_LIGHTGBM_CALLABLES = _collect_module_callables(skip_members=["register_op"])
 _install_cls_members(MARS_LIGHTGBM_CALLABLES)
-attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier)
-attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor)
-attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker)
+
+if lightgbm is not None:
+ attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier)
+ attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor)
+ attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker)