From dd46556e657a05a89a8ed76eb6a0b01bfc7dd0e0 Mon Sep 17 00:00:00 2001 From: Dawnfz <2912706234@qq.com> Date: Thu, 19 Sep 2024 19:08:05 +0800 Subject: [PATCH] DOC: update CI environment dependencies for doc build (#814) Signed-off-by: Dawnfz <2912706234@qq.com> --- .github/workflows/python.yaml | 202 +++++++++--------- python/xorbits/lightgbm/__init__.py | 6 +- python/xorbits/lightgbm/mars_adapters/core.py | 13 +- 3 files changed, 117 insertions(+), 104 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 5f8dcc702..e420b2f85 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -129,7 +129,7 @@ jobs: minikube-version: 1.31.2 - name: Install ucx dependencies - if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}} + if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.module != 'doc-build') }} run: | conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py - name: Install libomp (macOS) @@ -141,108 +141,114 @@ jobs: PYTHON: ${{ matrix.python-version }} if: ${{ matrix.module != 'gpu' }} run: | - pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar" - pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr + if [[ "$MODULE" == "doc-build" ]]; then + pip install --upgrade --no-cache-dir pip setuptools + pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext + pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]" + else + pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar" + pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr - if [[ "$MODULE" == "xorbits/pandas" ]]; then - pip install openpyxl - fi - if [[ "$MODULE" == "mars-core" ]]; then - pip install oss2 - fi - if [[ "$MODULE" == "kubernetes" ]]; then - pip install kubernetes - fi - if [[ "$MODULE" == "kubernetes-juicefs" ]]; then - pip install kubernetes - if helm version --short >/dev/null 2>&1; then - echo "Helm installed. Skipping installation." - else - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + if [[ "$MODULE" == "xorbits/pandas" ]]; then + pip install openpyxl fi - if helm repo list | grep "juicefs" >/dev/null 2>&1; then - echo "Repo juicefs existed. Skipping adding to helm repos." - else - helm repo add juicefs https://juicedata.github.io/charts/ - helm repo update - helm fetch --untar juicefs/juicefs-csi-driver + if [[ "$MODULE" == "mars-core" ]]; then + pip install oss2 fi - if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then - echo "Juicefs-csi-driver installed. Skipping installation." - else - helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml + if [[ "$MODULE" == "kubernetes" ]]; then + pip install kubernetes fi - sleep 60 - kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml - sleep 60 - fi - if [[ "$MODULE" == "hadoop" ]]; then - ../CI/install-hadoop.sh - echo "import coverage; coverage.process_startup()" > \ - $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth - conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0 - fi - if [[ "$MODULE" == "vineyard" ]]; then - pip install vineyard - mkdir -p /tmp/etcd-download-test - export ETCD_VER=v3.4.13 - export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download - curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz - tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 - sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/ - sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/ - rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test - fi - if [[ "$MODULE" == "external-storage" ]]; then - export CUR_DIR=$(pwd) - mkdir -p /tmp/alluxio-download-test - cd /tmp/alluxio-download-test - wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz - tar -xzf alluxio-2.9.3-bin.tar.gz - cd alluxio-2.9.3 - cp conf/alluxio-env.sh.template conf/alluxio-env.sh - cp conf/alluxio-site.properties.template conf/alluxio-site.properties - echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties - cd $CUR_DIR - sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/ - rm -R /tmp/alluxio-download-test - unset CUR_DIR - export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3" - ${ALLUXIO_HOME}/bin/alluxio format - ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount - curl -sSL https://d.juicefs.com/install | sh - - curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list - sudo apt-get update - sudo apt-get install redis - export JUICEFS_HOME="/usr/local/bin/juicefs" - fi - if [[ "$PYTHON" == '3.9' ]]; then - pip install h5py zarr matplotlib fastparquet - conda install -n test --quiet --yes -c conda-forge python=$PYTHON \ - "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true - fi - if [[ "$MODULE" == "_mars/learn" ]]; then - pip install scikit-learn - pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \ - statsmodels tsfresh dask[complete] mimesis\<9.0.0 - fi - if [[ "$MODULE" == "learn" ]]; then - pip install xgboost lightgbm - fi - if [[ "$MODULE" == "compatibility" ]]; then - # test if compatible with older versions - pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2" - fi - if [[ "$MODULE" == "jax" ]]; then - # test jax - pip install "jax>=0.4.0" "jaxlib>=0.4.0" - fi - if [[ "$MODULE" == "datasets" ]]; then - # test huggingface datasets - pip install datasets torch + if [[ "$MODULE" == "kubernetes-juicefs" ]]; then + pip install kubernetes + if helm version --short >/dev/null 2>&1; then + echo "Helm installed. Skipping installation." + else + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + fi + if helm repo list | grep "juicefs" >/dev/null 2>&1; then + echo "Repo juicefs existed. Skipping adding to helm repos." + else + helm repo add juicefs https://juicedata.github.io/charts/ + helm repo update + helm fetch --untar juicefs/juicefs-csi-driver + fi + if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then + echo "Juicefs-csi-driver installed. Skipping installation." + else + helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml + fi + sleep 60 + kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml + sleep 60 + fi + if [[ "$MODULE" == "hadoop" ]]; then + ../CI/install-hadoop.sh + echo "import coverage; coverage.process_startup()" > \ + $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth + conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0 + fi + if [[ "$MODULE" == "vineyard" ]]; then + pip install vineyard + mkdir -p /tmp/etcd-download-test + export ETCD_VER=v3.4.13 + export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download + curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz + tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 + sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/ + sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/ + rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test + fi + if [[ "$MODULE" == "external-storage" ]]; then + export CUR_DIR=$(pwd) + mkdir -p /tmp/alluxio-download-test + cd /tmp/alluxio-download-test + wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz + tar -xzf alluxio-2.9.3-bin.tar.gz + cd alluxio-2.9.3 + cp conf/alluxio-env.sh.template conf/alluxio-env.sh + cp conf/alluxio-site.properties.template conf/alluxio-site.properties + echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties + cd $CUR_DIR + sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/ + rm -R /tmp/alluxio-download-test + unset CUR_DIR + export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3" + ${ALLUXIO_HOME}/bin/alluxio format + ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount + curl -sSL https://d.juicefs.com/install | sh - + curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list + sudo apt-get update + sudo apt-get install redis + export JUICEFS_HOME="/usr/local/bin/juicefs" + fi + if [[ "$PYTHON" == '3.9' ]]; then + pip install h5py zarr matplotlib fastparquet + conda install -n test --quiet --yes -c conda-forge python=$PYTHON \ + "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true + fi + if [[ "$MODULE" == "_mars/learn" ]]; then + pip install scikit-learn + pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \ + statsmodels tsfresh dask[complete] mimesis\<9.0.0 + fi + if [[ "$MODULE" == "learn" ]]; then + pip install xgboost lightgbm + fi + if [[ "$MODULE" == "compatibility" ]]; then + # test if compatible with older versions + pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2" + fi + if [[ "$MODULE" == "jax" ]]; then + # test jax + pip install "jax>=0.4.0" "jaxlib>=0.4.0" + fi + if [[ "$MODULE" == "datasets" ]]; then + # test huggingface datasets + pip install datasets torch + fi + pip install -e ".[dev,extra,aws]" fi - pip install -e ".[dev,extra,aws]" working-directory: ./python - name: Install on JuiceFsCI diff --git a/python/xorbits/lightgbm/__init__.py b/python/xorbits/lightgbm/__init__.py index 822e710a4..b7f4bc313 100644 --- a/python/xorbits/lightgbm/__init__.py +++ b/python/xorbits/lightgbm/__init__.py @@ -22,7 +22,8 @@ def __dir__(): try: import lightgbm except ImportError: # pragma: no cover - raise AttributeError("lightgbm is required but not installed.") + # raise AttributeError("lightgbm is required but not installed.") + lightgbm = None from .mars_adapters import MARS_LIGHTGBM_CALLABLES return list(MARS_LIGHTGBM_CALLABLES.keys()) @@ -34,7 +35,8 @@ def __getattr__(name: str): try: import lightgbm except ImportError: # pragma: no cover - raise AttributeError("lightgbm is required but not installed.") + # raise AttributeError("lightgbm is required but not installed.") + lightgbm = None from .mars_adapters import MARS_LIGHTGBM_CALLABLES if name in MARS_LIGHTGBM_CALLABLES: diff --git a/python/xorbits/lightgbm/mars_adapters/core.py b/python/xorbits/lightgbm/mars_adapters/core.py index 5a35b1856..72829e5e0 100644 --- a/python/xorbits/lightgbm/mars_adapters/core.py +++ b/python/xorbits/lightgbm/mars_adapters/core.py @@ -16,7 +16,10 @@ import inspect from typing import Callable, Dict, List, Optional -import lightgbm +try: + import lightgbm +except ImportError: # pragma: no cover + lightgbm = None from ..._mars.learn.contrib import lightgbm as mars_lightgbm from ..._mars.learn.contrib.lightgbm.classifier import ( @@ -97,6 +100,8 @@ def _install_cls_members(module_callables: Dict[str, Callable]): MARS_LIGHTGBM_CALLABLES = _collect_module_callables(skip_members=["register_op"]) _install_cls_members(MARS_LIGHTGBM_CALLABLES) -attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier) -attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor) -attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker) + +if lightgbm is not None: + attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier) + attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor) + attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker)