replace conda with pip

Signed-off-by: Peixin Li <[email protected]>

jenkins/Dockerfile-blossom.integration.rocky

@@ -35,27 +35,25 @@ RUN yum update -y && \
     yum install epel-release -y && \
     yum install -y java-1.8.0-openjdk-devel java-11-openjdk-devel java-17-openjdk-devel wget expect rsync zip unzip procps
 
+RUN yum install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel make
+ARG PY_VER=3.10.15
+RUN wget -q https://www.python.org/ftp/python/${PY_VER}/Python-${PY_VER}.tgz && tar xzf Python-${PY_VER}.tgz
+RUN cd /Python-${PY_VER} && ./configure && make -j 4 \
+    && make altinstall && rm -rf /Python-${PY_VER} Python-${PY_VER}.tgz
+RUN ln -s /usr/local/bin/python3.10 /usr/bin/python
+
+RUN wget -q https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py
+# install pytest plugins for xdist parallel run
+RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==2024.5.0 pytest requests sre_yield
+RUN export CUDA_V=`echo ${CUDA_VER} | cut -d '.' -f 1` && pip install \
+        --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \
+        "cudf-cu${CUDA_V}>=${CUDF_VER}.0a0,<=${CUDF_VER}"
+
 # The plugin: net.alchim31.maven requires a higher mvn version.
 ENV MAVEN_HOME "/usr/local/apache-maven-3.6.3"
-RUN wget ${URM_URL}/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz -P /usr/local && \
+RUN wget -q ${URM_URL}/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz -P /usr/local && \
     tar xzvf $MAVEN_HOME-bin.tar.gz -C /usr/local && \
     rm -f $MAVEN_HOME-bin.tar.gz
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm -f ~/miniconda.sh
-ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}"
-RUN conda init && conda config --set solver classic
-
-# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below
-RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
-    conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.10 cuda-version=${CUDA_VER} && \
-    conda install -y spacy && python -m spacy download en_core_web_sm && \
-    conda install -y -c anaconda pytest requests && \
-    conda install -y -c conda-forge sre_yield && \
-    conda clean -ay
-# install pytest plugins for xdist parallel run
-RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==2024.5.0
-
 # Set default java as 1.8.0
 ENV JAVA_HOME "/usr/lib/jvm/java-1.8.0-openjdk"

jenkins/Dockerfile-blossom.integration.ubuntu

@@ -34,41 +34,33 @@ ARG UBUNTU_VER
 # https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771
 RUN UB_VER=$(echo ${UBUNTU_VER} | tr -d '.') && \
     apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UB_VER}/x86_64/3bf863cc.pub || true
-# Install jdk-8, jdk-11, jdk-17, docker image
+
 RUN apt-get update -y && \
     apt-get install -y software-properties-common rsync zip unzip
 
+ENV DEBIAN_FRONTEND "noninteractive"
 RUN add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update -y && \
-    DEBIAN_FRONTEND="noninteractive" apt-get install -y \
+    apt-get install -y \
     openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk tzdata git wget
 
+RUN apt-get install -y python3.10 python3.10-distutils
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
+RUN wget -q https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py
+RUN python -m CUDA_V install findspark pytest-xdist pytest-order fastparquet==2024.5.0 pytest requests sre_yield
+RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1` && pip install \
+        --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \
+        "cudf-cu${CUDA_V}>=${CUDF_VER}.0a0,<=${CUDF_VER}"
+
 # Install maven 3.8+ to be compatible with JDK17 on Ubuntu
 ARG MAVEN_VERSION=3.8.4
-RUN wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
+RUN wget -q https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
     tar xf apache-maven-$MAVEN_VERSION-bin.tar.gz -C /usr/local/ && \
     rm -rf apache-maven-$MAVEN_VERSION-bin.tar.gz && \
     ln -sf /usr/local/apache-maven-$MAVEN_VERSION/bin/mvn /usr/bin/mvn
-
 # Set default jdk as 1.8.0
 RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm -f ~/miniconda.sh
-ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}"
-RUN conda init && conda config --set solver classic
-
-# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below
-RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
-    conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.10 cuda-version=${CUDA_VER} && \
-    conda install -y spacy && python -m spacy download en_core_web_sm && \
-    conda install -y -c anaconda pytest requests && \
-    conda install -y -c conda-forge sre_yield && \
-    conda clean -ay
-# install pytest plugins for xdist parallel run
-RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==2024.5.0
-
 RUN apt install -y inetutils-ping expect
 
 # install locale and generate UTF-8, but don't set it as default, the test cases should specify it somehow

jenkins/spark-tests.sh

@@ -108,12 +108,12 @@ rm -rf $TMP_PYTHON && mkdir -p $TMP_PYTHON && cp -r $SPARK_HOME/python $TMP_PYTH
 PY4J_FILE=$(find $TMP_PYTHON/python/lib -type f -iname "py4j*.zip")
 export PYTHONPATH=$TMP_PYTHON/python:$TMP_PYTHON/python/pyspark/:$PY4J_FILE
 
-# Extract 'value' from conda config string 'key: value'
-CONDA_ROOT=`conda config --show root_prefix | cut -d ' ' -f2`
-if [[ x"$CONDA_ROOT" != x ]]; then
+if command -v conda >/dev/null 2>&1; then
+  # Extract 'value' from conda config string 'key: value'
+  CONDA_ROOT=$(conda config --show root_prefix | cut -d ' ' -f2)
   # Put conda package path ahead of the env 'PYTHONPATH',
   # to import the right pandas from conda instead of spark binary path.
-  PYTHON_VER=`conda config --show default_python | cut -d ' ' -f2`
+  PYTHON_VER=$(conda config --show default_python | cut -d ' ' -f2)
   export PYTHONPATH="$CONDA_ROOT/lib/python$PYTHON_VER/site-packages:$PYTHONPATH"
 else
   # if no conda, then try with default python