apacheGH-43518: [Python][Packaging][CI] Drop Python 3.8 support (apac…

…he#43970) ### Rationale for this change Python 3.8 is End of Support on 31st October 2023. We can drop support for pyarrow 18.0.0. See: https://endoflife.date/python ### What changes are included in this PR? Remove support for 3.8. Update minimum required Python version and update some CI jobs to use the minimum Python supported version ### Are these changes tested? They will be tested via Archery ### Are there any user-facing changes? Yes in terms of not supporting older versions of Python. This will be a breaking change for some users but not for the API itself * GitHub Issue: apache#43518 Authored-by: Raúl Cumplido <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
CurtHagenlocher · Sep 16, 2024 · aa6ab95 · aa6ab95
1 parent 3600db8
commit aa6ab95
Show file tree

Hide file tree

Showing 43 changed files with 124 additions and 135 deletions.
diff --git a/.env b/.env
@@ -70,8 +70,8 @@ NODE=18
 NUMBA=latest
 NUMPY=latest
 PANDAS=latest
-PYTHON=3.8
-PYTHON_IMAGE_TAG=3.8
+PYTHON=3.9
+PYTHON_IMAGE_TAG=3.9
 R=4.4
 SPARK=master
 TURBODBC=latest

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -66,6 +66,7 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
+          UBUNTU: 22.04
         run: |
           source ci/scripts/util_enable_core_dumps.sh
           archery docker run -e GITHUB_ACTIONS=true ubuntu-lint

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -211,7 +211,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build
@@ -251,7 +251,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -93,7 +93,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -80,7 +80,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -74,7 +74,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build
@@ -116,7 +116,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
@@ -58,7 +58,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -58,41 +58,41 @@ jobs:
       matrix:
         name:
           - conda-python-docs
-          - conda-python-3.9-nopandas
-          - conda-python-3.8-pandas-1.0
-          - conda-python-3.10-pandas-latest
-          - conda-python-3.10-no-numpy
+          - conda-python-3.10-nopandas
+          - conda-python-3.9-pandas-1.1.3
+          - conda-python-3.11-pandas-latest
+          - conda-python-3.11-no-numpy
         include:
           - name: conda-python-docs
-            cache: conda-python-3.9
+            cache: conda-python-3.10
             image: conda-python-docs
-            title: AMD64 Conda Python 3.9 Sphinx & Numpydoc
-            python: 3.9
-          - name: conda-python-3.9-nopandas
-            cache: conda-python-3.9
+            title: AMD64 Conda Python 3.10 Sphinx & Numpydoc
+            python: "3.10"
+          - name: conda-python-3.10-nopandas
+            cache: conda-python-3.10
             image: conda-python
-            title: AMD64 Conda Python 3.9 Without Pandas
-            python: 3.9
-          - name: conda-python-3.8-pandas-1.0
-            cache: conda-python-3.8
+            title: AMD64 Conda Python 3.10 Without Pandas
+            python: "3.10"
+          - name: conda-python-3.9-pandas-1.1.3
+            cache: conda-python-3.9
             image: conda-python-pandas
-            title: AMD64 Conda Python 3.8 Pandas 1.0
-            python: 3.8
-            pandas: "1.0"
-            numpy: 1.16
-          - name: conda-python-3.10-pandas-latest
-            cache: conda-python-3.10
+            title: AMD64 Conda Python 3.9 Pandas 1.1.3
+            python: 3.9
+            pandas: "1.1.3"
+            numpy: 1.19.5
+          - name: conda-python-3.11-pandas-latest
+            cache: conda-python-3.11
             image: conda-python-pandas
-            title: AMD64 Conda Python 3.10 Pandas latest
-            python: "3.10"
+            title: AMD64 Conda Python 3.11 Pandas latest
+            python: "3.11"
             pandas: latest
-          - name: conda-python-3.10-no-numpy
-            cache: conda-python-3.10
+          - name: conda-python-3.11-no-numpy
+            cache: conda-python-3.11
             image: conda-python-no-numpy
-            title: AMD64 Conda Python 3.10 without NumPy
-            python: "3.10"
+            title: AMD64 Conda Python 3.11 without NumPy
+            python: "3.11"
     env:
-      PYTHON: ${{ matrix.python || 3.8 }}
+      PYTHON: ${{ matrix.python || 3.9 }}
       UBUNTU: ${{ matrix.ubuntu || 20.04 }}
       PANDAS: ${{ matrix.pandas || 'latest' }}
       NUMPY: ${{ matrix.numpy || 'latest' }}
@@ -111,7 +111,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -150,7 +150,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build
@@ -209,7 +209,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -87,7 +87,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

diff --git a/ci/docker/conda-python-cpython-debug.dockerfile b/ci/docker/conda-python-cpython-debug.dockerfile
@@ -17,11 +17,11 @@
 
 ARG repo
 ARG arch
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 # (Docker oddity: ARG needs to be repeated after FROM)
-ARG python=3.8
+ARG python=3.9
 RUN mamba install -y "conda-forge/label/python_debug::python=${python}[build=*_cpython]" && \
     mamba clean --all
 # Quick check that we do have a debug mode CPython

diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 RUN mamba install -q -y "cython<3" && \

diff --git a/ci/docker/conda-python-dask.dockerfile b/ci/docker/conda-python-dask.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch=amd64
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG dask=latest

diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch=amd64
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=11

diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch=amd64
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=11

diff --git a/ci/docker/conda-python-pandas.dockerfile b/ci/docker/conda-python-pandas.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch=amd64
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG pandas=latest

diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile
@@ -17,7 +17,7 @@
 
 ARG repo
 ARG arch=amd64
-ARG python=3.8
+ARG python=3.9
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=11

diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile
@@ -20,7 +20,7 @@ ARG arch
 FROM ${repo}:${arch}-conda-cpp
 
 # install python specific packages
-ARG python=3.8
+ARG python=3.9
 COPY ci/conda_env_python.txt \
      /arrow/ci/
 # If the Python version being tested is the same as the Python used by the system gdb,

diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile
@@ -23,6 +23,7 @@ COPY python/requirements-build.txt \
      /arrow/python/
 
 ENV ARROW_PYTHON_VENV /arrow-dev
+
 RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
     . ${ARROW_PYTHON_VENV}/bin/activate && \
     pip install -U pip setuptools wheel && \

diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile
@@ -47,7 +47,7 @@ RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget
 # on manylinux_2_28, no system python is installed.
 # We therefore override the PATH with Python 3.8 in /opt/python
 # so that we have a consistent Python version across base images.
-ENV CPYTHON_VERSION=cp38
+ENV CPYTHON_VERSION=cp39
 ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH}
 
 # Install CMake
@@ -104,7 +104,7 @@ RUN vcpkg install \
 RUN pipx upgrade auditwheel
 
 # Configure Python for applications running in the bash shell of this Dockerfile
-ARG python=3.8
+ARG python=3.9
 ENV PYTHON_VERSION=${python}
 RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-cp${PYTHON_VERSION/./}) && \
     echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -48,13 +48,12 @@ COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
 RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 
-# Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)
-ARG python=3.8
-RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
+# Define the full version number otherwise choco falls back to patch number 0 (3.9 => 3.9.0)
+ARG python=3.9
+RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
     (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11") & \
     (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4") & \
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5") & \
     (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1")
 
 # Install archiver to extract xz archives

diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile b/ci/docker/python-wheel-windows-vs2019.dockerfile
@@ -78,13 +78,12 @@ RUN vcpkg install \
 RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
     rm -rf Python*
 
-# Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)
-ARG python=3.8
-RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+# Define the full version number otherwise choco falls back to patch number 0 (3.9 => 3.9.0)
+ARG python=3.9
+RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
     (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
     (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") & \
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") & \
     (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH "%PATH%;C:\Python313;C:\Python313\Scripts")
 RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
 RUN python -m pip install -U pip setuptools

diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
@@ -25,11 +25,10 @@ platforms=([windows]=Windows
            [linux]=Linux)
 
 declare -A versions
-versions=([3.8]=3.8.10
-          [3.9]=3.9.13
+versions=([3.9]=3.9.13
           [3.10]=3.10.11
           [3.11]=3.11.9
-          [3.12]=3.12.4
+          [3.12]=3.12.5
           [3.13]=3.13.0)
 
 if [ "$#" -ne 2 ]; then
@@ -73,4 +72,5 @@ if [ $platform = "macOS" ]; then
     $pip install -U pip setuptools
 else
     echo "Unsupported platform: $platform"
+    exit 1
 fi
diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh
@@ -54,7 +54,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T
     case "$PACKAGE_MANAGER" in
       zypper)
         # python3 is Python 3.6 on OpenSUSE 15.3.
-        # PyArrow supports Python 3.8 or later.
+        # PyArrow supports Python 3.9 or later.
         $PACKAGE_MANAGER install -y python39-pip
         ln -s /usr/bin/python3.9 /usr/local/bin/python
         ln -s /usr/bin/pip3.9 /usr/local/bin/pip

diff --git a/dev/archery/setup.py b/dev/archery/setup.py
@@ -21,12 +21,8 @@
 import sys
 from setuptools import setup, find_packages
 
-# pygit2>=1.14.0 requires python 3.9, so crossbow and all
-# both technically require python 3.9 — however we still need to
-# support 3.8 when using docker. When 3.8 is EOLed and we bump
-# to Python 3.9 this will resolve itself.
-if sys.version_info < (3, 8):
-    sys.exit('Python < 3.8 is not supported')
+if sys.version_info < (3, 9):
+    sys.exit('Python < 3.9 is not supported')
 
 # For pathlib.Path compatibility
 jinja_req = 'jinja2>=2.11'
@@ -57,7 +53,7 @@
     maintainer_email='[email protected]',
     packages=find_packages(),
     include_package_data=True,
-    python_requires='>=3.8',
+    python_requires='>=3.9',
     install_requires=['click>=7'],
     tests_require=['pytest', 'responses'],
     extras_require=extras,