From 8be5f9c6b41ddd840939602016811aa9c739f0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 10 Oct 2024 12:01:08 +0200 Subject: [PATCH 1/8] GH-43846: [Python][Packaging] Remove numpy dependency from pyarrow packaging (#44148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## WIP Creating the PR to validate if / what are the CI failures ### Rationale for this change Once the issue has been merged: - https://github.com/apache/arrow/issues/25118 We do not require numpy as a pyarrow dependency. ### What changes are included in this PR? Remove numpy as a required dependency ### Are these changes tested? Via CI ### Are there any user-facing changes? Yes, pyarrow won't install numpy as a required dependency. * GitHub Issue: #43846 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- docs/source/python/install.rst | 5 +---- python/pyproject.toml | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst index 9727a68f7424b..33355b64159f8 100644 --- a/docs/source/python/install.rst +++ b/docs/source/python/install.rst @@ -71,12 +71,9 @@ See :ref:`python-development`. Dependencies ------------ -Required dependency - -* **NumPy 1.16.6** or higher. - Optional dependencies +* **NumPy 1.16.6** or higher. * **pandas 1.0** or higher, * **cffi**. diff --git a/python/pyproject.toml b/python/pyproject.toml index 932210044399e..8748fe4052be1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -35,9 +35,6 @@ build-backend = "setuptools.build_meta" name = "pyarrow" dynamic = ["version"] requires-python = ">=3.9" -dependencies = [ - "numpy >= 1.16.6" -] description = "Python library for Apache Arrow" readme = {file = "README.md", content-type = "text/markdown"} license = {text = "Apache Software License"} From 6fe0ff0c7f61b1e4bcf279b82e3da6831a43662c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 10 Oct 2024 18:47:29 +0200 Subject: [PATCH 2/8] GH-43519: [Python][CI][Packaging] Use released versions to build and test wheels on Python 3.13 (#44193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Now that numpy, cython and pandas all have releases that support Python 3.13, we can remove the extra index and ensure that we are building our wheels with the released version instead of nightly dev version. * GitHub Issue: #43519 Authored-by: Joris Van den Bossche Signed-off-by: Raúl Cumplido --- ci/docker/python-wheel-manylinux.dockerfile | 5 +++++ ci/scripts/python_wheel_macos_build.sh | 9 ++++++++- python/requirements-wheel-build.txt | 5 ----- python/requirements-wheel-test.txt | 6 ------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index d22a70a2d777b..452809ad09c6b 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -114,5 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG SHELL ["/bin/bash", "-i", "-c"] ENTRYPOINT ["/bin/bash", "-i", "-c"] +# Remove once there are released Cython wheels for 3.13 free-threaded available +RUN if [ "${python_abi_tag}" = "cp313t" ]; then \ + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \ + fi + COPY python/requirements-wheel-build.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index d2c392e6b9db3..91925e7abe8b0 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -49,8 +49,15 @@ fi echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +# Remove once there are released Cython wheels for 3.13 free-threaded available +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary +fi +# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching +pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58" + pip install \ - --upgrade \ --only-binary=:all: \ --target $PIP_SITE_PACKAGES \ -r ${source_dir}/python/requirements-wheel-build.txt diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 2d448004768ce..faa078d3d7fe7 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,8 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0; python_version>='3.9' diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index d129ac753ab37..c051efb0aaab1 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -1,9 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---prefer-binary ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cffi cython hypothesis From e0d1f5105ba0dece2b74ed4a2e1530fccd0eef0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 11 Oct 2024 03:01:11 +0200 Subject: [PATCH 3/8] GH-44369: [CI][Python] Remove ds requirement from test collection on test_dataset.py (#44370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change If pyarrow has been built without dataset test collection errors due to pytest.mark.parametrize requiring dataset. ### What changes are included in this PR? Move dataset requirement outside of test collection by removing it from the parametrization. ### Are these changes tested? Via archery ### Are there any user-facing changes? No * GitHub Issue: #44369 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- python/pyarrow/tests/test_dataset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ea6cb88074ab7..0d3a0fbd3bec7 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -737,20 +737,20 @@ def test_partitioning_pickling(pickle_module): @pytest.mark.parametrize( "flavor, expected_defined_partition, expected_undefined_partition", [ - (ds.HivePartitioning, (r"foo=A/bar=ant%20bee", ""), ("", "")), - (ds.DirectoryPartitioning, (r"A/ant bee", ""), ("", "")), - (ds.FilenamePartitioning, ("", r"A_ant bee_"), ("", "_")), + ("HivePartitioning", (r"foo=A/bar=ant%20bee", ""), ("", "")), + ("DirectoryPartitioning", (r"A/ant bee", ""), ("", "")), + ("FilenamePartitioning", ("", r"A_ant bee_"), ("", "_")), ], ) def test_dataset_partitioning_format( - flavor: "ds.Partitioning", + flavor: str, expected_defined_partition: tuple, expected_undefined_partition: tuple, ): partitioning_schema = pa.schema([("foo", pa.string()), ("bar", pa.string())]) - partitioning = flavor(schema=partitioning_schema) + partitioning = getattr(ds, flavor)(schema=partitioning_schema) # test forward transformation (format) assert ( @@ -782,7 +782,7 @@ def test_dataset_partitioning_format( == expected_undefined_partition ) - if flavor != ds.HivePartitioning: + if flavor != "HivePartitioning": # Raises error upon filtering for lower level partition without filtering for # higher level partition with pytest.raises( From deee9acec0e3ba77088c36a025847d9b4952abf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 11 Oct 2024 03:06:32 +0200 Subject: [PATCH 4/8] GH-44314: [Packaging][Python] Use macOS 12 as deployment target to have macOS 12 pyarrow wheels (#44315) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change We did just bump MACOSX_DEPLOYMENT_TARGET from 10 to 12 for pyarrow wheels on 18.0.0, we probably should wait a little before dropping. ### What changes are included in this PR? Revert moving macOS deployment target to 12 ### Are these changes tested? Will trigger wheel jobs on archery ### Are there any user-facing changes? Yes, wheels will be available for macOS 12. * GitHub Issue: #44314 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- dev/tasks/tasks.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d38b822ffd853..d7ea2fd85014e 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -416,10 +416,7 @@ tasks: {############################## Wheel macOS ####################################} -{% for macos_version, macos_codename in [("13.0", "ventura")] %} - {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} - - wheel-macos-{{ macos_codename }}-{{ python_tag }}-{{ abi_tag }}-amd64: + wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-amd64: ci: github template: python-wheels/github.osx.yml params: @@ -427,13 +424,11 @@ tasks: arrow_jemalloc: "ON" python_version: "{{ python_version }}" python_abi_tag: "{{ abi_tag }}" - macos_deployment_target: "{{ macos_version }}" + macos_deployment_target: "12.0" runs_on: "macos-13" vcpkg_arch: "amd64" artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - -{% endfor %} + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-macosx_12_0_x86_64.whl wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-arm64: ci: github From dcc1ee5b1d4851870724ab5e4cf475bcac007b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 11 Oct 2024 04:17:09 +0200 Subject: [PATCH 5/8] GH-44373: [Packaging][Java] Fix brew link to Python 3.13 on macOS (#44374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Java-jars are currently failing on macOS to install Python 3.13 because `python@ 3.13` (`python` is an alias of `python@ 3.13` now) isn't installed yet. Our link failure workaround is updating already installed `python@*` with `--overwrite` explicitly. `python@ 3.13` (`python`) isn't installed on GitHub Actions runner yet. So `brew bundle` installs `python` without `--overwrite`. ### What changes are included in this PR? Install `python` with `--overwrite` explicitly before `brew bundle`. ### Are these changes tested? Via CI ### Are there any user-facing changes? No * GitHub Issue: #44373 Lead-authored-by: Sutou Kouhei Co-authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- dev/tasks/java-jars/github.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 9910daa21ef37..e83552cc7c47c 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -117,6 +117,7 @@ jobs: for python_package in $(brew list | grep python@); do brew install --overwrite ${python_package} done + brew install --overwrite python brew bundle --file=arrow/cpp/Brewfile # We want to link aws-sdk-cpp statically but Homebrew's From 45b36976a31a4f363c0c471c2da1abd5bee97fd2 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 11 Oct 2024 11:13:30 +0200 Subject: [PATCH 6/8] GH-43519: [Python][CI] Add Python 3.13 conda test build (#44192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change When adding Python 3.13 support (https://github.com/apache/arrow/pull/43539), conda was not yet ready to support Python 3.13. But over the last weeks most conda packages having being updated, so trying again. * GitHub Issue: #43519 Authored-by: Joris Van den Bossche Signed-off-by: Raúl Cumplido --- ci/docker/conda-python.dockerfile | 2 +- dev/tasks/tasks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3897a7217d975..c08b69e6ef813 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -28,7 +28,7 @@ COPY ci/conda_env_python.txt \ RUN mamba install -q -y \ --file arrow/ci/conda_env_python.txt \ $([ "$python" == $(gdb --batch --eval-command 'python import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') ] && echo "gdb") \ - "python=${python}.*=*_cpython" \ + "python=${python}.*=*_cp*" \ nomkl && \ mamba clean --all diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d7ea2fd85014e..9137181466383 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1208,7 +1208,7 @@ tasks: UBUNTU: 22.04 image: ubuntu-cpp-emscripten -{% for python_version in ["3.9", "3.10", "3.11", "3.12"] %} +{% for python_version in ["3.9", "3.10", "3.11", "3.12", "3.13"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml From 5638169313ba09fff26f7dc11bce0f6ce3eefed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sat, 12 Oct 2024 12:16:45 +0200 Subject: [PATCH 7/8] GH-44381: [Ruby][Release] Pin not only glib but also python on verification jobs (#44382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Verification job is currently failing for Ruby. ### What changes are included in this PR? We can't use Python 3.13 and `glib==2.80.5`. So we use old Python as a workaround. ### Are these changes tested? Via archery ### Are there any user-facing changes? No * GitHub Issue: #44381 Lead-authored-by: Sutou Kouhei Co-authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- dev/release/verify-release-candidate.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 4e5593525477e..178d8cd458048 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -775,9 +775,9 @@ test_glib() { show_header "Build and test C GLib libraries" # Build and test C GLib - # We can remove '==2.80.5' once https://github.com/conda-forge/glib-feedstock/issues/191 - # is fixed. - maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja ruby + # We can remove '==2.80.5' and 'python<3.13' once + # https://github.com/conda-forge/glib-feedstock/issues/191 is fixed. + maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja 'python<3.13' ruby maybe_setup_virtualenv meson # Install bundler if doesn't exist From 8a7224d21fb7ac1938cb039cc6dcfd38db94519e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sat, 12 Oct 2024 22:28:22 +0200 Subject: [PATCH 8/8] GH-44386: [Integration][Release] Pin Python 3.12 for Integration verification when using Conda (#44388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Conda Python 3.13 fails when running verification tasks for Java. It's related to JPype. See also: * GH-44389 * https://discuss.python.org/t/api-for-python-3-13-prevents-use-of-3rd-party-gc-allocators/62709/5 ### What changes are included in this PR? Pin Python 3.12 on Conda integration jobs because JPype doesn't work with Python 3.13. We can use Python 3.13 again once JPype supports Python 3.13. ### Are these changes tested? Yes via CI ### Are there any user-facing changes? No * GitHub Issue: #44386 Lead-authored-by: Raúl Cumplido Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/tasks/verify-rc/github.macos.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/tasks/verify-rc/github.macos.yml b/dev/tasks/verify-rc/github.macos.yml index e2bc7895c6d05..37a3e6f3fd517 100644 --- a/dev/tasks/verify-rc/github.macos.yml +++ b/dev/tasks/verify-rc/github.macos.yml @@ -66,6 +66,14 @@ jobs: TEST_{{ target|upper }}: 1 {% if use_conda %} USE_CONDA: 1 + {% if target == "integration" %} + # JPype doesn't work with Python 3.13. + # See also: + # * https://discuss.python.org/t/api-for-python-3-13-prevents-use-of-3rd-party-gc-allocators/62709/5 + # * GH-44386 + # * GH-44389 + PYTHON_VERSION: "3.12" + {% endif %} {% endif %} run: | arrow/dev/release/verify-release-candidate.sh {{ release|default("") }} {{ rc|default("") }}