Merge remote-tracking branch 'origin/main' into vector-placement

ci/docker/conda-python.dockerfile

@@ -28,7 +28,7 @@ COPY ci/conda_env_python.txt \
 RUN mamba install -q -y \
         --file arrow/ci/conda_env_python.txt \
         $([ "$python" == $(gdb --batch --eval-command 'python import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') ] && echo "gdb") \
-        "python=${python}.*=*_cpython" \
+        "python=${python}.*=*_cp*" \
         nomkl && \
     mamba clean --all
 

ci/docker/python-wheel-manylinux.dockerfile

@@ -114,5 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG
 SHELL ["/bin/bash", "-i", "-c"]
 ENTRYPOINT ["/bin/bash", "-i", "-c"]
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+RUN if [ "${python_abi_tag}" = "cp313t" ]; then \
+      pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \
+    fi
+
 COPY python/requirements-wheel-build.txt /arrow/python/
 RUN pip install -r /arrow/python/requirements-wheel-build.txt

ci/scripts/python_wheel_macos_build.sh

@@ -49,8 +49,15 @@ fi
 echo "=== (${PYTHON_VERSION}) Install Python build dependencies ==="
 export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])')
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")"
+if [[ $FREE_THREADED_BUILD == "True"  ]]; then
+  pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary
+fi
+# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching
+pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58"
+
 pip install \
-  --upgrade \
   --only-binary=:all: \
   --target $PIP_SITE_PACKAGES \
   -r ${source_dir}/python/requirements-wheel-build.txt

dev/release/verify-release-candidate.sh

@@ -775,9 +775,9 @@ test_glib() {
   show_header "Build and test C GLib libraries"
 
   # Build and test C GLib
-  # We can remove '==2.80.5' once https://github.com/conda-forge/glib-feedstock/issues/191
-  # is fixed.
-  maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja ruby
+  # We can remove '==2.80.5' and 'python<3.13' once
+  # https://github.com/conda-forge/glib-feedstock/issues/191 is fixed.
+  maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja 'python<3.13' ruby
   maybe_setup_virtualenv meson
 
   # Install bundler if doesn't exist

dev/tasks/java-jars/github.yml

@@ -117,6 +117,7 @@ jobs:
           for python_package in $(brew list | grep python@); do
             brew install --overwrite ${python_package}
           done
+          brew install --overwrite python
 
           brew bundle --file=arrow/cpp/Brewfile
           # We want to link aws-sdk-cpp statically but Homebrew's

dev/tasks/tasks.yml

@@ -416,24 +416,19 @@ tasks:
 
 {############################## Wheel macOS ####################################}
 
-{% for macos_version, macos_codename in [("13.0", "ventura")] %}
-  {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %}
-
-  wheel-macos-{{ macos_codename }}-{{ python_tag }}-{{ abi_tag }}-amd64:
+  wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-amd64:
     ci: github
     template: python-wheels/github.osx.yml
     params:
       arch: "x86_64"
       arrow_jemalloc: "ON"
       python_version: "{{ python_version }}"
       python_abi_tag: "{{ abi_tag }}"
-      macos_deployment_target: "{{ macos_version }}"
+      macos_deployment_target: "12.0"
       runs_on: "macos-13"
       vcpkg_arch: "amd64"
     artifacts:
-      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl
-
-{% endfor %}
+      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-macosx_12_0_x86_64.whl
 
   wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-arm64:
     ci: github
@@ -1213,7 +1208,7 @@ tasks:
         UBUNTU: 22.04
       image: ubuntu-cpp-emscripten
 
-{% for python_version in ["3.9", "3.10", "3.11", "3.12"] %}
+{% for python_version in ["3.9", "3.10", "3.11", "3.12", "3.13"] %}
   test-conda-python-{{ python_version }}:
     ci: github
     template: docker-tests/github.linux.yml

dev/tasks/verify-rc/github.macos.yml

@@ -66,6 +66,14 @@ jobs:
           TEST_{{ target|upper }}: 1
         {% if use_conda %}
           USE_CONDA: 1
+          {% if target == "integration" %}
+          # JPype doesn't work with Python 3.13.
+          # See also:
+          # * https://discuss.python.org/t/api-for-python-3-13-prevents-use-of-3rd-party-gc-allocators/62709/5
+          # * GH-44386
+          # * GH-44389
+          PYTHON_VERSION: "3.12"
+          {% endif %}
         {% endif %}
         run: |
           arrow/dev/release/verify-release-candidate.sh {{ release|default("") }} {{ rc|default("") }}

docs/source/python/install.rst

@@ -71,12 +71,9 @@ See :ref:`python-development`.
 Dependencies
 ------------
 
-Required dependency
-
-* **NumPy 1.16.6** or higher.
-
 Optional dependencies
 
+* **NumPy 1.16.6** or higher.
 * **pandas 1.0** or higher,
 * **cffi**.
 

python/pyarrow/tests/test_dataset.py

@@ -737,20 +737,20 @@ def test_partitioning_pickling(pickle_module):
 @pytest.mark.parametrize(
     "flavor, expected_defined_partition, expected_undefined_partition",
     [
-        (ds.HivePartitioning, (r"foo=A/bar=ant%20bee", ""), ("", "")),
-        (ds.DirectoryPartitioning, (r"A/ant bee", ""), ("", "")),
-        (ds.FilenamePartitioning, ("", r"A_ant bee_"), ("", "_")),
+        ("HivePartitioning", (r"foo=A/bar=ant%20bee", ""), ("", "")),
+        ("DirectoryPartitioning", (r"A/ant bee", ""), ("", "")),
+        ("FilenamePartitioning", ("", r"A_ant bee_"), ("", "_")),
     ],
 )
 def test_dataset_partitioning_format(
-    flavor: "ds.Partitioning",
+    flavor: str,
     expected_defined_partition: tuple,
     expected_undefined_partition: tuple,
 ):
 
     partitioning_schema = pa.schema([("foo", pa.string()), ("bar", pa.string())])
 
-    partitioning = flavor(schema=partitioning_schema)
+    partitioning = getattr(ds, flavor)(schema=partitioning_schema)
 
     # test forward transformation (format)
     assert (
@@ -782,7 +782,7 @@ def test_dataset_partitioning_format(
         == expected_undefined_partition
     )
 
-    if flavor != ds.HivePartitioning:
+    if flavor != "HivePartitioning":
         # Raises error upon filtering for lower level partition without filtering for
         # higher level partition
         with pytest.raises(

python/pyproject.toml

@@ -35,9 +35,6 @@ build-backend = "setuptools.build_meta"
 name = "pyarrow"
 dynamic = ["version"]
 requires-python = ">=3.9"
-dependencies = [
-    "numpy >= 1.16.6"
-]
 description = "Python library for Apache Arrow"
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {text = "Apache Software License"}

python/requirements-wheel-build.txt

@@ -1,8 +1,3 @@
-# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
-# on PyPI
---pre
---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-
 cython>=0.29.31
 oldest-supported-numpy>=0.14; python_version<'3.9'
 numpy>=2.0.0; python_version>='3.9'

python/requirements-wheel-test.txt

@@ -1,9 +1,3 @@
-# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
-# on PyPI
---pre
---prefer-binary
---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-
 cffi
 cython
 hypothesis