Remove aws-sdk-pinning and revert to arrow 12.0.1 (#14319)

The aws-sdk-pinning is proving to be far too problematic to maintain. It causes conflicts in many environments due to its common usage across many other packages in the conda-forge ecosystem that have since updated their pinning to require newer versions than the 1.10.* that we've pinned to. This reversion will unblock most of RAPIDS CI. We will search for alternative fixes to the dask-cuda/distributed issues that we're observing (in particular, resolution of the underlying issues apache/arrow#38364 and aws/aws-sdk-cpp#2681). Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) - Bradley Dice (https://github.com/bdice) - Lawrence Mitchell (https://github.com/wence-) - Peter Andreas Entschev (https://github.com/pentschev) URL: #14319
rapidsai · Oct 24, 2023 · b390bca · b390bca
1 parent 630982a
commit b390bca
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 26 deletions.
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -9,7 +9,6 @@ channels:
 - nvidia
 dependencies:
 - aiobotocore>=2.2.0
-- aws-sdk-cpp<1.11
 - benchmark==1.8.0
 - boto3>=1.21.21
 - botocore>=1.24.21
@@ -40,7 +39,7 @@ dependencies:
 - hypothesis
 - identify>=2.5.20
 - ipython
-- libarrow==12.0.0.*
+- libarrow==12.0.1.*
 - libcufile-dev=1.4.0.31
 - libcufile=1.4.0.31
 - libcurand-dev=10.3.0.86
@@ -69,7 +68,7 @@ dependencies:
 - pre-commit
 - protobuf>=4.21,<5
 - ptxcompiler
-- pyarrow==12.0.0.*
+- pyarrow==12.0.1.*
 - pydata-sphinx-theme
 - pyorc
 - pytest

diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml
@@ -9,7 +9,6 @@ channels:
 - nvidia
 dependencies:
 - aiobotocore>=2.2.0
-- aws-sdk-cpp<1.11
 - benchmark==1.8.0
 - boto3>=1.21.21
 - botocore>=1.24.21
@@ -42,7 +41,7 @@ dependencies:
 - hypothesis
 - identify>=2.5.20
 - ipython
-- libarrow==12.0.0.*
+- libarrow==12.0.1.*
 - libcufile-dev
 - libcurand-dev
 - libkvikio==23.12.*
@@ -67,7 +66,7 @@ dependencies:
 - pip
 - pre-commit
 - protobuf>=4.21,<5
-- pyarrow==12.0.0.*
+- pyarrow==12.0.1.*
 - pydata-sphinx-theme
 - pyorc
 - pytest

diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml
@@ -61,7 +61,7 @@ requirements:
     - scikit-build >=0.13.1
     - setuptools
     - dlpack >=0.5,<0.6.0a0
-    - pyarrow =12.0.0
+    - pyarrow ==12.0.1.*
     - libcudf ={{ version }}
     - rmm ={{ minor_version }}
     {% if cuda_major == "11" %}

diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml
@@ -22,11 +22,8 @@ gbench_version:
 gtest_version:
   - ">=1.13.0"
 
-aws_sdk_cpp_version:
-  - "<1.11"
-
-libarrow:
-  - "==12.0.0"
+libarrow_version:
+  - "==12.0.1"
 
 dlpack_version:
   - ">=0.5,<0.6.0a0"

diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
@@ -65,7 +65,7 @@ requirements:
     {% endif %}
     - cuda-version ={{ cuda_version }}
     - nvcomp {{ nvcomp_version }}
-    - libarrow {{ libarrow }}
+    - libarrow {{ libarrow_version }}
     - dlpack {{ dlpack_version }}
     - librdkafka {{ librdkafka_version }}
     - fmt {{ fmt_version }}
@@ -74,7 +74,6 @@ requirements:
     - gtest {{ gtest_version }}
     - gmock {{ gtest_version }}
     - zlib {{ zlib_version }}
-    - aws-sdk-cpp {{ aws_sdk_cpp_version }}
 
 outputs:
   - name: libcudf
@@ -104,11 +103,10 @@ outputs:
         - nvcomp {{ nvcomp_version }}
         - librmm ={{ minor_version }}
         - libkvikio ={{ minor_version }}
-        - libarrow {{ libarrow }}
+        - libarrow {{ libarrow_version }}
         - dlpack {{ dlpack_version }}
         - gtest {{ gtest_version }}
         - gmock {{ gtest_version }}
-        - aws-sdk-cpp {{ aws_sdk_cpp_version }}
     test:
       commands:
         - test -f $PREFIX/lib/libcudf.so

diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake
@@ -411,7 +411,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow)
   set(CUDF_VERSION_Arrow
       # This version must be kept in sync with the libarrow version pinned for builds in
       # dependencies.yaml.
-      12.0.0
+      12.0.1
       CACHE STRING "The version of Arrow to find (or build)"
   )
 endif()

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -62,7 +62,6 @@ files:
     includes:
       - cudatoolkit
       - docs
-      - libarrow_run
       - py_version
   py_build_cudf:
     output: pyproject
@@ -219,14 +218,13 @@ dependencies:
           - libkvikio==23.12.*
       - output_types: conda
         packages:
-          - aws-sdk-cpp<1.11
           - fmt>=9.1.0,<10
           - &gbench benchmark==1.8.0
           - &gtest gtest>=1.13.0
           - &gmock gmock>=1.13.0
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
-          - &libarrow libarrow==12.0.0.*
+          - libarrow==12.0.1.*
           - librdkafka>=1.9.0,<1.10.0a0
           # Align nvcomp version with rapids-cmake
           - nvcomp==2.6.1
@@ -244,7 +242,7 @@ dependencies:
           - cython>=3.0.0
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
-          - &pyarrow pyarrow==12.0.0.*
+          - pyarrow==12.0.1.*
           # TODO: Pin to numpy<1.25 until cudf requires pandas 2
           - &numpy numpy>=1.21,<1.25
   build_python:
@@ -263,12 +261,14 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - *libarrow
+          # Allow runtime version to float up to minor version
+          - libarrow==12.*
   pyarrow_run:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - *pyarrow
+          # Allow runtime version to float up to minor version
+          - pyarrow==12.*
   cudatoolkit:
     specific:
       - output_types: conda

diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml
@@ -8,7 +8,7 @@ requires = [
     "ninja",
     "numpy>=1.21,<1.25",
     "protoc-wheel",
-    "pyarrow==12.0.0.*",
+    "pyarrow==12.0.1.*",
     "rmm==23.12.*",
     "scikit-build>=0.13.1",
     "setuptools",
@@ -38,7 +38,7 @@ dependencies = [
     "pandas>=1.3,<1.6.0dev0",
     "protobuf>=4.21,<5",
     "ptxcompiler",
-    "pyarrow==12.0.0.*",
+    "pyarrow==12.*",
     "rmm==23.12.*",
     "typing_extensions>=4.0.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml
@@ -5,7 +5,7 @@
 requires = [
     "cython>=3.0.0",
     "numpy>=1.21,<1.25",
-    "pyarrow==12.0.0.*",
+    "pyarrow==12.0.1.*",
     "setuptools",
     "wheel",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.