From b502418bd3570f6b7d54c9987ab6fe5e3f301a8e Mon Sep 17 00:00:00 2001 From: MithunR Date: Fri, 25 Oct 2024 16:11:24 -0700 Subject: [PATCH 1/2] Fix `orc_write_test.py` for Databricks 14.3 Fixes #11529. This commit fixes the failure of `orc_write_test.py::test_orc_do_not_lowercase_columns` as described in #11529. The failure was the result of a change in the exception text in Databricks 14.3. (It differs from Spark 3.5, on which it is based, and more closely resembles Spark 4.0.) Signed-off-by: MithunR --- integration_tests/src/main/python/orc_write_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/orc_write_test.py b/integration_tests/src/main/python/orc_write_test.py index f4928196c82..858d7d533bf 100644 --- a/integration_tests/src/main/python/orc_write_test.py +++ b/integration_tests/src/main/python/orc_write_test.py @@ -360,8 +360,8 @@ def test_orc_do_not_lowercase_columns(spark_tmp_path): # The wording of the `is not exists` error message in Spark 4.x is unfortunate, but accurate: # https://github.com/apache/spark/blob/4501285a49e4c0429c9cf2c105f044e1c8a93d21/python/pyspark/errors/error-conditions.json#L487 - expected_error_message = "No StructField named acol" if is_before_spark_400() else \ - "Key `acol` is not exists." + expected_error_message = "Key `acol` is not exists." if is_spark_400_or_later() or is_databricks_version_or_later(14, 3) \ + else "No StructField named acol" assert_gpu_and_cpu_writes_are_equal_collect( # column is uppercase lambda spark, path: spark.range(0, 1000).select(col("id").alias("Acol")).write.orc(path), From 5956fc4af745213efaeca268a0fd72ce197b70c6 Mon Sep 17 00:00:00 2001 From: MithunR Date: Mon, 28 Oct 2024 10:16:31 -0700 Subject: [PATCH 2/2] Fixed imports. Signed-off-by: MithunR --- integration_tests/src/main/python/orc_write_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/orc_write_test.py b/integration_tests/src/main/python/orc_write_test.py index 858d7d533bf..ddb69524ac4 100644 --- a/integration_tests/src/main/python/orc_write_test.py +++ b/integration_tests/src/main/python/orc_write_test.py @@ -15,7 +15,7 @@ import pytest from asserts import assert_gpu_and_cpu_writes_are_equal_collect, assert_gpu_fallback_write -from spark_session import is_before_spark_320, is_before_spark_400, is_spark_321cdh, is_spark_cdh, with_cpu_session, with_gpu_session +from spark_session import is_before_spark_320, is_databricks_version_or_later, is_spark_321cdh, is_spark_400_or_later, is_spark_cdh, with_cpu_session, with_gpu_session from conftest import is_not_utc from datetime import date, datetime, timezone from data_gen import *