Skip to content

Commit

Permalink
[SPARK-50071][SQL][PYTHON] Add try_make_timestamp(_ltz and _ntz) and …
Browse files Browse the repository at this point in the history
…related tests

### What changes were proposed in this pull request?
This PR adds try_make_timestamp, try_make_timestamp_ntz and try_make_timestamp_ltz expressions that set failOnError to false by default (ANSI compliant versions of related functions).

### Why are the changes needed?
We need these functions in order to provide an alternative for the standard functions, that do not throw an error when ANSI is enabled.

### Does this PR introduce _any_ user-facing change?
Yes, new expressions added.

### How was this patch tested?
Unit tests were added.

### Was this patch authored or co-authored using generative AI tooling?
No.

Closes #48624 from markonik-db/SPARK-50071-tryMakeTimestamp.

Authored-by: Marko Nikacevic <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
  • Loading branch information
markonik-db authored and MaxGekk committed Oct 29, 2024
1 parent 87f5a86 commit 9968a32
Show file tree
Hide file tree
Showing 26 changed files with 1,050 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1977,6 +1977,58 @@ class PlanGenerationTestSuite
fn.col("b"))
}

functionTest("try_make_timestamp with timezone") {
fn.try_make_timestamp(
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("b"),
fn.col("g"))
}

functionTest("try_make_timestamp without timezone") {
fn.try_make_timestamp(
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("b"))
}

functionTest("try_make_timestamp_ltz with timezone") {
fn.try_make_timestamp_ltz(
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("b"),
fn.col("g"))
}

functionTest("try_make_timestamp_ltz without timezone") {
fn.try_make_timestamp_ltz(
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("b"))
}

functionTest("try_make_timestamp_ntz") {
fn.try_make_timestamp_ntz(
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("a"),
fn.col("b"))
}

functionTest("make_ym_interval years months") {
fn.make_ym_interval(fn.col("a"), fn.col("a"))
}
Expand Down
4 changes: 4 additions & 0 deletions docs/sql-ref-ansi-compliance.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,10 @@ When ANSI mode is on, it throws exceptions for invalid operations. You can use t
- `try_element_at`: identical to the function `element_at`, except that it returns `NULL` result instead of throwing an exception on array's index out of bound.
- `try_to_timestamp`: identical to the function `to_timestamp`, except that it returns `NULL` result instead of throwing an exception on string parsing error.
- `try_parse_url`: identical to the function `parse_url`, except that it returns `NULL` result instead of throwing an exception on url parsing error.
- `try_make_timestamp`: identical to the function `make_timestamp`, except that it returns `NULL` result instead of throwing an exception on error.
- `try_make_timestamp_ltz`: identical to the function `make_timestamp_ltz`, except that it returns `NULL` result instead of throwing an exception on error.
- `try_make_timestamp_ntz`: identical to the function `make_timestamp_ntz`, except that it returns `NULL` result instead of throwing an exception on error.


### SQL Keywords (optional, disabled by default)

Expand Down
3 changes: 3 additions & 0 deletions python/docs/source/reference/pyspark.sql/functions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,9 @@ Date and Timestamp Functions
to_unix_timestamp
to_utc_timestamp
trunc
try_make_timestamp
try_make_timestamp_ltz
try_make_timestamp_ntz
try_to_timestamp
unix_date
unix_micros
Expand Down
60 changes: 60 additions & 0 deletions python/pyspark/sql/connect/functions/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -3759,6 +3759,28 @@ def make_timestamp(
make_timestamp.__doc__ = pysparkfuncs.make_timestamp.__doc__


def try_make_timestamp(
years: "ColumnOrName",
months: "ColumnOrName",
days: "ColumnOrName",
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
timezone: Optional["ColumnOrName"] = None,
) -> Column:
if timezone is not None:
return _invoke_function_over_columns(
"try_make_timestamp", years, months, days, hours, mins, secs, timezone
)
else:
return _invoke_function_over_columns(
"try_make_timestamp", years, months, days, hours, mins, secs
)


try_make_timestamp.__doc__ = pysparkfuncs.try_make_timestamp.__doc__


def make_timestamp_ltz(
years: "ColumnOrName",
months: "ColumnOrName",
Expand All @@ -3781,6 +3803,28 @@ def make_timestamp_ltz(
make_timestamp_ltz.__doc__ = pysparkfuncs.make_timestamp_ltz.__doc__


def try_make_timestamp_ltz(
years: "ColumnOrName",
months: "ColumnOrName",
days: "ColumnOrName",
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
timezone: Optional["ColumnOrName"] = None,
) -> Column:
if timezone is not None:
return _invoke_function_over_columns(
"try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone
)
else:
return _invoke_function_over_columns(
"try_make_timestamp_ltz", years, months, days, hours, mins, secs
)


try_make_timestamp_ltz.__doc__ = pysparkfuncs.try_make_timestamp_ltz.__doc__


def make_timestamp_ntz(
years: "ColumnOrName",
months: "ColumnOrName",
Expand All @@ -3797,6 +3841,22 @@ def make_timestamp_ntz(
make_timestamp_ntz.__doc__ = pysparkfuncs.make_timestamp_ntz.__doc__


def try_make_timestamp_ntz(
years: "ColumnOrName",
months: "ColumnOrName",
days: "ColumnOrName",
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
) -> Column:
return _invoke_function_over_columns(
"try_make_timestamp_ntz", years, months, days, hours, mins, secs
)


try_make_timestamp_ntz.__doc__ = pysparkfuncs.try_make_timestamp_ntz.__doc__


def make_ym_interval(
years: Optional["ColumnOrName"] = None,
months: Optional["ColumnOrName"] = None,
Expand Down
Loading

0 comments on commit 9968a32

Please sign in to comment.