Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(targets): Parse dates with datetime.fromisoformat/backports.datetime_fromisoformat #2070

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 3 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ fs-s3fs = {version = ">=1.1.1", optional = true}
# Testing dependencies installed as optional 'testing' extras
pytest = {version=">=7.2.1", optional = true}
pytest-durations = {version = ">=1.2.0", optional = true}
backports-datetime-fromisoformat = { version = ">=2.0.1", python = "<3.11" }

[tool.poetry.extras]
docs = [
Expand Down Expand Up @@ -200,12 +201,9 @@ warn_return_any = true
[[tool.mypy.overrides]]
ignore_missing_imports = true
module = [
"bcrypt.*",
"joblib.*",
"pyarrow.*",
"backports.datetime_fromisoformat.*",
"joblib.*", # TODO: Remove when https://github.com/joblib/joblib/issues/1516 is shipped
"jsonpath_ng.*",
"samples.*",
"sqlalchemy.*",
]

[build-system]
Expand Down
23 changes: 18 additions & 5 deletions singer_sdk/sinks/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@
import copy
import datetime
import json
import sys
import time
import typing as t
from gzip import GzipFile
from gzip import open as gzip_open
from types import MappingProxyType

from dateutil import parser
from jsonschema import Draft7Validator, FormatChecker
from jsonschema import Draft7Validator

from singer_sdk.exceptions import MissingKeyPropertiesError
from singer_sdk.helpers._batch import (
Expand All @@ -34,6 +34,11 @@

from singer_sdk.target_base import Target

if sys.version_info < (3, 11):
from backports.datetime_fromisoformat import MonkeyPatch

MonkeyPatch.patch_fromisoformat()

JSONSchemaValidator = Draft7Validator


Expand Down Expand Up @@ -90,7 +95,10 @@ def __init__(
self._batch_records_read: int = 0
self._batch_dupe_records_merged: int = 0

self._validator = Draft7Validator(schema, format_checker=FormatChecker())
self._validator = Draft7Validator(
schema,
format_checker=Draft7Validator.FORMAT_CHECKER,
)

def _get_context(self, record: dict) -> dict: # noqa: ARG002
"""Return an empty dictionary by default.
Expand Down Expand Up @@ -374,8 +382,13 @@ def _parse_timestamps_in_record(
date_val = value
try:
if value is not None:
date_val = parser.parse(date_val)
except parser.ParserError as ex:
if datelike_type == "time":
date_val = datetime.time.fromisoformat(date_val)
elif datelike_type == "date":
date_val = datetime.date.fromisoformat(date_val)
else:
date_val = datetime.datetime.fromisoformat(date_val)
except ValueError as ex:
date_val = handle_invalid_timestamp_in_record(
record,
[key],
Expand Down
14 changes: 14 additions & 0 deletions tests/core/sinks/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ def test_validate_record():
"properties": {
"id": {"type": "integer"},
"created_at": {"type": "string", "format": "date-time"},
"created_at_date": {"type": "string", "format": "date"},
"created_at_time": {"type": "string", "format": "time"},
"invalid_datetime": {"type": "string", "format": "date-time"},
},
},
Expand All @@ -27,6 +29,8 @@ def test_validate_record():
record = {
"id": 1,
"created_at": "2021-01-01T00:00:00+00:00",
"created_at_date": "2021-01-01",
"created_at_time": "00:01:00+00:00",
"missing_datetime": "2021-01-01T00:00:00+00:00",
"invalid_datetime": "not a datetime",
}
Expand All @@ -40,6 +44,16 @@ def test_validate_record():
0,
tzinfo=datetime.timezone.utc,
)
assert updated_record["created_at_date"] == datetime.date(
2021,
1,
1,
)
assert updated_record["created_at_time"] == datetime.time(
0,
1,
tzinfo=datetime.timezone.utc,
)
assert updated_record["missing_datetime"] == "2021-01-01T00:00:00+00:00"
assert updated_record["invalid_datetime"] == "9999-12-31 23:59:59.999999"

Expand Down