diff --git a/fennel/sources/sources.py b/fennel/sources/sources.py index 6a7f99f5c..8d2f28bd4 100644 --- a/fennel/sources/sources.py +++ b/fennel/sources/sources.py @@ -28,7 +28,7 @@ def source( conn: DataConnector, every: Optional[Duration] = None, - starting_from: Optional[datetime] = None, + since: Optional[datetime] = None, lateness: Optional[Duration] = None, cdc: Optional[str] = None, ) -> Callable[[T], Any]: @@ -40,14 +40,14 @@ def source( f"{', '.join(conn.required_fields())}." ) - if starting_from is not None and not isinstance(starting_from, datetime): + if since is not None and not isinstance(since, datetime): raise TypeError("starting_from must be of type datetime") def decorator(dataset_cls: T): conn.every = every if every is not None else DEFAULT_EVERY conn.lateness = lateness if lateness is not None else DEFAULT_LATENESS conn.cdc = cdc if cdc is not None else DEFAULT_CDC - conn.starting_from = starting_from + conn.starting_from = since if hasattr(dataset_cls, SOURCE_FIELD): raise Exception( "Multiple sources are not supported in dataset `%s`." diff --git a/fennel/sources/test_invalid_sources.py b/fennel/sources/test_invalid_sources.py index 2f4a1c5b4..62cd4b9d6 100644 --- a/fennel/sources/test_invalid_sources.py +++ b/fennel/sources/test_invalid_sources.py @@ -298,7 +298,7 @@ def test_invalid_starting_from(): bigquery.table("users_bq", cursor="added_on"), every="1h", lateness="2h", - starting_from=datetime.now(), + since=datetime.now(), ) @dataset class UserInfoDatasetBigQuery: @@ -325,7 +325,7 @@ class UserInfoDatasetBigQuery: @source( s3.bucket(bucket_name="bucket", prefix="prefix"), every="1h", - starting_from="2020-01-01T00:00:00Z", + since="2020-01-01T00:00:00Z", ) @meta(owner="aditya@fennel.ai") @dataset diff --git a/fennel/sources/test_sources.py b/fennel/sources/test_sources.py index 9053671d1..46493b27b 100644 --- a/fennel/sources/test_sources.py +++ b/fennel/sources/test_sources.py @@ -235,9 +235,7 @@ def test_multiple_sources(): ), every="1h", lateness="2d", - starting_from=datetime.strptime( - "2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ" - ), + since=datetime.strptime("2021-08-10T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) @dataset class UserInfoDatasetS3: