Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TDL-6619: Enable leads stream #180

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

TDL-6619: Enable leads stream #180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0e20576
Enabled leads stream
savan-chovatiya Dec 2, 2021
e57b70d
Added code comments
savan-chovatiya Dec 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions tap_facebook/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
'ads_insights_region',
'ads_insights_dma',
'ads_insights_hourly_advertiser',
#'leads',
'leads',
]

REQUIRED_CONFIG_KEYS = ['start_date', 'account_id', 'access_token']
Expand Down Expand Up @@ -486,7 +486,9 @@ def sync_batches(self, stream_objects):

# Ensure the final batch is executed
api_batch.execute()
return str(pendulum.parse(latest_lead[self.replication_key]))
# Return lead with maximum replication_key for bookmark if any leads found
if latest_lead:
return str(pendulum.parse(latest_lead[self.replication_key]))
Comment on lines +490 to +491
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added above condition because the tap was breaking(parsing None as a date error) when no data is available for leads.
The above function returns the maximum replication key's value out of all leads for writing bookmark.


# Added retry_pattern to handle AttributeError raised from account.get_ads() below
@retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
Expand Down
10 changes: 5 additions & 5 deletions tests/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,11 @@ def expected_metadata(self):
self.REPLICATION_METHOD: self.INCREMENTAL,
self.REPLICATION_KEYS: {"date_start"}
},
# "leads": {
# self.PRIMARY_KEYS: {"id"},
# self.REPLICATION_METHOD: self.INCREMENTAL,
# self.REPLICATION_KEYS: {"created_time"}
# },
"leads": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.REPLICATION_KEYS: {"created_time"}
},
}


Expand Down
2 changes: 2 additions & 0 deletions tests/test_facebook_automatic_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ def test_run(self):
"""

expected_streams = self.expected_streams()
# Don't have data for leads and it's required some chargeable actions so as per discussion skipping the 'leads' stream.(TDL-6619)
expected_streams = expected_streams - {"leads"}

# instantiate connection
conn_id = connections.ensure_connection(self)
Expand Down
2 changes: 2 additions & 0 deletions tests/test_facebook_bookmarks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ def is_expected_date_format(self, date):
def test_run(self):
expected_streams = self.expected_streams()
non_insight_streams = {stream for stream in expected_streams if not self.is_insight(stream)}
# Don't have data for leads and it's required some chargeable actions so as per discussion skipping the 'leads' stream.(TDL-6619)
non_insight_streams = non_insight_streams - {"leads"}
insight_streams = {stream for stream in expected_streams if self.is_insight(stream)}

# Testing against ads insights objects
Expand Down
5 changes: 3 additions & 2 deletions tests/test_facebook_field_selection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def expected_check_streams():
'ads_insights_region',
'ads_insights_dma',
"ads_insights_hourly_advertiser",
#'leads',
'leads',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is enabled as it's used in discover mode.

}

@staticmethod
Expand Down Expand Up @@ -97,7 +97,8 @@ def test_run(self):
all_excluded_fields = {}
# select all catalogs
for c in found_catalogs:
if c['stream_name'] == 'ads':
# Don't have data for leads and it's required some chargeable actions so as per discussion skipping the 'leads' stream.(TDL-6619)
if c['stream_name'] == 'ads' or c['stream_name'] == 'leads':
continue

discovered_schema = menagerie.get_annotated_schema(conn_id, c['stream_id'])['annotated-schema']
Expand Down
2 changes: 2 additions & 0 deletions tests/test_facebook_start_date.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ def test_run(self):
self.start_date = self.start_date_1

expected_streams = self.expected_streams()
# Don't have data for leads and it's required some chargeable actions so as per discussion skipping the 'leads' stream.(TDL-6619)
expected_streams = expected_streams - {"leads"}

##########################################################################
### First Sync
Expand Down