Skip to content

Commit

Permalink
Fixed some edge case errors and incorrect bookmarking
Browse files Browse the repository at this point in the history
  • Loading branch information
justedro committed Jun 3, 2019
1 parent dd1b7f9 commit 1420303
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 5 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ Sample config:
"key_file": "./AuthKey_AAAAAAAAA.p8",
"issuer_id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
"vendor": "3333333",
"start_date": "2019-02-01"
"start_date": "2019-02-01T00:00:00Z"
}
```
18 changes: 14 additions & 4 deletions tap_appstore/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def discover():
return {'streams': streams}


def tsv_to_list(tsv, column_name_modifier = None):
def tsv_to_list(tsv):
lines = tsv.split('\n')
header = [s.lower().replace(' ', '_') for s in lines[0].split('\t')]

Expand Down Expand Up @@ -147,12 +147,23 @@ def query_report(api):
extraction_time = singer.utils.now()

iterator = bookmark
singer.write_bookmark(
Context.state,
stream_name,
'start_date',
iterator.strftime(BOOKMARK_DATE_FORMAT)
)

with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer:
while iterator + delta <= extraction_time:

iterator_str = iterator.strftime("%Y-%m-%d")
rep_tsv = api.sales_report('SALES', 'SUMMARY', 'DAILY', Context.config['vendor'], iterator_str, '1_0')
rep = tsv_to_list(rep_tsv)
if isinstance(rep_tsv, dict):
LOGGER.warning("Received a JSON response instead of the report: %s", str(rep_tsv))
break
else:
rep = tsv_to_list(rep_tsv)

for index, line in enumerate(rep, start=1):
data = line
Expand All @@ -171,11 +182,10 @@ def query_report(api):
Context.state,
stream_name,
'start_date',
iterator.strftime(BOOKMARK_DATE_FORMAT)
(iterator + delta).strftime(BOOKMARK_DATE_FORMAT)
)

singer.write_state(Context.state)

iterator += delta

singer.write_state(Context.state)
Expand Down

0 comments on commit 1420303

Please sign in to comment.