fix some bugs

README.md

@@ -13,11 +13,12 @@ all the tables for you, and avoids having to sample the data to determine the sc
 ```javascript
 {
  "start_date": "2015-01-01T00:00:00Z",
- // A path to a JSON credentials file for a service account. Make sure
- // you give that user permissions by inviting them to your Play Store
- // account and giving them "View app information and download bulk reports (read-only)"
- // and "View financial data, orders, and cancellation survey responses" permissions.
+ // A path to a JSON credentials file for a service account. Make sure you give that
+ // user permissions by inviting them to your Play Store account and giving them
+ // "View app information and download bulk reports (read-only)" and
+ // "View financial data, orders, and cancellation survey responses" permissions.
  // These permissions must be granted account-wide, not just for one app.
+ // It can take a few hours before the grant takes effect.
  "credentials_path": "./client_secret.json",
  // The path to the bucket Google sends reports to. You can get this from the
  // "Copy Cloud Storage URI" button from the reports page.

setup.py

@@ -11,7 +11,7 @@
  py_modules=["tap_playstore"],
  install_requires=[
  'singer-python>=5.12.1',
- 'tap-gcs-csv @ git+https://github.com/fixdauto/tap-gcs-csv.git@9cb3c67561ba4a67512526e064248ba018695db3',
+ 'tap-gcs-csv @ git+https://github.com/fixdauto/tap-gcs-csv.git@v0.0.3',
  ],
  entry_points="""
  [console_scripts]

tap_playstore/__init__.py

@@ -36,7 +36,22 @@ def get_abs_path(path):
  "search_prefix": "sales",
  "format": "csv",
  "encoding": "utf-8",
- "compression": "zip"
+ "compression": "zip",
+ "schema_overrides": {
+ # these number fields have commas in them, and float("1,000.00") does not work.
+ "item_price": {
+ "type": ["null", "string"],
+ "_conversion_type": "string"
+ },
+ "taxes_collected": {
+ "type": ["null", "string"],
+ "_conversion_type": "string"
+ },
+ "charged_amount": {
+ "type": ["null", "string"],
+ "_conversion_type": "string"
+ }
+ }
  },
  "earnings": {
  "pattern": "earnings/earnings_.*\\.zip",
@@ -47,10 +62,7 @@ def get_abs_path(path):
  "schema_overrides": {
  # force post code to be treated as a string instead of an integer
  "buyer_postal_code": {
- "type": [
- "null",
- "string"
- ],
+ "type": ["null", "string"],
  "_conversion_type": "string"
  }
  }
@@ -82,9 +94,9 @@ def get_abs_path(path):
  "encoding": "utf-16"
  }
 
-def discover():
+def discover(cached=True):
  # return the cached catalog
- if os.getenv('TAP_PLAYSTORE_GENERATE_CATALOG') != 'true':
+ if cached:
  return Catalog.load(get_abs_path('cached_catalog.json'))
  # This generates the catalog using tap-gcs-csv sampling
  # and caches it to the tap.
@@ -107,15 +119,16 @@ def main():
  # Parse command line arguments
  args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
 
+ cached_catalog = args.config.get('cached_catalog', True)
  if args.discover:
- catalog = discover()
+ catalog = discover(cached_catalog)
  catalog.dump()
  # Otherwise run in sync mode
  else:
  if args.catalog:
  catalog = args.catalog
  else:
- catalog = discover()
+ catalog = discover(cached_catalog)
 
  tap_gcs_csv.do_sync(args.config, args.state, catalog)
 

tap_playstore/cached_catalog.json

@@ -709,19 +709,19 @@
  "item_price": {
  "type": [
  "null",
- "number"
+ "string"
  ]
  },
  "taxes_collected": {
  "type": [
  "null",
- "number"
+ "string"
  ]
  },
  "charged_amount": {
  "type": [
  "null",
- "number"
+ "string"
  ]
  },
  "city_of_buyer": {
@@ -898,7 +898,7 @@
  ],
  "metadata": {
  "inclusion": "available",
- "conversion_type": "number"
+ "conversion_type": "string"
  }
  },
  {
@@ -908,7 +908,7 @@
  ],
  "metadata": {
  "inclusion": "available",
- "conversion_type": "number"
+ "conversion_type": "string"
  }
  },
  {
@@ -918,7 +918,7 @@
  ],
  "metadata": {
  "inclusion": "available",
- "conversion_type": "number"
+ "conversion_type": "string"
  }
  },
  {