Add Missing Type Information in JSON Schema

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
             source /usr/local/share/virtualenvs/tap-s3-csv/bin/activate
             pip install .
             pip install pylint
-            pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access
+            pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access,broad-exception-raised
       - run:
           name: 'Unit Tests'
           command: |

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 1.3.8
+  * Add Missing Type Information in JSON Schema
+  * [#55](https://github.com/singer-io/tap-s3-csv/pull/62)
+
 ## 1.3.7
   * Remove Backoff for Access Denied errors
   * Add unitttest for access denied giveup

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-s3-csv',
-      version='1.3.7',
+      version='1.3.8',
       description='Singer.io tap for extracting CSV files from S3',
       author='Stitch',
       url='https://singer.io',

tap_s3_csv/conversion.py

@@ -110,7 +110,7 @@ def generate_schema(samples, table_spec):
         elif datatype == 'list':
             counts[key] = {
                 'anyOf': [
-                    {'type': 'array', 'items': ['null', 'string']},
+                    {'type': 'array', 'items': {'type': ['null', 'string']}},
                     {'type': ['null', 'string']}
                 ]
             }

tests/unittests/test_conversion.py

@@ -102,5 +102,5 @@ def test_generate_schema(self,mock_count_sample):
         samples = [{'name': 'test', 'id': 3, 'marks': [45.85, 25.38], 'students': {'no': 5, 'col': 6}, 'created_at': '20-05-2021', 'tota': []}]
         table_spec = {'search_prefix': '', 'search_pattern': 'test\\/.*\\.jsonl', 'table_name': 'jsonl_table', 'key_properties': ['id'],'date_overrides': ['created_at'], 'delimiter': ','}
         res = conversion.generate_schema(samples, table_spec)
-        expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': ['null', 'string']}, {'type': ['null', 'string']}]}}
+        expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'string']}}, {'type': ['null', 'string']}]}}
         self.assertEqual(res, expected_result)