From 5a52901716ecbf07b5d4d7d135dfef1f91ea192c Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi Date: Fri, 2 Feb 2024 12:10:13 +0000 Subject: [PATCH 1/4] Add the missing keyword- for datatype array --- tap_s3_csv/conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_s3_csv/conversion.py b/tap_s3_csv/conversion.py index 3805545..fb15633 100644 --- a/tap_s3_csv/conversion.py +++ b/tap_s3_csv/conversion.py @@ -110,7 +110,7 @@ def generate_schema(samples, table_spec): elif datatype == 'list': counts[key] = { 'anyOf': [ - {'type': 'array', 'items': ['null', 'string']}, + {'type': 'array', 'items': {'type': ['null', 'string']}}, {'type': ['null', 'string']} ] } From 0b92a8464ce9ec16b8ee8ef9b2018d8f26149145 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi Date: Fri, 2 Feb 2024 12:13:54 +0000 Subject: [PATCH 2/4] make circleci working --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 040010f..6a02bc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: source /usr/local/share/virtualenvs/tap-s3-csv/bin/activate pip install . pip install pylint - pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access + pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access,broad-exception-raised - run: name: 'Unit Tests' command: | From 8813f534822a50b9d9dc38fdee1952a876f97a35 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi Date: Fri, 2 Feb 2024 12:24:45 +0000 Subject: [PATCH 3/4] fix the unit test --- tests/unittests/test_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_conversion.py b/tests/unittests/test_conversion.py index 680fa8d..77c19ee 100644 --- a/tests/unittests/test_conversion.py +++ b/tests/unittests/test_conversion.py @@ -102,5 +102,5 @@ def test_generate_schema(self,mock_count_sample): samples = [{'name': 'test', 'id': 3, 'marks': [45.85, 25.38], 'students': {'no': 5, 'col': 6}, 'created_at': '20-05-2021', 'tota': []}] table_spec = {'search_prefix': '', 'search_pattern': 'test\\/.*\\.jsonl', 'table_name': 'jsonl_table', 'key_properties': ['id'],'date_overrides': ['created_at'], 'delimiter': ','} res = conversion.generate_schema(samples, table_spec) - expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': ['null', 'string']}, {'type': ['null', 'string']}]}} + expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'string']}}, {'type': ['null', 'string']}]}} self.assertEqual(res, expected_result) From d1c1d3c0b91d96247f625474d610de2d9d6b311d Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi Date: Fri, 2 Feb 2024 13:31:12 +0000 Subject: [PATCH 4/4] update setup and changelog --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384fb7a..b4b79a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.3.8 + * Add Missing Type Information in JSON Schema + * [#55](https://github.com/singer-io/tap-s3-csv/pull/62) + ## 1.3.7 * Remove Backoff for Access Denied errors * Add unitttest for access denied giveup diff --git a/setup.py b/setup.py index c951d34..3dd26d6 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup(name='tap-s3-csv', - version='1.3.7', + version='1.3.8', description='Singer.io tap for extracting CSV files from S3', author='Stitch', url='https://singer.io',