From ab0eff755294f5733cc8302ca98d75d29bb690c8 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Thu, 28 Sep 2023 07:16:56 +0530 Subject: [PATCH 1/4] return each record inside {data} --- .../source_kobotoolbox/source.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py b/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py index b0b03aff4a07..276b3a44477a 100644 --- a/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py +++ b/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py @@ -24,10 +24,11 @@ "null", ] }, - "endtime": {"type": ["string", "null"]} + "_submission_time": {"type": ["string", "null"]}, }, } + class KoboToolStream(HttpStream, IncrementalMixin): primary_key = "_id" cursor_field = "_submission_time" @@ -40,11 +41,11 @@ def __init__(self, config: Mapping[str, Any], form_id, schema, name, pagination_ self.auth_token = auth_token self.schema = schema self.stream_name = name - self.base_url = config['base_url'] + self.base_url = config["base_url"] self.PAGINATION_LIMIT = pagination_limit self._cursor_value = None self.start_time = config["start_time"] - self.exclude_fields = config['exclude_fields'] if 'exclude_fields' in config else [] + self.exclude_fields = config["exclude_fields"] if "exclude_fields" in config else [] @property def url_base(self) -> str: @@ -58,7 +59,7 @@ def name(self) -> str: s = s.strip() return s if len(s) > 0 else self.form_id - # State will be a dict : {'endtime': '2023-03-15T00:00:00.000+05:30'} + # State will be a dict : {'_submission_time': '2023-03-15T00:00:00.000+05:30'} @property def state(self) -> Mapping[str, Any]: @@ -77,7 +78,6 @@ def get_json_schema(self): def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"start": 0, "limit": self.PAGINATION_LIMIT, "sort": json.dumps({self.cursor_field: 1})} params["query"] = json.dumps({self.cursor_field: {"$gte": self.state[self.cursor_field]}}) @@ -111,7 +111,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for to_remove_field in self.exclude_fields: if to_remove_field in record: record.pop(to_remove_field) - yield record + retval = {"_id": record["_id"], "data": record} + retval[self.cursor_field] = record[self.cursor_field] + yield retval def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: for record in super().read_records(*args, **kwargs): @@ -134,7 +136,7 @@ def _check_credentials(cls, config: Mapping[str, Any]) -> Tuple[bool, Any]: return False, "password in credentials is not provided" return True, None - + def get_access_token(self, config) -> Tuple[str, any]: token_url = f"{config['base_url']}/token/?format=json" @@ -162,7 +164,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - # Fetch all assets(forms) url = f"{config['base_url']}/api/v2/assets.json" response = requests.get(url, auth=(config["username"], config["password"])) @@ -177,7 +178,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # Generate array of stream objects streams = [] for form_dict in key_list: - if form_dict['has_deployment']: + if form_dict["has_deployment"]: stream = KoboToolStream( config=config, form_id=form_dict["uid"], @@ -187,5 +188,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth_token=auth_token, ) streams.append(stream) - + return streams From b2dd317058c545c62fcce0be5b6920a54c94ef68 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Thu, 28 Sep 2023 11:00:26 +0530 Subject: [PATCH 2/4] upgrade version tag --- airbyte-integrations/connectors/source-kobotoolbox/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile b/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile index 88aad6a5fb05..55d96c315d18 100644 --- a/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile +++ b/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile @@ -31,5 +31,5 @@ COPY source_kobotoolbox ./source_kobotoolbox ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-kobotoolbox From 95667cba342678134204bac967e4a1cab146b36a Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 29 Sep 2023 08:17:52 +0530 Subject: [PATCH 3/4] renamed image --- airbyte-integrations/connectors/source-kobotoolbox/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile b/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile index 55d96c315d18..4a255fc9d125 100644 --- a/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile +++ b/airbyte-integrations/connectors/source-kobotoolbox/Dockerfile @@ -31,5 +31,5 @@ COPY source_kobotoolbox ./source_kobotoolbox ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/source-kobotoolbox +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=tech4dev/source-kobotoolbox From 65ae3cff7082e55fe23f77eb3baa842445f3ac89 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 29 Sep 2023 08:18:05 +0530 Subject: [PATCH 4/4] put data into the stream schema --- .../connectors/source-kobotoolbox/source_kobotoolbox/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py b/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py index 276b3a44477a..096a67181e59 100644 --- a/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py +++ b/airbyte-integrations/connectors/source-kobotoolbox/source_kobotoolbox/source.py @@ -24,6 +24,9 @@ "null", ] }, + "data": { + "type": "object", + }, "_submission_time": {"type": ["string", "null"]}, }, }