From df8f7b327961e02c63bda3dc1234a270c5a5d0af Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Tue, 14 May 2024 12:38:59 +0530 Subject: [PATCH 1/4] configurable api endpoint --- .../source-glific/source_glific/source.py | 29 ++++++++++--------- .../source-glific/source_glific/spec.yaml | 8 +++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-glific/source_glific/source.py b/airbyte-integrations/connectors/source-glific/source_glific/source.py index 9451781587af..4e8a1f9111bb 100644 --- a/airbyte-integrations/connectors/source-glific/source_glific/source.py +++ b/airbyte-integrations/connectors/source-glific/source_glific/source.py @@ -4,17 +4,16 @@ from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator - -import requests +from typing import Any, Iterable, List, Mapping, Optional, Tuple import json +from datetime import datetime +import requests + +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream, IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.core import StreamData -from datetime import datetime stream_json_schema = { @@ -75,6 +74,7 @@ def __init__(self, stream_name: str, url_base: str, pagination_limit: int, crede self.start_time = config["start_time"] self.offset = 0 self.last_record = None + self.state = {} @property def url_base(self) -> str: @@ -100,8 +100,8 @@ def path( def update_state(self) -> None: if self.latest_updated_date: - if self.latest_updated_date > self.state["updated_at"]: - self.state = {self.cursor_field: self.latest_updated_date} + if not self.state or self.latest_updated_date > self.state["updated_at"]: + self.state = {"updated_at": self.latest_updated_date} self.latest_updated_date = None return None @@ -204,7 +204,6 @@ def state(self, value: Mapping[str, Any]): class SourceGlific(AbstractSource): """Glific source""" - API_URL = "https://api.staging.tides.coloredcow.com/api" PAGINATION_LIMIT = 500 def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -226,7 +225,9 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: logger.info("Password missing") return False, "Password missing" - endpoint = f"{self.API_URL}/v1/session" + api_url = config["glific_url"] + + endpoint = f"{api_url}/v1/session" auth_payload = {"user": {"phone": config["phone"], "password": config["password"]}} response = requests.post(endpoint, json=auth_payload, timeout=30) @@ -243,8 +244,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ + api_url = config["glific_url"] + # authenticate and get the credentials for all streams - endpoint = f"{self.API_URL}/v1/session" + endpoint = f"{api_url}/v1/session" auth_payload = {"user": {"phone": config["phone"], "password": config["password"]}} try: response = requests.post(endpoint, json=auth_payload, timeout=30) @@ -255,7 +258,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [] # fetch the export config for organization/client/user - endpoint = f"{self.API_URL}" + endpoint = api_url headers = {"authorization": credentials["access_token"]} try: @@ -274,7 +277,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: export_config = json.loads(data["data"]["organizationExportConfig"]["data"]) streams = [] for table in export_config["tables"]: - stream_obj = IncrementalGlificStream(table, self.API_URL, self.PAGINATION_LIMIT, credentials, config) + stream_obj = IncrementalGlificStream(table, api_url, self.PAGINATION_LIMIT, credentials, config) streams.append(stream_obj) return streams diff --git a/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml b/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml index 2e649192c6f2..7e3ef2135e65 100644 --- a/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml +++ b/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml @@ -26,3 +26,11 @@ connectionSpecification: order: 2 default: "2023-01-26T11:11:11Z" pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + glific_url: + type: string + title: Glific URL + description: URL of the Glific instance + order: 3 + default: "https://api.staging.glific.com/api" + pattern: ^https://[a-zA-Z0-9.-]+$ + From 1f48a4d53b98544604f00c5bef2898c492a9d5d9 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Tue, 14 May 2024 12:51:27 +0530 Subject: [PATCH 2/4] allow forward slash in the url pattern escaped the backslashes and the dash --- .../connectors/source-glific/source_glific/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml b/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml index 7e3ef2135e65..55732522f247 100644 --- a/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml +++ b/airbyte-integrations/connectors/source-glific/source_glific/spec.yaml @@ -32,5 +32,5 @@ connectionSpecification: description: URL of the Glific instance order: 3 default: "https://api.staging.glific.com/api" - pattern: ^https://[a-zA-Z0-9.-]+$ + pattern: ^https:\/\/[a-zA-Z0-9.\-\/]+$ From 97a3ef15bf42874152a299ac686feb2926b1de79 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Tue, 14 May 2024 13:07:51 +0530 Subject: [PATCH 3/4] state can't be {} --- .../connectors/source-glific/source_glific/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-glific/source_glific/source.py b/airbyte-integrations/connectors/source-glific/source_glific/source.py index 4e8a1f9111bb..b894a2c27b38 100644 --- a/airbyte-integrations/connectors/source-glific/source_glific/source.py +++ b/airbyte-integrations/connectors/source-glific/source_glific/source.py @@ -74,7 +74,7 @@ def __init__(self, stream_name: str, url_base: str, pagination_limit: int, crede self.start_time = config["start_time"] self.offset = 0 self.last_record = None - self.state = {} + self.state = {"updated_at": "2020-01-01T00:00:00Z"} @property def url_base(self) -> str: @@ -100,7 +100,7 @@ def path( def update_state(self) -> None: if self.latest_updated_date: - if not self.state or self.latest_updated_date > self.state["updated_at"]: + if self.latest_updated_date > self.state["updated_at"]: self.state = {"updated_at": self.latest_updated_date} self.latest_updated_date = None return None From ff804da4b3e9e9ae0f04b4e36418668c0558f8a9 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Tue, 14 May 2024 14:24:24 +0530 Subject: [PATCH 4/4] final tweaks --- .../connectors/source-glific/source_glific/source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-glific/source_glific/source.py b/airbyte-integrations/connectors/source-glific/source_glific/source.py index b894a2c27b38..cd14e2efb6c8 100644 --- a/airbyte-integrations/connectors/source-glific/source_glific/source.py +++ b/airbyte-integrations/connectors/source-glific/source_glific/source.py @@ -74,7 +74,6 @@ def __init__(self, stream_name: str, url_base: str, pagination_limit: int, crede self.start_time = config["start_time"] self.offset = 0 self.last_record = None - self.state = {"updated_at": "2020-01-01T00:00:00Z"} @property def url_base(self) -> str: @@ -196,7 +195,7 @@ def state(self) -> Mapping[str, Any]: @state.setter def state(self, value: Mapping[str, Any]): - self.cursor_value = value[self.cursor_field] + self.cursor_value = value.get(self.cursor_field) self._state = value