diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py index d866d9361d1b..6e652147b0e7 100644 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ b/airbyte-integrations/connectors/source-recurly/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "recurly==4.10.0", "requests"] +MAIN_REQUIREMENTS = ["airbyte-cdk==0.67", "recurly==4.10.0", "requests"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py b/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py index f7526476f4ae..ea3d0b82b000 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py @@ -3,6 +3,7 @@ # import re +import time from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from airbyte_cdk.models import SyncMode @@ -128,11 +129,16 @@ def read_records( if self.end_time: params.update({END_TIME_PARAM: self.end_time}) - items = getattr(self._client, self.client_method_name)(params=params).items() + pages = getattr(self._client, self.client_method_name)(params=params).pages() # Call the Recurly client methods - for item in items: - yield self._item_to_dict(item) + for page in pages: + for item in page: + yield self._item_to_dict(item) + # slow down the API calls to avoid rate limiting + # https://recurly.com/developers/api-v2/v2.2/#section/Rate-Limits + # making 5 API calls per second / 300 API calls per minute / 1500 API calls per 5 minutes + time.sleep(0.2) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """