📝 Genesys source - document the delay in between API calls (#1045)

* 🐛 fixed unmatching reports.

* 🚨 Apply linting

---------

Co-authored-by: trymzet <[email protected]>

src/viadot/orchestration/prefect/flows/genesys_to_adls.py

@@ -1,6 +1,5 @@
 """Download data from Genesys Cloud and upload it to Azure Data Lake Storage."""
 
-import time
 from typing import Any
 
 from prefect import flow
@@ -28,6 +27,7 @@ def genesys_to_adls(  # noqa: PLR0913
     view_type: str | None = None,
     view_type_time_sleep: int | None = None,
     post_data_list: list[dict[str, Any]] | None = None,
+    time_between_api_call: float = 0.5,
     normalization_sep: str = ".",
     drop_duplicates: bool = False,
     validate_df_dict: dict[str, Any] | None = None,
@@ -58,6 +58,8 @@ def genesys_to_adls(  # noqa: PLR0913
             from Genesys Cloud API. Defaults to None.
         post_data_list (Optional[List[Dict[str, Any]]], optional): List of string
             templates to generate json body in POST calls to the API. Defaults to None.
+        time_between_api_call (int, optional): The time, in seconds, to sleep the call
+            to the API. Defaults to 0.5.
         normalization_sep (str, optional): Nested records will generate names separated
             by sep. Defaults to ".".
         drop_duplicates (bool, optional): Remove duplicates from the DataFrame.
@@ -96,14 +98,12 @@ def genesys_to_adls(  # noqa: PLR0913
         view_type=view_type,
         view_type_time_sleep=view_type_time_sleep,
         post_data_list=post_data_list,
+        time_between_api_call=time_between_api_call,
         normalization_sep=normalization_sep,
         drop_duplicates=drop_duplicates,
         validate_df_dict=validate_df_dict,
     )
 
-    # ???
-    time.sleep(0.5)
-
     return df_to_adls(
         df=data_frame,
         path=adls_path,

src/viadot/orchestration/prefect/tasks/genesys.py

@@ -22,6 +22,7 @@ def genesys_to_df(  # noqa: PLR0913
     view_type: str | None = None,
     view_type_time_sleep: int | None = None,
     post_data_list: list[dict[str, Any]] | None = None,
+    time_between_api_call: float = 0.5,
     normalization_sep: str = ".",
     drop_duplicates: bool = False,
     validate_df_dict: dict[str, Any] | None = None,
@@ -48,6 +49,8 @@ def genesys_to_df(  # noqa: PLR0913
             from Genesys Cloud API. Defaults to None.
         post_data_list (Optional[List[Dict[str, Any]]], optional): List of string
             templates to generate json body in POST calls to the API. Defaults to None.
+        time_between_api_call (int, optional): The time, in seconds, to sleep the call
+            to the API. Defaults to 0.5.
         normalization_sep (str, optional): Nested records will generate names separated
             by sep. Defaults to ".".
         drop_duplicates (bool, optional): Remove duplicates from the DataFrame.
@@ -98,6 +101,7 @@ def genesys_to_df(  # noqa: PLR0913
         view_type=view_type,
         view_type_time_sleep=view_type_time_sleep,
         post_data_list=post_data_list,
+        time_between_api_call=time_between_api_call,
         normalization_sep=normalization_sep,
     )
     logger.info("running `to_df` method:\n")

src/viadot/sources/genesys.py

@@ -175,7 +175,7 @@ def _api_call(
         post_data_list: list[str],
         method: str,
         params: dict[str, Any] | None = None,
-        sleep_time: float = 0.5,
+        time_between_api_call: float = 0.5,
     ) -> dict[str, Any]:
         """General method to connect to Genesys Cloud API and generate the response.
 
@@ -185,8 +185,8 @@ def _api_call(
             method (str): Type of connection to the API. Defaults to "POST".
             params (Optional[Dict[str, Any]], optional): Parameters to be passed into
                 the POST call. Defaults to None.
-            sleep_time (int, optional): The time, in seconds, to sleep the call to the
-                API. Defaults to 0.5.
+            time_between_api_call (int, optional): The time, in seconds, to sleep the
+                call to the API. Defaults to 0.5.
 
         Raises:
             RuntimeError: There is no current event loop in asyncio thread.
@@ -238,7 +238,7 @@ async def generate_post():
 
                                 semaphore.release()
 
-                await asyncio.sleep(sleep_time)
+                await asyncio.sleep(time_between_api_call)
 
         try:
             loop = asyncio.get_event_loop()
@@ -593,6 +593,7 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
         view_type: str | None = None,
         view_type_time_sleep: int = 10,
         post_data_list: list[dict[str, Any]] | None = None,
+        time_between_api_call: float = 0.5,
         normalization_sep: str = ".",
     ) -> None:
         """General method to connect to Genesys Cloud API and generate the response.
@@ -614,6 +615,8 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
             post_data_list (Optional[List[Dict[str, Any]]], optional): List of string
                 templates to generate json body in POST calls to the API.
                 Defaults to None.
+            time_between_api_call (int, optional): The time, in seconds, to sleep the
+                call to the API. Defaults to 0.5.
             normalization_sep (str, optional): Nested records will generate names
                 separated by sep. Defaults to ".".
 
@@ -632,6 +635,7 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
             self._api_call(
                 endpoint=endpoint,
                 post_data_list=post_data_list,
+                time_between_api_call=time_between_api_call,
                 method="POST",
             )
 
@@ -645,8 +649,14 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
             request_json = self._load_reporting_exports()
             entities = request_json["entities"]
 
-            if isinstance(entities, list) and len(entities) == len(post_data_list):
+            if isinstance(entities, list):
                 ids, urls = self._get_reporting_exports_url(entities)
+                if len(entities) != len(post_data_list):
+                    self.logger.warning(
+                        f"There are {len(entities)} available reports in Genesys, "
+                        f"and where sent {len(post_data_list)} reports. "
+                        "Unsed reports will be removed."
+                    )
             else:
                 APIError(
                     "There are no reports to be downloaded."
@@ -703,6 +713,7 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
                 report = self._api_call(
                     endpoint=endpoint,
                     post_data_list=post_data_list,
+                    time_between_api_call=time_between_api_call,
                     method="POST",
                 )
 
@@ -742,6 +753,7 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
                 response = self._api_call(
                     endpoint=endpoint,
                     post_data_list=post_data_list,
+                    time_between_api_call=time_between_api_call,
                     method="GET",
                     params=params,
                 )
@@ -773,6 +785,7 @@ def api_connection(  # noqa: PLR0912, PLR0915, C901.
                         endpoint=f"routing/queues/{qid}/members",
                         params={"pageSize": 100, "pageNumber": page},
                         post_data_list=post_data_list,
+                        time_between_api_call=time_between_api_call,
                         method="GET",
                     )
 

tests/unit/test_genesys.py

@@ -413,6 +413,7 @@ def test_api_connection_reporting_exports(
         endpoint="analytics/reporting/exports",
         post_data_list=variables["mock_post_data_list"],
         method="POST",
+        time_between_api_call=0.5,
     )
     mock_load.assert_called_once()
     mock_get_url.assert_called_once()