Feature/dss13 sc 210707 pi system add pagination to asset search

CHANGELOG.md

@@ -5,6 +5,7 @@
 - Add summaryDuration selector (duration of each summary interval)
 - Fix issue when using `Starting After` search option in browse PI event frames
 - Fix issue with `Value` and `End value` data types in Search attributes dataset
+- Add pagination on element retrieval
 
 ## [Version 1.2.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.2) - Feature release - 2023-12-11
 

python-lib/osisoft_client.py

@@ -10,7 +10,7 @@
 from osisoft_plugin_common import (
     assert_server_url_ok, build_requests_params,
     is_filtered_out, is_server_throttling, escape, epoch_to_iso,
-    iso_to_epoch, RecordsLimit, is_iso8601
+    iso_to_epoch, RecordsLimit, is_iso8601, get_next_page_url
 )
 from osisoft_pagination import OffsetPagination
 from safe_logger import SafeLogger
@@ -763,9 +763,15 @@ def search_attributes(self, database_webid, **kwargs):
         json_response = self.get(url=search_attributes_base_url, headers=headers, params=params)
         if OSIsoftConstants.DKU_ERROR_KEY in json_response:
             yield json_response
-        items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
-        for item in items:
-            yield item
+        while json_response:
+            next_page_url = get_next_page_url(json_response)
+            items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
+            for item in items:
+                yield item
+            if next_page_url:
+                json_response = self.get(url=next_page_url, headers={}, params={})
+            else:
+                json_response = None
 
     def build_element_query(self, **kwargs):
         element_query_keys = {

python-lib/osisoft_plugin_common.py

@@ -451,6 +451,17 @@ def fields_selector(data_type):
         return "Links%3BItems.Timestamp%3BItems.Value%3BItems.Type"
 
 
+def get_next_page_url(json):
+    if not json:
+        return None
+    next_page_url = json.get("Links", {}).get("Next", "").replace('&', '&')
+    if next_page_url:
+        logger.info("Next page's url is {}".format(next_page_url))
+    else:
+        logger.info("No more pages available")
+    return next_page_url
+
+
 class RecordsLimit():
     def __init__(self, records_limit=-1):
         self.has_no_limit = (records_limit == -1)