client: rm per_request and retry

mpcontribs-client/mpcontribs/client/__init__.py

@@ -694,9 +694,11 @@ def _get_per_page_default_max(self, op: str = "query", resource: str = "contribu
         return param_spec["default"], param_spec["maximum"]
 
     def _get_per_page(
-        self, per_page: int, op: str = "query", resource: str = "contributions"
+        self, per_page: int = -1, op: str = "query", resource: str = "contributions"
     ) -> int:
-        _, per_page_max = self._get_per_page_default_max(op=op, resource=resource)
+        per_page_default, per_page_max = self._get_per_page_default_max(op=op, resource=resource)
+        if per_page < 0:
+            per_page = per_page_default
         return min(per_page_max, per_page)
 
     def _split_query(
@@ -1592,8 +1594,6 @@ def submit_contributions(
         self,
         contributions: List[dict],
         ignore_dupes: bool = False,
-        retry: bool = False,
-        per_request: int = 100,
         timeout: int = -1,
         skip_dupe_check: bool = False
     ):
@@ -1621,8 +1621,6 @@ def submit_contributions(
         Args:
             contributions (list): list of contribution dicts to submit
             ignore_dupes (bool): force duplicate components to be submitted
-            retry (bool): keep trying until all contributions successfully submitted
-            per_request (int): number of contributions to submit per request
             timeout (int): cancel remaining requests if timeout exceeded (in seconds)
             skip_dupe_check (bool): skip duplicate check for contribution identifiers
         """
@@ -1635,7 +1633,6 @@ def submit_contributions(
         project_names = set()
         collect_ids = []
         require_one_of = {"data"} | set(COMPONENTS)
-        per_page = self._get_per_page(per_request)
 
         for idx, c in enumerate(contributions):
             has_keys = require_one_of & c.keys()
@@ -1816,6 +1813,7 @@ def submit_contributions(
 
         # submit contributions
         if contribs:
+            per_page = self._get_per_page()
             total, total_processed = 0, 0
 
             def post_future(track_id, payload):
@@ -1878,13 +1876,14 @@ def put_future(pk, payload):
                         break  # nothing to do
 
                     responses = _run_futures(
-                        futures, total=ncontribs, timeout=timeout, desc="Submit"
+                        futures, total=ncontribs-total_processed, timeout=timeout, desc="Submit"
                     )
                     processed = sum(r.get("count", 0) for r in responses.values())
                     total_processed += processed
 
-                    if processed != ncontribs and retry and retries < RETRIES and \
+                    if total_processed != ncontribs and retries < RETRIES and \
                             unique_identifiers.get(project_name):
+                        logger.info(f"{total_processed}/{ncontribs} processed -> retrying ...")
                         existing[project_name] = self.get_all_ids(
                             dict(project=project_name), include=COMPONENTS, timeout=timeout
                         ).get(project_name, {"identifiers": set()})
@@ -1896,10 +1895,11 @@ def put_future(pk, payload):
                             c for c in contribs[project_name]
                             if c["identifier"] not in existing_ids
                         ]
+                        per_page = int(per_page / 2)
                         retries += 1
                     else:
                         contribs[project_name] = []  # abort retrying
-                        if processed != ncontribs and retry:
+                        if total_processed != ncontribs:
                             if retries >= RETRIES:
                                 logger.error(f"{project_name}: Tried {RETRIES} times - abort.")
                             elif not unique_identifiers.get(project_name):