Don't assume that PMHC extracts will succeed

Previously, we assumed that all PMHC extracts would eventually change
from status=Processing to status=Completed, at which point we could
download the extract file. However, it is possible that the PMHC server
may error, which results in status=Error. We did not detect this Error
state previously, so would loop indefinitely in the false hope that the
extract file would eventually be created.

Now, we correctly detect status=Error, and raise a suitable exception.

src/pmhclib/pmhc.py

@@ -44,6 +44,14 @@ class CouldNotFindPmhcUpload(Exception):
  """Custom error handler for when an upload cannot be found on PMHC"""
 
 
+class PmhcServerError(Exception):
+ """Custom exception for when a PMHC Server error is encountered"""
+
+
+class MaxRetriesExceeded(Exception):
+ """Custom exception for when the maximum retries is exceeded"""
+
+
 class SecureString(str):
  """Show `'***'` instead of string value in tracebacks"""
 
@@ -383,6 +391,7 @@ def download_pmhc_mds(
  "all_episode_children": int(matched_episodes),
  "spec_type": specification.term,
  }
+
  download_request = self.page.request.get(
  "https://pmhc-mds.net/api/extract/csv",
  params=params,
@@ -400,10 +409,75 @@ def download_pmhc_mds(
  )
  raise err
 
+ # Both PMHC server errors and incomplete processing extracts
+ # return the same HTTP status (400) and JSON response when
+ # trying to fetch the extract by UUID:
+ # https://pmhc-mds.net/api/extract/{download_uuid}/fetch
+ # {
+ # "errors": {
+ # "export_fetch": "Can not fetch extract for uuid
+ # [123...]. Extract is not complete. Extract has
+ # expired."
+ # }
+ # }
+ #
+ # For this reason, it's not sufficient to simply try the
+ # download URL until we get a success code. If there is
+ # a PMHC server error, we will end up retrying forever.
+ #
+ # Instead, we need to fetch the list of extracts and filter
+ # for one with the required uuid. We can then check the
+ # extract status explicitly, which should be one of the
+ # following values:
+ #
+ # - Completed
+ # - Processing
+ # - Queued
+ # - Error
+ #
+ # If Completed, we can download the extract.
+ # If Processing or Queued, keep looping and waiting.
+ # If Error, the extract has failed. Exit.
+
+ # Wait for extract to be ready
  progress.update(extract_task, description="Waiting for extract...")
+ retries = 0
+ while retries <= max_retries:
+ time.sleep(30)
+ try:
+ extracts_request = self.page.request.get(
+ "https://pmhc-mds.net/api/extract?sort=-date"
+ )
+ except playwright.sync_api.Error as err:
+ if "Request timed out" in err.message:
+ retries += 1
+ logging.warning(
+ f"Request timed out ({retries} of {max_retries}). Retrying."
+ )
+ else:
+ raise err
+
+ extracts = extracts_request.json()
+ extract = next(
+ filter(lambda item: item.get("uuid") == download_uuid, extracts)
+ )
+ status = extract["status"]
+
+ if status == "Completed":
+ break
+ elif status == "Error":
+ logging.error(f"PMHC extract with uuid {download_uuid} has failed.")
+ logging.error("See PMHC Server error:")
+ logging.error(extract["stash"]["error"])
+ raise PmhcServerError("The PMHC extract has failed on the server.")
+ else:
+ raise MaxRetriesExceeded(
+ f"Tried fetching PMHC extract list {retries - 1} times"
+ )
+
  # We know the URL which will give us the final download URL,
- # as we have the uuid. However, the URL doesn't exist
- # immediately. We loop until we get a success code.
+ # as we have the uuid. We have confirmed above that the
+ # extract is completed.
  request_ok = False
  retries = 0
  while not request_ok and retries <= max_retries:
@@ -422,6 +496,11 @@ def download_pmhc_mds(
  else:
  raise err
 
+ if retries > max_retries:
+ raise MaxRetriesExceeded(
+ f"Tried fetching PMHC extract {retries - 1} times."
+ )
+
  download_url_json = download_url_request.json()
  download_url = download_url_json["location"]