add contexts to harvest_erddap

HakaiInstitute · Apr 17, 2024 · 0b38b52 · 0b38b52
1 parent 1e44e16
commit 0b38b52
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/harvester/cde_harvester/__main__.py b/harvester/cde_harvester/__main__.py
@@ -15,7 +15,7 @@
     unescape_ascii,
     unescape_ascii_list,
 )
-from cde_harvester.harvest_erddap import harvest_erddap
+from cde_harvester.harvest_erddap import harvest_erddap_contextualized
 from cde_harvester.utils import cf_standard_names, supported_standard_names
 from dotenv import load_dotenv
 from loguru import logger
@@ -109,7 +109,7 @@ def main(erddaps, cache_requests, folder: Path, max_workers: int):
 
     def worker():
         while True:
-            harvest_erddap(*q.get())
+            harvest_erddap_contextualized(*q.get())
             time.sleep(1)
             q.task_done()
 

diff --git a/harvester/cde_harvester/harvest_erddap.py b/harvester/cde_harvester/harvest_erddap.py
@@ -88,6 +88,11 @@ def get_datasets_to_skip():
     return {}
 
 
+def harvest_erddap_contextualized(erddap_conn, result, cache_requests=False):
+    with logger.contextualize( erddap_url=erddap_conn["url"]):
+        return harvest_erddap(erddap_conn, result, cache_requests)
+
+
 def harvest_erddap(erddap_conn, result, cache_requests=False):
     # """ """
     skipped_datasets_reasons = []