scraper: Increase the read Timeout to 30 seconds

This should reduce retrying by a good margin.
seareport · Nov 15, 2023 · 48b07b8 · 48b07b8
1 parent f1ec7f6
commit 48b07b8
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/observer/ioc/scraper.py b/observer/ioc/scraper.py
@@ -110,7 +110,8 @@ def scrape_ioc_station(
     urls = generate_urls(ioc_code, start_date, end_date)
     logger.debug("%s: There are %d urls", ioc_code, len(urls))
     logger.debug("%s:\n%s", ioc_code, "\n".join(urls))
-    with httpx.Client() as client:
+    timeout = httpx.Timeout(timeout=10, read=30)
+    with httpx.Client(timeout=timeout) as client:
         func_kwargs = [dict(url=url, client=client, rate_limit=rate_limit) for url in urls]
         logger.debug("%s: Starting data retrieval", ioc_code)
         results = multifutures.multithread(fetch_url, func_kwargs, check=True)
@@ -151,5 +152,4 @@ def scrape_ioc_station(
     logger.info("%s: Finished scraping: %s - %s", ioc_code, start_date, end_date)
     return df
 
-
     return df