Adds ultimate error handling to get_compound_htmls

IvanChernyshov · Sep 23, 2024 · 9f3c732 · 9f3c732
1 parent 4fafe14
commit 9f3c732
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/update/get_compound_htmls.py b/update/get_compound_htmls.py
@@ -58,9 +58,11 @@ def download_compound_htmls(df: pd.core.frame.DataFrame, dir_html: str,
             download_compound_html(url, path_html, path_err)
             n_errs = 0
             time.sleep(crawl_delay)
-        except TimeoutError:
+        except (KeyboardInterrupt, SystemError, SystemExit):
+            raise
+        except:
             n_errs += 1
-            time.sleep(max(30, 10*crawl_delay))
+            time.sleep(max(120, 10*crawl_delay))
 
     return