fix: reduce scraper page load timeout and increase retry attempts

Flexicon · Nov 11, 2024 · 3ff03c2 · 3ff03c2
1 parent 85b38e0
commit 3ff03c2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/.github/workflows/run_scraper.yml b/.github/workflows/run_scraper.yml
@@ -23,7 +23,7 @@ jobs:
           MONGODB_URI: ${{ secrets.MONGODB_URI }}
         with:
           timeout_minutes: 1
-          max_attempts: 5
+          max_attempts: 10
           retry_on: error
           command: |
             docker run -e DB_NAME -e MONGODB_URI \

diff --git a/scraper/helpers.py b/scraper/helpers.py
@@ -14,7 +14,7 @@ def __init__(self) -> None:
         chrome_options.add_argument('--no-sandbox')
         chrome_options.add_argument('--headless')
         self.driver = webdriver.Chrome(options=chrome_options)
-        self.driver.set_page_load_timeout(15)
+        self.driver.set_page_load_timeout(10)
         self.driver.implicitly_wait(5)
 
     def fetch_content_html(self, url: str, *, selector: str = '.main') -> str: