Skip to content

Commit

Permalink
feat(cl_back_scrape_opinions): add optional wait argument
Browse files Browse the repository at this point in the history
Solves #4564
  • Loading branch information
grossir committed Oct 11, 2024
1 parent 9f2dcd7 commit d8aad31
Showing 1 changed file with 16 additions and 0 deletions.
16 changes: 16 additions & 0 deletions cl/scrapers/management/commands/cl_back_scrape_opinions.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import time

from juriscraper import AbstractSite
from juriscraper.AbstractSite import logger
from juriscraper.lib.importer import site_yielder
Expand All @@ -20,6 +22,13 @@ def add_arguments(self, parser):
dest="backscrape_end",
help="End value for backscraper iterable creation.",
)
parser.add_argument(
"--backscrape-wait",
type=int,
default=0,
help="Seconds to wait after consuming each element "
"of the backscrape iterable",
)

def parse_and_scrape_site(
self,
Expand Down Expand Up @@ -50,5 +59,12 @@ def parse_and_scrape_site(
site.parse()
self.scrape_court(site, full_crawl=True)

if wait := options["backscrape_wait"]:
logger.info(
"Sleeping for %s seconds before continuing backscrape",
wait,
)
time.sleep(wait)

def save_everything(self, items, index=False, backscrape=True):
super().save_everything(items, index, backscrape)

0 comments on commit d8aad31

Please sign in to comment.