diff --git a/cl/scrapers/management/commands/cl_back_scrape_opinions.py b/cl/scrapers/management/commands/cl_back_scrape_opinions.py index f5c3a0dd0..bba640d2e 100644 --- a/cl/scrapers/management/commands/cl_back_scrape_opinions.py +++ b/cl/scrapers/management/commands/cl_back_scrape_opinions.py @@ -1,3 +1,5 @@ +import time + from juriscraper import AbstractSite from juriscraper.AbstractSite import logger from juriscraper.lib.importer import site_yielder @@ -20,6 +22,13 @@ def add_arguments(self, parser): dest="backscrape_end", help="End value for backscraper iterable creation.", ) + parser.add_argument( + "--backscrape-wait", + type=int, + default=0, + help="Seconds to wait after consuming each element " + "of the backscrape iterable", + ) def parse_and_scrape_site( self, @@ -50,5 +59,12 @@ def parse_and_scrape_site( site.parse() self.scrape_court(site, full_crawl=True) + if wait := options["backscrape_wait"]: + logger.info( + "Sleeping for %s seconds before continuing backscrape", + wait, + ) + time.sleep(wait) + def save_everything(self, items, index=False, backscrape=True): super().save_everything(items, index, backscrape)