Limit scraping for testing

datamade · Dec 18, 2023 · 9a20f44 · 9a20f44
1 parent 8c60d03
commit 9a20f44
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/courtscraper/spiders/chancery.py b/courtscraper/spiders/chancery.py
@@ -20,7 +20,11 @@ def get_case_numbers(self, year):
  yield case_number
 
  def start_requests(self):
+ count = 0
  for case_number in self.case_numbers:
+ if count > 10:
+ break
+ count += 1
  yield Request(
  ChancerySpider.url,
  meta={

diff --git a/courtscraper/spiders/civil.py b/courtscraper/spiders/civil.py
@@ -12,7 +12,11 @@ def __init__(self, division="2", year=2023, **kwargs):
  super().__init__(**kwargs)
 
  def start_requests(self):
+ count = 0
  for case_number in self.case_numbers:
+ if count > 10:
+ break
+ count += 1
  yield Request(
  CivilSpider.url,
  meta={