From 9a20f44c3c0fa0c74050ac5b37a0f99a5bd8b5c5 Mon Sep 17 00:00:00 2001 From: msj Date: Mon, 18 Dec 2023 16:28:49 -0500 Subject: [PATCH] Limit scraping for testing --- courtscraper/spiders/chancery.py | 4 ++++ courtscraper/spiders/civil.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/courtscraper/spiders/chancery.py b/courtscraper/spiders/chancery.py index 685f980..e63a24b 100644 --- a/courtscraper/spiders/chancery.py +++ b/courtscraper/spiders/chancery.py @@ -20,7 +20,11 @@ def get_case_numbers(self, year): yield case_number def start_requests(self): + count = 0 for case_number in self.case_numbers: + if count > 10: + break + count += 1 yield Request( ChancerySpider.url, meta={ diff --git a/courtscraper/spiders/civil.py b/courtscraper/spiders/civil.py index a248d76..b6f14e2 100644 --- a/courtscraper/spiders/civil.py +++ b/courtscraper/spiders/civil.py @@ -12,7 +12,11 @@ def __init__(self, division="2", year=2023, **kwargs): super().__init__(**kwargs) def start_requests(self): + count = 0 for case_number in self.case_numbers: + if count > 10: + break + count += 1 yield Request( CivilSpider.url, meta={