Fix - Spider de Caçapava-SP [Fixes #1131]

data_collection/gazette/spiders/sp/sp_cacapava.py

@@ -1,4 +1,5 @@
 from datetime import date, datetime
+from urllib.parse import parse_qsl, urlencode, urlparse
 
 from scrapy.http import Request
 
@@ -19,19 +20,18 @@ def start_requests(self):
         url = f"https://cacapava.sp.gov.br/diario-oficial?dataDe={data_de}&dataAte={data_ate}"
         yield Request(url)
 
-    def parse(self, response):
-        num_pages = int(
-            response.css(".pagination__label::text").re_first(r"\/ (\d+)") or "1"
-        )
-        if num_pages > 1:
-            for page in range(1, num_pages + 1):
-                yield Request(f"{response.url}&pagina={page}")
+    def pagination_url(self, url: str, page: str):
+        url_parsed = urlparse(url)
+        url_query = dict(parse_qsl(url_parsed.query))
+        url_query.update({"pagina": page})
+        return url_parsed._replace(query=urlencode(url_query)).geturl()
 
+    def parse(self, response):
         for gazette in response.css(".list-item__info"):
             edition_number = gazette.css("h3::text").re_first(r"Edição nº (\d+)")
             gazette_raw_date = gazette.css("p::text").re_first(r"\d{2}/\d{2}/\d{4}")
             gazette_date = datetime.strptime(gazette_raw_date, "%d/%m/%Y").date()
-            gazette_url = gazette.css("a::attr(href)")
+            gazette_url = gazette.css("a::attr(href)").get()
 
             yield Gazette(
                 date=gazette_date,
@@ -40,3 +40,7 @@ def parse(self, response):
                 power="executive_legislative",
                 file_urls=[gazette_url],
             )
+
+        pages = response.css(".pagination__select option::text").getall()
+        for page in pages[1:]:
+            yield Request(self.pagination_url(response.url, page))