Fix failure in appearances crawler (#82)

Fixes #81
dcaribou · Jan 10, 2024 · 34a2a3e · 34a2a3e
1 parent 266d6f6
commit 34a2a3e
Show file tree

Hide file tree

Showing 6 changed files with 571 additions and 403 deletions.
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.11.7
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -10,7 +10,7 @@ python = "^3.8"
 inflection = "^0.5.1"
 Protego = "^0.2.1"
 python-dateutil = "^2.8.2"
-Scrapy = "2.7.1"
+scrapy = "^2.11.0"
 
 [tool.poetry.dev-dependencies]
 

diff --git a/samples/players.json b/samples/players.json
diff --git a/tfmkt/spiders/appearances.py b/tfmkt/spiders/appearances.py
@@ -93,8 +93,9 @@ def parse_stats_elem(elem):
 
     # stats tables are 'responsive-tables' (except the first one, which is
     # a summary table)
+
     competitions = response.css(
-        'div.table-header > a::attr(name)'
+        'div.content-box-headline > a::attr(name)'
     ).getall()
     stats_tables = response.css('div.responsive-table')[1:]
     assert(len(competitions) == len(stats_tables))

diff --git a/tfmkt/spiders/competitions.py b/tfmkt/spiders/competitions.py
@@ -70,7 +70,7 @@ def parse_competitions(self, response, base):
     """Parse competitions from the country competitions page.
 
     @url https://www.transfermarkt.co.uk/wettbewerbe/national/wettbewerbe/157
-    @returns items 2 2
+    @returns items 3 3
     @cb_kwargs {"base": {"href": "some_href/3", "type": "competition", "parent": {}, "country_id": 1, "country_name": "n", "country_code": "CC"}}
     @scrapes type href parent country_id country_name country_code competition_type
     """