If Retry has a message, use it as retry reason (#202)

scrapy_poet/spidermiddlewares.py

@@ -24,7 +24,7 @@ def process_spider_exception(
  new_request_or_none = get_retry_request(
  response.request,
  spider=spider,
- reason="page_object_retry",
+ reason=str(exception) or "page_object_retry",
  )
  if not new_request_or_none:
  return []

tests/test_retries.py

@@ -53,6 +53,40 @@ def parse(self, response, page: SamplePage):
  _assert_all_unique_instances(page_response_instances)
 
 
+@inlineCallbacks
+def test_retry_reason():
+ retries = deque([True, False])
+ items, page_instances, page_response_instances = [], [], []
+
+ with MockServer(EchoResource) as server:
+
+ class SamplePage(WebPage):
+ def to_item(self):
+ page_instances.append(self)
+ page_response_instances.append(self.response)
+ if retries.popleft():
+ raise Retry("foo")
+ return {"foo": "bar"}
+
+ class TestSpider(BaseSpider):
+ def start_requests(self):
+ yield Request(server.root_url, callback=self.parse)
+
+ def parse(self, response, page: SamplePage):
+ items.append(page.to_item())
+
+ crawler = make_crawler(TestSpider)
+ yield crawler.crawl()
+
+ assert items == [{"foo": "bar"}]
+ assert crawler.stats.get_value("downloader/request_count") == 2
+ assert crawler.stats.get_value("retry/count") == 1
+ assert crawler.stats.get_value("retry/reason_count/foo") == 1
+ assert crawler.stats.get_value("retry/max_reached") is None
+ _assert_all_unique_instances(page_instances)
+ _assert_all_unique_instances(page_response_instances)
+
+
 @inlineCallbacks
 def test_retry_max():
  # The default value of the RETRY_TIMES Scrapy setting is 2.

tox.ini

@@ -94,7 +94,7 @@ commands = pre-commit run --all-files --show-diff-on-failure
 [testenv:twinecheck]
 basepython = python3
 deps =
- twine==4.0.2
+ twine==5.0.0
  build==0.10.0
 commands =
  python -m build --sdist