From 011cd9d141c0fde4476ace175fddd753fa944fc0 Mon Sep 17 00:00:00 2001 From: bayu Date: Mon, 2 Oct 2017 18:48:41 +0700 Subject: [PATCH] add DropItem for incomplete item --- scraper/code/crawler/pipelines/completeness.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scraper/code/crawler/pipelines/completeness.py b/scraper/code/crawler/pipelines/completeness.py index 3c4e27f..e56c785 100644 --- a/scraper/code/crawler/pipelines/completeness.py +++ b/scraper/code/crawler/pipelines/completeness.py @@ -46,5 +46,6 @@ def process_item(self, item, spider): if not complete: self.exporter.export_item(item) + raise DropItem("Missing one or more element in %s" % item['url']) return item