Updated for Scrapy v1.0 API changes

scrapy_webdriver/download.py

@@ -1,5 +1,7 @@
-from scrapy import log, version_info
-from scrapy.utils.decorator import inthread
+import logging
+
+from scrapy import version_info
+from scrapy.utils.decorators import inthread
 from scrapy.utils.misc import load_object
 
 from .http import WebdriverActionRequest, WebdriverRequest, WebdriverResponse
@@ -37,13 +39,13 @@ def download_request(self, request, spider):
     @inthread
     def _download_request(self, request, spider):
         """Download a request URL using webdriver."""
-        log.msg('Downloading %s with webdriver' % request.url, level=log.DEBUG)
+        logging.debug(u'Downloading %s with webdriver' % request.url)
         request.manager.webdriver.get(request.url)
         return WebdriverResponse(request.url, request.manager.webdriver)
 
     @inthread
     def _do_action_request(self, request, spider):
         """Perform an action on a previously webdriver-loaded page."""
-        log.msg('Running webdriver actions %s' % request.url, level=log.DEBUG)
+        logging.debug(u'Running webdriver actions %s' % request.url)
         request.actions.perform()
         return WebdriverResponse(request.url, request.manager.webdriver)

scrapy_webdriver/tests/test_manager.py

@@ -1,5 +1,6 @@
 from scrapy.crawler import Crawler
 from scrapy.settings import Settings
+from scrapy.spiders import CrawlSpider
 from selenium import webdriver
 
 from scrapy_webdriver.manager import WebdriverManager
@@ -29,19 +30,16 @@ class TestBrowser(object):
             pass
 
         settings = self.settings(WEBDRIVER_BROWSER='Firefox')
-        crawler = Crawler(Settings(values=settings))
-        crawler.configure()
+        crawler = Crawler(CrawlSpider, settings=Settings(values=settings))
         browser = WebdriverManager(crawler)
         assert issubclass(browser._browser, webdriver.Firefox)
 
         settings = self.settings(WEBDRIVER_BROWSER=TestBrowser)
-        crawler = Crawler(Settings(values=settings))
-        crawler.configure()
+        crawler = Crawler(CrawlSpider, settings=Settings(values=settings))
         browser = WebdriverManager(crawler)
         assert issubclass(browser._browser, TestBrowser)
 
         settings = self.settings(WEBDRIVER_BROWSER=TestBrowser())
-        crawler = Crawler(Settings(values=settings))
-        crawler.configure()
+        crawler = Crawler(CrawlSpider, settings=Settings(values=settings))
         browser = WebdriverManager(crawler)
         assert isinstance(browser._webdriver, TestBrowser)

scrapy_webdriver/tests/test_request_queue.py

@@ -1,14 +1,13 @@
 from functools import partial
+import logging
 from time import sleep
 
 from mock import call, Mock
-from scrapy.crawler import Crawler
+from scrapy.crawler import CrawlerProcess
 from scrapy.http import Request
-from scrapy import log, signals
+from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.spider import BaseSpider
-from scrapy.xlib.pydispatch import dispatcher
-from twisted.internet import reactor
 
 from scrapy_webdriver.http import WebdriverRequest
 
@@ -32,9 +31,6 @@ def settings(self, **options):
         settings.update(**options)
         return settings
 
-    def _stop_reactor(self):
-        reactor.stop()
-
     def _wait(self, url, *args, **kwargs):
         sleep(0.1)
 
@@ -44,15 +40,10 @@ def test_priorization(self):
         webdriver.get.side_effect = self._wait
         webdriver.page_source = u''
 
-        dispatcher.connect(self._stop_reactor, signal=signals.spider_closed)
-
-        crawler = Crawler(Settings(values=settings))
-        crawler.configure()
-        spider = self.Spider(name='test', domain='testdomain')
-        crawler.crawl(spider)
-        crawler.start()
-        log.start(loglevel='ERROR')
-        reactor.run()
+        process = CrawlerProcess(Settings(values=settings))
+        process.crawl(self.Spider, name='test', domain='testdomain')
+        logging.getLogger('scrapy').setLevel(logging.ERROR)
+        process.start()
 
         assert webdriver.get.mock_calls == [
             call('http://testdomain/path?wr=0'),