Run Scrapy crawler in a thread


/ Published in: Python
Save to your folder(s)



Copy this code and paste it in your HTML
  1. # When you run the Scrapy crawler from a program, the code blocks until the Scrapy crawler is finished. This is due to how Twisted (the underlying asynchronous network library) works. This prevents using the Scrapy crawler from scripts or other code.
  2. #
  3. # To circumvent this issue you can run the Scrapy crawler in a thread with this code.
  4. #
  5. # Keep in mind that this code is mainly for illustrative purposes and far from production ready.
  6. #
  7. # Also the code was only tested with Scrapy 0.8, and will probably need some adjustments for newer versions (since the core API isn't stable yet), but you get the idea.
  8.  
  9. """
  10. Code to run Scrapy crawler in a thread - works on Scrapy 0.8
  11. """
  12.  
  13. import threading, Queue
  14.  
  15. from twisted.internet import reactor
  16.  
  17. from scrapy.xlib.pydispatch import dispatcher
  18. from scrapy.core.manager import scrapymanager
  19. from scrapy.core.engine import scrapyengine
  20. from scrapy.core import signals
  21.  
  22. class CrawlerThread(threading.Thread):
  23.  
  24. def __init__(self):
  25. threading.Thread.__init__(self)
  26. self.running = False
  27.  
  28. def run(self):
  29. self.running = True
  30. scrapymanager.configure(control_reactor=False)
  31. scrapymanager.start()
  32. reactor.run(installSignalHandlers=False)
  33.  
  34. def crawl(self, *args):
  35. if not self.running:
  36. raise RuntimeError("CrawlerThread not running")
  37. self._call_and_block_until_signal(signals.spider_closed, \
  38. scrapymanager.crawl, *args)
  39.  
  40. def stop(self):
  41. reactor.callFromThread(scrapyengine.stop)
  42.  
  43. def _call_and_block_until_signal(self, signal, f, *a, **kw):
  44. q = Queue.Queue()
  45. def unblock():
  46. q.put(None)
  47. dispatcher.connect(unblock, signal=signal)
  48. reactor.callFromThread(f, *a, **kw)
  49. q.get()
  50.  
  51.  
  52. # Usage example below:
  53.  
  54. import os
  55. os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'myproject.settings')
  56.  
  57. from scrapy.xlib.pydispatch import dispatcher
  58. from scrapy.core import signals
  59. from scrapy.conf import settings
  60. from scrapy.crawler import CrawlerThread
  61.  
  62. settings.overrides['LOG_ENABLED'] = False # avoid log noise
  63.  
  64. def item_passed(item):
  65. print "Just scraped item:", item
  66.  
  67. dispatcher.connect(item_passed, signal=signals.item_passed)
  68.  
  69. crawler = CrawlerThread()
  70. print "Starting crawler thread..."
  71. crawler.start()
  72. print "Crawling somedomain.com...."
  73. crawler.crawl('somedomain.com) # blocking call
  74. print "Crawling anotherdomain.com..."
  75. crawler.crawl('anotherdomain.com') # blocking call
  76. print "Stopping crawler thread..."
  77. crawler.stop()
  78.  
  79. # Snippet imported from snippets.scrapy.org (which no longer works)
  80. # author: pablo
  81. # date : Aug 11, 2010
  82.  

Report this snippet


Comments

RSS Icon Subscribe to comments

You need to login to post a comment.