Return to Snippet

Revision: 59341
at September 1, 2012 07:15 by scrapy

Initial Code
# This is a tac file that can be used for launching Scrapy as a service (in the [Twisted Application Framework]( ) using twistd.
# You can start the service with:
#     twistd -ny scrapy-service.tac
# And then schedule spiders with:
#     scrapy queue add myspider

from twisted.application.service import Service, Application
from twisted.python import log as txlog

from scrapy import log
from scrapy.crawler import Crawler
from scrapy.conf import settings

class CrawlerService(Service):

    def startService(self):
        settings.overrides['QUEUE_CLASS'] = settings['SERVER_QUEUE_CLASS']
        self.crawler = Crawler(settings)

    def stopService(self):
        return self.crawler.stop()

def get_application(logfile, loglevel=log.DEBUG):
    app = Application("Scrapy")
    app.setComponent(txlog.ILogObserver, \
        log.ScrapyFileLogObserver(open(logfile, 'a'), loglevel).emit)
    return app

application = get_application('scrapy.log')

# Snippet imported from (which no longer works)
# author: pablo
# date  : Aug 26, 2010

Initial URL

Initial Description

Initial Title
scrapy-service.tac (Twisted Application Framework)

Initial Tags

Initial Language