Return to Snippet

Revision: 59324
at September 1, 2012 07:15 by scrapy

Initial Code
# This recipe was taken from the old wiki.
# You can make Scrapy send HTTP/1.1 requests by overriding the Scrapy HTTP Client Factory, with the following (undocumented) setting:
#     DOWNLOADER_HTTPCLIENTFACTORY = 'myproject.downloader.HTTPClientFactory'

from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory, ScrapyHTTPPageGetter

class PageGetter(ScrapyHTTPPageGetter):

    def sendCommand(self, command, path):
        self.transport.write('%s %s HTTP/1.1
' % (command, path))

class HTTPClientFactory(ScrapyHTTPClientFactory):

     protocol = PageGetter

# Snippet imported from (which no longer works)
# author: pablo
# date  : Sep 16, 2011

Initial URL

Initial Description

Initial Title
Spoof requests as HTTP/1.1

Initial Tags

Initial Language