Advanced Search

Return to Snippet

Revision: 59326
at September 1, 2012 07:15 by scrapy
Initial Code 
# This is a downloader middleware that can be used to get rendered javascript pages using webkit.
# 
# this could be extended to handle form requests and load errors, but this is the bare bones code to get it done.
# 
# the advantage over the selenium based approachs I've seen is that it only makes one request and you don't have to set up selenium.

from scrapy.http import Request, FormRequest, HtmlResponse

import gtk
import webkit
import jswebkit

class WebkitDownloader( object ):
    def process_request( self, request, spider ):
        if( type(request) is not FormRequest ):
            webview = webkit.WebView()
            webview.connect( 'load-finished', lambda v,f: gtk.main_quit() )
            webview.load_uri( request.url )
            gtk.main()
            js = jswebkit.JSContext( webview.get_main_frame().get_global_context() )
            renderedBody = str( js.EvaluateScript( 'document.documentElement.innerHTML' ) )
            return HtmlResponse( request.url, body=renderedBody )

# Snippet imported from snippets.scrapy.org (which no longer works)
# author: jdwilson
# date  : Sep 06, 2011
Initial URL 

                                

                                                            

                                    

                                        

                                        Initial Description
                                        

                                    

                                    

                                

                                                            

                                    

                                        

                                        Initial Title
                                        

                                    

                                    
Rendered/interactive javascript with gtk/webkit/jswebkit

                                

                                                            

                                    

                                        

                                        Initial Tags
                                        

                                    

                                    
javascript

                                

                                                            

                                    

                                        

                                        Initial Language
                                        

                                    

                                    
Python