Advanced Search

Middleware to avoid re-visiting already visited items

/ Published in: Python
Save to your folder(s)

Expand | Embed | Plain Text

Copy this code and paste it in your HTML 
  1. # This middleware can be used to avoid re-visiting already visited items, which can be useful for speeding up the scraping for projects with immutable items, ie. items that, once scraped, don't change.
  2.  
  3. from scrapy import log
  4. from scrapy.http import Request
  5. from scrapy.item import BaseItem
  6. from scrapy.utils.request import request_fingerprint
  7.  
  8. from myproject.items import MyItem
  9.  
  10. class IgnoreVisitedItems(object):
  11.     """Middleware to ignore re-visiting item pages if they were already visited
  12.     before. The requests to be filtered by have a meta['filter_visited'] flag
  13.     enabled and optionally define an id to use for identifying them, which
  14.     defaults the request fingerprint, although you'd want to use the item id,
  15.     if you already have it beforehand to make it more robust.
  16.     """
  17.  
  18.     FILTER_VISITED = 'filter_visited'
  19.     VISITED_ID = 'visited_id'
  20.     CONTEXT_KEY = 'visited_ids'
  21.  
  22.     def process_spider_output(self, response, result, spider):
  23.         context = getattr(spider, 'context', {})
  24.         visited_ids = context.setdefault(self.CONTEXT_KEY, {})
  25.         ret = []
  26.         for x in result:
  27.             visited = False
  28.             if isinstance(x, Request):
  29.                 if self.FILTER_VISITED in x.meta:
  30.                     visit_id = self._visited_id(x)
  31.                     if visit_id in visited_ids:
  32.                         log.msg("Ignoring already visited: %s" % x.url,
  33.                                 level=log.INFO, spider=spider)
  34.                         visited = True
  35.             elif isinstance(x, BaseItem):
  36.                 visit_id = self._visited_id(response.request)
  37.                 if visit_id:
  38.                     visited_ids[visit_id] = True
  39.                     x['visit_id'] = visit_id
  40.                     x['visit_status'] = 'new'
  41.             if visited:
  42.                 ret.append(MyItem(visit_id=visit_id, visit_status='old'))
  43.             else:
  44.                 ret.append(x)
  45.         return ret
  46.  
  47.     def _visited_id(self, request):
  48.         return request.meta.get(self.VISITED_ID) or request_fingerprint(request)
  49.  
  50. # Snippet imported from snippets.scrapy.org (which no longer works)
  51. # author: pablo
  52. # date  : Aug 10, 2010
  53.  

Report this snippet


Comments

RSS Icon Subscribe to comments

You need to login to post a comment.