RobotsTxtMiddleware: remove unused attribute

DamonGuo · Sep 22, 2014 · fe6f3ef · fe6f3ef
1 parent 7be3479
commit fe6f3ef
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -22,7 +22,6 @@ def __init__(self, crawler):
         self.crawler = crawler
         self._useragent = crawler.settings.get('USER_AGENT')
         self._parsers = {}
-        self._spider_netlocs = set()
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -45,7 +44,6 @@ def robot_parser(self, request, spider):
             robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
             dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
-            self._spider_netlocs.add(netloc)
         return self._parsers[netloc]
 
     def _parse_robots(self, response):