From fe6f3efe95eb9f0a33154d239f9b1a6b1c02028d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 22 Sep 2014 22:56:54 +0600 Subject: [PATCH] RobotsTxtMiddleware: remove unused attribute --- scrapy/contrib/downloadermiddleware/robotstxt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index f1e8012e731..f9d1396fb11 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -22,7 +22,6 @@ def __init__(self, crawler): self.crawler = crawler self._useragent = crawler.settings.get('USER_AGENT') self._parsers = {} - self._spider_netlocs = set() @classmethod def from_crawler(cls, crawler): @@ -45,7 +44,6 @@ def robot_parser(self, request, spider): robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) - self._spider_netlocs.add(netloc) return self._parsers[netloc] def _parse_robots(self, response):