We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 2a1822f commit 864b07aCopy full SHA for 864b07a
misc/recurse.py
@@ -50,13 +50,13 @@ def recurse(url):
50
recurse(link)
51
52
if __name__ == '__main__':
53
- url = 'http://www.google.com/'
54
- url = 'http://www.suse.com/'
+ #url = 'http://www.google.com/'
+ #url = 'http://www.suse.com/'
55
url = 'http://www.redhat.com/'
56
site = urlparse(url).netloc
57
- #http = httplib2.Http()
58
- proxy = httplib2.proxy_info_from_url('http://squid:3128')
59
- http = httplib2.Http(proxy_info=proxy)
+ http = httplib2.Http()
+ #proxy = httplib2.proxy_info_from_url('http://squid:3128')
+ #http = httplib2.Http(proxy_info=proxy)
60
browser = webbrowser.get();
61
robots = robotparser.RobotFileParser(urljoin(url, '/robots.txt'))
62
robots.read()
0 commit comments