Merge pull request scrapy#832 from ivannotes/master

Bugfix for leaking Proxy-Authorization header to targeted host

scrapy/core/downloader/handlers/http11.py

@@ -166,6 +166,8 @@ def download_request(self, request):
         url = urldefrag(request.url)[0]
         method = request.method
         headers = TxHeaders(request.headers)
+        if isinstance(agent, self._TunnelingAgent):
+            headers.removeHeader('Proxy-Authorization')
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
 
         start_time = time()

tests/test_proxy_connect.py

@@ -1,5 +1,5 @@
+import json
 import os
-import subprocess
 import time
 
 from threading import Thread
@@ -9,13 +9,11 @@
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import get_testlog, docrawl
-from tests.spiders import SimpleSpider
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
 from tests.mockserver import MockServer
 
 
-
-
-
 class HTTPSProxy(controller.Master, Thread):
 
     def __init__(self, port):
@@ -79,6 +77,15 @@ def test_https_tunnel_auth_error(self):
         self._assert_got_tunnel_error()
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
+    @defer.inlineCallbacks
+    def test_https_tunnel_without_leak_proxy_authorization_header(self):
+        request = Request("https://localhost:8999/echo")
+        spider = SingleRequestSpider(seed=request)
+        yield docrawl(spider)
+        self._assert_got_response_code(200)
+        echo = json.loads(spider.meta['responses'][0].body)
+        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect'