diff --git a/tests/mockserver.py b/tests/mockserver.py index 2c0ad66fba5..b73208c5cad 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,10 +1,10 @@ from __future__ import print_function import sys, time, random, urllib, os, json +import six from subprocess import Popen, PIPE -from twisted.web.server import Site, NOT_DONE_YET, GzipEncoderFactory -from twisted.web.resource import Resource, EncodingResourceWrapper +from twisted.web.server import Site, NOT_DONE_YET +from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl -from twisted.web.test.test_webclient import PayloadResource from scrapy import twisted_version @@ -168,8 +168,13 @@ def __init__(self): self.putChild("drop", Drop()) self.putChild("raw", Raw()) self.putChild("echo", Echo()) - self.putChild('payload', PayloadResource()) - self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + + if six.PY2 and twisted_version > (12, 3, 0): + from twisted.web.test.test_webclient import PayloadResource + from twisted.web.server import GzipEncoderFactory + from twisted.web.resource import EncodingResourceWrapper + self.putChild('payload', PayloadResource()) + self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 127c4a4bda2..9021af3b403 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,6 @@ import os import twisted +import six from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -287,23 +288,28 @@ def test_download(self): @defer.inlineCallbacks def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider) - body = '1'*100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) - yield crawler.crawl(seed=request) - failure = crawler.spider.meta['failure'] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') + if six.PY2 and twisted_version > (12, 3, 0): + + crawler = get_crawler(SingleRequestSpider) + body = '1'*100 # PayloadResource requires body length to be 100 + request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) + yield crawler.crawl(seed=request) + failure = crawler.spider.meta['failure'] + # download_maxsize < 100, hence the CancelledError + self.assertIsInstance(failure.value, defer.CancelledError) + + request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') + yield crawler.crawl(seed=request) + + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') + else: + raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x") class UriResource(resource.Resource):