to 0011

Author: Germey

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Gerapy Pyppeteer Changelog
 
+## 0.0.11 (2020-08-05)
+
+### Bug Fixes
+
+* Fix bug about `asyncio` in Python 3.8 on Windows [https://github.com/Gerapy/GerapyPyppeteer/issues/5](https://github.com/Gerapy/GerapyPyppeteer/issues/5)
+* Fix bug of setting cookies [https://github.com/Gerapy/GerapyPyppeteer/issues/11](https://github.com/Gerapy/GerapyPyppeteer/issues/11)
+
+### Features
+
+* Add settings of `GERAPY_ENABLE_REQUEST_INTERCEPTION` [https://github.com/Gerapy/GerapyPyppeteer/issues/6](https://github.com/Gerapy/GerapyPyppeteer/issues/6)
+
 ## 0.0.10 (2020-08-01)
 
 ### Features

example/example/spiders/movie.py

@@ -22,7 +22,10 @@ def start_requests(self):
         for page in range(1, self.max_page + 1):
             url = f'{self.base_url}/page/{page}'
             logger.debug('start url %s', url)
-            yield PyppeteerRequest(url, callback=self.parse_index, priority=10, wait_for='.item', pretend=True)
+            cookies = {
+                'name': 'germey'
+            }
+            yield PyppeteerRequest(url, callback=self.parse_index, priority=10, wait_for='.item', pretend=True, cookies=cookies)
 
     def parse_index(self, response):
         """

gerapy_pyppeteer/__version__.py

@@ -1,3 +1,3 @@
-VERSION = (0, 0, '10')
+VERSION = (0, 0, '11')
 
 version = __version__ = '.'.join(map(str, VERSION))

gerapy_pyppeteer/downloadermiddlewares.py

@@ -1,7 +1,6 @@
 import sys
 import asyncio
 from io import BytesIO
-
 from pyppeteer.errors import PageError, TimeoutError
 from scrapy.http import HtmlResponse
 import twisted.internet
@@ -11,6 +10,10 @@
 from pyppeteer import launch
 from gerapy_pyppeteer.pretend import SCRIPTS as PRETEND_SCRIPTS
 from gerapy_pyppeteer.settings import *
+import urllib.parse
+
+if sys.platform == 'win32':
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
 reactor = AsyncioSelectorReactor(asyncio.get_event_loop())
 
@@ -117,6 +120,8 @@ def from_crawler(cls, crawler):
         cls.screenshot = settings.get('GERAPY_PYPPETEER_SCREENSHOT', GERAPY_PYPPETEER_SCREENSHOT)
         cls.pretend = settings.get('GERAPY_PYPPETEER_PRETEND', GERAPY_PYPPETEER_PRETEND)
         cls.sleep = settings.get('GERAPY_PYPPETEER_SLEEP', GERAPY_PYPPETEER_SLEEP)
+        cls.enable_request_interception = settings.getbool('GERAPY_ENABLE_REQUEST_INTERCEPTION',
+                                                           GERAPY_ENABLE_REQUEST_INTERCEPTION)
         cls.retry_enabled = settings.getbool('RETRY_ENABLED')
         cls.max_retry_times = settings.getint('RETRY_TIMES')
         cls.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
@@ -198,16 +203,20 @@ async def _process_request(self, request, spider):
                 await page.evaluateOnNewDocument(script)
 
         # set cookies
+        parse_result = urllib.parse.urlsplit(request.url)
+        domain = parse_result.hostname
+        _cookies = []
         if isinstance(request.cookies, dict):
-            await page.setCookie(*[
-                {'name': k, 'value': v}
-                for k, v in request.cookies.items()
-            ])
+            _cookies = [{'name': k, 'value': v, 'domain': domain}
+                        for k, v in request.cookies.items()]
         else:
-            await page.setCookie(request.cookies)
+            for _cookie in _cookies:
+                if isinstance(_cookie, dict) and 'domain' not in _cookie.keys():
+                    _cookie['domain'] = domain
+        await page.setCookie(*_cookies)
 
         # the headers must be set using request interception
-        await page.setRequestInterception(True)
+        await page.setRequestInterception(self.enable_request_interception)
 
         @page.on('request')
         async def _handle_interception(pu_request):

gerapy_pyppeteer/settings.py

@@ -38,3 +38,6 @@
 GERAPY_PYPPETEER_IGNORE_RESOURCE_TYPES = []
 GERAPY_PYPPETEER_SCREENSHOT = None
 GERAPY_PYPPETEER_SLEEP = 1
+GERAPY_ENABLE_REQUEST_INTERCEPTION = True
+
+