Merge branch 'develop'

Author: tijme

.gitignore

@@ -1,6 +1,7 @@
 # Custom
 acstis/.semver
 ghostdriver.log
+extended_test.py
 
 # OS
 Thumbs.db

.semver

@@ -1 +1 @@
-3.0.1
+3.0.2

acstis/Driver.py

@@ -33,6 +33,7 @@
 from nyawc.CrawlerActions import CrawlerActions
 from nyawc.http.Request import Request
 from nyawc.http.Response import Response
+from nyawc.helpers.HTTPRequestHelper import HTTPRequestHelper
 from acstis.helpers.BrowserHelper import BrowserHelper
 from acstis.helpers.PackageHelper import PackageHelper
 from acstis.Scanner import Scanner
@@ -90,9 +91,12 @@ def __signal_handler(self, signum, frame):
 
         colorlog.getLogger().warning("Received SIGINT, stopping the crawling threads safely. This could take up to 30 seconds (the thread timeout).")
 
-    def __set_angular_version(self):
+    def __set_angular_version(self, startpoint):
         """Find and set the AngularJS version as class attribute
 
+        Args:
+            startpoint (:class:`nyawc.http.Request`): The startpoint request.
+
         Returns:
             str: True if found and set, False otherwise.
 
@@ -107,7 +111,7 @@ def __set_angular_version(self):
         colorlog.getLogger().info("Waiting until DOM is completely loaded.")
 
         self.__angular_version = BrowserHelper.javascript(
-            QueueItem(Request(self.__args.domain), Response(self.__args.domain)),
+            QueueItem(startpoint, Response(self.__args.domain)),
             "return angular.version.full"
         )
 
@@ -122,11 +126,13 @@ def __set_angular_version(self):
     def start(self):
         """Start the crawler."""
 
-        if self.__set_angular_version():
+        startpoint = Request(self.__args.domain)
+        HTTPRequestHelper.patch_with_options(startpoint, self.__options)
+
+        if self.__set_angular_version(startpoint):
             crawler = Crawler(self.__options)
             signal.signal(signal.SIGINT, self.__signal_handler)
 
-            startpoint = Request(self.__args.domain)
             crawler.start_with(startpoint)
 
         # Exit the process with the correct status code

acstis/helpers/BrowserHelper.py

@@ -28,10 +28,12 @@
 import stat
 import ctypes
 import colorlog
+import requests
 import requests.cookies
 
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from nyawc.helpers.HTTPRequestHelper import HTTPRequestHelper
 from nyawc.helpers.URLHelper import URLHelper
 from nyawc.http.Request import Request
 
@@ -116,25 +118,37 @@ def __get_browser(queue_item=None):
 
         if queue_item:
 
-            # Authentication
+            # Add authentication header to request
             if queue_item.request.auth:
                 queue_item.request.auth(queue_item.request)
 
-            # Headers
+            # Add cookie header to request
+            if queue_item.request.cookies:
+                cookie_string = HTTPRequestHelper.get_cookie_header(queue_item)
+                queue_item.request.headers["Cookie"] = cookie_string
+
+            # Add headers to PhantomJS
             if queue_item.request.headers:
+                default_headers = requests.utils.default_headers()
                 for (key, value) in queue_item.request.headers.items():
                     if key.lower() == "user-agent":
                         capabilities["phantomjs.page.settings.userAgent"] = value
                     else:
-                        capabilities["phantomjs.page.settings." + key] = value
+                        
+                        # PhantomJS has issues with executing JavaScript on pages with GZIP encoding.
+                        # See link for more information (https://github.com/detro/ghostdriver/issues/489).
+                        if key == "Accept-Encoding" and "gzip" in value:
+                            continue
+
+                        capabilities["phantomjs.page.customHeaders." + key] = value
 
             # Proxies
             if queue_item.request.proxies:
                 service.extend(BrowserHelper.__proxies_to_service_args(queue_item.request.proxies))
 
-        driver = BrowserHelper.__get_phantomjs_driver()
+        driver_path = BrowserHelper.__get_phantomjs_driver()
         return webdriver.PhantomJS(
-            executable_path=driver,
+            executable_path=driver_path,
             desired_capabilities=capabilities,
             service_args=service
         )

acstis/phantomjs/mac-2.1.1

@@ -1,5 +1,5 @@
 colorlog==2.10.0
-nyawc==1.7.8
+nyawc==1.7.10
 requests==2.18.1
 requests_toolbelt==0.8.0
 selenium==3.4.3