ChromeDriverManager & Selenium4+ support for SeleniumMiddleware when no SELENIUM_DRIVER_EXECUTABLE_PATH | SELENIUM_COMMAND_EXECUTOR

Author: ton77v

scrapy_selenium/middlewares.py

@@ -9,7 +9,6 @@
 
 from .http import SeleniumRequest
 
-
 class SeleniumMiddleware:
     """Scrapy middleware handling the requests using selenium"""
 
@@ -64,6 +63,17 @@ def __init__(self, driver_name, driver_executable_path,
             capabilities = driver_options.to_capabilities()
             self.driver = webdriver.Remote(command_executor=command_executor,
                                            desired_capabilities=capabilities)
+        # webdriver-manager
+        else:
+            # selenium4+ & webdriver-manager
+            from selenium import webdriver
+            from webdriver_manager.chrome import ChromeDriverManager
+            from selenium.webdriver.chrome.service import Service as ChromeService
+            if driver_name and driver_name.lower() == 'chrome':
+                # options = webdriver.ChromeOptions()
+                # options.add_argument(o)
+                self.driver = webdriver.Chrome(options=driver_options,
+                                               service=ChromeService(ChromeDriverManager().install()))
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -78,7 +88,8 @@ def from_crawler(cls, crawler):
         if driver_name is None:
             raise NotConfigured('SELENIUM_DRIVER_NAME must be set')
 
-        if driver_executable_path is None and command_executor is None:
+        # let's use webdriver-manager when nothing specified instead | RN just for Chrome
+        if (driver_name.lower() != 'chrome') and (driver_executable_path is None and command_executor is None):
             raise NotConfigured('Either SELENIUM_DRIVER_EXECUTABLE_PATH '
                                 'or SELENIUM_COMMAND_EXECUTOR must be set')
 

tests/test_cases.py

@@ -21,9 +21,16 @@ def parse(self, response):
     def setUpClass(cls):
         """Create a scrapy process and a spider class to use in the tests"""
 
+        '''
         cls.settings = {
             'SELENIUM_DRIVER_NAME': 'firefox',
             'SELENIUM_DRIVER_EXECUTABLE_PATH': which('geckodriver'),
             'SELENIUM_DRIVER_ARGUMENTS': ['-headless']
         }
+        '''
+        cls.settings = {
+            'SELENIUM_DRIVER_NAME': 'chrome',
+            'SELENIUM_DRIVER_EXECUTABLE_PATH': None,
+            'SELENIUM_DRIVER_ARGUMENTS': ['--headless']
+        }
         cls.spider_klass = cls.SimpleSpider