Merge pull request #74 from kayqueGovetri/ENH/undetectable-chrome

ENH: Implement undetectable chrome

Author: kayqueGovetri

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.10"]
-        browser: ["firefox", "chrome", "edge"]
+        browser: ["firefox", "chrome", "edge", "undetected_chrome"]
         headless: [true]
         exclude:
           # Can't install firefox using setup-firefox on Windows

botcity/web/bot.py

@@ -259,16 +259,41 @@ def check_driver():
         self.capabilities = cap
         driver_path = self.driver_path or check_driver()
         self.driver_path = driver_path
-        if compat.version_selenium_is_larger_than_four():
-            service = BROWSER_CONFIGS.get(self.browser).get("service")
-            service = service(executable_path=self.driver_path)
-            service.desired_capabilities = cap
-
-            self._driver = driver_class(options=opt, service=service)
-        else:
-            self._driver = driver_class(options=opt, desired_capabilities=cap, executable_path=driver_path)
+        self._driver = driver_class(**self._get_parameters_to_driver())
+        self._others_configurations()
         self.set_screen_resolution()
 
+    def _get_parameters_to_driver(self):
+        if self.browser == Browser.UNDETECTED_CHROME:
+            return {"driver_executable_path": self.driver_path, "options": self.options,
+                    "desired_capabilities": self.capabilities}
+        if compat.version_selenium_is_larger_than_four():
+            return {"options": self.options, "service": self._get_service()}
+
+        return {"options": self.options, "desired_capabilities": self.capabilities,
+                "executable_path": self.driver_path}
+
+    def _get_service(self):
+        service = BROWSER_CONFIGS.get(self.browser).get("service")
+        service = service(executable_path=self.driver_path)
+        service.desired_capabilities = self.capabilities
+        return service
+
+    def _others_configurations(self):
+        if self.browser == Browser.UNDETECTED_CHROME:
+            """
+            There is a problem in undetected chrome that prevents downloading files even passing
+            download_folder_path in preferences.
+            This solution is taken from the following issue
+            https://github.com/ultrafunkamsterdam/undetected-chromedriver/issues/260#issuecomment-901276808.
+            It will be a temporary solution.
+            """
+            params = {
+                "behavior": "allow",
+                "downloadPath": self.download_folder_path
+            }
+            self.driver.execute_cdp_cmd("Page.setDownloadBehavior", params)
+
     def stop_browser(self):
         """
         Stops the Chrome browser and clean up the User Data Directory.
@@ -1128,7 +1153,7 @@ def wait_for_downloads(self, timeout: int = 120000):
         Args:
             timeout (int, optional): Timeout in millis. Defaults to 120000.
         """
-        if self.browser in [Browser.CHROME, Browser.EDGE] and self.headless:
+        if self.browser in [Browser.CHROME, Browser.UNDETECTED_CHROME, Browser.EDGE] and self.headless:
             start_time = time.time()
             while True:
                 elapsed_time = (time.time() - start_time) * 1000

botcity/web/browsers/__init__.py

@@ -4,6 +4,7 @@
 from . import firefox
 from . import edge
 from . import ie
+from . import undetected_chrome
 
 
 class Browser(str, enum.Enum):
@@ -20,6 +21,7 @@ class Browser(str, enum.Enum):
     FIREFOX = "firefox"
     EDGE = "edge"
     IE = "ie"
+    UNDETECTED_CHROME = 'undetected_chrome'
 
 
 class PageLoadStrategy(str, enum.Enum):
@@ -55,6 +57,14 @@ class PageLoadStrategy(str, enum.Enum):
         "wait_for_downloads": firefox.wait_for_downloads,
         "service": firefox.FirefoxService
     },
+    Browser.UNDETECTED_CHROME: {
+        "driver": "chromedriver",
+        "class": undetected_chrome.Chrome,  # noqa: F401, F403
+        "options": undetected_chrome.default_options,
+        "capabilities": undetected_chrome.default_capabilities,
+        "wait_for_downloads": undetected_chrome.wait_for_downloads,
+        "service": undetected_chrome.ChromeService
+    },
     Browser.EDGE: {
         "driver": "msedgedriver",
         "class": edge.Edge,

botcity/web/browsers/undetected_chrome.py

@@ -0,0 +1,138 @@
+import atexit
+import json
+import os
+import tempfile
+from typing import Dict
+
+from undetected_chromedriver import Chrome  # noqa: F401, F403
+from undetected_chromedriver.options import ChromeOptions
+from undetected_chromedriver import Service as ChromeService  # noqa: F401, F403
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from ..util import cleanup_temp_dir
+
+
+def default_options(headless=False, download_folder_path=None, user_data_dir=None,
+                    page_load_strategy="normal") -> ChromeOptions:
+    """Retrieve the default options for this browser curated by BotCity.
+
+    Args:
+        headless (bool, optional): Whether or not to use the headless mode. Defaults to False.
+        download_folder_path (str, optional): The default path in which to save files.
+            If None, the current directory is used. Defaults to None.
+        user_data_dir ([type], optional): The directory to use as user profile.
+            If None, a new temporary directory is used. Defaults to None.
+        page_load_strategy (str, optional): The page load strategy. Defaults to "normal".
+
+    Returns:
+        ChromeOptions: The Chrome options.
+    """
+    chrome_options = ChromeOptions()
+    try:
+        page_load_strategy = page_load_strategy.value
+    except AttributeError:
+        page_load_strategy = page_load_strategy
+    chrome_options.page_load_strategy = page_load_strategy
+    chrome_options.add_argument("--remote-debugging-port=0")
+    chrome_options.add_argument("--no-first-run")
+    chrome_options.add_argument("--no-default-browser-check")
+    chrome_options.add_argument("--disable-background-networking")
+    chrome_options.add_argument("--disable-background-timer-throttling")
+    chrome_options.add_argument("--disable-client-side-phishing-detection")
+    chrome_options.add_argument("--disable-default-apps")
+    chrome_options.add_argument("--disable-hang-monitor")
+    chrome_options.add_argument("--disable-popup-blocking")
+    chrome_options.add_argument("--disable-prompt-on-repost")
+    chrome_options.add_argument("--disable-syncdisable-translate")
+    chrome_options.add_argument("--metrics-recording-only")
+    chrome_options.add_argument("--safebrowsing-disable-auto-update")
+
+    # Disable What's New banner for new chrome installs
+    chrome_options.add_argument("--disable-features=ChromeWhatsNewUI")
+
+    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
+
+    # Disable banner for Browser being remote-controlled
+    # chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
+    # chrome_options.add_experimental_option('useAutomationExtension', False)
+
+    if headless:
+        chrome_options.add_argument("--headless")
+        chrome_options.add_argument("--disable-gpu")
+        chrome_options.add_argument("--hide-scrollbars")
+        chrome_options.add_argument("--mute-audio")
+
+    # Check if user is root
+    try:
+        # This is only valid with Unix
+        if os.geteuid() == 0:
+            chrome_options.add_argument("--no-sandbox")
+    except AttributeError:
+        pass
+
+    if not user_data_dir:
+        temp_dir = tempfile.TemporaryDirectory(prefix="botcity_")
+        user_data_dir = temp_dir.name
+        atexit.register(cleanup_temp_dir, temp_dir)
+
+    chrome_options.add_argument(f"--user-data-dir={user_data_dir}")
+
+    if not download_folder_path:
+        download_folder_path = os.getcwd()
+
+    app_state = {
+        'recentDestinations': [{
+            'id': 'Save as PDF',
+            'origin': 'local',
+            'account': ''
+        }],
+        'selectedDestinationId': 'Save as PDF',
+        'version': 2
+    }
+
+    # Set the Downloads default folder
+    prefs = {
+        "printing.print_preview_sticky_settings.appState": json.dumps(app_state),
+        "download.default_directory": download_folder_path,
+        "savefile.default_directory": download_folder_path,
+        "printing.default_destination_selection_rules": {
+            "kind": "local",
+            "namePattern": "Save as PDF",
+        },
+        "safebrowsing.enabled": True,
+        "credentials_enable_service": False,
+        "profile.password_manager_enabled": False,
+        "plugins.always_open_pdf_externally": True
+    }
+
+    chrome_options.add_experimental_option("prefs", prefs)
+    chrome_options.add_argument(
+        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+    )
+
+    chrome_options.add_argument("--kiosk-printing")
+
+    return chrome_options
+
+
+def default_capabilities() -> Dict:
+    """Fetch the default capabilities for this browser.
+
+    Returns:
+        Dict: Dictionary with the default capabilities defined.
+    """
+    return DesiredCapabilities.CHROME.copy()
+
+
+def wait_for_downloads(driver):
+    """Wait for all downloads to finish.
+    *Important*: This method overwrites the current page with the downloads page.
+    """
+    if not driver.current_url.startswith("chrome://downloads"):
+        driver.get("chrome://downloads/")
+    return driver.execute_script("""
+        var items = document.querySelector('downloads-manager')
+            .shadowRoot.getElementById('downloadsList').items;
+        if (items.every(e => e.state === "COMPLETE"))
+            return items.map(e => e.fileUrl || e.file_url);
+        """)

conftest.py

@@ -2,6 +2,7 @@
 import json
 import shutil
 import tempfile
+import time
 
 import pytest
 import typing
@@ -18,6 +19,12 @@
 TEST_PAGE = "https://lf2a.github.io/webpage-test/test.html"
 INDEX_PAGE = "https://lf2a.github.io/webpage-test/"
 
+platforms = {
+    "Linux": "linux",
+    "Darwin": "mac",
+    "Windows": "windows"
+}
+
 
 def get_fake_bin_path(web: WebBot) -> str:
     return os.path.join(web.download_folder_path, 'fake.bin')
@@ -32,6 +39,15 @@ def setup_chrome(headless: bool, tmp_folder: str, download_driver: str) -> WebBo
     return web
 
 
+def setup_undetected_chrome(headless: bool, tmp_folder: str, download_driver: str) -> WebBot:
+    web = WebBot(headless)
+    web.browser = Browser.UNDETECTED_CHROME
+
+    web.driver_path = download_driver
+    web.download_folder_path = tmp_folder
+    return web
+
+
 def setup_firefox(headless: bool, tmp_folder: str, download_driver: str) -> WebBot:
     web = WebBot(headless)
     web.browser = Browser.FIREFOX
@@ -49,11 +65,6 @@ def setup_edge(headless: bool, tmp_folder: str, download_driver: str) -> WebBot:
     web.driver_path = download_driver
     web.download_folder_path = tmp_folder
     opt = browsers.edge.default_options(headless=headless, download_folder_path=tmp_folder)
-    platforms = {
-        "Linux": "linux",
-        "Darwin": "mac",
-        "Windows": "windows"
-    }
     platform_name = platforms.get(platform.system())
 
     opt.platform_name = platform_name
@@ -66,7 +77,8 @@ def factory_setup_browser(browser: str, is_headless: bool, tmp_folder: str, down
     dict_browsers = {
         'chrome': setup_chrome,
         'firefox': setup_firefox,
-        'edge': setup_edge
+        'edge': setup_edge,
+        'undetected_chrome': setup_undetected_chrome
     }
 
     setup_browser = dict_browsers.get(browser, None)
@@ -81,7 +93,8 @@ def factory_driver_manager(browser: str):
     dict_driver_manager = {
         'chrome': ChromeDriverManager,
         'firefox': GeckoDriverManager,
-        'edge': EdgeChromiumDriverManager
+        'edge': EdgeChromiumDriverManager,
+        'undetected_chrome': ChromeDriverManager
     }
 
     driver_manager = dict_driver_manager.get(browser, None)
@@ -106,6 +119,9 @@ def download_driver(request):
     manager = factory_driver_manager(browser=browser)
     installed_driver = manager(path=folder_driver).install()
     yield installed_driver
+    # Issue: https://github.com/ultrafunkamsterdam/undetected-chromedriver/issues/551
+    if platforms.get(platform.system()) == "windows" and browser == Browser.UNDETECTED_CHROME:
+        time.sleep(3)
     shutil.rmtree(folder_driver)
 
 
@@ -128,4 +144,3 @@ def get_event_result(id_event: str, web: WebBot) -> typing.Dict:
 def pytest_addoption(parser):
     parser.addoption('--headless', action='store', default="true")
     parser.addoption('--browser', action='store', default='chrome')
-

requirements.txt

@@ -3,4 +3,6 @@ beautifulsoup4
 numpy
 opencv-python
 pillow
-selenium==4.6.1
+selenium>=4.6.1,<5
+undetected-chromedriver
+packaging

tests/test_browser.py

@@ -1,9 +1,11 @@
 import os
+import platform
+
 import pytest
 import conftest
 
 from PIL import Image
-from botcity.web import WebBot, By
+from botcity.web import WebBot, By, Browser
 
 
 def test_context(web: WebBot):
@@ -201,8 +203,12 @@ def test_leave_iframe(web: WebBot):
 def test_get_view_port_size(web: WebBot):
     web.browse(conftest.INDEX_PAGE)
     size = web.get_viewport_size()
-
-    element = web.find_element('window-size', By.ID).text.split('x')
+    if web.browser == Browser.UNDETECTED_CHROME and conftest.platforms.get(platform.system()) == 'mac':
+        width = web.execute_javascript("return window.innerWidth")
+        height = web.execute_javascript("return window.innerHeight")
+        element = [width, height]
+    else:
+        element = web.find_element('window-size', By.ID).text.split('x')
     assert size == tuple(int(e) for e in element)