Merge pull request #73 from kayqueGovetri/ENH/selenium-4

ENH: Implement selenium 4

Author: kayqueGovetri

botcity/web/__init__.py

@@ -1,7 +1,11 @@
-from .bot import WebBot, Browser, BROWSER_CONFIGS, By, PageLoadStrategy  # noqa: F401, F403
-from .parsers import table_to_dict, data_from_row, sanitize_header  # noqa: F401, F403
-from .util import element_as_select  # noqa: F401, F403
+# PATCH Selenium for compat
+from .compat import patch_selenium  # noqa: F401, F403, E402
+patch_selenium()  # noqa: F401, F403, E402
 
-from botcity.web._version import get_versions
+from .bot import WebBot, Browser, BROWSER_CONFIGS, By, PageLoadStrategy  # noqa: F401, F403, E402
+from .parsers import table_to_dict, data_from_row, sanitize_header  # noqa: F401, F403, E402
+from .util import element_as_select  # noqa: F401, F403, E402
+
+from botcity.web._version import get_versions  # noqa: F401, F403, E402
 __version__ = get_versions()['version']
 del get_versions

botcity/web/bot.py

@@ -26,7 +26,7 @@
 from selenium.webdriver.support.wait import WebDriverWait, TimeoutException, NoSuchElementException
 from selenium.webdriver.support import expected_conditions as EC
 
-from . import config, cv2find
+from . import config, cv2find, compat
 from .browsers import BROWSER_CONFIGS, Browser, PageLoadStrategy
 
 try:
@@ -235,7 +235,6 @@ def start_browser(self):
         """
         Starts the selected browser.
         """
-
         def check_driver():
             # Look for driver
             driver_name = BROWSER_CONFIGS.get(self.browser).get("driver")
@@ -260,8 +259,14 @@ def check_driver():
         self.capabilities = cap
         driver_path = self.driver_path or check_driver()
         self.driver_path = driver_path
+        if compat.version_selenium_is_larger_than_four():
+            service = BROWSER_CONFIGS.get(self.browser).get("service")
+            service = service(executable_path=self.driver_path)
+            service.desired_capabilities = cap
 
-        self._driver = driver_class(options=opt, desired_capabilities=cap, executable_path=driver_path)
+            self._driver = driver_class(options=opt, service=service)
+        else:
+            self._driver = driver_class(options=opt, desired_capabilities=cap, executable_path=driver_path)
         self.set_screen_resolution()
 
     def stop_browser(self):
@@ -1368,14 +1373,14 @@ def mouse_move(self, x, y):
         if self.browser == Browser.FIREFOX:
             # Reset coordinates if the page has gone stale. Only required for Firefox
             if self._html_elem is None:
-                self._html_elem = self._driver.find_element_by_tag_name('body')
+                self._html_elem = self._driver.find_element(By.TAG_NAME, 'body')
                 self._x = 0
                 self._y = 0
             else:
                 try:
                     self._html_elem.is_enabled()
                 except StaleElementReferenceException:
-                    self._html_elem = self._driver.find_element_by_tag_name('body')
+                    self._html_elem = self._driver.find_element(By.TAG_NAME, 'body')
                     self._x = 0
                     self._y = 0
 

botcity/web/browsers/__init__.py

@@ -44,27 +44,31 @@ class PageLoadStrategy(str, enum.Enum):
         "class": chrome.Chrome,
         "options": chrome.default_options,
         "capabilities": chrome.default_capabilities,
-        "wait_for_downloads": chrome.wait_for_downloads
+        "wait_for_downloads": chrome.wait_for_downloads,
+        "service": chrome.ChromeService,
     },
     Browser.FIREFOX: {
         "driver": "geckodriver",
         "class": firefox.Firefox,
         "options": firefox.default_options,
         "capabilities": firefox.default_capabilities,
-        "wait_for_downloads": firefox.wait_for_downloads
+        "wait_for_downloads": firefox.wait_for_downloads,
+        "service": firefox.FirefoxService
     },
     Browser.EDGE: {
         "driver": "msedgedriver",
         "class": edge.Edge,
         "options": edge.default_options,
         "capabilities": edge.default_capabilities,
-        "wait_for_downloads": edge.wait_for_downloads
+        "wait_for_downloads": edge.wait_for_downloads,
+        "service": edge.EdgeService
     },
     Browser.IE: {
         "driver": "IEDriverServer",
         "class": ie.Ie,
         "options": ie.default_options,
         "capabilities": ie.default_capabilities,
-        "wait_for_downloads": ie.wait_for_downloads
+        "wait_for_downloads": ie.wait_for_downloads,
+        "service": ie.IeService
     },
 }

botcity/web/browsers/chrome.py

@@ -7,7 +7,7 @@
 from selenium.webdriver import Chrome  # noqa: F401, F403
 from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-
+from selenium.webdriver.chrome.service import Service as ChromeService  # noqa: F401, F403
 from ..util import cleanup_temp_dir
 
 

botcity/web/browsers/edge.py

@@ -5,9 +5,9 @@
 import time
 from typing import Dict
 
-from msedge.selenium_tools import Edge, EdgeOptions  # noqa: F401, F403
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-
+from selenium.webdriver import Edge, EdgeOptions  # noqa: F401, F403
+from selenium.webdriver.edge.service import Service as EdgeService  # noqa: F401, F403
 from ..util import cleanup_temp_dir
 
 

botcity/web/browsers/firefox.py

@@ -3,10 +3,10 @@
 import tempfile
 from typing import Dict
 
-from selenium import webdriver
 from selenium.webdriver import Firefox  # noqa: F401, F403
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.firefox.service import Service as FirefoxService  # noqa: F401, F403
 
 from ..util import cleanup_temp_dir
 
@@ -362,23 +362,19 @@ def default_options(headless=False, download_folder_path=None, user_data_dir=Non
         temp_dir = tempfile.TemporaryDirectory(prefix="botcity_")
         user_data_dir = temp_dir.name
         atexit.register(cleanup_temp_dir, temp_dir)
-    firefox_profile = webdriver.FirefoxProfile(user_data_dir)
-    firefox_profile.set_preference("security.default_personal_cert", "Select Automatically")
-    firefox_profile.set_preference('browser.download.folderList', 2)
-    firefox_profile.set_preference('browser.download.manager.showWhenStarting', False)
+    firefox_options.set_preference("profile", user_data_dir)
+    firefox_options.set_preference("security.default_personal_cert", "Select Automatically")
+    firefox_options.set_preference('browser.download.folderList', 2)
+    firefox_options.set_preference('browser.download.manager.showWhenStarting', False)
     if not download_folder_path:
         download_folder_path = os.getcwd()
-    firefox_profile.set_preference('browser.download.dir', download_folder_path)
-    firefox_profile.set_preference('general.warnOnAboutConfig', False)
+    firefox_options.set_preference('browser.download.dir', download_folder_path)
+    firefox_options.set_preference('general.warnOnAboutConfig', False)
 
     mimetypes_to_download = ",".join(FIREFOX_MIMETYPES_TO_DOWNLOAD)
-    firefox_profile.set_preference("pdfjs.disabled", True)
-    firefox_profile.set_preference("plugin.disable_full_page_plugin_for_types", mimetypes_to_download)
-    firefox_profile.set_preference('browser.helperApps.neverAsk.saveToDisk', mimetypes_to_download)
-
-    firefox_profile.update_preferences()
-    firefox_options.profile = firefox_profile
-
+    firefox_options.set_preference("pdfjs.disabled", True)
+    firefox_options.set_preference("plugin.disable_full_page_plugin_for_types", mimetypes_to_download)
+    firefox_options.set_preference('browser.helperApps.neverAsk.saveToDisk', mimetypes_to_download)
     return firefox_options
 
 

botcity/web/browsers/ie.py

@@ -3,6 +3,7 @@
 from selenium.webdriver import Ie  # noqa: F401, F403
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.ie.options import Options
+from selenium.webdriver.ie.service import Service as IeService  # noqa: F401, F403
 
 
 def default_options(headless=False, download_folder_path=None, user_data_dir=None,

botcity/web/compat.py

@@ -0,0 +1,45 @@
+import warnings
+
+
+def patch_selenium():
+    if version_selenium_is_larger_than_four():
+        from selenium import webdriver
+
+        web_element = webdriver.remote.webelement.WebElement
+
+        remote_driver = webdriver.remote.webdriver.WebDriver
+
+        bys = ['id', 'class name', 'xpath', 'link text', 'partial link text', 'name', 'css selector', 'tag name']
+        for by in bys:
+            name = '_'.join(by.split(' '))
+            find_element_by_function_name = f"find_element_by_{name}"
+            find_elements_by_function_name = f"find_elements_by_{name}"
+            setattr(web_element, find_element_by_function_name, find_element_by(by))
+            setattr(web_element, find_elements_by_function_name, find_elements_by(by))
+            setattr(remote_driver, find_element_by_function_name, find_element_by(by))
+            setattr(remote_driver, find_elements_by_function_name, find_elements_by(by))
+
+
+def find_element_by(by):
+    def func(self, value):
+        warnings.warn("This function is deprecated in version 4, please review the usage for find_element.",
+                      category=DeprecationWarning, stacklevel=2)
+        return self.find_element(by=by, value=value)
+    return func
+
+
+def find_elements_by(by):
+    def func(self, value):
+        warnings.warn("This function is deprecated in version 4, please review the usage for find_elements.",
+                      category=DeprecationWarning, stacklevel=2)
+        return self.find_elements(by=by, value=value)
+    return func
+
+
+def version_selenium_is_larger_than_four():
+    import selenium
+    from packaging import version
+    se_version = version.parse(selenium.__version__)
+
+    needs_patch = se_version >= version.parse("4.0")
+    return needs_patch

conftest.py

@@ -49,7 +49,14 @@ def setup_edge(headless: bool, tmp_folder: str, download_driver: str) -> WebBot:
     web.driver_path = download_driver
     web.download_folder_path = tmp_folder
     opt = browsers.edge.default_options(headless=headless, download_folder_path=tmp_folder)
-    opt.set_capability('platform', 'ANY')
+    platforms = {
+        "Linux": "linux",
+        "Darwin": "mac",
+        "Windows": "windows"
+    }
+    platform_name = platforms.get(platform.system())
+
+    opt.platform_name = platform_name
 
     web.options = opt
     return web

requirements.txt

@@ -1,7 +1,6 @@
 botcity-framework-base>=0.2.2
 beautifulsoup4
-msedge-selenium-tools==3.141.3
 numpy
 opencv-python
 pillow
-selenium==3.141
+selenium==4.6.1