Fix flaky functional tests

ci-jobs/functional_test.yml

@@ -5,8 +5,6 @@ parameters:
   xcodeForIOS: 11.3
   CI: true
 
-# [Android] Need to fix and add flaky tests for activities_tests, find_by_uiautomator_tests
-
 jobs:
   - template: ./functional/run_ios_test.yml
     parameters:
@@ -70,7 +68,7 @@ jobs:
       name: 'func_test_android6'
       vmImage: ${{ parameters.vmImage }}
       pytestOpt: ${{ parameters.pytestOpt }}
-      testFiles: 'common_tests.py'
+      testFiles: 'common_tests.py multi_action_tests.py webelement_tests.py'
       sdkVer: ${{ parameters.androidSdkVer }}
       CI: ${{ parameters.ci }}
   - template: ./functional/run_android_test.yml
@@ -86,6 +84,6 @@ jobs:
       name: 'func_test_android8'
       vmImage: ${{ parameters.vmImage }}
       pytestOpt: ${{ parameters.pytestOpt }}
-      testFiles: 'network_connection_tests.py log_event_tests.py'
+      testFiles: 'network_connection_tests.py log_event_tests.py activities_tests.py hw_actions_tests.py touch_action_tests.py'
       sdkVer: ${{ parameters.androidSdkVer }}
       CI: ${{ parameters.ci }}

test/functional/android/applications_tests.py

@@ -13,11 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
 from time import sleep
 
 from appium.webdriver.applicationstate import ApplicationState
 
+from .helper.desired_capabilities import PATH
 from .helper.test_helper import APIDEMO_PKG_NAME, BaseTestCase
 
 
@@ -33,9 +35,8 @@ def test_is_app_installed(self):
         self.assertTrue(self.driver.is_app_installed(APIDEMO_PKG_NAME))
 
     def test_install_app(self):
-        self.skipTest('This causes the server to crash. no idea why')
         self.assertFalse(self.driver.is_app_installed('io.selendroid.testapp'))
-        self.driver.install_app('/Users/isaac/code/python-client/test/apps/selendroid-test-app.apk')
+        self.driver.install_app(PATH(os.path.join('../..', 'apps', 'selendroid-test-app.apk')))
         self.assertTrue(self.driver.is_app_installed('io.selendroid.testapp'))
 
     def test_remove_app(self):

test/functional/android/common_tests.py

@@ -39,9 +39,6 @@ def test_end_test_coverage(self):
         sleep(5)
 
     def test_open_notifications(self):
-        if is_ci():
-            # TODO Due to unexpected dialog, "System UI isn't responding"
-            self.skipTest('Need to fix flaky test during running on CI.')
         for word in ['App', 'Notification', 'Status Bar', ':-|']:
             wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR,
                              'new UiSelector().text("{}")'.format(word)).click()

test/functional/android/context_switching_tests.py

@@ -22,28 +22,31 @@
 from .helper import desired_capabilities
 
 
-@pytest.mark.skip(reason="Need to fix broken test")
 class ContextSwitchingTests(unittest.TestCase):
     def setUp(self):
         desired_caps = desired_capabilities.get_desired_capabilities('selendroid-test-app.apk')
         self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
 
     def test_contexts_list(self):
+        self.skipTest('Need to replace with apk which has WEBVIEW as context')
         self._enter_webview()
         contexts = self.driver.contexts
         self.assertEqual(2, len(contexts))
 
     def test_move_to_correct_context(self):
+        self.skipTest('Need to replace with apk which has WEBVIEW as context')
         self._enter_webview()
         self.assertEqual('WEBVIEW_io.selendroid.testapp', self.driver.current_context)
 
     def test_actually_in_webview(self):
+        self.skipTest('Need to replace with apk which has WEBVIEW as context')
         self._enter_webview()
         self.driver.find_element_by_css_selector('input[type=submit]').click()
         el = self.driver.find_element_by_xpath("//h1[contains(., 'This is my way')]")
         self.assertIsNot(None, el)
 
     def test_move_back_to_native_context(self):
+        self.skipTest('Need to replace with apk which has WEBVIEW as context')
         self._enter_webview()
         self.driver.switch_to.context(None)
         self.assertEqual('NATIVE_APP', self.driver.current_context)
@@ -55,7 +58,7 @@ def tearDown(self):
         self.driver.quit()
 
     def _enter_webview(self):
-        btn = self.driver.find_element_by_name('buttonStartWebviewCD')
+        btn = self.driver.find_element_by_accessibility_id('buttonStartWebviewCD')
         btn.click()
         self.driver.switch_to.context('WEBVIEW')
 

test/functional/android/multi_action_tests.py

@@ -19,29 +19,13 @@
 from appium.webdriver.common.multi_action import MultiAction
 from appium.webdriver.common.touch_action import TouchAction
 
-from .helper.test_helper import BaseTestCase, wait_for_element
+from .helper.test_helper import BaseTestCase, is_ci, wait_for_element
 
 
 class MultiActionTests(BaseTestCase):
     def test_parallel_actions(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-        self.driver.scroll(el1, el2)
-
-        el = self.driver.find_element_by_accessibility_id('Views')
-        action = TouchAction(self.driver)
-        action.tap(el).perform()
-
-        # simulate a swipe/scroll
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
-        action.press(el).move_to(x=100, y=-1000).release().perform()
-        el = self.driver.find_element_by_accessibility_id('Layouts')
-        action.press(el).move_to(x=100, y=-1000).release().perform()
+        self._move_to_splitting_touches_accros_views()
 
-        el = self.driver.find_element_by_accessibility_id('Splitting Touches across Views')
-        action.tap(el).perform()
-
-        wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.ListView')
         els = self.driver.find_elements_by_class_name('android.widget.ListView')
         a1 = TouchAction()
         a1.press(els[0]) \
@@ -56,24 +40,8 @@ def test_parallel_actions(self):
         ma.perform()
 
     def test_actions_with_waits(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-        self.driver.scroll(el1, el2)
-
-        el = self.driver.find_element_by_accessibility_id('Views')
-        action = TouchAction(self.driver)
-        action.tap(el).perform()
-
-        # simulate a swipe/scroll
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
-        action.press(el).move_to(x=100, y=-1000).release().perform()
-        el = self.driver.find_element_by_accessibility_id('Layouts')
-        action.press(el).move_to(x=100, y=-1000).release().perform()
-
-        el = self.driver.find_element_by_accessibility_id('Splitting Touches across Views')
-        action.tap(el).perform()
+        self._move_to_splitting_touches_accros_views()
 
-        wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.ListView')
         els = self.driver.find_elements_by_class_name('android.widget.ListView')
         a1 = TouchAction()
         a1.press(els[0]) \
@@ -95,7 +63,29 @@ def test_actions_with_waits(self):
         ma.add(a1, a2)
         ma.perform()
 
+    def _move_to_splitting_touches_accros_views(self):
+        el1 = self.driver.find_element_by_accessibility_id('Content')
+        el2 = self.driver.find_element_by_accessibility_id('Animation')
+        self.driver.scroll(el1, el2)
+
+        el = self.driver.find_element_by_accessibility_id('Views')
+        action = TouchAction(self.driver)
+        action.tap(el).perform()
+
+        # simulate a swipe/scroll
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
+        action.press(el).move_to(x=100, y=-1000).release().perform()
+        el = self.driver.find_element_by_accessibility_id('Layouts')
+        action.press(el).move_to(x=100, y=-1000).release().perform()
+
+        el = self.driver.find_element_by_accessibility_id('Splitting Touches across Views')
+        action.tap(el).perform()
+
+        wait_for_element(self.driver, MobileBy.ID, 'io.appium.android.apis:id/list1')
+
     def test_driver_multi_tap(self):
+        if is_ci():
+            self.skipTest('Skip since the test must be watched to check if it works')
         el = self.driver.find_element_by_accessibility_id('Graphics')
         action = TouchAction(self.driver)
         action.tap(el).perform()

test/functional/android/network_connection_tests.py

@@ -17,7 +17,6 @@
 
 from appium.webdriver.connectiontype import ConnectionType
 
-from ..test_helper import is_ci
 from .helper.test_helper import BaseTestCase
 
 
@@ -27,8 +26,6 @@ def test_get_network_connection(self):
         self.assertIsInstance(nc, int)
 
     def test_set_network_connection(self):
-        if is_ci():
-            self.skipTest('Need to fix flaky test during running on CI')
         nc = self.driver.set_network_connection(ConnectionType.DATA_ONLY)
         self.assertIsInstance(nc, int)
         self.assertEqual(nc, ConnectionType.DATA_ONLY)

test/functional/android/search_context/find_by_accessibility_id_tests.py

@@ -19,7 +19,6 @@
     BaseTestCase,
     wait_for_element
 )
-from test.functional.test_helper import is_ci
 
 
 class FindByAccessibilityIDTests(BaseTestCase):
@@ -35,12 +34,10 @@ def test_find_multiple_elements(self):
         self.assertIsInstance(els, list)
 
     def test_element_find_single_element(self):
-        if is_ci():
-            self.skipTest('Need to fix flaky test during running on CI')
         wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click()
         wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR,
                          'new UiSelector().text("Accessibility Node Querying")').click()
-        el = wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.ListView')
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Task Take out Trash')
 
         sub_el = el.find_element_by_accessibility_id('Task Take out Trash')
         self.assertIsNotNone(sub_el)

test/functional/android/search_context/find_by_uiautomator_tests.py

@@ -19,7 +19,6 @@
 from test.functional.android.helper.test_helper import BaseTestCase
 
 
-@pytest.mark.skip(reason="Need to fix flaky test")
 class FindByUIAutomatorTests(BaseTestCase):
     def test_find_single_element(self):
         el = self.driver.find_element_by_android_uiautomator('new UiSelector().text("Animation")')

test/functional/android/touch_action_tests.py

@@ -22,6 +22,7 @@
 from .helper.test_helper import (
     APIDEMO_PKG_NAME,
     BaseTestCase,
+    is_ci,
     wait_for_element
 )
 
@@ -73,20 +74,8 @@ def test_press_and_immediately_release_x_y(self):
         self.assertIsNotNone(el)
 
     def test_press_and_wait(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-
+        self._move_to_custom_adapter()
         action = TouchAction(self.driver)
-        action.press(el1).move_to(el2).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Custom Adapter')
-        action.tap(el).perform()
 
         el = wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR,
                               'new UiSelector().text("People Names")')
@@ -118,20 +107,8 @@ def test_press_and_moveto_x_y(self):
         self.assertIsNotNone(el)
 
     def test_long_press(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-
+        self._move_to_custom_adapter()
         action = TouchAction(self.driver)
-        action.press(el1).move_to(el2).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Custom Adapter')
-        action.tap(el).perform()
 
         el = wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR,
                               'new UiSelector().text("People Names")')
@@ -143,20 +120,10 @@ def test_long_press(self):
         self.assertIsNotNone(el)
 
     def test_long_press_x_y(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-
+        if is_ci():
+            self.skipTest("Skip since this check is low robust due to hard-coded position.")
+        self._move_to_custom_adapter()
         action = TouchAction(self.driver)
-        action.press(el1).move_to(el2).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
-        action.tap(el).perform()
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Custom Adapter')
-        action.tap(el).perform()
 
         # the element "People Names" is located at 430:310 (top left corner)
         # location can be changed by phone resolusion, OS version
@@ -168,13 +135,8 @@ def test_long_press_x_y(self):
         self.assertIsNotNone(el)
 
     def test_drag_and_drop(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-        self.driver.scroll(el1, el2)
-
-        el = self.driver.find_element_by_accessibility_id('Views')
+        self._move_to_views()
         action = TouchAction(self.driver)
-        action.tap(el).perform()
 
         el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Drag and Drop')
         action.tap(el).perform()
@@ -185,17 +147,12 @@ def test_drag_and_drop(self):
         # dnd is stimulated by longpress-move_to-release
         action.long_press(dd3).move_to(dd2).release().perform()
 
-        el = wait_for_element(self.driver, MobileBy.ID, '{}:id/drag_text'.format(APIDEMO_PKG_NAME))
-        self.assertTrue('drag_dot_3' in el.text)
+        el = wait_for_element(self.driver, MobileBy.ID, '{}:id/drag_result_text'.format(APIDEMO_PKG_NAME))
+        self.assertTrue('Dropped!' in el.text)
 
     def test_driver_drag_and_drop(self):
-        el1 = self.driver.find_element_by_accessibility_id('Content')
-        el2 = self.driver.find_element_by_accessibility_id('Animation')
-        self.driver.scroll(el1, el2)
-
-        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views')
+        self._move_to_views()
         action = TouchAction(self.driver)
-        action.tap(el).perform()
 
         el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Drag and Drop')
         action.tap(el).perform()
@@ -205,20 +162,40 @@ def test_driver_drag_and_drop(self):
 
         self.driver.drag_and_drop(dd3, dd2)
 
-        el = wait_for_element(self.driver, MobileBy.ID, '{}:id/drag_text'.format(APIDEMO_PKG_NAME))
-        self.assertTrue('drag_dot_3' in el.text)
+        el = wait_for_element(self.driver, MobileBy.ID, '{}:id/drag_result_text'.format(APIDEMO_PKG_NAME))
+        self.assertTrue('Dropped!' in el.text)
 
     def test_driver_swipe(self):
         el = self.driver.find_element_by_accessibility_id('Views')
         action = TouchAction(self.driver)
         action.tap(el).perform()
 
+        wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Animation')
         self.assertRaises(NoSuchElementException, self.driver.find_element_by_accessibility_id, 'ImageView')
 
         self.driver.swipe(100, 1000, 100, 100, 800)
-        el = self.driver.find_element_by_accessibility_id('ImageView')
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'ImageView')
         self.assertIsNotNone(el)
 
+    def _move_to_views(self):
+        el1 = self.driver.find_element_by_accessibility_id('Content')
+        el2 = self.driver.find_element_by_accessibility_id('Animation')
+        self.driver.scroll(el1, el2)
+
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views')
+        action = TouchAction(self.driver)
+        action.tap(el).perform()
+
+    def _move_to_custom_adapter(self):
+        self._move_to_views()
+        action = TouchAction(self.driver)
+
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists')
+        action.tap(el).perform()
+
+        el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Custom Adapter')
+        action.tap(el).perform()
+
 
 if __name__ == '__main__':
     suite = unittest.TestLoader().loadTestsFromTestCase(TouchActionTests)