Skip flaky tests on Chrome 71 + Windows 10 (ampproject#19926)

* Skip flaky tests on Chrome 71/Windows. * Skip Windows on amp-image-slider integration test.
cathyxz · Dec 17, 2018 · ed89100 · ed89100
1 parent 046034f
commit ed89100
Show file tree

Hide file tree

Showing 8 changed files with 538 additions and 507 deletions.
diff --git a/extensions/amp-image-slider/0.1/test/integration/test-amp-image-slider.js b/extensions/amp-image-slider/0.1/test/integration/test-amp-image-slider.js
@@ -16,8 +16,11 @@
 
 import {createPointerEvent} from '../../../../../testing/test-helper';
 
-const config = describe.configure().ifChrome();
-config.run('amp-image-slider', function() {
+const t = describe.configure()
+    .ifChrome()
+    .skipWindows(); // TODO(#19647): Flaky on Chrome 71 on Windows 10.
+
+t.run('amp-image-slider', function() {
   this.timeout(20000);
   const DEFAULT_TIMEOUT = 1600;
 

diff --git a/src/service/platform-impl.js b/src/service/platform-impl.js
@@ -110,6 +110,14 @@ export class Platform {
     return /WebKit/i.test(this.navigator_.userAgent) && !this.isEdge();
   }
 
+  /**
+   * Whether the current browser is running on Windows.
+   * @return {boolean}
+   */
+  isWindows() {
+    return /Windows/i.test(this.navigator_.userAgent);
+  }
+
   /**
    * Whether the current browser is isStandalone.
    * @return {boolean}

diff --git a/test/_init_tests.js b/test/_init_tests.js
@@ -190,6 +190,10 @@ class TestConfig {
     });
   }
 
+  skipWindows() {
+    return this.skip(() => this.platform.isWindows());
+  }
+
   enableIe() {
     this.skipMatchers.splice(this.skipMatchers.indexOf(this.runOnIe), 1);
     return this;

diff --git a/test/integration/test-amp-carousel.js b/test/integration/test-amp-carousel.js
@@ -14,8 +14,12 @@
  * limitations under the License.
  */
 
-const config = describe.configure().ifChrome().skipSinglePass();
-config.run('amp-carousel', function() {
+const t = describe.configure()
+    .ifChrome()
+    .skipSinglePass()
+    .skipWindows(); // TODO(#19647): Flaky on Chrome 71 on Windows 10.
+
+t.run('amp-carousel', function() {
   this.timeout(10000);
 
   const extensions = ['amp-carousel'];

diff --git a/test/integration/test-errors.js b/test/integration/test-errors.js
@@ -23,10 +23,12 @@ import {
 /** @const {number} */
 const TIMEOUT = window.ampTestRuntimeConfig.mochaTimeout;
 
-// TODO(@cramforce): Find out why it does not work with obfuscated
-// props.
-const t = describe.configure().retryOnSaucelabs()
-    .skipIfPropertiesObfuscated();
+const t = describe.configure()
+    .retryOnSaucelabs()
+    // TODO(@cramforce): Find out why it does not work with obfuscated props.
+    .skipIfPropertiesObfuscated()
+    .skipWindows(); // TODO(#19647): Flaky on Chrome 71 on Windows 10.
+
 t.run('error page', function() {
   this.timeout(TIMEOUT);
 

diff --git a/test/integration/test-user-error-reporting.js b/test/integration/test-user-error-reporting.js
@@ -16,104 +16,107 @@
 
 import {RequestBank} from '../../testing/test-helper';
 
-describe.configure().skipIfPropertiesObfuscated()
-    .skipSafari().skipEdge().run('user-error', function() {
-      //TODO(zhouyx, #11459): Unskip the test on safari.
+const t = describe.configure()
+    .skipIfPropertiesObfuscated()
+    .skipSafari() // TODO(zhouyx, #11459): Unskip the test on safari.
+    .skipEdge()
+    .skipWindows(); // TODO(#19647): Flaky on Chrome 71 on Windows 10.
 
-      describes.integration('user-error integration test', {
-        extensions: ['amp-analytics'],
-        hash: 'log=0',
-        experiments: ['user-error-reporting'],
-        body: `
-        <amp-analytics>
-          <script type="application/json">
-            {
-              "requests": {
-                "error": "${RequestBank.getUrl()}"
-              },
-              "triggers": {
-                "userError": {
-                  "on": "user-error",
-                  "request": "error"
-                }
-              }
+t.run('user-error', function() {
+  describes.integration('user-error integration test', {
+    extensions: ['amp-analytics'],
+    hash: 'log=0',
+    experiments: ['user-error-reporting'],
+    body: `
+    <amp-analytics>
+      <script type="application/json">
+        {
+          "requests": {
+            "error": "${RequestBank.getUrl()}"
+          },
+          "triggers": {
+            "userError": {
+              "on": "user-error",
+              "request": "error"
             }
-          </script>
-        </amp-analytics>
+          }
+        }
+      </script>
+    </amp-analytics>
 
-        <amp-pixel src="https://foo.com/tracker/foo"
-               referrerpolicy="fail-referrer">
-               `,
-      }, () => {
-        it('should ping correct host with amp-pixel user().assert err', () => {
-          return RequestBank.withdraw();
-        });
-      });
+    <amp-pixel src="https://foo.com/tracker/foo"
+            referrerpolicy="fail-referrer">
+            `,
+  }, () => {
+    it('should ping correct host with amp-pixel user().assert err', () => {
+      return RequestBank.withdraw();
+    });
+  });
 
-      describes.integration('user-error integration test', {
-        extensions: ['amp-analytics'],
-        hash: 'log=0',
-        experiments: ['user-error-reporting'],
+  describes.integration('user-error integration test', {
+    extensions: ['amp-analytics'],
+    hash: 'log=0',
+    experiments: ['user-error-reporting'],
 
-        body: `
-        <amp-img
-          src="../../examples/img/sea@1x.jpg"
-          width="360" height="216" layout="responsive"
-          role='img'>
-        </amp-img>
+    body: `
+    <amp-img
+      src="../../examples/img/sea@1x.jpg"
+      width="360" height="216" layout="responsive"
+      role='img'>
+    </amp-img>
 
-        <amp-analytics>
-          <script type="application/json">
-            {
-              "requests": {
-                "error": "${RequestBank.getUrl()}"
-              },
-              "triggers": {
-                "userError": {
-                  "on": "user-error",
-                  "request": "error"
-                }
-              }
+    <amp-analytics>
+      <script type="application/json">
+        {
+          "requests": {
+            "error": "${RequestBank.getUrl()}"
+          },
+          "triggers": {
+            "userError": {
+              "on": "user-error",
+              "request": "error"
             }
-          </script>
-        </amp-analytics>`,
-      }, () => {
-        it('should ping correct host with amp-img user().error err', () => {
-          return RequestBank.withdraw();
-        });
-      });
+          }
+        }
+      </script>
+    </amp-analytics>`,
+  }, () => {
+    it('should ping correct host with amp-img user().error err', () => {
+      return RequestBank.withdraw();
+    });
+  });
 
-      describes.integration('3p user-error integration test', {
-        extensions: ['amp-analytics', 'amp-ad'],
-        hash: 'log=0',
-        experiments: ['user-error-reporting'],
+  describes.integration('3p user-error integration test', {
+    extensions: ['amp-analytics', 'amp-ad'],
+    hash: 'log=0',
+    experiments: ['user-error-reporting'],
 
-        body: `
-        <amp-ad width=300 height=250
-            type="_ping_"
-            data-url='not-exist'
-            data-valid='false'
-            data-error='true'>
-        </amp-ad>
+    body: `
+    <amp-ad width=300 height=250
+        type="_ping_"
+        data-url='not-exist'
+        data-valid='false'
+        data-error='true'>
+    </amp-ad>
 
-        <amp-analytics>
-          <script type="application/json">
-            {
-              "requests": {
-                "error": "${RequestBank.getUrl()}"
-              },
-              "triggers": {
-                "userError": {
-                  "on": "user-error",
-                  "request": "error"
-                }
-              }
+    <amp-analytics>
+      <script type="application/json">
+        {
+          "requests": {
+            "error": "${RequestBank.getUrl()}"
+          },
+          "triggers": {
+            "userError": {
+              "on": "user-error",
+              "request": "error"
             }
-          </script>
-        </amp-analytics>`,
-      }, () => {
-        it('should ping correct host with 3p error message', () => {
-          return RequestBank.withdraw();
-        });
-      });
+          }
+        }
+      </script>
+    </amp-analytics>`,
+  }, () => {
+    it('should ping correct host with 3p error message', () => {
+      return RequestBank.withdraw();
     });
+  });
+});
diff --git a/test/integration/test-video-players-helper.js b/test/integration/test-video-players-helper.js
@@ -136,8 +136,11 @@ export function runVideoPlayerIntegrationTests(
     afterEach(cleanUp);
   });
 
-  describe.configure().skipIfPropertiesObfuscated()
-      .ifChrome().run('Analytics Triggers', function() {
+  describe.configure()
+      .skipIfPropertiesObfuscated()
+      .ifChrome()
+      .skipWindows() // TODO(#19647): Flaky on Chrome 71 on Windows 10.
+      .run('Analytics Triggers', function() {
         this.timeout(TIMEOUT);
         let video;
 
@@ -342,6 +345,7 @@ export function runVideoPlayerIntegrationTests(
 
         afterEach(cleanUp);
       });
+
   const t = describe.configure().ifChrome().skipIfPropertiesObfuscated();
   t.run('Autoplay', function() {
     this.timeout(TIMEOUT);