fix: skip unloaded iframes for all apis (#752)

* fix: skip unloaded iframes for all apis

* no loaded

* Update packages/webdriverio/src/index.ts

Co-authored-by: Wilco Fiers <WilcoFiers@users.noreply.github.com>

* Update packages/webdriverio/src/index.ts

Co-authored-by: Stephen Mathieson <571265+stephenmathieson@users.noreply.github.com>

* reject

* assert properly

---------

Co-authored-by: Wilco Fiers <WilcoFiers@users.noreply.github.com>
Co-authored-by: Stephen Mathieson <571265+stephenmathieson@users.noreply.github.com>

Author: 3 people

packages/playwright/package-lock.json

@@ -437,6 +437,24 @@ describe('@axe-core/playwright', () => {
       assert.equal(res?.status(), 200);
       assert.strictEqual(count, 9);
     });
+
+    it('handles unloaded iframes (e.g. loading=lazy)', async () => {
+      const res = await page.goto(`${addr}/external/lazy-loaded-iframe.html`);
+
+      const results = await new AxeBuilder({ page })
+        .options({ runOnly: ['label', 'frame-tested'] })
+        .analyze();
+
+      assert.equal(res?.status(), 200);
+      assert.lengthOf(results.incomplete, 0);
+      assert.equal(results.violations[0].id, 'label');
+      assert.lengthOf(results.violations[0].nodes, 1);
+      assert.deepEqual(results.violations[0].nodes[0].target, [
+        '#ifr-lazy',
+        '#lazy-baz',
+        'input'
+      ]);
+    })
   });
 
   describe('include/exclude', () => {

packages/playwright/test/axe-playwright.spec.ts

@@ -1,3 +1,4 @@
+import assert from 'assert';
 import * as fs from 'fs';
 import * as Axe from 'axe-core';
 import { Frame } from 'puppeteer';
@@ -73,14 +74,18 @@ export async function getChildFrame(
 export async function assertFrameReady(frame: Frame): Promise<void> {
   // Wait so that we know there is an execution context.
   // Assume that if we have an html node we have an execution context.
-  // Check if the page is loaded.
-  let pageReady = false;
   try {
-    pageReady = await frame.evaluate(pageIsLoaded);
+    // Puppeteer freezes on unloaded iframes. Set a race timeout in order to handle that.
+    // @see https://github.com/dequelabs/axe-core-npm/issues/727
+    const timeoutPromise = new Promise((resolve, reject) => {
+      setTimeout(() => {
+        reject();
+      }, 1000)
+    });
+    const evaluatePromise = frame.evaluate(pageIsLoaded);
+    const readyState = await Promise.race([timeoutPromise, evaluatePromise]);
+    assert(readyState);
   } catch {
-    /* ignore */
-  }
-  if (!pageReady) {
     throw new Error('Page/Frame is not ready');
   }
 }

packages/puppeteer/package-lock.json

@@ -872,6 +872,25 @@ describe('AxePuppeteer', function () {
       assert.equal(res?.status(), 200);
       assert.deepEqual(pageResults, frameResults);
     });
+
+    it('skips unloaded iframes (e.g. loading=lazy)', async () => {
+      const res = await page.goto(`${addr}/external/lazy-loaded-iframe.html`);
+      const results = await new AxePuppeteer(page)
+        .options({ runOnly: ['label', 'frame-tested'] })
+        .analyze();
+
+      assert.equal(res?.status(), 200);
+      assert.equal(results.incomplete[0].id, 'frame-tested');
+      assert.lengthOf(results.incomplete[0].nodes, 1);
+      assert.deepEqual(results.incomplete[0].nodes[0].target, ['#ifr-lazy', '#lazy-iframe']);
+      assert.equal(results.violations[0].id, 'label');
+      assert.lengthOf(results.violations[0].nodes, 1);
+      assert.deepEqual(results.violations[0].nodes[0].target, [
+        '#ifr-lazy',
+        '#lazy-baz',
+        'input'
+      ]);
+    })
   });
 
   describe('axe.finishRun errors', () => {

packages/puppeteer/src/utils.ts

@@ -10,7 +10,8 @@ import {
   axeRunPartial,
   axeFinishRun,
   axeRunLegacy,
-  configureAllowedOrigins
+  configureAllowedOrigins,
+  FRAME_LOAD_TIMEOUT
 } from './utils';
 import { getFilename } from 'cross-dirname';
 import { pathToFileURL } from 'url';
@@ -241,7 +242,21 @@ export default class AxeBuilder {
       return await this.runLegacy(context);
     }
 
-    const partials = await this.runPartialRecursive(context);
+    // ensure we fail quickly if an iframe cannot be loaded (instead of waiting
+    // the default length of 30 seconds)
+    const { pageLoad } = await this.client.getTimeouts();
+    this.client.setTimeout({
+      pageLoad: FRAME_LOAD_TIMEOUT,
+    });
+
+    let partials: PartialResults | null
+    try {
+      partials = await this.runPartialRecursive(context);
+    } finally {
+      this.client.setTimeout({
+        pageLoad,
+      });
+    }
 
     try {
       return await this.finishRun(partials);
@@ -300,7 +315,8 @@ export default class AxeBuilder {
    */
 
   private async runPartialRecursive(
-    context: SerialContextObject
+    context: SerialContextObject,
+    frameStack: Element[] = []
   ): Promise<PartialResults> {
     const frameContexts = await axeGetFrameContext(this.client, context);
     const partials: PartialResults = [
@@ -313,10 +329,16 @@ export default class AxeBuilder {
         assert(frame, `Expect frame of "${frameSelector}" to be defined`);
         await this.client.switchToFrame(frame);
         await axeSourceInject(this.client, this.script);
-        partials.push(...(await this.runPartialRecursive(frameContext)));
+        partials.push(...(await this.runPartialRecursive(frameContext, [...frameStack, frame])));
       } catch (error) {
+        const [topWindow] = await this.client.getWindowHandles()
+        await this.client.switchToWindow(topWindow)
+
+        for (const frameElm of frameStack) {
+          await this.client.switchToFrame(frameElm);
+        }
+
         partials.push(null);
-        await this.client.switchToParentFrame();
       }
     }
     await this.client.switchToParentFrame();

packages/puppeteer/test/axe-puppeteer.spec.ts

@@ -10,6 +10,8 @@ import type {
   SerialContextObject
 } from 'axe-core';
 
+export const FRAME_LOAD_TIMEOUT = 1000
+
 /**
  * Validates that the client provided is WebdriverIO v5 or v6.
  */
@@ -83,6 +85,7 @@ export const axeSourceInject = async (
   client: Browser,
   axeSource: string
 ): Promise<{ runPartialSupported: boolean }> => {
+  await assertFrameReady(client);
   return promisify(
     // Had to use executeAsync() because we could not use multiline statements in client.execute()
     // we were able to return a single boolean in a line but not when assigned to a variable.
@@ -98,6 +101,34 @@ export const axeSourceInject = async (
   );
 };
 
+async function assertFrameReady(client: Browser): Promise<void> {
+  // Wait so that we know there is an execution context.
+  // Assume that if we have an html node we have an execution context.
+  try {
+    /*
+      When using the devtools protocol trying to call
+      client.execute() on an unloaded iframe would cause
+      the code to hang indefinitely since it is using
+      Puppeteer which freezes on unloaded iframes. Set a
+      race timeout in order to handle that. Code taken
+      from our @axe-core/puppeteer utils function.
+      @see https://github.com/dequelabs/axe-core-npm/issues/727
+    */
+    const timeoutPromise = new Promise((resolve, reject) => {
+      setTimeout(() => {
+        reject();
+      }, FRAME_LOAD_TIMEOUT)
+    });
+    const executePromise = client.execute(() => {
+      return document.readyState === 'complete'
+    });
+    const readyState = await Promise.race([timeoutPromise, executePromise]);
+    assert(readyState);
+  } catch {
+    throw new Error('Page/Frame is not ready');
+  }
+}
+
 export const axeRunPartial = (
   client: Browser,
   context?: SerialContextObject,

packages/webdriverio/package-lock.json

@@ -850,6 +850,40 @@ describe('@axe-core/webdriverio', () => {
             normalResults.testEngine.name = legacyResults.testEngine.name;
             assert.deepEqual(normalResults, legacyResults);
           });
+
+          it('handles unloaded iframes (e.g. loading=lazy)', async () => {
+            await client.url(`${addr}/lazy-loaded-iframe.html`);
+            const title = await client.getTitle();
+
+            const results = await new AxeBuilder({client})
+              .options({ runOnly: ['label', 'frame-tested'] })
+              .analyze();
+
+            assert.notEqual(title, 'Error');
+            assert.equal(results.incomplete[0].id, 'frame-tested');
+            assert.lengthOf(results.incomplete[0].nodes, 1);
+            assert.deepEqual(results.incomplete[0].nodes[0].target, ['#ifr-lazy', '#lazy-iframe']);
+            assert.equal(results.violations[0].id, 'label');
+            assert.lengthOf(results.violations[0].nodes, 1);
+            assert.deepEqual(results.violations[0].nodes[0].target, [
+              '#ifr-lazy',
+              '#lazy-baz',
+              'input'
+            ]);
+          });
+
+          it('resets pageLoad timeout to user setting', async () => {
+            await client.url(`${addr}/lazy-loaded-iframe.html`);
+            client.setTimeout({ pageLoad: 500 })
+            const title = await client.getTitle();
+
+            const results = await new AxeBuilder({client})
+              .options({ runOnly: ['label', 'frame-tested'] })
+              .analyze();
+
+            const timeout = await client.getTimeouts();
+            assert.equal(timeout.pageLoad, 500);
+          });
         });
 
         describe('logOrRethrowError', () => {

packages/webdriverio/src/index.ts

@@ -398,7 +398,7 @@ describe('@axe-core/webdriverjs', () => {
     });
 
     it('skips unloaded iframes (e.g. loading=lazy)', async () => {
-      await driver.get(`${addr}/lazy-loaded-iframe.html`);
+      await driver.get(`${addr}/external/lazy-loaded-iframe.html`);
       const title = await driver.getTitle();
 
       const results = await new AxeBuilder(driver)
@@ -408,18 +408,18 @@ describe('@axe-core/webdriverjs', () => {
       assert.notEqual(title, 'Error');
       assert.equal(results.incomplete[0].id, 'frame-tested');
       assert.lengthOf(results.incomplete[0].nodes, 1);
-      assert.deepEqual(results.incomplete[0].nodes[0].target, ['#parent', '#lazy-iframe']);
+      assert.deepEqual(results.incomplete[0].nodes[0].target, ['#ifr-lazy', '#lazy-iframe']);
       assert.equal(results.violations[0].id, 'label');
       assert.lengthOf(results.violations[0].nodes, 1);
       assert.deepEqual(results.violations[0].nodes[0].target, [
-        '#parent',
-        '#child',
+        '#ifr-lazy',
+        '#lazy-baz',
         'input'
       ]);
     })
 
     it('resets pageLoad timeout to user setting', async () => {
-      await driver.get(`${addr}/lazy-loaded-iframe.html`);
+      await driver.get(`${addr}/external/lazy-loaded-iframe.html`);
       driver.manage().setTimeouts({ pageLoad: 500 })
       const title = await driver.getTitle();