feat: incomplete with node on which an error occurred (#4863)

Part of #4860

If a check or matches method throws, catch the error and add a node with
an `error-occurred` none check. This way incomplete results are always
reported.

This only happens for axe.run / axe.runPartial. `rule.runSync()` still
throws if an occurs in a rule or check.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Steven Lambert <2433219+straker@users.noreply.github.com>
Co-authored-by: Dan Bjorge <dan.bjorge@deque.com>
Co-authored-by: WilcoFiers <530687+WilcoFiers@users.noreply.github.com>

Author: 4 people

axe.d.ts

@@ -155,7 +155,7 @@ declare namespace axe {
     toolOptions: RunOptions;
     passes: Result[];
     violations: Result[];
-    incomplete: Result[];
+    incomplete: IncompleteResult[];
     inapplicable: Result[];
   }
   interface Result {
@@ -167,6 +167,9 @@ declare namespace axe {
     tags: TagValue[];
     nodes: NodeResult[];
   }
+  interface IncompleteResult extends Result {
+    error?: Omit<SupportError, 'errorNode'>;
+  }
   interface NodeResult {
     html: string;
     impact?: ImpactValue;
@@ -204,6 +207,21 @@ declare namespace axe {
     fail: string | { [key: string]: string };
     incomplete?: string | { [key: string]: string };
   }
+  interface SupportError {
+    name: string;
+    message: string;
+    stack: string;
+    ruleId?: string;
+    method?: string;
+    cause?: SerialError;
+    errorNode?: DqElement;
+  }
+  interface SerialError {
+    message: string;
+    stack: string;
+    name: string;
+    cause?: SerialError;
+  }
   interface CheckLocale {
     [key: string]: CheckMessages;
   }
@@ -461,7 +479,13 @@ declare namespace axe {
     isLabelledShadowDomSelector: (
       selector: unknown
     ) => selector is LabelledShadowDomSelector;
-
+    SupportError: (
+      error: Error,
+      ruleId?: string,
+      method?: string,
+      errorNode?: DqElement
+    ) => SupportError;
+    serializeError: (error: Error) => SerialError;
     DqElement: DqElementConstructor;
     uuid: (
       options?: { random?: Uint8Array | Array<number> },

lib/checks/shared/error-occurred.json

@@ -0,0 +1,10 @@
+{
+  "id": "error-occurred",
+  "evaluate": "exists-evaluate",
+  "metadata": {
+    "messages": {
+      "pass": "",
+      "incomplete": "Axe encountered an error; test the page for this type of problem manually"
+    }
+  }
+}

lib/core/base/audit.js

@@ -9,7 +9,8 @@ import {
   preload,
   findBy,
   ruleShouldRun,
-  performanceTimer
+  performanceTimer,
+  serializeError
 } from '../utils';
 import { doT } from '../imports';
 import constants from '../constants';
@@ -368,14 +369,24 @@ export default class Audit {
   after(results, options) {
     const rules = this.rules;
     return results.map(ruleResult => {
+      if (ruleResult.error) {
+        return ruleResult;
+      }
       const rule = findBy(rules, 'id', ruleResult.id);
       if (!rule) {
         // If you see this, you're probably running the Mocha tests with the axe extension installed
         throw new Error(
           'Result for unknown rule. You may be running mismatch axe-core versions'
         );
       }
-      return rule.after(ruleResult, options);
+      try {
+        return rule.after(ruleResult, options);
+      } catch (err) {
+        if (options.debug) {
+          throw err;
+        }
+        return createIncompleteErrorResult(rule, err);
+      }
     });
   }
   /**
@@ -732,36 +743,37 @@ function getDefferedRule(rule, context, options) {
     rule.run(
       context,
       options,
-      // resolve callback for rule `run`
-      ruleResult => {
-        // resolve
-        resolve(ruleResult);
-      },
-      // reject callback for rule `run`
+      ruleResult => resolve(ruleResult),
       err => {
-        // if debug - construct error details
-        if (!options.debug) {
-          const errResult = Object.assign(new RuleResult(rule), {
-            result: constants.CANTTELL,
-            description: 'An error occured while running this rule',
-            message: err.message,
-            stack: err.stack,
-            error: err,
-            // Add a serialized reference to the node the rule failed on for easier debugging.
-            // See https://github.com/dequelabs/axe-core/issues/1317.
-            errorNode: err.errorNode
-          });
-          // resolve
-          resolve(errResult);
-        } else {
-          // reject
+        if (options.debug) {
           reject(err);
+        } else {
+          resolve(createIncompleteErrorResult(rule, err));
         }
       }
     );
   };
 }
 
+function createIncompleteErrorResult(rule, error) {
+  const { errorNode } = error;
+  const serialError = serializeError(error);
+  const none = [
+    {
+      id: 'error-occurred',
+      result: undefined,
+      data: serialError,
+      relatedNodes: []
+    }
+  ];
+  const node = errorNode || new DqElement(document.documentElement);
+  return Object.assign(new RuleResult(rule), {
+    error: serialError,
+    result: constants.CANTTELL,
+    nodes: [{ any: [], all: [], none, node }]
+  });
+}
+
 /**
  * For all the rules, create the helpUrl and add it to the data for that rule
  */

lib/core/base/rule.js

@@ -1,4 +1,3 @@
-/*global SupportError */
 import { createExecutionContext } from './check';
 import RuleResult from './rule-result';
 import {
@@ -8,7 +7,8 @@ import {
   queue,
   DqElement,
   select,
-  assert
+  assert,
+  RuleError
 } from '../utils';
 import { isVisibleToScreenReaders } from '../../commons/dom';
 import constants from '../constants';
@@ -181,7 +181,16 @@ Rule.prototype.runChecks = function runChecks(
     const check = self._audit.checks[c.id || c];
     const option = getCheckOption(check, self.id, options);
     checkQueue.defer((res, rej) => {
-      check.run(node, option, context, res, rej);
+      check.run(node, option, context, res, error => {
+        rej(
+          new RuleError({
+            ruleId: self.id,
+            method: `${check.id}#evaluate`,
+            errorNode: new DqElement(node),
+            error
+          })
+        );
+      });
     });
   });
 
@@ -235,8 +244,7 @@ Rule.prototype.run = function run(context, options = {}, resolve, reject) {
     // Matches throws an error when it lacks support for document methods
     nodes = this.gatherAndMatchNodes(context, options);
   } catch (error) {
-    // Exit the rule execution if matches fails
-    reject(new SupportError({ cause: error, ruleId: this.id }));
+    reject(error);
     return;
   }
 
@@ -312,15 +320,7 @@ Rule.prototype.runSync = function runSync(context, options = {}) {
   }
 
   const ruleResult = new RuleResult(this);
-  let nodes;
-
-  try {
-    nodes = this.gatherAndMatchNodes(context, options);
-  } catch (error) {
-    // Exit the rule execution if matches fails
-    throw new SupportError({ cause: error, ruleId: this.id });
-  }
-
+  const nodes = this.gatherAndMatchNodes(context, options);
   if (options.performanceTimer) {
     this._logGatherPerformance(nodes);
   }
@@ -451,7 +451,18 @@ Rule.prototype.gatherAndMatchNodes = function gatherAndMatchNodes(
     performanceTimer.mark(markMatchesStart);
   }
 
-  nodes = nodes.filter(node => this.matches(node.actualNode, node, context));
+  nodes = nodes.filter(node => {
+    try {
+      return this.matches(node.actualNode, node, context);
+    } catch (error) {
+      throw new RuleError({
+        ruleId: this.id,
+        method: `#matches`,
+        errorNode: new DqElement(node),
+        error
+      });
+    }
+  });
 
   if (options.performanceTimer) {
     performanceTimer.mark(markMatchesEnd);
@@ -542,12 +553,20 @@ function sanitizeNodes(result) {
  */
 Rule.prototype.after = function after(result, options) {
   const afterChecks = findAfterChecks(this);
-  const ruleID = this.id;
   afterChecks.forEach(check => {
     const beforeResults = findCheckResults(result.nodes, check.id);
-    const checkOption = getCheckOption(check, ruleID, options);
-
-    const afterResults = check.after(beforeResults, checkOption.options);
+    const checkOption = getCheckOption(check, this.id, options);
+    let afterResults;
+    try {
+      afterResults = check.after(beforeResults, checkOption.options);
+    } catch (error) {
+      throw new RuleError({
+        ruleId: this.id,
+        method: `${check.id}#after`,
+        errorNode: result.nodes?.[0]?.node,
+        error
+      });
+    }
 
     if (this.reviewOnFail) {
       afterResults.forEach(checkResult => {

lib/core/constants.js

@@ -47,7 +47,15 @@ const constants = {
     timeout: 10000
   }),
   allOrigins: '<unsafe_all_origins>',
-  sameOrigin: '<same_origin>'
+  sameOrigin: '<same_origin>',
+  serializableErrorProps: Object.freeze([
+    'message',
+    'stack',
+    'name',
+    'code',
+    'ruleId',
+    'method'
+  ])
 };
 
 definitions.forEach(definition => {

lib/core/index.js

@@ -25,16 +25,3 @@ if (typeof window.getComputedStyle === 'function') {
 }
 // local namespace for common functions
 let commons;
-
-function SupportError(error) {
-  this.name = 'SupportError';
-  this.cause = error.cause;
-  this.message = `\`${error.cause}\` - feature unsupported in your environment.`;
-  if (error.ruleId) {
-    this.ruleId = error.ruleId;
-    this.message += ` Skipping ${this.ruleId} rule.`;
-  }
-  this.stack = new Error().stack;
-}
-SupportError.prototype = Object.create(Error.prototype);
-SupportError.prototype.constructor = SupportError;

lib/core/utils/index.js

@@ -87,6 +87,8 @@ export { default as ruleShouldRun } from './rule-should-run';
 export { default as filterHtmlAttrs } from './filter-html-attrs';
 export { default as select } from './select';
 export { default as sendCommandToFrame } from './send-command-to-frame';
+export { default as serializeError } from './serialize-error';
+export { default as RuleError } from './rule-error';
 export { default as setScrollState } from './set-scroll-state';
 export { default as shadowSelect } from './shadow-select';
 export { default as shadowSelectAll } from './shadow-select-all';

lib/core/utils/merge-results.js

@@ -95,6 +95,9 @@ function mergeResults(frameResults, options) {
         if (ruleResult.nodes.length) {
           spliceNodes(res.nodes, ruleResult.nodes);
         }
+        if (ruleResult.error) {
+          res.error ??= ruleResult.error;
+        }
       }
     });
   });

lib/core/utils/rule-error.js

@@ -0,0 +1,23 @@
+import serializeError from './serialize-error';
+
+export default class RuleError extends Error {
+  constructor({ error, ruleId, method, errorNode }) {
+    super();
+    this.name = error.name ?? 'RuleError';
+    this.message = error.message;
+    this.stack = error.stack;
+    if (error.cause) {
+      this.cause = serializeError(error.cause);
+    }
+    if (ruleId) {
+      this.ruleId = ruleId;
+      this.message += ` Skipping ${this.ruleId} rule.`;
+    }
+    if (method) {
+      this.method = method;
+    }
+    if (errorNode) {
+      this.errorNode = errorNode;
+    }
+  }
+}

lib/core/utils/serialize-error.js

@@ -0,0 +1,24 @@
+import constants from '../constants';
+
+/**
+ * Serializes an error to a JSON object
+ * @param e - The error to serialize
+ * @returns A JSON object representing the error
+ */
+export default function serializeError(err, iteration = 0) {
+  if (typeof err !== 'object' || err === null) {
+    return { message: String(err) };
+  }
+  const serial = {};
+  for (const prop of constants.serializableErrorProps) {
+    if (['string', 'number', 'boolean'].includes(typeof err[prop])) {
+      serial[prop] = err[prop];
+    }
+  }
+  // Recursively serialize cause up to 10 levels deep
+  if (err.cause) {
+    serial.cause =
+      iteration < 10 ? serializeError(err.cause, iteration + 1) : '...';
+  }
+  return serial;
+}