feat: add new rule require-test-case-name (#553)

* feat(test-case-name-property): add rule to require `name` for test cases

This change adds a new rule to require `name` be present in test cases under certain conditions.  This rule aims to ensure test suites are producing logs in a form that make it easy to identify failing test, when they happen.
For thoroughly tested rules, it's not uncommon to have the same `code` across multiple test cases, with only `options` or `settings` differing between them.
Requiring these test cases to have a `name` helps ensure the test output is meaningful and distinct.

* Address copilot feedbac

* address some of the lint violations

* fix more lint violations

* rename rule

* update meta.docs.url

* fix more lint violations

* fix more lint violations

* fix more lint violations

* added additional tests

Author: michaelfaith

README.md

@@ -105,6 +105,7 @@ export default [
 | [no-identical-tests](docs/rules/no-identical-tests.md)                   | disallow identical tests                                                     | ✅  | 🔧  |     |     |
 | [no-only-tests](docs/rules/no-only-tests.md)                             | disallow the test case property `only`                                       | ✅  |     | 💡  |     |
 | [prefer-output-null](docs/rules/prefer-output-null.md)                   | disallow invalid RuleTester test cases where the `output` matches the `code` | ✅  | 🔧  |     |     |
+| [require-test-case-name](docs/rules/require-test-case-name.md)           | require test cases to have a `name` property under certain conditions        |     |     |     |     |
 | [test-case-property-ordering](docs/rules/test-case-property-ordering.md) | require the properties of a test case to be placed in a consistent order     |     | 🔧  |     |     |
 | [test-case-shorthand-strings](docs/rules/test-case-shorthand-strings.md) | enforce consistent usage of shorthand strings for test cases with no options |     | 🔧  |     |     |
 

docs/rules/require-test-case-name.md

@@ -0,0 +1,63 @@
+# Require test cases to have a `name` property under certain conditions (`eslint-plugin/require-test-case-name`)
+
+<!-- end auto-generated rule header -->
+
+This rule enforces that test cases include a `name` property, under certain circumstances based on the configuration.
+
+## Rule Details
+
+This rule aims to ensure test suites are producing logs in a form that make it easy to identify failing test, when they happen.
+For thoroughly tested rules, it's not uncommon to have the same `code` across multiple test cases, with only `options` or `settings` differing between them.
+Requiring these test cases to have a `name` helps ensure the test output is meaningful and distinct.
+
+### Options
+
+This rule has one option.
+
+#### `require: 'always' | 'objects' | 'objects-with-config'`
+
+- `always`: all test cases should have a `name` property (this means that no shorthand string test cases are allowed as a side effect)
+- `objects`: requires that a `name` property is present in all `object`-based test cases.
+- `objects-with-config` (default): requires that test cases that have `options` or `settings` defined, should also have a `name` property.
+
+Examples of **incorrect** code for this rule:
+
+```js
+// invalid; require: objects-with-config (default)
+const testCase1 = {
+  code: 'foo',
+  options: ['baz'],
+};
+
+// invalid; require: objects
+const testCase2 = {
+  code: 'foo',
+};
+
+// invalid; require: always
+const testCase3 = 'foo';
+```
+
+Examples of **correct** code for this rule:
+
+```js
+// require: objects-with-config, objects
+const testCase1 = 'foo';
+
+// require: objects-with-config, objects, always
+const testCase2 = {
+  code: 'foo',
+  options: ['baz'],
+  name: "foo (option: ['baz'])",
+};
+
+// require: objects-with-config, objects, always
+const testCase4 = {
+  code: 'foo',
+  name: 'foo without options',
+};
+```
+
+## When Not to Use It
+
+If you aren't concerned with the nature of the test logs or don't want to require `name` on test cases.

lib/index.ts

@@ -36,6 +36,7 @@ import requireMetaHasSuggestions from './rules/require-meta-has-suggestions.ts';
 import requireMetaSchemaDescription from './rules/require-meta-schema-description.ts';
 import requireMetaSchema from './rules/require-meta-schema.ts';
 import requireMetaType from './rules/require-meta-type.ts';
+import requireTestCaseName from './rules/require-test-case-name.ts';
 import testCasePropertyOrdering from './rules/test-case-property-ordering.ts';
 import testCaseShorthandStrings from './rules/test-case-shorthand-strings.ts';
 
@@ -115,6 +116,7 @@ const allRules = {
   'require-meta-schema-description': requireMetaSchemaDescription,
   'require-meta-schema': requireMetaSchema,
   'require-meta-type': requireMetaType,
+  'require-test-case-name': requireTestCaseName,
   'test-case-property-ordering': testCasePropertyOrdering,
   'test-case-shorthand-strings': testCaseShorthandStrings,
 } satisfies Record<string, Rule.RuleModule>;

lib/rules/require-test-case-name.ts

@@ -0,0 +1,141 @@
+import type { Rule } from 'eslint';
+
+import { evaluateObjectProperties, getKeyName, getTestInfo } from '../utils.ts';
+import type { TestInfo } from '../types.ts';
+
+type TestCaseData = {
+  node: NonNullable<TestInfo['valid'][number]>;
+  isObject: boolean;
+  hasName: boolean;
+  hasConfig: boolean;
+};
+
+const violationFilters = {
+  always: (testCase: TestCaseData) => !testCase.hasName,
+  objects: (testCase: TestCaseData) => testCase.isObject && !testCase.hasName,
+  'objects-with-config': (testCase: TestCaseData) =>
+    testCase.isObject && testCase.hasConfig && !testCase.hasName,
+} satisfies Record<Options['require'], (testCase: TestCaseData) => boolean>;
+
+const violationMessages = {
+  always: 'nameRequiredAlways',
+  objects: 'nameRequiredObjects',
+  'objects-with-config': 'nameRequiredObjectsWithConfig',
+} satisfies Record<Options['require'], string>;
+
+type Options = {
+  require: 'always' | 'objects' | 'objects-with-config';
+};
+
+const rule: Rule.RuleModule = {
+  meta: {
+    type: 'suggestion',
+    docs: {
+      description:
+        'require test cases to have a `name` property under certain conditions',
+      category: 'Tests',
+      recommended: false,
+      url: 'https://github.com/eslint-community/eslint-plugin-eslint-plugin/tree/HEAD/docs/rules/require-test-case-name.md',
+    },
+    schema: [
+      {
+        additionalProperties: false,
+        properties: {
+          require: {
+            description:
+              'When should the name property be required on a test case object.',
+            enum: ['always', 'objects', 'objects-with-config'],
+          },
+        },
+        type: 'object',
+      },
+    ],
+    defaultOptions: [{ require: 'objects-with-config' }],
+    messages: {
+      nameRequiredAlways:
+        'This test case is missing the `name` property.  All test cases should have a name property.',
+      nameRequiredObjects:
+        'This test case is missing the `name` property.  Test cases defined as objects should have a name property.',
+      nameRequiredObjectsWithConfig:
+        'This test case is missing the `name` property.  Test cases defined as objects with additional configuration should have a name property.',
+    },
+  },
+
+  create(context) {
+    const { require: requireOption = 'objects-with-config' }: Options =
+      context.options[0] || {};
+    const sourceCode = context.sourceCode;
+
+    /**
+     * Validates test cases and reports them if found in violation
+     * @param cases A list of test case nodes
+     */
+    function validateTestCases(cases: TestInfo['valid']): void {
+      // Gather all of the information from each test case
+      const testCaseData: TestCaseData[] = cases
+        .filter((testCase) => !!testCase)
+        .map((testCase) => {
+          if (
+            testCase.type === 'Literal' ||
+            testCase.type === 'TemplateLiteral'
+          ) {
+            return {
+              node: testCase,
+              isObject: false,
+              hasName: false,
+              hasConfig: false,
+            };
+          }
+          if (testCase.type === 'ObjectExpression') {
+            let hasName = false;
+            let hasConfig = false;
+
+            // evaluateObjectProperties is used here to expand spread elements
+            for (const property of evaluateObjectProperties(
+              testCase,
+              sourceCode.scopeManager,
+            )) {
+              if (property.type === 'Property') {
+                const keyName = getKeyName(
+                  property,
+                  sourceCode.getScope(testCase),
+                );
+                if (keyName === 'name') {
+                  hasName = true;
+                } else if (keyName === 'options' || keyName === 'settings') {
+                  hasConfig = true;
+                }
+              }
+            }
+
+            return {
+              node: testCase,
+              isObject: true,
+              hasName,
+              hasConfig,
+            };
+          }
+          return null;
+        })
+        .filter((testCase) => !!testCase);
+
+      const violations = testCaseData.filter(violationFilters[requireOption]);
+      for (const violation of violations) {
+        context.report({
+          node: violation.node,
+          messageId: violationMessages[requireOption],
+        });
+      }
+    }
+
+    return {
+      Program(ast) {
+        getTestInfo(context, ast)
+          .map((testRun) => [...testRun.valid, ...testRun.invalid])
+          .forEach(validateTestCases);
+      },
+    };
+  },
+};
+
+export default rule;

tests/lib/rules/consistent-output.ts

@@ -67,6 +67,7 @@ ruleTester.run('consistent-output', rule, {
         });
       `,
       options: ['always'],
+      name: 'test case with code, output, and errors (options: always)',
     },
     `
     new NotRuleTester().run('foo', bar, {
@@ -118,6 +119,7 @@ ruleTester.run('consistent-output', rule, {
       `,
       options: ['always'],
       errors: [ERROR],
+      name: 'invalid test case missing output (options: always)',
     },
   ],
 });

tests/lib/rules/meta-property-ordering.ts

@@ -65,6 +65,7 @@ ruleTester.run('test-case-property-ordering', rule, {
           create() {},
         };`,
       options: [['schema', 'docs']],
+      name: 'custom order (options: [schema, docs])',
     },
     `
     module.exports = {
@@ -179,6 +180,7 @@ ruleTester.run('test-case-property-ordering', rule, {
           data: { order: ['type', 'docs', 'fixable'].join(', ') },
         },
       ],
+      name: 'custom order with extra prop (options: [type, docs, fixable])',
     },
   ],
 });

tests/lib/rules/no-property-in-node.ts

@@ -110,6 +110,7 @@ ruleTester.run('no-property-in-node', rule, {
           additionalNodeTypeFiles: [/not-found/],
         },
       ],
+      name: 'additionalNodeTypeFiles with no matches',
     },
   ],
   invalid: [
@@ -204,6 +205,7 @@ ruleTester.run('no-property-in-node', rule, {
           messageId: 'in',
         },
       ],
+      name: 'additionalNodeTypeFiles with matches',
     },
   ],
 });