feat: add testIsolation option and support describe-only overrides (#…

…23040)

Co-authored-by: Lachlan Miller <lachlan.miller.1990@outlook.com>
Co-authored-by: Bill Glesias <bglesias@gmail.com>
Co-authored-by: Matt Henkes <mjhenkes@gmail.com>

cli/types/cypress.d.ts

@@ -2784,6 +2784,13 @@ declare namespace Cypress {
      * @default "cypress/support/{e2e|component}.js"
      */
     supportFile: string | false
+    /**
+     * The test isolation level applied to ensure a clean slate between tests.
+     *   - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test.
+     *   - strict - applies all resets/clears from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test.
+     * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict"
+     */
+    testIsolation: 'legacy' | 'strict'
     /**
      * Path to folder where videos will be saved after a headless or CI run
      * @default "cypress/videos"

packages/config/__snapshots__/index.spec.ts.js

@@ -67,6 +67,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1
   "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}",
   "supportFolder": false,
   "taskTimeout": 60000,
+  "testIsolation": "legacy",
   "trashAssetsBeforeRuns": true,
   "userAgent": null,
   "video": true,
@@ -77,6 +78,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1
   "viewportWidth": 1000,
   "waitForAnimations": true,
   "watchForFileChanges": true,
+  "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}",
   "additionalIgnorePattern": [],
   "autoOpen": false,
   "browsers": [],
@@ -146,6 +148,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f
   "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}",
   "supportFolder": false,
   "taskTimeout": 60000,
+  "testIsolation": "legacy",
   "trashAssetsBeforeRuns": true,
   "userAgent": null,
   "video": true,
@@ -156,6 +159,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f
   "viewportWidth": 1000,
   "waitForAnimations": true,
   "watchForFileChanges": true,
+  "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}",
   "additionalIgnorePattern": [],
   "autoOpen": false,
   "browsers": [],
@@ -172,8 +176,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f
   "socketId": null,
   "socketIoCookie": "__socket",
   "socketIoRoute": "/__socket",
-  "xhrRoute": "/xhrs/",
-  "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}"
+  "xhrRoute": "/xhrs/"
 }
 
 exports['config/src/index .getPublicConfigKeys returns list of public config keys 1'] = [
@@ -221,6 +224,7 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key
   "supportFile",
   "supportFolder",
   "taskTimeout",
+  "testIsolation",
   "trashAssetsBeforeRuns",
   "userAgent",
   "video",
@@ -235,6 +239,5 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key
   "browsers",
   "hosts",
   "isInteractive",
-  "modifyObstructiveCode",
-  "specPattern"
+  "modifyObstructiveCode"
 ]

packages/config/src/browser.ts

@@ -1,30 +1,41 @@
 import _ from 'lodash'
 import Debug from 'debug'
-import { defaultSpecPattern, options, breakingOptions, breakingRootOptions, testingTypeBreakingOptions, additionalOptionsToResolveConfig } from './options'
-import type { BreakingOption, BreakingOptionErrorKey } from './options'
+import {
+  defaultSpecPattern,
+  options,
+  breakingOptions,
+  breakingRootOptions,
+  testingTypeBreakingOptions,
+} from './options'
+
 import type { TestingType } from '@packages/types'
+import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel } from './options'
+import type { ErrResult } from './validation'
 
 // this export has to be done in 2 lines because of a bug in babel typescript
 import * as validation from './validation'
 
 export {
   defaultSpecPattern,
-  validation,
   options,
   breakingOptions,
   BreakingOption,
   BreakingOptionErrorKey,
+  ErrResult,
+  validation,
 }
 
 const debug = Debug('cypress:config:browser')
 
 const dashesOrUnderscoresRe = /^(_-)+/
 
 // takes an array and creates an index object of [keyKey]: [valueKey]
-function createIndex<T extends Record<string, any>> (arr: Array<T>, keyKey: keyof T, valueKey: keyof T) {
+function createIndex<T extends Record<string, any>> (arr: Array<T>, keyKey: keyof T, valueKey: keyof T, defaultValueFallback?: any) {
   return _.reduce(arr, (memo: Record<string, any>, item) => {
     if (item[valueKey] !== undefined) {
-      memo[item[keyKey] as string] = item[valueKey]
+      memo[item[keyKey]] = item[valueKey]
+    } else {
+      memo[item[keyKey]] = defaultValueFallback
     }
 
     return memo
@@ -33,13 +44,20 @@ function createIndex<T extends Record<string, any>> (arr: Array<T>, keyKey: keyo
 
 const breakingKeys = _.map(breakingOptions, 'name')
 const defaultValues = createIndex(options, 'name', 'defaultValue')
-const publicConfigKeys = _([...options, ...additionalOptionsToResolveConfig]).reject({ isInternal: true }).map('name').value()
+const publicConfigKeys = _(options).reject({ isInternal: true }).map('name').value()
 const validationRules = createIndex(options, 'name', 'validation')
-const testConfigOverrideOptions = createIndex(options, 'name', 'canUpdateDuringTestTime')
+
+export const testOverrideLevels = createIndex(options, 'name', 'overrideLevel', 'never')
+
 const restartOnChangeOptionsKeys = _.filter(options, 'requireRestartOnChange')
 
 const issuedWarnings = new Set()
 
+export type InvalidTestOverrideResult = {
+  invalidConfigKey: string
+  supportedOverrideLevel: string
+}
+
 export type BreakingErrResult = {
   name: string
   newName?: string
@@ -137,7 +155,7 @@ export const matchesConfigKey = (key: string) => {
   return
 }
 
-export const validate = (cfg: any, onErr: (property: string) => void) => {
+export const validate = (cfg: any, onErr: (property: ErrResult | string) => void) => {
   debug('validating configuration')
 
   return _.each(cfg, (value, key) => {
@@ -172,16 +190,22 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof
   return validateNoBreakingOptions(options, cfg, onWarning, onErr, testingType)
 }
 
-export const validateNoReadOnlyConfig = (config: any, onErr: (property: string) => void) => {
-  let errProperty
+export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: boolean, onErr: (result: InvalidTestOverrideResult) => void) => {
+  Object.keys(config).some((configKey) => {
+    const overrideLevel: OverrideLevel = testOverrideLevels[configKey]
 
-  Object.keys(config).some((option) => {
-    return errProperty = testConfigOverrideOptions[option] === false ? option : undefined
-  })
+    if (!overrideLevel) {
+      // non-cypress configuration option. skip validation
+      return
+    }
 
-  if (errProperty) {
-    return onErr(errProperty)
-  }
+    if (overrideLevel === 'never' || (overrideLevel === 'suite' && !isSuiteLevelOverride)) {
+      onErr({
+        invalidConfigKey: configKey,
+        supportedOverrideLevel: overrideLevel,
+      })
+    }
+  })
 }
 
 export const validateNeedToRestartOnChange = (cachedConfig: any, updatedConfig: any) => {