chore(launchpad): work on infra for scaffold tests (#20818)

* chore: work on infra for scaffold tests

* revert

* wip

* ignore scaffolded files in linting

* add snapshot for plain TS project

* add more tests

* linting

* add temp-dir dep to data-context

* exclude scaffolded assets from linting

* bump server tsconfig lib to es2019

* bump graphql tsconfig lib

* add missing dependency

* use task to avoid test code leaking into data-context prod

* remove unused code

* remove deps

* make currentProject cross platform with path.posix and path.sep

* posix.join

* use path.posix everywhere

* update types

* correct paths and add ct framework to expected dir

* fix scaffolded files from indexHtmlFile merge

Co-authored-by: Zachary Williams <ZachJW34@gmail.com>

Author: lmiller1990

.eslintignore

@@ -91,3 +91,5 @@ system-tests/projects/nuxtjs-vue2-unconfigured/**/*
 system-tests/projects/nuxtjs-vue2-configured/**/*
 
 system-tests/projects/react-app-webpack-5-unconfigured/**/*
+
+system-tests/projects/**/*/expected-cypress*/**/*

packages/graphql/tsconfig.json

@@ -9,6 +9,7 @@
     "script"
   ],
   "compilerOptions": {
+    "lib": ["es2020"],
     "strict": true,
     "allowJs": false,
     "noImplicitAny": true,
@@ -19,4 +20,4 @@
     "importsNotUsedAsValues": "error",
     "types": []
   },
-}
+}

packages/launchpad/cypress.config.ts

@@ -1,6 +1,7 @@
 import { defineConfig } from 'cypress'
 import getenv from 'getenv'
 import { devServer } from '@cypress/vite-dev-server'
+import { snapshotCypressDirectory } from './cypress/tasks/snapshotsScaffold'
 
 const CYPRESS_INTERNAL_CLOUD_ENV = getenv('CYPRESS_INTERNAL_CLOUD_ENV', process.env.CYPRESS_INTERNAL_ENV || 'development')
 
@@ -36,6 +37,10 @@ export default defineConfig({
       process.env.CYPRESS_INTERNAL_E2E_TESTING_SELF = 'true'
       const { e2ePluginSetup } = require('@packages/frontend-shared/cypress/e2e/e2ePluginSetup')
 
+      on('task', {
+        snapshotCypressDirectory,
+      })
+
       return await e2ePluginSetup(on, config)
     },
   },

packages/launchpad/cypress/e2e/scaffold-project.cy.ts

@@ -0,0 +1,117 @@
+import type { FRONTEND_FRAMEWORKS } from '@packages/scaffold-config'
+import type { SnapshotScaffoldTestResult } from '@packages/launchpad/cypress/tasks/snapshotsScaffold'
+
+// The tests in this file take an existing project without Cypress Configured
+// and add Cypress using the launchpad setup wizard.
+//
+// See `system-tests/projects/pristine` for an example.
+//
+// After adding a test for a project for the first time and running
+// this spec, it will see what files are scaffolded and save them
+// in a directory named `expected-cypress-{lang}-{testingType}`.
+// For example, when configuring the `pristine` project using
+// plain JS for E2E testing, it will make
+// a new directory in `pristine` named `expected-cypress-js-e2e`
+// containing the scaffolded files.
+//
+// Each subsequent run will compare the scaffolded files in the
+// `expected-cypress-js-e2e` directory to the newly created ones.
+//
+// If there is a discrepancy, the test will fail and show the diff in the command log.
+//
+// To update your expected files, just delete the `expected-cypress` and re-run the test,
+// or modify them by hand.
+function scaffoldAndOpenE2EProject (
+  name: Parameters<typeof cy.scaffoldProject>[0],
+  language: 'js' | 'ts',
+  args?: Parameters<typeof cy.openProject>[1],
+) {
+  cy.scaffoldProject(name)
+  cy.openProject(name, args)
+
+  cy.visitLaunchpad()
+
+  cy.contains('Welcome to Cypress!').should('be.visible')
+  cy.contains('[data-cy-testingtype="e2e"]', 'Not Configured')
+  cy.contains('[data-cy-testingtype="component"]', 'Not Configured')
+  cy.contains('E2E Testing').click()
+  cy.contains(language === 'js' ? 'JavaScript' : 'TypeScript').click()
+  cy.contains('Next').click()
+  cy.contains('We added the following files to your project.')
+  cy.contains('Continue').click()
+  cy.contains('Choose a Browser')
+}
+
+function scaffoldAndOpenCTProject (
+  name: Parameters<typeof cy.scaffoldProject>[0],
+  language: 'js' | 'ts',
+  framework: typeof FRONTEND_FRAMEWORKS[number]['name'],
+  bundler?: typeof FRONTEND_FRAMEWORKS[number]['supportedBundlers'][number]['name'],
+  args?: Parameters<typeof cy.openProject>[1],
+) {
+  cy.scaffoldProject(name)
+  cy.openProject(name, args)
+
+  cy.visitLaunchpad()
+
+  cy.contains('Welcome to Cypress!').should('be.visible')
+  cy.contains('[data-cy-testingtype="e2e"]', 'Not Configured')
+  cy.contains('[data-cy-testingtype="component"]', 'Not Configured')
+  cy.contains('Component Testing').click()
+  cy.contains(language === 'js' ? 'JavaScript' : 'TypeScript').click()
+
+  cy.contains('Pick a framework').click()
+  cy.contains(framework).click()
+  if (bundler) {
+    cy.contains(bundler).click()
+  }
+
+  cy.contains('Next Step').click()
+  cy.contains('Skip').click()
+  cy.contains('We added the following files to your project.')
+  cy.contains('Continue').click()
+  cy.contains('Choose a Browser')
+}
+
+function assertScaffoldedFilesAreCorrect (language: 'js' | 'ts', testingType: Cypress.TestingType, ctFramework?: string) {
+  cy.withCtx((ctx) => ctx.currentProject).then((currentProject) => {
+    cy.task<SnapshotScaffoldTestResult>('snapshotCypressDirectory', {
+      currentProject,
+      testingType,
+      language,
+      ctFramework,
+    })
+    .then((result) => {
+      if (result.status === 'ok') {
+        return result
+      }
+
+      throw new Error(result.message)
+    })
+    .then((res) => {
+      cy.log(`✅ ${res.message}`)
+    })
+  })
+}
+
+describe('scaffolding new projects', () => {
+  it('scaffolds E2E for a JS project', () => {
+    scaffoldAndOpenE2EProject('pristine', 'js')
+    assertScaffoldedFilesAreCorrect('js', 'e2e')
+  })
+
+  it('scaffolds E2E for a TS project', () => {
+    scaffoldAndOpenE2EProject('pristine', 'ts')
+    assertScaffoldedFilesAreCorrect('ts', 'e2e')
+  })
+
+  it('scaffolds CT for a JS project', () => {
+    scaffoldAndOpenCTProject('pristine', 'js', 'Create React App (v5)')
+    assertScaffoldedFilesAreCorrect('js', 'component', 'Create React App (v5)')
+  })
+
+  it('scaffolds CT for a TS project', () => {
+    scaffoldAndOpenCTProject('pristine', 'ts', 'Create React App (v5)')
+    assertScaffoldedFilesAreCorrect('ts', 'component', 'Create React App (v5)')
+  })
+})

packages/launchpad/cypress/tasks/snapshotsScaffold.ts

@@ -0,0 +1,135 @@
+import path from 'path'
+import globby from 'globby'
+import fs from 'fs-extra'
+import disparity from 'disparity'
+import dedent from 'dedent'
+import { cyTmpDir } from '@tooling/system-tests/lib/fixtures'
+
+const systemTestsDir = path.join(__dirname, '..', '..', '..', '..', 'system-tests', 'projects')
+
+export interface SnapshotScaffoldTestResult {
+  status: 'ok' | 'fail'
+  message: string
+}
+
+interface FileToDiff {
+  actual: string
+  expected: string
+}
+
+async function snapshotScaffoldedFiles (expectedScaffoldDir: string, filesToDiff: FileToDiff[]): Promise<SnapshotScaffoldTestResult> {
+  // see if existing snapshots exist compared to source
+  // project in `system-tests/projects`
+  // expected-cypress-{'js' | 'ts'}
+  fs.mkdirSync(expectedScaffoldDir)
+
+  await Promise.all(filesToDiff.map(async (files) => {
+    const { expected, actual } = files
+
+    await fs.ensureFile(expected)
+    await fs.copy(actual, expected)
+    await fs.copy(actual, expected)
+  }))
+
+  return {
+    status: 'ok',
+    message: `Created expected files based on test`,
+  }
+}
+
+function compareScaffoldedFiles (filesToDiff: FileToDiff[]): SnapshotScaffoldTestResult {
+  for (const { actual, expected } of filesToDiff) {
+    try {
+      const read = (f: string) => fs.readFileSync(f, 'utf-8')
+      const actualContent = read(actual).trim()
+      const expectedContent = read(expected).trim()
+      const diff = disparity.unifiedNoColor(actualContent, expectedContent, {})
+
+      if (diff !== '') {
+        return {
+          status: 'fail',
+          message: `Expected contents of ${actual} and ${expected} to match. Diff: ${diff}`,
+        }
+      }
+    } catch (err) {
+      const e = err as NodeJS.ErrnoException
+
+      if (e.code === 'ENOENT') {
+        return {
+          status: 'fail',
+          message: `Expected ${e.path} to exist, but it was not found`,
+        }
+      }
+
+      return {
+        status: 'fail',
+        message: `Unexpected error: ${e.message}`,
+      }
+    }
+  }
+
+  return {
+    status: 'ok',
+    message: 'Scaffolded files matched expected files',
+  }
+}
+
+interface SnapshotCypressDirectoryOptions {
+  currentProject: string
+  language: 'js' | 'ts'
+  testingType: Cypress.TestingType
+  ctFramework?: string
+}
+
+function removeHyphensAndBrackets (str: string) {
+  return str.toLowerCase().replaceAll(' ', '-').replaceAll('(', '').replaceAll(')', '')
+}
+
+export async function snapshotCypressDirectory ({ currentProject, language, testingType, ctFramework }: SnapshotCypressDirectoryOptions): Promise<SnapshotScaffoldTestResult> {
+  if (!currentProject.startsWith(cyTmpDir)) {
+    throw Error(dedent`
+      snapshotCypressDirectory is designed to be used with system-tests infrastructure.
+      It should not be used with projects that are not created in a temporary directory
+      by the system-tests infrastructure.
+      
+      Expected currentProject to be in /tmp. currentProject found at path ${currentProject}`)
+  }
+
+  const projectDir = currentProject.replace(cyTmpDir, systemTestsDir)
+  const projectName = projectDir.replace(systemTestsDir, '').slice(1)
+
+  let expectedScaffoldDir = path.join(projectDir, `expected-cypress-${language}-${testingType}`)
+
+  if (ctFramework) {
+    expectedScaffoldDir += `-${removeHyphensAndBrackets(ctFramework)}`
+  }
+
+  const joinPosix = (...s: string[]) => path.join(...s).split(path.sep).join(path.posix.sep)
+
+  const files = (
+    await Promise.all([
+      globby(joinPosix(currentProject, 'cypress'), { onlyFiles: true }),
+      globby(joinPosix(currentProject, 'cypress.config.*'), { onlyFiles: true }),
+    ])
+  ).reduce((acc, curr) => {
+    return [acc, curr].flat(2)
+  }, [])
+
+  const actualRelativeFiles = files.map((file) => {
+    return file.replace(cyTmpDir, '').slice(projectName.length + 2)
+  })
+
+  const filesToDiff = actualRelativeFiles.map<FileToDiff>((file) => {
+    return {
+      actual: path.join(currentProject, file),
+      expected: path.join(expectedScaffoldDir, file),
+    }
+  })
+
+  if (fs.existsSync(expectedScaffoldDir)) {
+    // compare!
+    return compareScaffoldedFiles(filesToDiff)
+  }
+
+  return snapshotScaffoldedFiles(expectedScaffoldDir, filesToDiff)
+}

packages/launchpad/package.json

@@ -30,7 +30,10 @@
     "@percy/cypress": "^3.1.0",
     "@purge-icons/generated": "0.8.1",
     "@testing-library/cypress": "8.0.0",
+    "@tooling/system-tests": "0.0.0-development",
     "@toycode/markdown-it-class": "1.2.3",
+    "@types/dedent": "^0.7.0",
+    "@types/fs-extra": "^8.0.1",
     "@urql/core": "2.3.1",
     "@urql/devtools": "2.0.3",
     "@urql/vue": "0.4.3",
@@ -46,6 +49,10 @@
     "cross-env": "6.0.3",
     "cypress-plugin-tab": "1.0.5",
     "cypress-real-events": "1.6.0",
+    "dedent": "^0.7.0",
+    "disparity": "^3.0.0",
+    "fs-extra": "8.1.0",
+    "globby": "^11.0.1",
     "graphql": "^15.5.1",
     "graphql-tag": "^2.12.5",
     "gravatar": "1.8.0",

packages/server/tsconfig.json

@@ -8,6 +8,7 @@
     "./../ts/index.d.ts"
   ],
   "compilerOptions": {
+    "lib": ["es2020"],
     "types": [
       "mocha",
       "node"
@@ -16,4 +17,4 @@
     "noUnusedLocals": false,
     "importHelpers": true
   }
-}
+}

system-tests/lib/fixtures.ts

@@ -14,7 +14,9 @@ export const cyTmpDir = _path.join(tempDir, 'cy-projects')
 const safeRemove = (path) => {
   try {
     fs.removeSync(path)
-  } catch (err) {
+  } catch (_err) {
+    const err = _err as NodeJS.ErrnoException
+
     // Windows does not like the en masse deleting of files, since the AV will hold
     // a lock on files when they are written. This skips deleting if the lock is
     // encountered.

system-tests/projects/pristine/expected-cypress-js-component-create-react-app-v5/cypress.config.js

@@ -0,0 +1,7 @@
+const { devServer } = require('@cypress/react/plugins/react-scripts')
+
+module.exports = {
+  component: {
+    devServer,
+  },
+}

system-tests/projects/pristine/expected-cypress-js-component-create-react-app-v5/cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}