test: node_modules installs for system-tests, other improvements (#18574)

Co-authored-by: Emily Rohrbough <emilyrohrbough@users.noreply.github.com>

Author: flotwig

circle.yml

@@ -149,6 +149,55 @@ commands:
           name: Restore all node_modules to proper workspace folders
           command: node scripts/circle-cache.js --action unpack
 
+  restore_cached_system_tests_deps:
+    description: 'Restore the cached node_modules for projects in "system-tests/projects/**"'
+    steps:
+      - run:
+          name: Generate Circle Cache key for system tests
+          command: ./system-tests/scripts/cache-key.sh > system_tests_cache_key
+      - restore_cache:
+          name: Restore system tests node_modules cache
+          keys:
+            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
+
+  update_cached_system_tests_deps:
+    description: 'Update the cached node_modules for projects in "system-tests/projects/**"'
+    steps:
+      - run:
+          name: Generate Circle Cache key for system tests
+          command: ./system-tests/scripts/cache-key.sh > system_tests_cache_key
+      - restore_cache:
+          name: Restore cache state, to check for known modules cache existence
+          keys:
+            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
+      - run:
+          name: Bail if specific cache exists
+          command: |
+            if [[ -f "system_tests_node_modules_installed" ]]; then
+              echo "No updates to system tests node modules, exiting"
+              circleci-agent step halt
+            fi
+      - restore_cache:
+          name: Restore system tests node_modules cache
+          keys:
+            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
+      - run:
+          name: Update system-tests node_modules cache
+          command: yarn workspace @tooling/system-tests projects:yarn:install
+      - save_cache:
+          name: Save system tests node_modules cache
+          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+          paths:
+            - ~/.cache/cy-system-tests-node-modules
+      - run: touch system_tests_node_modules_installed
+      - save_cache:
+          name: Save system tests node_modules cache state key
+          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
+          paths:
+            - system_tests_node_modules_installed
+
   caching-dependency-installer:
     description: 'Installs & caches the dependencies based on yarn lock & package json dependencies'
     parameters:
@@ -427,6 +476,7 @@ commands:
         type: string
     steps:
       - restore_cached_workspace
+      - restore_cached_system_tests_deps
       - run:
           name: Run system tests
           command: |
@@ -1083,6 +1133,12 @@ jobs:
           path: /tmp/artifacts
       - store-npm-logs
 
+  system-tests-node-modules-install:
+    <<: *defaults
+    steps:
+      - restore_cached_workspace
+      - update_cached_system_tests_deps
+
   system-tests-chrome:
     <<: *defaults
     resource_class: medium
@@ -2007,19 +2063,22 @@ linux-workflow: &linux-workflow
     - server-performance-tests:
         requires:
           - build
-    - system-tests-chrome:
+    - system-tests-node-modules-install:
         requires:
           - build
+    - system-tests-chrome:
+        requires:
+          - system-tests-node-modules-install
     - system-tests-electron:
         requires:
-          - build
+          - system-tests-node-modules-install
     - system-tests-firefox:
         requires:
-          - build
+          - system-tests-node-modules-install
     - system-tests-non-root:
         executor: non-root-docker-user
         requires:
-          - build
+          - system-tests-node-modules-install
     - driver-integration-tests-chrome:
         requires:
           - build

npm/webpack-preprocessor/lib/typescript-overrides.ts

@@ -17,7 +17,15 @@ export const overrideSourceMaps = (sourceMap: boolean, typescriptPath?: string)
       return
     }
 
-    const typescript = require(typescriptPath || 'typescript') as typeof import('typescript')
+    // when using webpack-preprocessor as a local filesystem dependency (`file:...`),
+    // require(typescript) will resolve to this repo's `typescript` devDependency, not the
+    // targeted project's `typescript`, which breaks monkeypatching. resolving from the
+    // CWD avoids this issue.
+    const projectTsPath = require.resolve(typescriptPath || 'typescript', {
+      paths: [process.cwd()],
+    })
+
+    const typescript = require(projectTsPath) as typeof import('typescript')
     const { createProgram } = typescript
 
     debug('typescript found, overriding typescript.createProgram()')

npm/webpack-preprocessor/test/unit/typescript-overrides.spec.ts

@@ -147,7 +147,7 @@ describe('./lib/typescript-overrides', () => {
       const err = typescriptOverrides.overrideSourceMaps(true)
 
       expect(err).to.be.instanceOf(Error)
-      expect(err.message).to.eq(`Cannot find module 'typescript'`)
+      expect(err.message).to.match(/Cannot find module '.*typescript\.js'/)
     })
   })
 })

packages/server/lib/plugins/index.js

@@ -42,7 +42,7 @@ const init = (config, options) => {
 
   // test and warn for incompatible plugin
   try {
-    const retriesPluginPath = path.dirname(resolve.sync('cypress-plugin-retries', {
+    const retriesPluginPath = path.dirname(resolve.sync('cypress-plugin-retries/package.json', {
       basedir: options.projectRoot,
     }))
 

packages/server/test/integration/http_requests_spec.js

@@ -42,7 +42,7 @@ zlib = Promise.promisifyAll(zlib)
 // force supertest-session to use promises provided in supertest
 const session = proxyquire('supertest-session', { supertest })
 
-const absolutePathRegex = /"\/[^{}]*?\.projects/g
+const absolutePathRegex = /"\/[^{}]*?cy-projects/g
 let sourceMapRegex = /\n\/\/# sourceMappingURL\=.*/
 
 const replaceAbsolutePaths = (content) => {

packages/server/test/integration/plugins_spec.js

@@ -6,8 +6,9 @@ const Fixtures = require('@tooling/system-tests/lib/fixtures')
 const pluginsFile = Fixtures.projectPath('plugin-before-browser-launch-deprecation/cypress/plugins/index.js')
 
 describe('lib/plugins', () => {
-  beforeEach(() => {
-    Fixtures.scaffold()
+  beforeEach(async () => {
+    Fixtures.scaffoldProject('plugin-before-browser-launch-deprecation')
+    await Fixtures.scaffoldCommonNodeModules()
   })
 
   afterEach(() => {

packages/server/test/unit/files_spec.js

@@ -25,7 +25,7 @@ describe('lib/files', () => {
       return files.readFile(this.projectRoot, 'tests/_fixtures/message.txt').then(({ contents, filePath }) => {
         expect(contents).to.eq('foobarbaz')
 
-        expect(filePath).to.include('/.projects/todos/tests/_fixtures/message.txt')
+        expect(filePath).to.include('/cy-projects/todos/tests/_fixtures/message.txt')
       })
     })
 
@@ -69,7 +69,7 @@ describe('lib/files', () => {
         return files.readFile(this.projectRoot, '.projects/write_file.txt').then(({ contents, filePath }) => {
           expect(contents).to.equal('foo')
 
-          expect(filePath).to.include('/.projects/todos/.projects/write_file.txt')
+          expect(filePath).to.include('/cy-projects/todos/.projects/write_file.txt')
         })
       })
     })

scripts/binary/smoke.js

@@ -159,21 +159,24 @@ const runFailingProjectTest = function (buildAppExecutable, e2e) {
   .then(verifyScreenshots)
 }
 
-const test = function (buildAppExecutable) {
-  Fixtures.scaffold()
-
+const test = async function (buildAppExecutable) {
+  await Fixtures.scaffoldCommonNodeModules()
+  Fixtures.scaffoldProject('e2e')
   const e2e = Fixtures.projectPath('e2e')
 
-  return runSmokeTest(buildAppExecutable)
-  .then(() => {
-    return runProjectTest(buildAppExecutable, e2e)
-  }).then(() => {
-    return runFailingProjectTest(buildAppExecutable, e2e)
-  }).then(() => {
-    return Fixtures.remove()
-  })
+  await runSmokeTest(buildAppExecutable)
+  await runProjectTest(buildAppExecutable, e2e)
+  await runFailingProjectTest(buildAppExecutable, e2e)
+  Fixtures.remove()
 }
 
 module.exports = {
   test,
 }
+
+if (require.main === module) {
+  const buildAppExecutable = path.join(__dirname, `../../build/${os.platform()}-unpacked/Cypress`)
+
+  console.log('Script invoked directly, running smoke tests.')
+  test(buildAppExecutable)
+}

system-tests/README.md

@@ -7,7 +7,7 @@ These tests launch the [Cypress server](../packages/server) process for each tes
 
 These tests run in CI in Electron, Chrome, and Firefox under the `system-tests` job family.
 
-## Running system tests
+## Running System Tests
 
 ```bash
 yarn test <path/to/test>
@@ -28,14 +28,14 @@ To debug the Cypress process under test, you can pass `--cypress-inspect-brk`:
 yarn test test/go_spec.js --browser chrome --no-exit
 ```
 
-## Developing tests
+## Developing Tests
 
 System tests cover the entire Cypress run, so they are good for testing features that do not fit into a normal integration or unit test. However, they do take more resources to run, so consider carefully if you really *need* to write a system test, or if you could achieve 100% coverage via an integration or unit test instead.
 
 There are two parts to a system test:
 
 1. A test written using the [`systemTests`](./lib/system-tests) Mocha wrapper that lives in [`./test`](./test), and
-2. A matching Cypress project that lives in the [`./projects`](./projects) directory.
+2. A matching Cypress [test project](#Test-Projects) that lives in the [`./projects`](./projects) directory.
 
 For example, if you initialized a new project in `./projects/my-new-project`, and you wanted to assert that 2 tests fail and take a snapshot of the `stdout`, you'd write a test like this:
 
@@ -61,10 +61,32 @@ From here, you could run this test with `yarn test my-new-project`.
 
 There are many more options available for `systemTests.it` and `systemTests.setup`. You can massage the stdout, do pre-run tasks, set up HTTP/S servers, and more. Explore the typedocs in [`./lib/system-tests`](./lib/system-tests) for more information.
 
-## Updating snaphots
+### Updating Snaphots
 
 Prepend `SNAPSHOT_UPDATE=1` to any test command. See [`snap-shot-it` instructions](https://github.com/bahmutov/snap-shot-it#advanced-use) for more info.
 
 ```bash
 SNAPSHOT_UPDATE=1 yarn test go_spec
 ```
+
+### Test Projects
+
+Every folder in [`./projects`](./lib/projects) represents a self-contained Cypress project. When you pass the `project` property to `systemTests.it` or `systemTests.exec`, Cypress launches using this project.
+
+If a test project has a `package.json` file, the `systemTests.exec` helper will attempt to install the correct `node_modules` by running `yarn install` against the project. This is cached in CI and locally to speed up test times.
+
+`systemTests.exec` *copies* the project directory to a temporary folder outside of the monorepo root. This means that temporary projects will not inherit the `node_modules` from this package or the monorepo. So, you must add the dependencies required for your project in `dependencies` or `devDependencies`.
+
+The exception is some commonly used packages that are scaffolded for all projects, like `lodash` and `debug`. You can see the list by looking at `scaffoldCommonNodeModules` in [`./lib/fixtures.ts`](./lib/fixtures.ts) These packages do not need to be added to a test project's `package.json`.
+
+You can also set special properties in a test project's `package.json` to influence the helper's behavior when running `yarn`:
+
+`package.json` Property Name | Type | Description
+--- | --- | ---
+`_cySkipYarnInstall` | `boolean` | If `true`, skip the automatic `yarn install` for this package, even though it has a `package.json`.
+`_cyYarnV2` | `boolean` | Run the yarn v2-style install command instead of yarn v1-style.
+`_cyRunScripts` | `boolean` | By default, the automatic `yarn install` will not run postinstall scripts. This option, if set, will cause postinstall scripts to run for this project.
+
+Run `yarn projects:yarn:install` to run `yarn install` for all projects with a `package.json`.
+
+Use the `UPDATE_YARN_LOCK=1` environment variable with `yarn test` or `yarn projects:yarn:install` to allow the `yarn.lock` to be updated and synced back to the monorepo from the temp dir.

system-tests/__snapshots__/busted_support_file_spec.js

@@ -25,6 +25,8 @@ Error: Webpack Compilation Error
 ./cypress/support/index.js
 Module not found: Error: Can't resolve './does/not/exist' in '/foo/bar/.projects/busted-support-file/cypress/support'
 Looked for and couldn't find the file at the following paths:
+[/foo/bar/.projects/busted-support-file/cypress/support/package.json]
+[/foo/bar/.projects/busted-support-file/cypress/support/does/not/exist/package.json]
 [/foo/bar/.projects/busted-support-file/cypress/support/does/not/exist]
 [/foo/bar/.projects/busted-support-file/cypress/support/does/not/exist.js]
 [/foo/bar/.projects/busted-support-file/cypress/support/does/not/exist.json]