test(system-tests): support npm for test projects (#20664)

Author: flotwig

.gitignore

@@ -101,7 +101,6 @@ system-tests/fixtures/large-img
 
 # Building app binary
 scripts/support
-package-lock.json
 binary-url.json
 
 # Allows us to dynamically create eslint rules that override the default for Decaffeinate scripts

.vscode/settings.json

@@ -39,4 +39,6 @@
   // These are commented out because they slow down node development
   // "volar.autoCompleteRefs": false,
   "volar.takeOverMode.enabled": true,
+
+  "editor.tabSize": 2,
 }

circle.yml

@@ -312,6 +312,7 @@ commands:
           key: v{{ .Environment.CACHE_VERSION }}-{{ checksum "platform_key" }}-deps-root-weekly-{{ checksum "cache_date" }}
           paths:
             - ~/.yarn
+            - ~/.cy-npm-cache
 
   verify-build-setup:
     description: Common commands run when setting up for build or yarn install

packages/server/test/integration/plugins_spec.js

@@ -2,13 +2,14 @@ require('../spec_helper')
 
 const plugins = require('../../lib/plugins')
 const Fixtures = require('@tooling/system-tests/lib/fixtures')
+const { scaffoldCommonNodeModules } = require('@tooling/system-tests/lib/dep-installer')
 
 const pluginsFile = Fixtures.projectPath('plugin-before-browser-launch-deprecation/cypress/plugins/index.js')
 
 describe('lib/plugins', () => {
   beforeEach(async () => {
     Fixtures.scaffoldProject('plugin-before-browser-launch-deprecation')
-    await Fixtures.scaffoldCommonNodeModules()
+    await scaffoldCommonNodeModules()
   })
 
   afterEach(() => {

scripts/binary/smoke.js

@@ -7,6 +7,7 @@ const Promise = require('bluebird')
 const os = require('os')
 const verify = require('../../cli/lib/tasks/verify')
 const Fixtures = require('@tooling/system-tests/lib/fixtures')
+const { scaffoldCommonNodeModules } = require('@tooling/system-tests/lib/dep-installer')
 
 const fs = Promise.promisifyAll(fse)
 
@@ -160,7 +161,7 @@ const runFailingProjectTest = function (buildAppExecutable, e2e) {
 }
 
 const test = async function (buildAppExecutable) {
-  await Fixtures.scaffoldCommonNodeModules()
+  await scaffoldCommonNodeModules()
   Fixtures.scaffoldProject('e2e')
   const e2e = Fixtures.projectPath('e2e')
 

system-tests/README.md

@@ -103,20 +103,20 @@ SNAPSHOT_UPDATE=1 yarn test go_spec
 
 Every folder in [`./projects`](./lib/projects) represents a self-contained Cypress project. When you pass the `project` property to `systemTests.it` or `systemTests.exec`, Cypress launches using this project.
 
-If a test project has a `package.json` file, the `systemTests.exec` helper will attempt to install the correct `node_modules` by running `yarn install` against the project. This is cached in CI and locally to speed up test times.
+If a test project has a `package.json` file, the `systemTests.exec` helper will attempt to install the correct `node_modules` by running `yarn install` or `npm install` (depending on which lockfile is present) against the project. This is cached in CI and locally to speed up test times.
 
 `systemTests.exec` *copies* the project directory to a temporary folder outside of the monorepo root. This means that temporary projects will not inherit the `node_modules` from this package or the monorepo. So, you must add the dependencies required for your project in `dependencies` or `devDependencies`.
 
 The exception is some commonly used packages that are scaffolded for all projects, like `lodash` and `debug`. You can see the list by looking at `scaffoldCommonNodeModules` in [`./lib/fixtures.ts`](./lib/fixtures.ts) These packages do not need to be added to a test project's `package.json`.
 
-You can also set special properties in a test project's `package.json` to influence the helper's behavior when running `yarn`:
+You can also set special properties in a test project's `package.json` to influence the helper's behavior when running `yarn` or `npm`:
 
 `package.json` Property Name | Type | Description
 --- | --- | ---
-`_cySkipYarnInstall` | `boolean` | If `true`, skip the automatic `yarn install` for this package, even though it has a `package.json`.
+`_cySkipDepInstall` | `boolean` | If `true`, skip the automatic `yarn install` or `npm install` for this package, even though it has a `package.json`.
 `_cyYarnV311` | `boolean` | Run the yarn v3.1.1-style install command instead of yarn v1-style.
-`_cyRunScripts` | `boolean` | By default, the automatic `yarn install` will not run postinstall scripts. This option, if set, will cause postinstall scripts to run for this project.
+`_cyRunScripts` | `boolean` | By default, the automatic install will not run postinstall scripts. This option, if set, will cause postinstall scripts to run for this project.
 
-Run `yarn projects:yarn:install` to run `yarn install` for all projects with a `package.json`.
+Run `yarn projects:yarn:install` to run `yarn install`/`npm install` for all applicable projects.
 
-Use the `UPDATE_YARN_LOCK=1` environment variable with `yarn test` or `yarn projects:yarn:install` to allow the `yarn.lock` to be updated and synced back to the monorepo from the temp dir.
+Use the `UPDATE_LOCK_FILE=1` environment variable with `yarn test` or `yarn projects:yarn:install` to allow the `yarn.lock` or `package-lock.json` to be updated and synced back to the monorepo from the temp dir.

system-tests/lib/dep-installer/index.ts

@@ -0,0 +1,310 @@
+import fs from 'fs-extra'
+import path from 'path'
+import cachedir from 'cachedir'
+import execa from 'execa'
+import { cyTmpDir, projectPath, projects, root } from '../fixtures'
+import { getYarnCommand } from './yarn'
+import { getNpmCommand } from './npm'
+
+type Dependencies = Record<string, string>
+
+/**
+ * Type for package.json files for system-tests example projects.
+ */
+type SystemTestPkgJson = {
+  /**
+   * By default, scaffolding will run install if there is a `package.json`.
+   * This option, if set, disables that.
+   */
+  _cySkipDepInstall?: boolean
+  /**
+   * Run the yarn v3-style install command instead of yarn v1-style.
+   */
+  _cyYarnV311?: boolean
+  /**
+   * By default, the automatic install will not run postinstall scripts. This
+   * option, if set, will cause postinstall scripts to run for this project.
+   */
+  _cyRunScripts?: boolean
+  dependencies?: Dependencies
+  devDependencies?: Dependencies
+  optionalDependencies?: Dependencies
+}
+
+const log = (...args) => console.log('📦', ...args)
+
+/**
+* Given a package name, returns the path to the module directory on disk.
+*/
+function pathToPackage (pkg: string): string {
+  return path.dirname(require.resolve(`${pkg}/package.json`))
+}
+
+async function ensureCacheDir (cacheDir: string) {
+  try {
+    await fs.stat(cacheDir)
+  } catch (err) {
+    log(`Creating a new node_modules cache dir at ${cacheDir}`)
+    await fs.mkdirp(cacheDir)
+  }
+}
+
+/**
+ * Symlink the cached `node_modules` directory to the temp project directory's `node_modules`.
+ */
+async function symlinkNodeModulesFromCache (tmpNodeModulesDir: string, cacheDir: string): Promise<void> {
+  await fs.symlink(cacheDir, tmpNodeModulesDir, 'junction')
+
+  log(`node_modules symlink created at ${tmpNodeModulesDir}`)
+}
+
+/**
+ * Copy the cached `node_modules` to the temp project directory's `node_modules`.
+ *
+ * @returns a callback that will copy changed `node_modules` back to the cached `node_modules`.
+ */
+async function copyNodeModulesFromCache (tmpNodeModulesDir: string, cacheDir: string): Promise<() => Promise<void>> {
+  await fs.copy(cacheDir, tmpNodeModulesDir, { dereference: true })
+
+  log(`node_modules copied to ${tmpNodeModulesDir} from cache dir ${cacheDir}`)
+
+  return async () => {
+    try {
+      await fs.copy(tmpNodeModulesDir, cacheDir, { dereference: true })
+    } catch (err) {
+      if (err.message === 'Source and destination must not be the same') return
+
+      throw err
+    }
+
+    log(`node_modules copied from ${tmpNodeModulesDir} to cache dir ${cacheDir}`)
+  }
+}
+
+async function getLockFilename (dir: string) {
+  const hasYarnLock = !!await fs.stat(path.join(dir, 'yarn.lock')).catch(() => false)
+  const hasNpmLock = !!await fs.stat(path.join(dir, 'package-lock.json')).catch(() => false)
+
+  if (hasYarnLock && hasNpmLock) throw new Error(`The example project at '${dir}' has conflicting lockfiles. Only use one package manager's lockfile per project.`)
+
+  if (hasNpmLock) return 'package-lock.json'
+
+  // default to yarn
+  return 'yarn.lock'
+}
+
+function getRelativePathToProjectDir (projectDir: string) {
+  return path.relative(projectDir, path.join(root, '..'))
+}
+
+async function restoreLockFileRelativePaths (opts: { projectDir: string, lockFilePath: string, relativePathToMonorepoRoot: string }) {
+  const relativePathToProjectDir = getRelativePathToProjectDir(opts.projectDir)
+  const lockFileContents = (await fs.readFile(opts.lockFilePath, 'utf8'))
+  .replaceAll(opts.relativePathToMonorepoRoot, relativePathToProjectDir)
+
+  await fs.writeFile(opts.lockFilePath, lockFileContents)
+}
+
+async function normalizeLockFileRelativePaths (opts: { project: string, projectDir: string, lockFilePath: string, lockFilename: string, relativePathToMonorepoRoot: string }) {
+  const relativePathToProjectDir = getRelativePathToProjectDir(opts.projectDir)
+  const lockFileContents = (await fs.readFile(opts.lockFilePath, 'utf8'))
+  .replaceAll(relativePathToProjectDir, opts.relativePathToMonorepoRoot)
+
+  // write back to the original project dir, not the tmp copy
+  await fs.writeFile(path.join(projects, opts.project, opts.lockFilename), lockFileContents)
+}
+
+/**
+ * Given a path to a `package.json`, convert any references to development
+ * versions of packages to absolute paths, so `yarn`/`npm` will not reach out to
+ * the Internet to obtain these packages once it runs in the temp dir.
+ * @returns a list of dependency names that were updated
+ */
+async function makeWorkspacePackagesAbsolute (pathToPkgJson: string): Promise<string[]> {
+  const pkgJson = await fs.readJson(pathToPkgJson)
+  const updatedDeps: string[] = []
+
+  for (const deps of [pkgJson.dependencies, pkgJson.devDependencies, pkgJson.optionalDependencies]) {
+    for (const dep in deps) {
+      const version = deps[dep]
+
+      if (version.startsWith('file:')) {
+        const absPath = pathToPackage(dep)
+
+        log(`Setting absolute path in package.json for ${dep}: ${absPath}.`)
+
+        deps[dep] = `file:${absPath}`
+        updatedDeps.push(dep)
+      }
+    }
+  }
+
+  await fs.writeJson(pathToPkgJson, pkgJson)
+
+  return updatedDeps
+}
+
+/**
+ * Given a `system-tests` project name, detect and install the `node_modules`
+ * specified in the project's `package.json`. No-op if no `package.json` is found.
+ * Will use `yarn` or `npm` based on the lockfile present.
+ */
+export async function scaffoldProjectNodeModules (project: string, updateLockFile: boolean = !!process.env.UPDATE_LOCK_FILE): Promise<void> {
+  const projectDir = projectPath(project)
+  const relativePathToMonorepoRoot = path.relative(
+    path.join(projects, project),
+    path.join(root, '..'),
+  )
+  const projectPkgJsonPath = path.join(projectDir, 'package.json')
+
+  const runCmd = async (cmd) => {
+    log(`Running "${cmd}" in ${projectDir}`)
+    await execa(cmd, { cwd: projectDir, stdio: 'inherit', shell: true })
+  }
+
+  const cacheNodeModulesDir = path.join(cachedir('cy-system-tests-node-modules'), project, 'node_modules')
+  const tmpNodeModulesDir = path.join(projectPath(project), 'node_modules')
+
+  async function removeWorkspacePackages (packages: string[]): Promise<void> {
+    for (const dep of packages) {
+      const depDir = path.join(tmpNodeModulesDir, dep)
+
+      log('Removing', depDir)
+      await fs.remove(depDir)
+    }
+  }
+
+  try {
+    // this will throw and exit early if the package.json does not exist
+    const pkgJson: SystemTestPkgJson = require(projectPkgJsonPath)
+
+    log(`Found package.json for project ${project}.`)
+
+    if (pkgJson._cySkipDepInstall) {
+      return log(`_cySkipDepInstall set in package.json, skipping dep-installer steps`)
+    }
+
+    if (!pkgJson.dependencies && !pkgJson.devDependencies && !pkgJson.optionalDependencies) {
+      return log(`No dependencies found, skipping dep-installer steps`)
+    }
+
+    const lockFilename = await getLockFilename(projectDir)
+    const hasYarnLock = lockFilename === 'yarn.lock'
+
+    // 1. Ensure there is a cache directory set up for this test project's `node_modules`.
+    await ensureCacheDir(cacheNodeModulesDir)
+
+    let persistCacheCb: () => Promise<void>
+
+    if (hasYarnLock) {
+      await symlinkNodeModulesFromCache(tmpNodeModulesDir, cacheNodeModulesDir)
+    } else {
+      // due to an issue in NPM, we cannot have `node_modules` be a symlink. fall back to copying.
+      // https://github.com/npm/npm/issues/10013
+      persistCacheCb = await copyNodeModulesFromCache(tmpNodeModulesDir, cacheNodeModulesDir)
+    }
+
+    // 2. Before running the package installer, resolve workspace deps to absolute paths.
+    // This is required to fix install for workspace-only packages.
+    const workspaceDeps = await makeWorkspacePackagesAbsolute(projectPkgJsonPath)
+
+    // 3. Delete cached workspace packages since the pkg manager will create a fresh symlink during install.
+    await removeWorkspacePackages(workspaceDeps)
+
+    // 4. Fix relative paths in temp dir's lockfile.
+    const lockFilePath = path.join(projectDir, lockFilename)
+
+    log(`Writing ${lockFilename} with fixed relative paths to temp dir`)
+    await restoreLockFileRelativePaths({ projectDir, lockFilePath, relativePathToMonorepoRoot })
+
+    // 5. Run `yarn/npm install`.
+    const getCommandFn = hasYarnLock ? getYarnCommand : getNpmCommand
+    const cmd = getCommandFn({
+      updateLockFile,
+      yarnV311: pkgJson._cyYarnV311,
+      isCI: !!process.env.CI,
+      runScripts: pkgJson._cyRunScripts,
+    })
+
+    await runCmd(cmd)
+
+    // 6. Now that the lockfile is up to date, update workspace dependency paths in the lockfile with monorepo
+    // relative paths so it can be the same for all developers
+    log(`Copying ${lockFilename} and fixing relative paths for ${project}`)
+    await normalizeLockFileRelativePaths({ project, projectDir, lockFilePath, lockFilename, relativePathToMonorepoRoot })
+
+    // 7. After install, we must now symlink *over* all workspace dependencies, or else
+    // `require` calls from installed workspace deps to peer deps will fail.
+    await removeWorkspacePackages(workspaceDeps)
+    for (const dep of workspaceDeps) {
+      const destDir = path.join(tmpNodeModulesDir, dep)
+      const targetDir = pathToPackage(dep)
+
+      log(`Symlinking workspace dependency: ${dep} (${destDir} -> ${targetDir})`)
+
+      await fs.mkdir(path.dirname(destDir), { recursive: true })
+      await fs.symlink(targetDir, destDir, 'junction')
+    }
+
+    // 8. If necessary, ensure that the `node_modules` cache is updated by copying `node_modules` back.
+    if (persistCacheCb) await persistCacheCb()
+  } catch (err) {
+    if (err.code === 'MODULE_NOT_FOUND') return
+
+    console.error(`⚠ An error occurred while installing the node_modules for ${project}.`)
+    console.error(err)
+    throw err
+  }
+}
+
+/**
+ * Create symlinks to very commonly used (in example projects) `node_modules`.
+ *
+ * This is done because many `projects` use the same modules, like `lodash`, and it's not worth it
+ * to increase CI install times just to have it explicitly specified by `package.json`. A symlink
+ * is faster than a real `npm install`.
+ *
+ * Adding modules here *decreases the quality of test coverage* because it allows test projects
+ * to make assumptions about what modules are available that don't hold true in the real world. So
+ * *do not add a module here* unless you are really sure that it should be available in every
+ * single test project.
+ */
+export async function scaffoldCommonNodeModules () {
+  await Promise.all([
+    '@cypress/code-coverage',
+    '@cypress/webpack-dev-server',
+    '@packages/socket',
+    '@packages/ts',
+    '@tooling/system-tests',
+    'bluebird',
+    'chai',
+    'dayjs',
+    'debug',
+    'execa',
+    'fs-extra',
+    'https-proxy-agent',
+    'jimp',
+    'lazy-ass',
+    'lodash',
+    'proxyquire',
+    'react',
+    'semver',
+    'systeminformation',
+    'tslib',
+    'typescript',
+  ].map(symlinkNodeModule))
+}
+
+async function symlinkNodeModule (pkg) {
+  const from = path.join(cyTmpDir, 'node_modules', pkg)
+  const to = pathToPackage(pkg)
+
+  await fs.ensureDir(path.dirname(from))
+  try {
+    await fs.symlink(to, from, 'junction')
+  } catch (err) {
+    if (err.code === 'EEXIST') return
+
+    throw err
+  }
+}