From 4f1cd09828549207fd8389d837b5a668f4a5d184 Mon Sep 17 00:00:00 2001 From: Daniel Choudhury Date: Thu, 18 Jan 2024 07:50:04 +0700 Subject: [PATCH] chore(build): Avoid prebuilding api side, instead use an esbuild plugin (#9767) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resurrecting this again, to see if I can get CI more stable this time **2nd edition Implementation:** With the second edition, I'm trying esbuild's watcher - rather than chokidar This is a no go ❌, because when new files are introduced the watcher isn't triggered. **3rd edition implementation**: Use esbuild's `rebuild` function that _may_ provide performance benefits Old PR #7672 --------- Co-authored-by: Tobbe Lundberg Co-authored-by: Dominic Saadi --- .github/workflows/ci.yml | 59 ++--- package.json | 1 + packages/api-server/src/watch.ts | 81 +++++-- packages/babel-config/jest.config.js | 2 +- packages/babel-config/package.json | 1 + .../babel-config/src/__tests__/api.test.ts | 19 +- .../src/__tests__/prebuildApiFile.test.ts | 202 +++++++++--------- packages/babel-config/src/api.ts | 67 ++---- packages/babel-config/src/common.ts | 4 +- packages/babel-config/src/index.ts | 1 - ...in-redwood-directory-named-imports.test.ts | 2 +- packages/babel-config/tsconfig.json | 3 + packages/cli/src/commands/buildHandler.js | 10 +- .../commands/deploy/__tests__/nftPack.test.js | 48 ++--- .../internal/src/__tests__/build_api.test.ts | 40 +++- packages/internal/src/build/api.ts | 114 +++++----- packages/vite/src/buildFeServer.ts | 4 +- tasks/e2e/cypress.config.js | 15 +- tasks/e2e/cypress/e2e/04-logger/logger.cy.js | 8 +- tasks/e2e/cypress/support/e2e.js | 1 + tasks/run-e2e | 3 +- yarn.lock | 13 ++ 22 files changed, 390 insertions(+), 308 deletions(-) create mode 100644 tasks/e2e/cypress/support/e2e.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07f9d0a4ebee..0a6d830aa149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,54 +179,55 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - name: 📁 Create a temporary directory - id: createpath - run: | - project_path=$(mktemp -d -t redwood.XXXXXX) - echo "::set-output name=project_path::$project_path" - framework_path=$(pwd) - echo "::set-output name=framework_path::$framework_path" + - name: 🔨 Build + run: yarn build + + - name: 🌲 Install Cypress + run: yarn cypress install - name: 🌲 Create a Redwood App - run: | - ./tasks/run-e2e ${{ steps.createpath.outputs.project_path }} \ - --no-start \ - --bundler ${{ matrix.bundler }} + id: crwa env: YARN_ENABLE_IMMUTABLE_INSTALLS: false - - - name: 🐙 Git init in the Redwood App directory run: | + project_path=$(mktemp -d -t redwood.XXXXXX) + echo "project-path=$project_path" >> $GITHUB_OUTPUT + git config --global user.email "you@example.com" git config --global user.name "Your Name" - git init --initial-branch main && git add . - git commit -a --message=init - working-directory: ${{ steps.createpath.outputs.project_path }} - - name: Start server in background - run: yarn rw dev --no-generate --fwd="--no-open" & - working-directory: ${{ steps.createpath.outputs.project_path }} + ./tasks/run-e2e "$project_path" \ + --bundler ${{ matrix.bundler }} \ + --no-build-framework \ + --no-start - - name: 🌲 Install Cypress - run: yarn run cypress install + - name: Start the dev server in the background + run: | + yarn rw dev --no-generate --fwd="--no-open" 2>&1 | tee dev_server.log & + working-directory: ${{ steps.crwa.outputs.project-path }} - - name: 🌲 Run cypress - uses: cypress-io/github-action@v5 + - name: 🌲 Run Cypress + uses: cypress-io/github-action@v6 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_RW_PATH: "${{ steps.createpath.outputs.project_path }}" + CYPRESS_RW_PATH: ${{ steps.crwa.outputs.project-path }} with: - # We've already installed dependencies. - install: false - env: true browser: chrome - record: false + env: true + install: false wait-on: 'http://[::1]:8910' working-directory: ./tasks/e2e spec: | cypress/e2e/01-tutorial/*.cy.js cypress/e2e/04-logger/*.cy.js + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ matrix.bundler }}-logs + path: | + ${{ steps.crwa.outputs.project-path }}/dev_server.log + ${{ steps.crwa.outputs.project-path }}/e2e.log + tutorial-e2e-skip: needs: detect-changes if: needs.detect-changes.outputs.onlydocs == 'true' diff --git a/package.json b/package.json index 1b8ac389f7bd..6c9b2dd030b1 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "boxen": "5.1.2", "core-js": "3.34.0", "cypress": "13.6.1", + "cypress-fail-fast": "7.1.0", "cypress-wait-until": "3.0.1", "dependency-cruiser": "15.5.0", "dotenv": "16.3.1", diff --git a/packages/api-server/src/watch.ts b/packages/api-server/src/watch.ts index 3de776455599..29fd4aedf5ed 100644 --- a/packages/api-server/src/watch.ts +++ b/packages/api-server/src/watch.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node -import { fork } from 'child_process' import type { ChildProcess } from 'child_process' +import { fork } from 'child_process' import fs from 'fs' import path from 'path' @@ -13,12 +13,16 @@ import { debounce } from 'lodash' import { hideBin } from 'yargs/helpers' import yargs from 'yargs/yargs' -import { buildApi } from '@redwoodjs/internal/dist/build/api' +import { + buildApi, + cleanApiBuild, + rebuildApi, +} from '@redwoodjs/internal/dist/build/api' import { loadAndValidateSdls } from '@redwoodjs/internal/dist/validateSchema' import { - getPaths, ensurePosixPath, getConfig, + getPaths, resolveFile, } from '@redwoodjs/project-config' @@ -43,9 +47,6 @@ dotenv.config({ path: rwjsPaths.base, }) -// TODO: -// 1. Move this file out of the HTTP server, and place it in the CLI? - let httpServerProcess: ChildProcess const killApiServer = () => { @@ -63,19 +64,32 @@ const validate = async () => { console.error(c.red(e?.message)) console.log(c.redBright('-'.repeat(40))) - delayRestartServer.cancel() + debouncedBuild.cancel() + debouncedRebuild.cancel() return false } } -const rebuildApiServer = () => { +const buildAndRestart = async ({ + rebuild = false, + clean = false, +}: { rebuild?: boolean; clean?: boolean } = {}) => { try { // Shutdown API server killApiServer() const buildTs = Date.now() process.stdout.write(c.dim(c.italic('Building... '))) - buildApi() + + if (clean) { + await cleanApiBuild() + } + + if (rebuild) { + await rebuildApi() + } else { + await buildApi() + } console.log(c.dim(c.italic('Took ' + (Date.now() - buildTs) + ' ms'))) const forkOpts = { @@ -148,11 +162,18 @@ const rebuildApiServer = () => { // this usually happens when running RedwoodJS generator commands. // Local writes are very fast, but writes in e2e environments are not, // so allow the default to be adjust with a env-var. -const delayRestartServer = debounce( - rebuildApiServer, +const debouncedRebuild = debounce( + () => buildAndRestart({ rebuild: true }), process.env.RWJS_DELAY_RESTART ? parseInt(process.env.RWJS_DELAY_RESTART, 10) - : 5 + : 500 +) + +const debouncedBuild = debounce( + () => buildAndRestart({ rebuild: false }), + process.env.RWJS_DELAY_RESTART + ? parseInt(process.env.RWJS_DELAY_RESTART, 10) + : 500 ) // NOTE: the file comes through as a unix path, even on windows @@ -165,7 +186,7 @@ const IGNORED_API_PATHS = [ ].map((path) => ensurePosixPath(path)) chokidar - .watch(rwjsPaths.api.base, { + .watch([rwjsPaths.api.src], { persistent: true, ignoreInitial: true, ignored: (file: string) => { @@ -188,7 +209,11 @@ chokidar }, }) .on('ready', async () => { - rebuildApiServer() + // First time + await buildAndRestart({ + clean: true, + rebuild: false, + }) await validate() }) .on('all', async (eventName, filePath) => { @@ -199,20 +224,30 @@ chokidar return } - // We validate here, so that developers will see the error - // As they're running the dev server - if (filePath.includes('.sdl')) { - const isValid = await validate() + if (eventName) { + if (filePath.includes('.sdl')) { + // We validate here, so that developers will see the error + // As they're running the dev server + const isValid = await validate() - // Exit early if not valid - if (!isValid) { - return + // Exit early if not valid + if (!isValid) { + return + } } } console.log( c.dim(`[${eventName}] ${filePath.replace(rwjsPaths.api.base, '')}`) ) - delayRestartServer.cancel() - delayRestartServer() + + if (eventName === 'add' || eventName === 'unlink') { + debouncedBuild.cancel() + debouncedRebuild.cancel() + debouncedBuild() + } else { + // If files have just changed, then rebuild + debouncedRebuild.cancel() + debouncedRebuild() + } }) diff --git a/packages/babel-config/jest.config.js b/packages/babel-config/jest.config.js index a191690e9a42..ca095a6ad325 100644 --- a/packages/babel-config/jest.config.js +++ b/packages/babel-config/jest.config.js @@ -1,3 +1,3 @@ module.exports = { - testPathIgnorePatterns: ['fixtures'], + testPathIgnorePatterns: ['fixtures', 'dist/*'], } diff --git a/packages/babel-config/package.json b/packages/babel-config/package.json index 11bca75dfbe5..1b3275eff0f9 100644 --- a/packages/babel-config/package.json +++ b/packages/babel-config/package.json @@ -47,6 +47,7 @@ "devDependencies": { "@types/babel-plugin-tester": "9.0.9", "@types/babel__core": "7.20.4", + "@types/node": "20.10.4", "babel-plugin-tester": "11.0.4", "esbuild": "0.19.9", "jest": "29.7.0" diff --git a/packages/babel-config/src/__tests__/api.test.ts b/packages/babel-config/src/__tests__/api.test.ts index a3c7af764927..6fef1a715cb6 100644 --- a/packages/babel-config/src/__tests__/api.test.ts +++ b/packages/babel-config/src/__tests__/api.test.ts @@ -1,7 +1,8 @@ import { vol } from 'memfs' -import { getPaths, ensurePosixPath } from '@redwoodjs/project-config' +import { ensurePosixPath, getPaths } from '@redwoodjs/project-config' +import type { PluginList } from '../api' import { getApiSideBabelConfigPath, getApiSideBabelPlugins, @@ -87,7 +88,7 @@ describe('api', () => { ) const apiSideBabelConfigPath = getApiSideBabelConfigPath() - expect(ensurePosixPath(apiSideBabelConfigPath)).toMatch( + expect(ensurePosixPath(apiSideBabelConfigPath || '')).toMatch( '/redwood-app/api/babel.config.js' ) }) @@ -185,9 +186,17 @@ describe('api', () => { }, ]) + type ModuleResolverConfig = { + root: string[] + alias: Record + cwd: string + loglevel: string + } + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const [_, babelPluginModuleResolverConfig] = apiSideBabelPlugins.find( (plugin) => plugin[0] === 'babel-plugin-module-resolver' - ) + )! as [any, ModuleResolverConfig, any] expect(babelPluginModuleResolverConfig).toMatchObject({ alias: { @@ -238,12 +247,12 @@ describe('api', () => { }) }) -function getPluginAliases(plugins) { +function getPluginAliases(plugins: PluginList) { return plugins.reduce((pluginAliases, plugin) => { if (plugin.length !== 3) { return pluginAliases } return [...pluginAliases, plugin[2]] - }, []) + }, [] as any) } diff --git a/packages/babel-config/src/__tests__/prebuildApiFile.test.ts b/packages/babel-config/src/__tests__/prebuildApiFile.test.ts index 3bfadf7a4368..24fd5126f0c3 100644 --- a/packages/babel-config/src/__tests__/prebuildApiFile.test.ts +++ b/packages/babel-config/src/__tests__/prebuildApiFile.test.ts @@ -2,25 +2,25 @@ import path from 'path' import compat from 'core-js-compat' -import { getPaths, getConfig } from '@redwoodjs/project-config' +import { getConfig } from '@redwoodjs/project-config' import { BABEL_PLUGIN_TRANSFORM_RUNTIME_OPTIONS, - getApiSideBabelPlugins, - prebuildApiFile, TARGETS_NODE, + getApiSideBabelPlugins, + transformWithBabel, } from '../api' const RWJS_CWD = path.join(__dirname, '__fixtures__/redwood-app') process.env.RWJS_CWD = RWJS_CWD -let code +let code: string describe('api prebuild ', () => { describe('polyfills unsupported functionality', () => { - beforeAll(() => { + beforeAll(async () => { const apiFile = path.join(RWJS_CWD, 'api/src/lib/polyfill.js') - code = prebuildApiFileWrapper(apiFile) + code = await prebuildApiFileWrapper(apiFile) }) describe('ES features', () => { @@ -371,7 +371,7 @@ describe('api prebuild ', () => { }) it('includes source maps', () => { - const sourceMaps = code.split('\n').pop() + const sourceMaps = code.split('\n').pop() as string const sourceMapsMatcher = '//# sourceMappingURL=data:application/json;charset=utf-8;base64,' @@ -386,16 +386,16 @@ describe('api prebuild ', () => { expect(sources).toMatchInlineSnapshot(` [ - "../../../../../api/src/lib/polyfill.js", + "polyfill.js", ] `) }) }) describe('uses core-js3 aliasing', () => { - beforeAll(() => { + beforeAll(async () => { const apiFile = path.join(RWJS_CWD, 'api/src/lib/transform.js') - code = prebuildApiFileWrapper(apiFile) + code = await prebuildApiFileWrapper(apiFile) }) it('works', () => { @@ -425,9 +425,9 @@ describe('api prebuild ', () => { }) describe('typescript', () => { - beforeAll(() => { + beforeAll(async () => { const apiFile = path.join(RWJS_CWD, 'api/src/lib/typescript.ts') - code = prebuildApiFileWrapper(apiFile) + code = await prebuildApiFileWrapper(apiFile) }) it('transpiles ts to js', () => { @@ -437,9 +437,9 @@ describe('api prebuild ', () => { }) describe('auto imports', () => { - beforeAll(() => { + beforeAll(async () => { const apiFile = path.join(RWJS_CWD, 'api/src/lib/autoImports.ts') - code = prebuildApiFileWrapper(apiFile) + code = await prebuildApiFileWrapper(apiFile) }) it('auto imports', () => { @@ -451,7 +451,7 @@ describe('api prebuild ', () => { test('core-js polyfill list', () => { const { list } = compat({ targets: { node: TARGETS_NODE }, - version: BABEL_PLUGIN_TRANSFORM_RUNTIME_OPTIONS.corejs.version, + version: BABEL_PLUGIN_TRANSFORM_RUNTIME_OPTIONS.corejs.version.toString(), }) /** @@ -463,105 +463,97 @@ describe('api prebuild ', () => { * Some "ES Next" polyfills have landed in v12+ Node.js versions. */ expect(list).toMatchInlineSnapshot(` - [ - "esnext.array.last-index", - "esnext.array.last-item", - "esnext.composite-key", - "esnext.composite-symbol", - "esnext.map.delete-all", - "esnext.map.every", - "esnext.map.filter", - "esnext.map.find", - "esnext.map.find-key", - "esnext.map.from", - "esnext.map.group-by", - "esnext.map.includes", - "esnext.map.key-by", - "esnext.map.key-of", - "esnext.map.map-keys", - "esnext.map.map-values", - "esnext.map.merge", - "esnext.map.of", - "esnext.map.reduce", - "esnext.map.some", - "esnext.map.update", - "esnext.math.clamp", - "esnext.math.deg-per-rad", - "esnext.math.degrees", - "esnext.math.fscale", - "esnext.math.iaddh", - "esnext.math.imulh", - "esnext.math.isubh", - "esnext.math.rad-per-deg", - "esnext.math.radians", - "esnext.math.scale", - "esnext.math.seeded-prng", - "esnext.math.signbit", - "esnext.math.umulh", - "esnext.number.from-string", - "esnext.observable", - "esnext.promise.try", - "esnext.reflect.define-metadata", - "esnext.reflect.delete-metadata", - "esnext.reflect.get-metadata", - "esnext.reflect.get-metadata-keys", - "esnext.reflect.get-own-metadata", - "esnext.reflect.get-own-metadata-keys", - "esnext.reflect.has-metadata", - "esnext.reflect.has-own-metadata", - "esnext.reflect.metadata", - "esnext.set.add-all", - "esnext.set.delete-all", - "esnext.set.difference", - "esnext.set.every", - "esnext.set.filter", - "esnext.set.find", - "esnext.set.from", - "esnext.set.intersection", - "esnext.set.is-disjoint-from", - "esnext.set.is-subset-of", - "esnext.set.is-superset-of", - "esnext.set.join", - "esnext.set.map", - "esnext.set.of", - "esnext.set.reduce", - "esnext.set.some", - "esnext.set.symmetric-difference", - "esnext.set.union", - "esnext.string.at", - "esnext.string.code-points", - "esnext.symbol.observable", - "esnext.symbol.pattern-match", - "esnext.weak-map.delete-all", - "esnext.weak-map.from", - "esnext.weak-map.of", - "esnext.weak-set.add-all", - "esnext.weak-set.delete-all", - "esnext.weak-set.from", - "esnext.weak-set.of", - ] - `) + [ + "esnext.array.last-index", + "esnext.array.last-item", + "esnext.composite-key", + "esnext.composite-symbol", + "esnext.map.delete-all", + "esnext.map.every", + "esnext.map.filter", + "esnext.map.find", + "esnext.map.find-key", + "esnext.map.from", + "esnext.map.group-by", + "esnext.map.includes", + "esnext.map.key-by", + "esnext.map.key-of", + "esnext.map.map-keys", + "esnext.map.map-values", + "esnext.map.merge", + "esnext.map.of", + "esnext.map.reduce", + "esnext.map.some", + "esnext.map.update", + "esnext.math.clamp", + "esnext.math.deg-per-rad", + "esnext.math.degrees", + "esnext.math.fscale", + "esnext.math.iaddh", + "esnext.math.imulh", + "esnext.math.isubh", + "esnext.math.rad-per-deg", + "esnext.math.radians", + "esnext.math.scale", + "esnext.math.seeded-prng", + "esnext.math.signbit", + "esnext.math.umulh", + "esnext.number.from-string", + "esnext.observable", + "esnext.promise.try", + "esnext.reflect.define-metadata", + "esnext.reflect.delete-metadata", + "esnext.reflect.get-metadata", + "esnext.reflect.get-metadata-keys", + "esnext.reflect.get-own-metadata", + "esnext.reflect.get-own-metadata-keys", + "esnext.reflect.has-metadata", + "esnext.reflect.has-own-metadata", + "esnext.reflect.metadata", + "esnext.set.add-all", + "esnext.set.delete-all", + "esnext.set.difference", + "esnext.set.every", + "esnext.set.filter", + "esnext.set.find", + "esnext.set.from", + "esnext.set.intersection", + "esnext.set.is-disjoint-from", + "esnext.set.is-subset-of", + "esnext.set.is-superset-of", + "esnext.set.join", + "esnext.set.map", + "esnext.set.of", + "esnext.set.reduce", + "esnext.set.some", + "esnext.set.symmetric-difference", + "esnext.set.union", + "esnext.string.at", + "esnext.string.code-points", + "esnext.symbol.observable", + "esnext.symbol.pattern-match", + "esnext.weak-map.delete-all", + "esnext.weak-map.from", + "esnext.weak-map.of", + "esnext.weak-set.add-all", + "esnext.weak-set.delete-all", + "esnext.weak-set.from", + "esnext.weak-set.of", + ] + `) }) }) /** - * A copy of prebuildApiFiles from packages/internal/src/build/api.ts - * This will be re-architected, but doing so now would introduce breaking changes. + * We no longer prebuild files as part of the build process + * This is so we can test the babel configuration in isolation */ -export const prebuildApiFileWrapper = (srcFile: string) => { - const redwoodProjectPaths = getPaths() - +export const prebuildApiFileWrapper = async (srcFile: string) => { const plugins = getApiSideBabelPlugins({ openTelemetry: getConfig().experimental.opentelemetry.enabled, }) - const relativePathFromSrc = path.relative(redwoodProjectPaths.base, srcFile) - - const dstPath = path - .join(redwoodProjectPaths.generated.prebuild, relativePathFromSrc) - .replace(/\.(ts)$/, '.js') - - const result = prebuildApiFile(srcFile, dstPath, plugins) + const result = await transformWithBabel(srcFile, plugins) if (!result?.code) { throw new Error(`Couldn't prebuild ${srcFile}`) diff --git a/packages/babel-config/src/api.ts b/packages/babel-config/src/api.ts index c719e9c30ef0..53c05cc56801 100644 --- a/packages/babel-config/src/api.ts +++ b/packages/babel-config/src/api.ts @@ -1,19 +1,20 @@ -import fs from 'fs' +import { existsSync } from 'fs' +import fs from 'fs/promises' import path from 'path' -import { transform } from '@babel/core' -import type { PluginItem, TransformOptions } from '@babel/core' +import type { PluginOptions, PluginTarget, TransformOptions } from '@babel/core' +import { transformAsync } from '@babel/core' import { getPaths } from '@redwoodjs/project-config' import type { RegisterHookOptions } from './common' import { - registerBabel, CORE_JS_VERSION, RUNTIME_CORE_JS_VERSION, getCommonPlugins, - parseTypeScriptConfigFiles, getPathsFromTypeScriptConfig, + parseTypeScriptConfigFiles, + registerBabel, } from './common' export const TARGETS_NODE = '20.10' @@ -65,17 +66,22 @@ export const BABEL_PLUGIN_TRANSFORM_RUNTIME_OPTIONS = { version: RUNTIME_CORE_JS_VERSION, } +// Plugin shape: [ ["Target", "Options", "name"] ], +// a custom "name" can be supplied so that user's do not accidentally overwrite +// Redwood's own plugins when they specify their own. +export type PluginList = Array +type PluginShape = + | [PluginTarget, PluginOptions, undefined | string] + | [PluginTarget, PluginOptions] + export const getApiSideBabelPlugins = ( { openTelemetry } = { openTelemetry: false, } ) => { - // Plugin shape: [ ["Target", "Options", "name"] ], - // a custom "name" is supplied so that user's do not accidentally overwrite - // Redwood's own plugins when they specify their own. const tsConfig = parseTypeScriptConfigFiles() - const plugins: TransformOptions['plugins'] = [ + const plugins: Array = [ ...getCommonPlugins(), // Needed to support `/** @jsxImportSource custom-jsx-library */` // comments in JSX files @@ -130,17 +136,17 @@ export const getApiSideBabelPlugins = ( undefined, 'rwjs-babel-otel-wrapping', ], - ].filter(Boolean) as PluginItem[] + ] - return plugins + return plugins.filter(Boolean) as PluginList // ts doesn't play nice with filter(Boolean) } export const getApiSideBabelConfigPath = () => { const p = path.join(getPaths().api.base, 'babel.config.js') - if (fs.existsSync(p)) { + if (existsSync(p)) { return p } else { - return undefined + return } } @@ -188,23 +194,17 @@ export const registerApiSideBabelHook = ({ }) } -export const prebuildApiFile = ( +export const transformWithBabel = async ( srcPath: string, - // we need to know dstPath as well - // so we can generate an inline, relative sourcemap - dstPath: string, plugins: TransformOptions['plugins'] ) => { - const code = fs.readFileSync(srcPath, 'utf-8') + const code = await fs.readFile(srcPath, 'utf-8') const defaultOptions = getApiSideDefaultBabelConfig() - const result = transform(code, { + const result = transformAsync(code, { ...defaultOptions, cwd: getPaths().api.base, filename: srcPath, - // we set the sourceFile (for the sourcemap) as a correct, relative path - // this is why this function (prebuildFile) must know about the dstPath - sourceFileName: path.relative(path.dirname(dstPath), srcPath), // we need inline sourcemaps at this level // because this file will eventually be fed to esbuild // when esbuild finds an inline sourcemap, it tries to "combine" it @@ -212,29 +212,6 @@ export const prebuildApiFile = ( sourceMaps: 'inline', plugins, }) - return result -} - -// TODO (STREAMING) I changed the prebuildApiFile function in https://github.com/redwoodjs/redwood/pull/7672/files -// but we had to revert. For this branch temporarily, I'm going to add a new function -// This is used in building routeHooks -export const transformWithBabel = ( - srcPath: string, - plugins: TransformOptions['plugins'] -) => { - const code = fs.readFileSync(srcPath, 'utf-8') - const defaultOptions = getApiSideDefaultBabelConfig() - const result = transform(code, { - ...defaultOptions, - cwd: getPaths().api.base, - filename: srcPath, - // we need inline sourcemaps at this level - // because this file will eventually be fed to esbuild - // when esbuild finds an inline sourcemap, it tries to "combine" it - // so the final sourcemap (the one that esbuild generates) combines both mappings - sourceMaps: 'inline', - plugins, - }) return result } diff --git a/packages/babel-config/src/common.ts b/packages/babel-config/src/common.ts index b05f18a42a16..5aeb57b5ad49 100644 --- a/packages/babel-config/src/common.ts +++ b/packages/babel-config/src/common.ts @@ -1,7 +1,7 @@ import fs from 'fs' import path from 'path' -import type { TransformOptions, PluginItem } from '@babel/core' +import type { PluginItem, PluginOptions, TransformOptions } from '@babel/core' import { parseConfigFileTextToJson } from 'typescript' import { getPaths } from '@redwoodjs/project-config' @@ -61,7 +61,7 @@ if (!RUNTIME_CORE_JS_VERSION) { ) } -export const getCommonPlugins = () => { +export const getCommonPlugins = (): Array<[string, PluginOptions]> => { return [ ['@babel/plugin-transform-class-properties', { loose: true }], // Note: The private method loose mode configuration setting must be the diff --git a/packages/babel-config/src/index.ts b/packages/babel-config/src/index.ts index 0c4d675cb117..7658f3e7a67c 100644 --- a/packages/babel-config/src/index.ts +++ b/packages/babel-config/src/index.ts @@ -9,7 +9,6 @@ export { getApiSideBabelPlugins, getApiSideBabelPresets, getApiSideDefaultBabelConfig, - prebuildApiFile, registerApiSideBabelHook, transformWithBabel, } from './api' diff --git a/packages/babel-config/src/plugins/__tests__/babel-plugin-redwood-directory-named-imports.test.ts b/packages/babel-config/src/plugins/__tests__/babel-plugin-redwood-directory-named-imports.test.ts index 6f50c9ee1de7..14ed1918c6df 100644 --- a/packages/babel-config/src/plugins/__tests__/babel-plugin-redwood-directory-named-imports.test.ts +++ b/packages/babel-config/src/plugins/__tests__/babel-plugin-redwood-directory-named-imports.test.ts @@ -65,7 +65,7 @@ describe('directory named imports', () => { }, ], ], - }).code + })?.code expect(babeled).toMatch(output) }) }) diff --git a/packages/babel-config/tsconfig.json b/packages/babel-config/tsconfig.json index 91b48264c7cf..3b0700b39db5 100644 --- a/packages/babel-config/tsconfig.json +++ b/packages/babel-config/tsconfig.json @@ -4,8 +4,11 @@ "baseUrl": ".", "rootDir": "src", "outDir": "dist", + "types": ["node", "jest"], }, "include": ["src"], + // Excluding cypress types, because it interferes with expect in tests + "exclude": ["node_modules/cypress/types/*", "**/__tests__/__fixtures__/*"], "references": [ { "path": "../project-config" } ] diff --git a/packages/cli/src/commands/buildHandler.js b/packages/cli/src/commands/buildHandler.js index 5c801f2dd2b4..9d3367d11ade 100644 --- a/packages/cli/src/commands/buildHandler.js +++ b/packages/cli/src/commands/buildHandler.js @@ -7,7 +7,7 @@ import { rimraf } from 'rimraf' import terminalLink from 'terminal-link' import { recordTelemetryAttributes } from '@redwoodjs/cli-helpers' -import { buildApi } from '@redwoodjs/internal/dist/build/api' +import { buildApi, cleanApiBuild } from '@redwoodjs/internal/dist/build/api' import { generate } from '@redwoodjs/internal/dist/generate/generate' import { loadAndValidateSdls } from '@redwoodjs/internal/dist/validateSchema' import { detectPrerenderRoutes } from '@redwoodjs/prerender/detection' @@ -100,8 +100,9 @@ export const handler = async ({ }, side.includes('api') && { title: 'Building API...', - task: () => { - const { errors, warnings } = buildApi() + task: async () => { + await cleanApiBuild() + const { errors, warnings } = await buildApi() if (errors.length) { console.error(errors) @@ -183,7 +184,10 @@ export const handler = async ({ 'file://' + rwjsPaths.web.routes )}.` ) + + return } + // Running a separate process here, otherwise it wouldn't pick up the // generated Prisma Client due to require module caching await execa('yarn rw prerender', { diff --git a/packages/cli/src/commands/deploy/__tests__/nftPack.test.js b/packages/cli/src/commands/deploy/__tests__/nftPack.test.js index ab3168fab06c..b1d90a728ce8 100644 --- a/packages/cli/src/commands/deploy/__tests__/nftPack.test.js +++ b/packages/cli/src/commands/deploy/__tests__/nftPack.test.js @@ -1,32 +1,32 @@ -import path from 'path' - -import fs from 'fs-extra' - -import { buildApi } from '@redwoodjs/internal/dist/build/api' import { findApiDistFunctions } from '@redwoodjs/internal/dist/files' import * as nftPacker from '../packing/nft' -const FIXTURE_PATH = path.resolve( - __dirname, - '../../../../../../__fixtures__/example-todo-main' -) - -let functionDistFiles - -beforeAll(() => { - process.env.RWJS_CWD = FIXTURE_PATH - - // Actually build the fixture, if we need it - if (!fs.existsSync(path.join(FIXTURE_PATH, 'api/dist/functions'))) { - buildApi() +jest.mock('@redwoodjs/internal/dist/files', () => { + return { + findApiDistFunctions: () => { + return [ + '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/api/dist/functions/graphql.js', + '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/api/dist/functions/healthz/healthz.js', + '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/api/dist/functions/invalid/x.js', + '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/api/dist/functions/nested/nested.js', + '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/api/dist/functions/x/index.js', + ] + }, } - - functionDistFiles = findApiDistFunctions() }) -afterAll(() => { - delete process.env.RWJS_CWD +jest.mock('@redwoodjs/project-config', () => { + return { + getPaths: () => { + return { + base: '/Users/carmack/dev/redwood/__fixtures__/example-todo-main/', + } + }, + ensurePosixPath: (path) => { + return path.replace(/\\/g, '/') + }, + } }) test('Check packager detects all functions', () => { @@ -40,7 +40,7 @@ test('Check packager detects all functions', () => { }) test('Creates entry file for nested functions correctly', () => { - const nestedFunction = functionDistFiles.find((fPath) => + const nestedFunction = findApiDistFunctions().find((fPath) => fPath.includes('nested') ) @@ -56,7 +56,7 @@ test('Creates entry file for nested functions correctly', () => { }) test('Creates entry file for top level functions correctly', () => { - const graphqlFunction = functionDistFiles.find((fPath) => + const graphqlFunction = findApiDistFunctions().find((fPath) => fPath.includes('graphql') ) diff --git a/packages/internal/src/__tests__/build_api.test.ts b/packages/internal/src/__tests__/build_api.test.ts index deda22f781aa..38945bf3d282 100644 --- a/packages/internal/src/__tests__/build_api.test.ts +++ b/packages/internal/src/__tests__/build_api.test.ts @@ -6,10 +6,11 @@ import * as babel from '@babel/core' import { getApiSideBabelPlugins, getApiSideDefaultBabelConfig, + transformWithBabel, } from '@redwoodjs/babel-config' import { ensurePosixPath, getPaths } from '@redwoodjs/project-config' -import { cleanApiBuild, prebuildApiFiles } from '../build/api' +import { cleanApiBuild } from '../build/api' import { findApiFiles } from '../files' const FIXTURE_PATH = path.resolve( @@ -17,6 +18,33 @@ const FIXTURE_PATH = path.resolve( '../../../../__fixtures__/example-todo-main' ) +// @NOTE: we no longer prebuild files into the .redwood/prebuild folder +// However, prebuilding in the tests is still helpful for us to validate +// that everything is working as expected. +export const prebuildApiFiles = async (srcFiles: string[]) => { + const rwjsPaths = getPaths() + const plugins = getApiSideBabelPlugins() + + return Promise.all( + srcFiles.map(async (srcPath) => { + const relativePathFromSrc = path.relative(rwjsPaths.base, srcPath) + const dstPath = path + .join(rwjsPaths.generated.prebuild, relativePathFromSrc) + .replace(/\.(ts)$/, '.js') + + const result = await transformWithBabel(srcPath, plugins) + if (!result?.code) { + throw new Error(`Could not prebuild ${srcPath}`) + } + + fs.mkdirSync(path.dirname(dstPath), { recursive: true }) + fs.writeFileSync(dstPath, result.code) + + return dstPath + }) + ) +} + const cleanPaths = (p) => { return ensurePosixPath(path.relative(FIXTURE_PATH, p)) } @@ -25,12 +53,12 @@ const cleanPaths = (p) => { let prebuiltFiles let relativePaths -beforeAll(() => { +beforeAll(async () => { process.env.RWJS_CWD = FIXTURE_PATH cleanApiBuild() const apiFiles = findApiFiles() - prebuiltFiles = prebuildApiFiles(apiFiles) + prebuiltFiles = await prebuildApiFiles(apiFiles) relativePaths = prebuiltFiles .filter((x) => typeof x !== 'undefined') @@ -80,6 +108,10 @@ test.skip('api prebuild transforms gql with `babel-plugin-graphql-tag`', () => { .filter((p) => p.endsWith('todos.sdl.js')) .pop() + if (!p) { + throw new Error('No built files') + } + const code = fs.readFileSync(p, 'utf-8') expect(code.includes('import gql from "graphql-tag";')).toEqual(false) expect(code.includes('gql`')).toEqual(false) @@ -99,7 +131,7 @@ test('jest mock statements also handle', () => { cwd: getPaths().api.base, // We override the plugins, to match packages/testing/config/jest/api/index.js plugins: getApiSideBabelPlugins({ forJest: true }), - }).code + })?.code // Step 2: check that output has correct import statement path expect(outputForJest).toContain('import dog from "../../lib/dog"') diff --git a/packages/internal/src/build/api.ts b/packages/internal/src/build/api.ts index 4ca1ba05a20a..e8770d6baf83 100644 --- a/packages/internal/src/build/api.ts +++ b/packages/internal/src/build/api.ts @@ -1,85 +1,91 @@ -import fs from 'fs' -import path from 'path' - -import * as esbuild from 'esbuild' -import { removeSync } from 'fs-extra' +import type { BuildContext, BuildOptions, PluginBuild } from 'esbuild' +import { build, context } from 'esbuild' +import { remove } from 'fs-extra' import { getApiSideBabelPlugins, - prebuildApiFile, + transformWithBabel, } from '@redwoodjs/babel-config' -import { getPaths, getConfig } from '@redwoodjs/project-config' +import { getConfig, getPaths } from '@redwoodjs/project-config' import { findApiFiles } from '../files' -export const buildApi = () => { - // TODO: Be smarter about caching and invalidating files, - // but right now we just delete everything. - cleanApiBuild() +let BUILD_CTX: BuildContext | null = null + +export const buildApi = async () => { + // Reset the build context for rebuilding + // No need to wait for promise to resolve + BUILD_CTX?.dispose() + BUILD_CTX = null + + return transpileApi(findApiFiles()) +} - const srcFiles = findApiFiles() +export const rebuildApi = async () => { + const apiFiles = findApiFiles() - const prebuiltFiles = prebuildApiFiles(srcFiles).filter( - (path): path is string => path !== undefined - ) + if (!BUILD_CTX) { + BUILD_CTX = await context(getEsbuildOptions(apiFiles)) + } - return transpileApi(prebuiltFiles) + return BUILD_CTX.rebuild() } -export const cleanApiBuild = () => { +export const cleanApiBuild = async () => { const rwjsPaths = getPaths() - removeSync(rwjsPaths.api.dist) - removeSync(path.join(rwjsPaths.generated.prebuild, 'api')) + return remove(rwjsPaths.api.dist) } -/** - * Remove RedwoodJS "magic" from a user's code leaving JavaScript behind. - */ -export const prebuildApiFiles = (srcFiles: string[]) => { - const rwjsPaths = getPaths() - const rwjsConfig = getConfig() - const plugins = getApiSideBabelPlugins({ - openTelemetry: - rwjsConfig.experimental.opentelemetry.enabled && - rwjsConfig.experimental.opentelemetry.wrapApi, - }) - - return srcFiles.map((srcPath) => { - const relativePathFromSrc = path.relative(rwjsPaths.base, srcPath) - const dstPath = path - .join(rwjsPaths.generated.prebuild, relativePathFromSrc) - .replace(/\.(ts)$/, '.js') - - const result = prebuildApiFile(srcPath, dstPath, plugins) - if (!result?.code) { - // TODO: Figure out a better way to return these programatically. - console.warn('Error:', srcPath, 'could not prebuilt.') - - return undefined - } - - fs.mkdirSync(path.dirname(dstPath), { recursive: true }) - fs.writeFileSync(dstPath, result.code) - - return dstPath - }) +const runRwBabelTransformsPlugin = { + name: 'rw-esbuild-babel-transform', + setup(build: PluginBuild) { + const rwjsConfig = getConfig() + + build.onLoad({ filter: /\.(js|ts|tsx|jsx)$/ }, async (args) => { + // @TODO Implement LRU cache? Unsure how much of a performance benefit its going to be + // Generate a CRC of file contents, then save it to LRU cache with a limit + // without LRU cache, the memory usage can become unbound + const transformedCode = await transformWithBabel( + args.path, + getApiSideBabelPlugins({ + openTelemetry: + rwjsConfig.experimental.opentelemetry.enabled && + rwjsConfig.experimental.opentelemetry.wrapApi, + }) + ) + + if (transformedCode?.code) { + return { + contents: transformedCode.code, + loader: 'js', + } + } + + throw new Error(`Could not transform file: ${args.path}`) + }) + }, +} + +export const transpileApi = async (files: string[]) => { + return build(getEsbuildOptions(files)) } -export const transpileApi = (files: string[], options = {}) => { +function getEsbuildOptions(files: string[]): BuildOptions { const rwjsPaths = getPaths() - return esbuild.buildSync({ + return { absWorkingDir: rwjsPaths.api.base, entryPoints: files, platform: 'node', target: 'node20', format: 'cjs', + allowOverwrite: true, bundle: false, + plugins: [runRwBabelTransformsPlugin], outdir: rwjsPaths.api.dist, // setting this to 'true' will generate an external sourcemap x.js.map // AND set the sourceMappingURL comment // (setting it to 'external' will ONLY generate the file, but won't add the comment) sourcemap: true, - ...options, - }) + } } diff --git a/packages/vite/src/buildFeServer.ts b/packages/vite/src/buildFeServer.ts index f5d01e254268..382e80dd5f48 100644 --- a/packages/vite/src/buildFeServer.ts +++ b/packages/vite/src/buildFeServer.ts @@ -80,9 +80,7 @@ export const buildFeServer = async ({ verbose, webDir }: BuildOptions = {}) => { name: 'rw-esbuild-babel-transform', setup(build: PluginBuild) { build.onLoad({ filter: /\.(js|ts|tsx|jsx)$/ }, async (args) => { - // Remove RedwoodJS "magic" from a user's code leaving JavaScript behind. - // TODO (STREAMING) We need the new transformWithBabel function in https://github.com/redwoodjs/redwood/pull/7672/files - const transformedCode = transformWithBabel(args.path, [ + const transformedCode = await transformWithBabel(args.path, [ ...getRouteHookBabelPlugins(), ]) diff --git a/tasks/e2e/cypress.config.js b/tasks/e2e/cypress.config.js index e7a3142f764c..67498e6087be 100644 --- a/tasks/e2e/cypress.config.js +++ b/tasks/e2e/cypress.config.js @@ -2,17 +2,24 @@ const { defineConfig } = require('cypress') module.exports = defineConfig({ e2e: { - excludeSpecPattern: ['**/codemods/*.js', '**/sharedTests.js'], specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}', - supportFile: false, + excludeSpecPattern: ['**/codemods/*.js', '**/sharedTests.js'], + testIsolation: false, + + setupNodeEvents(on, config) { + require('cypress-fail-fast/plugin')(on, config) + return config + }, }, + // `runMode` is for `cypress run`, `openMode` is for `cypress open`. // Locally, we use open. But in CI, we use run. retries: { - runMode: 5, - openMode: 0, + runMode: 3, + openMode: 1, }, + defaultCommandTimeout: 12_0000, execTimeout: 12_0000, pageLoadTimeout: 12_0000, diff --git a/tasks/e2e/cypress/e2e/04-logger/logger.cy.js b/tasks/e2e/cypress/e2e/04-logger/logger.cy.js index 66d9e18aa561..82a40611fd81 100644 --- a/tasks/e2e/cypress/e2e/04-logger/logger.cy.js +++ b/tasks/e2e/cypress/e2e/04-logger/logger.cy.js @@ -18,6 +18,8 @@ import 'cypress-wait-until' describe('The Redwood Logger - Basic Scaffold CRUD Logging', () => { const LOG_PATH = path.join(BASE_DIR, LOG_FILENAME) + const WAIT_TIMEOUT = 10_000 + it('1. Test Logging for CRUD', () => { // Empty log file. cy.writeFile(LOG_PATH, '') @@ -46,7 +48,7 @@ describe('The Redwood Logger - Basic Scaffold CRUD Logging', () => { console.log(str) return str.includes('> in posts()') }), - { interval: 2000, timeout: 2000 } + { interval: 2000, timeout: WAIT_TIMEOUT } ) // CREATE / SAVE @@ -110,7 +112,7 @@ describe('The Redwood Logger - Basic Scaffold CRUD Logging', () => { !str.includes('Slow Query performed in ') ) }), - { interval: 2000, timeout: 2000 } + { interval: 2000, timeout: WAIT_TIMEOUT } ) // With slow query logging. @@ -135,7 +137,7 @@ describe('The Redwood Logger - Basic Scaffold CRUD Logging', () => { console.log(str) return str.includes('Slow Query performed in ') }), - { interval: 2000, timeout: 2000 } + { interval: 2000, timeout: WAIT_TIMEOUT } ) }) }) diff --git a/tasks/e2e/cypress/support/e2e.js b/tasks/e2e/cypress/support/e2e.js new file mode 100644 index 000000000000..40971b0f6df1 --- /dev/null +++ b/tasks/e2e/cypress/support/e2e.js @@ -0,0 +1 @@ +import 'cypress-fail-fast' diff --git a/tasks/run-e2e b/tasks/run-e2e index 6e8cb303aca0..d0b31c409455 100755 --- a/tasks/run-e2e +++ b/tasks/run-e2e @@ -58,7 +58,8 @@ const createRedwoodJSApp = ({ typescript, bundler }) => { '--no-yarn-install', `--typescript ${typescript}`, '--no-telemetry', - '--no-git', + '--git', + '-m "first"', ].filter(Boolean), { cwd: path.join(REDWOODJS_FRAMEWORK_PATH, 'packages/create-redwood-app'), diff --git a/yarn.lock b/yarn.lock index 0a3617838424..19c5f8e6d538 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7888,6 +7888,7 @@ __metadata: "@redwoodjs/project-config": "npm:6.0.7" "@types/babel-plugin-tester": "npm:9.0.9" "@types/babel__core": "npm:7.20.4" + "@types/node": "npm:20.10.4" babel-plugin-auto-import: "npm:1.1.0" babel-plugin-graphql-tag: "npm:3.3.0" babel-plugin-module-resolver: "npm:5.0.0" @@ -15964,6 +15965,17 @@ __metadata: languageName: node linkType: hard +"cypress-fail-fast@npm:7.1.0": + version: 7.1.0 + resolution: "cypress-fail-fast@npm:7.1.0" + dependencies: + chalk: "npm:4.1.2" + peerDependencies: + cypress: ">=8.0.0" + checksum: a523c0dac14c8c6a7297282e0dfd275f9bbcf242661932a449a5a2a02521886ceb70d420ed81d042a1d69bc62dc96209310225a9e54a2c44cf9c929dd2b8d7e4 + languageName: node + linkType: hard + "cypress-wait-until@npm:3.0.1": version: 3.0.1 resolution: "cypress-wait-until@npm:3.0.1" @@ -29195,6 +29207,7 @@ __metadata: boxen: "npm:5.1.2" core-js: "npm:3.34.0" cypress: "npm:13.6.1" + cypress-fail-fast: "npm:7.1.0" cypress-wait-until: "npm:3.0.1" dependency-cruiser: "npm:15.5.0" dotenv: "npm:16.3.1"