chore(tasks): Benchmark tests

package.json

@@ -11,6 +11,7 @@
     "packages/studio/web"
   ],
   "scripts": {
+    "benchmark": "tsx ./tasks/benchmark/run-benchmarks.mts",
     "build": "lerna run build",
     "build:clean": "yarn clean:prisma && rimraf \"packages/**/dist\" --glob",
     "build:clean:super": "git clean -fdx && yarn && yarn build",
@@ -99,6 +100,7 @@
     "ora": "6.3.1",
     "prompts": "2.4.2",
     "rimraf": "5.0.1",
+    "tsx": "3.12.7",
     "typescript": "5.1.6",
     "yargs": "17.7.2",
     "zx": "7.2.2"

tasks/benchmark/README.md

@@ -0,0 +1,64 @@
+# Benchmark Task
+
+This task is used to benchmark the performance of the Redwood api locally.
+
+## Tests
+
+The `tests` folder contains the k6 tests which will be executed to benchmark the api.
+
+Please remember that these are run by the k6 program and so you can't treat them exactly like any old javascript file.
+
+The tests should in most cases be hitting the `http://localhost:8911` endpoint as the test runner is running `yarn rw serve api` to start the api.
+
+## Setups
+
+The `setups` folder contains the setup scripts which will be executed before the tests are run to setup the correct environment.
+
+These setup scripts should be in a named folder and contain a `setup.mts` file which will be executed to perform the setup. This file must export the following:
+* A `setup` function which will be executed to perform the setup.
+* A `validForTests` string array which contains the names of the tests which this setup is valid for.
+
+## Running
+
+To run the benchmark tests you can run the following command:
+
+```bash
+yarn benchmark
+```
+
+This will need the k6 program to be installed on your machine. You can find information about it here: https://k6.io/docs/getting-started/installation
+
+Running the benchmarks will do the following:
+* Create a temporary redwood test project within the `/tmp/redwood-benchmark` folder - or windows equivalent.
+* Link the current state of your framework into the test project.
+* Run each of the setup scripts in the `setups` folder and for each one:
+  * Run the `setup` function.
+  * Build the api side.
+  * Start the api server.
+  * Run each of the appropriate tests in the `tests` folder.
+  * Stop the api server.
+  * Reset the project to its original state.
+
+
+**Filtering tests and setups**
+
+You can filter the tests and setups which are run by passing command line arguments to the benchmark command.
+
+To limit the setups which are run you can pass a list of setup names to the `--setup` argument. For example:
+
+```bash
+yarn benchmark --setup setup1 setup2
+```
+
+To limit the tests which are run you can pass a list of test names to the `--test` argument. For example:
+
+```bash
+yarn benchmark --test test1 test2
+```
+
+You can combine these like so:
+```bash
+yarn benchmark --setup setup1 setup2 --test test1 test2
+```
+
+By default all setups and tests will be run.

tasks/benchmark/run-benchmarks.mts

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+/* eslint-env node, es6*/
+
+import os from 'node:os'
+import path from 'node:path'
+import url from "node:url"
+
+import execa from 'execa'
+import fg  from 'fast-glob'
+import fs from 'fs-extra'
+import { hideBin } from 'yargs/helpers'
+import yargs from 'yargs/yargs'
+import { $ } from 'zx'
+
+import { buildRedwoodFramework, addFrameworkDepsToProject, cleanUp, copyFrameworkPackages, createRedwoodJSApp, initGit, runYarnInstall } from "./util/util.mjs"
+
+// useful consts
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+
+// Parse input
+const args = yargs(hideBin(process.argv))
+  .positional('project-directory', {
+    type: 'string',
+    describe: 'The project directory to run the benchmarks in',
+    demandOption: false,
+  })
+  .option('setup', { default: [], type: 'array', alias: 's' })
+  .option('test', { default: [], type: 'array', alias: 't' })
+  .option('clean-up', { default: true, type: 'boolean' })
+  .scriptName('run-benchmarks')
+  .example('run-benchmarks', 'Run all the benchmarks')
+  .example(
+    'run-benchmarks /tmp/redwood-app --setup someSetup --test somTest anotherTest',
+    "Run the benchmarks only for the setup 'someSetup' and the tests 'someTest' and 'anotherTest'"
+  )
+  .help()
+  .parseSync()
+
+const REDWOODJS_FRAMEWORK_PATH = path.join(__dirname, '..', '..')
+const REDWOOD_PROJECT_DIRECTORY =
+  args._?.[0]?.toString() ??
+  path.join(
+    os.tmpdir(),
+    'redwood-benchmark',
+    // ":" is problematic with paths
+    new Date().toISOString().split(':').join('-')
+  )
+
+const SETUPS_DIR = path.join(__dirname, 'setups')
+const TESTS_DIR = path.join(__dirname, 'tests')
+let cleanUpExecuted = false
+
+async function main() {
+  const divider = '~'.repeat(process.stdout.columns)
+
+  console.log(`${divider}\nBenchmark tests\n${divider}\n`)
+  console.log('Benchmark tests will be run in the following directory:')
+  console.log(`${REDWOOD_PROJECT_DIRECTORY}\n`)
+
+  fs.mkdirSync(REDWOOD_PROJECT_DIRECTORY, { recursive: true })
+
+  // Register clean up
+  if (args.cleanUp) {
+    console.log('The directory will be deleted after the tests are run')
+    process.on('SIGINT', () => {
+      if(!cleanUpExecuted){
+        cleanUp({
+          projectPath: REDWOOD_PROJECT_DIRECTORY,
+        })
+        cleanUpExecuted = true
+      }
+    })
+    process.on('exit', () => {
+      if(!cleanUpExecuted){
+        cleanUp({
+          projectPath: REDWOOD_PROJECT_DIRECTORY,
+        })
+        cleanUpExecuted = true
+      }
+    })
+  }
+
+  // Get all the setups
+  const setups = fg
+    .sync('*', {
+      onlyDirectories: true,
+      cwd: SETUPS_DIR,
+    })
+    .filter((setupDir) => {
+      return (
+        args.setup.length === 0 ||
+        args.setup.some((setup) => setupDir.includes(setup.toString()))
+      )
+    })
+
+  if (setups.length === 0) {
+    console.log('\nThere are no setups to run.')
+    process.exit(0)
+  }
+
+  // Create a test project and sync the current framework state
+  console.log('\nCreating a fresh project:')
+
+  buildRedwoodFramework({
+    frameworkPath: REDWOODJS_FRAMEWORK_PATH,
+  })
+  createRedwoodJSApp({
+    frameworkPath: REDWOODJS_FRAMEWORK_PATH,
+    projectPath: REDWOOD_PROJECT_DIRECTORY,
+    typescript: true
+  })
+  addFrameworkDepsToProject({
+    frameworkPath: REDWOODJS_FRAMEWORK_PATH,
+    projectPath: REDWOOD_PROJECT_DIRECTORY,
+  })
+  runYarnInstall({
+    projectPath: REDWOOD_PROJECT_DIRECTORY,
+  })
+  copyFrameworkPackages({
+    frameworkPath: REDWOODJS_FRAMEWORK_PATH,
+    projectPath: REDWOOD_PROJECT_DIRECTORY,
+  })
+  initGit({
+    projectPath: REDWOOD_PROJECT_DIRECTORY,
+  })
+
+  // zx setup
+  $.verbose = false
+  $.cwd = REDWOOD_PROJECT_DIRECTORY
+  $.log = () => {}
+
+  console.log('\nThe following setups will be run:')
+  for(let i = 0; i < setups.length; i++){
+    console.log(`- ${setups[i]}`)
+  }
+
+  for (const setup of setups) {
+    // import the setup
+    const setupFile = path.join(SETUPS_DIR, setup, 'setup.mjs')
+    const setupModule = await import(setupFile)
+    const runForTests = args.test.length === 0 ?
+      setupModule.validForTests :
+      args.test.filter((test) => setupModule.validForTests.includes(test))
+
+    if(runForTests.length === 0){
+      console.log(`\nThere are no tests to run for setup ${setup}, skipping...`)
+      continue
+    }
+
+    // Clean up the project state
+    console.log(`\nCleaning up the project state...`)
+    await $`git reset --hard`
+    await $`git clean -fd`
+
+    // Run the setup
+    console.log(`\nRunning setup: ${setup}\n`)
+    await setupModule.setup({
+      projectPath: REDWOOD_PROJECT_DIRECTORY,
+    })
+
+    // Build the app
+    console.log('- Building the api...')
+    await execa('yarn', ['rw', 'build', 'api'], {
+      cwd: REDWOOD_PROJECT_DIRECTORY,
+      stdio: 'inherit'
+    })
+
+    // Run the tests
+    for(let i = 0; i < runForTests.length; i++){
+      console.log(`\nRunning test ${i+1}/${runForTests.length}: ${runForTests[i]}`)
+
+      // Start the server
+      const serverSubprocess = $`yarn rw serve api`
+
+      // Wait for the server to be ready
+      let ready = false
+      serverSubprocess.stdout?.on('data', (data) => {
+        const text = Buffer.from(data).toString()
+        if (text.includes('API listening on')) {
+          ready = true
+        }
+      })
+      while (!ready) {
+        await new Promise((resolve) => setTimeout(resolve, 100))
+      }
+
+      // Run k6 test
+      try {
+        await execa('k6', ['run', path.join(TESTS_DIR, `${runForTests[i]}.js`)], {
+          cwd: REDWOOD_PROJECT_DIRECTORY,
+          stdio: 'inherit'
+        })
+
+        // TODO: Consider collecting the results into some summary output?
+      } catch (_error) {
+        console.warn('The k6 test failed')
+      }
+
+      // Stop the server
+      serverSubprocess.kill("SIGINT")
+      try {
+        await serverSubprocess
+      } catch (_error) {
+        // ignore
+      }
+    }
+  }
+
+  process.emit('SIGINT')
+}
+
+main()

tasks/benchmark/setups/context_magic_number/setup.mts

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+/* eslint-env node, es6*/
+
+import path from "node:path"
+import url from "node:url"
+
+import fs from "fs-extra"
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+
+export const validForTests = [
+  "context_functions",
+  "context_graphql",
+]
+
+export function setup({ projectPath }: { projectPath: string }) {
+  // Copy over SDL
+  fs.copyFileSync(
+    path.join(__dirname, 'templates', 'benchmark.sdl.ts'),
+    path.join(projectPath, 'api', 'src', 'graphql', 'benchmark.sdl.ts')
+  )
+
+  // Copy over the service
+  const benchmarkServicePath = path.join(
+    projectPath,
+    'api',
+    'src',
+    'services',
+    'benchmarks'
+  )
+  fs.mkdirSync(benchmarkServicePath)
+  fs.copyFileSync(
+    path.join(__dirname, 'templates', 'benchmarks.ts'),
+    path.join(benchmarkServicePath, 'benchmarks.ts')
+  )
+
+  // Copy over the function
+  const benchmarkFunctionPath = path.join(
+    projectPath,
+    'api',
+    'src',
+    'functions'
+  )
+  fs.copyFileSync(
+    path.join(__dirname, 'templates', 'func.ts'),
+    path.join(benchmarkFunctionPath, 'func.ts')
+  )
+}

tasks/benchmark/setups/context_magic_number/templates/benchmark.sdl.ts

@@ -0,0 +1,8 @@
+export const schema = gql`
+  type MagicNumber {
+    value: Int!
+  }
+  type Mutation {
+    magicNumber(value: Int!): MagicNumber! @skipAuth
+  }
+`

tasks/benchmark/setups/context_magic_number/templates/benchmarks.ts

@@ -0,0 +1,27 @@
+import type { MutationResolvers } from 'types/graphql'
+
+import { setContext } from '@redwoodjs/graphql-server'
+
+import { logger } from 'src/lib/logger'
+
+export const magicNumber: MutationResolvers['magicNumber'] = async ({
+  value,
+}) => {
+  setContext({
+    // ...context,
+    magicNumber: value,
+  })
+  // context.magicNumber = value
+
+  const sleep = Math.random() * 200
+  // logger.info(`Sleeping for ${sleep}ms`)
+  await new Promise((resolve) => setTimeout(resolve, sleep))
+
+  const numberFromContext = (context.magicNumber ?? -1) as number
+  if (value !== numberFromContext) {
+    logger.error(`Expected ${value} but got ${numberFromContext}`)
+    // throw new Error(`Expected ${value} but got ${numberFromContext}`)
+  }
+
+  return { value: numberFromContext }
+}