diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6af7c985916e..3c0af74cbe00 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -121,22 +121,25 @@ pnpm exec changeset
### Running benchmarks
-We have benchmarks to keep performance under control. You can run these by running (from the project root):
+We have benchmarks to keep performance under control. They are located in the `benchmarks` directory, and it exposes a CLI you can use to run them.
+
+You can run all available benchmarks sequentially by running (from the project root):
```shell
-pnpm run benchmark --filter astro
+pnpm run benchmark
```
-Which will fail if the performance has regressed by **10%** or more.
-
-To update the times cd into the `packages/astro` folder and run the following:
+To run a specific benchmark only, you can add the name of the benchmark after the command:
```shell
-node test/benchmark/build.bench.js --save
-node test/benchmark/dev.bench.js --save
+pnpm run benchmark memory
```
-Which will update the build and dev benchmarks.
+Use `pnpm run benchmark --help` to see all available options.
+
+To run these benchmarks in a PR on GitHub instead of using the CLI, you can comment `!bench`. The benchmarks will run on both the PR branch and the `main` branch, and the results will be posted as a new comment.
+
+To run only a specific benchmark on CI, add its name after the command in your comment, for example, `!bench memory`.
## Code Structure
diff --git a/packages/astro/package.json b/packages/astro/package.json
index f901ddf785b7..2809376d6405 100644
--- a/packages/astro/package.json
+++ b/packages/astro/package.json
@@ -90,7 +90,6 @@
"build:ci": "pnpm run prebuild && astro-scripts build \"src/**/*.ts\"",
"dev": "astro-scripts dev --prebuild \"src/runtime/server/astro-island.ts\" --prebuild \"src/runtime/client/{idle,load,media,only,visible}.ts\" \"src/**/*.ts\"",
"postbuild": "astro-scripts copy \"src/**/*.astro\"",
- "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
"test:unit": "mocha --exit --timeout 30000 ./test/units/**/*.test.js",
"test:unit:match": "mocha --exit --timeout 30000 ./test/units/**/*.test.js -g",
"test": "pnpm run test:unit && mocha --exit --timeout 20000 --ignore **/lit-element.test.js && mocha --timeout 20000 **/lit-element.test.js",
diff --git a/packages/astro/test/benchmark/benchmark.js b/packages/astro/test/benchmark/benchmark.js
deleted file mode 100644
index d8b1c72fc39c..000000000000
--- a/packages/astro/test/benchmark/benchmark.js
+++ /dev/null
@@ -1,71 +0,0 @@
-import { promises as fsPromises, existsSync } from 'fs';
-import { performance } from 'perf_hooks';
-import * as assert from 'uvu/assert';
-
-const MUST_BE_AT_LEAST_PERC_OF = 90;
-
-const shouldSave = process.argv.includes('--save');
-
-export class Benchmark {
- constructor(options) {
- this.options = options;
- this.setup = options.setup || Function.prototype;
- }
-
- async execute() {
- const { run } = this.options;
- const start = performance.now();
- const end = await run(this.options);
- const time = Math.floor(end - start);
- return time;
- }
-
- async run() {
- const { file } = this.options;
-
- await this.setup();
- const time = await this.execute();
-
- if (existsSync(file)) {
- const raw = await fsPromises.readFile(file, 'utf-8');
- const data = JSON.parse(raw);
- if (Math.floor((data.time / time) * 100) > MUST_BE_AT_LEAST_PERC_OF) {
- this.withinPreviousRuns = true;
- } else {
- this.withinPreviousRuns = false;
- }
- }
- this.time = time;
- }
-
- report() {
- const { name } = this.options;
- console.log(name, 'took', this.time, 'ms');
- }
-
- check() {
- assert.ok(this.withinPreviousRuns !== false, `${this.options.name} ran too slowly`);
- }
-
- async save() {
- const { file, name } = this.options;
- const data = JSON.stringify(
- {
- name,
- time: this.time,
- },
- null,
- ' '
- );
- await fsPromises.writeFile(file, data, 'utf-8');
- }
-
- async test() {
- await this.run();
- if (shouldSave) {
- await this.save();
- }
- this.report();
- this.check();
- }
-}
diff --git a/packages/astro/test/benchmark/build-cached.json b/packages/astro/test/benchmark/build-cached.json
deleted file mode 100644
index d0ec5912ddef..000000000000
--- a/packages/astro/test/benchmark/build-cached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "name": "Snowpack Example Build Cached",
- "time": 8496
-}
diff --git a/packages/astro/test/benchmark/build-uncached.json b/packages/astro/test/benchmark/build-uncached.json
deleted file mode 100644
index 101621c720a7..000000000000
--- a/packages/astro/test/benchmark/build-uncached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "name": "Snowpack Example Build Uncached",
- "time": 16200
-}
diff --git a/packages/astro/test/benchmark/build.bench.js b/packages/astro/test/benchmark/build.bench.js
deleted file mode 100644
index a140082a325a..000000000000
--- a/packages/astro/test/benchmark/build.bench.js
+++ /dev/null
@@ -1,85 +0,0 @@
-/** @todo migrate these to use the independent docs repository at https://github.com/withastro/docs */
-
-import { fileURLToPath } from 'url';
-import { performance } from 'perf_hooks';
-import { build as astroBuild } from '#astro/build';
-import { loadConfig } from '#astro/config';
-import { Benchmark } from './benchmark.js';
-import { deleteAsync } from 'del';
-import { Writable } from 'stream';
-import { format as utilFormat } from 'util';
-
-const snowpackExampleRoot = new URL('../../../../docs/', import.meta.url);
-
-export const errorWritable = new Writable({
- objectMode: true,
- write(event, _, callback) {
- let dest = process.stderr;
- dest.write(utilFormat(...event.args));
- dest.write('\n');
-
- callback();
- },
-});
-
-let build;
-async function setupBuild() {
- const astroConfig = await loadConfig(fileURLToPath(snowpackExampleRoot));
-
- const logging = {
- level: 'error',
- dest: errorWritable,
- };
-
- build = () => astroBuild(astroConfig, logging);
-}
-
-async function runBuild() {
- await build();
- return performance.now();
-}
-
-const benchmarks = [
- new Benchmark({
- name: 'Snowpack Example Build Uncached',
- root: snowpackExampleRoot,
- file: new URL('./build-uncached.json', import.meta.url),
- async setup() {
- process.chdir(new URL('../../../../', import.meta.url).pathname);
- const spcache = new URL('../../node_modules/.cache/', import.meta.url);
- await Promise.all([deleteAsync(spcache.pathname, { force: true }), setupBuild()]);
- },
- run: runBuild,
- }),
- new Benchmark({
- name: 'Snowpack Example Build Cached',
- root: snowpackExampleRoot,
- file: new URL('./build-cached.json', import.meta.url),
- async setup() {
- process.chdir(new URL('../../../../', import.meta.url).pathname);
- await setupBuild();
- await this.execute();
- },
- run: runBuild,
- }),
- /*new Benchmark({
- name: 'Snowpack Example Dev Server Cached',
- root: snowpackExampleRoot,
- file: new URL('./dev-server-cached.json', import.meta.url),
- async setup() {
- // Execute once to make sure Snowpack is cached.
- await this.execute();
- }
- })*/
-];
-
-async function run() {
- for (const b of benchmarks) {
- await b.test();
- }
-}
-
-run().catch((err) => {
- console.error(err);
- process.exit(1);
-});
diff --git a/packages/astro/test/benchmark/dev-server-cached.json b/packages/astro/test/benchmark/dev-server-cached.json
deleted file mode 100644
index 16eb8a471643..000000000000
--- a/packages/astro/test/benchmark/dev-server-cached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "name": "Snowpack Example Dev Server Cached",
- "time": 1229
-}
diff --git a/packages/astro/test/benchmark/dev-server-uncached.json b/packages/astro/test/benchmark/dev-server-uncached.json
deleted file mode 100644
index c0d72650dd6f..000000000000
--- a/packages/astro/test/benchmark/dev-server-uncached.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "name": "Snowpack Example Dev Server Uncached",
- "time": 3913
-}
diff --git a/packages/astro/test/benchmark/dev.bench.js b/packages/astro/test/benchmark/dev.bench.js
deleted file mode 100644
index 134992634abc..000000000000
--- a/packages/astro/test/benchmark/dev.bench.js
+++ /dev/null
@@ -1,63 +0,0 @@
-/** @todo migrate these to use the independent docs repository at https://github.com/withastro/docs */
-
-import { performance } from 'perf_hooks';
-import { Benchmark } from './benchmark.js';
-import { runDevServer } from '../helpers.js';
-import { deleteAsync } from 'del';
-
-const docsExampleRoot = new URL('../../../../docs/', import.meta.url);
-
-async function runToStarted(root) {
- const args = [];
- const process = runDevServer(root, args);
-
- let started = null;
- process.stdout.setEncoding('utf8');
- for await (const chunk of process.stdout) {
- if (/Server started/.test(chunk)) {
- started = performance.now();
- break;
- }
- }
-
- process.kill();
- return started;
-}
-
-const benchmarks = [
- new Benchmark({
- name: 'Docs Site Example Dev Server Uncached',
- root: docsExampleRoot,
- file: new URL('./dev-server-uncached.json', import.meta.url),
- async setup() {
- const spcache = new URL('../../node_modules/.cache/', import.meta.url);
- await deleteAsync(spcache.pathname);
- },
- run({ root }) {
- return runToStarted(root);
- },
- }),
- new Benchmark({
- name: 'Docs Site Example Dev Server Cached',
- root: docsExampleRoot,
- file: new URL('./dev-server-cached.json', import.meta.url),
- async setup() {
- // Execute once to make sure Docs Site is cached.
- await this.execute();
- },
- run({ root }) {
- return runToStarted(root);
- },
- }),
-];
-
-async function run() {
- for (const b of benchmarks) {
- await b.test();
- }
-}
-
-run().catch((err) => {
- console.error(err);
- process.exit(1);
-});
diff --git a/packages/astro/test/benchmark/simple/astro.config.mjs b/packages/astro/test/benchmark/simple/astro.config.mjs
deleted file mode 100644
index 16a331be35b9..000000000000
--- a/packages/astro/test/benchmark/simple/astro.config.mjs
+++ /dev/null
@@ -1,7 +0,0 @@
-import { defineConfig } from 'astro/config';
-import nodejs from '@astrojs/node';
-
-export default defineConfig({
- output: 'server',
- adapter: nodejs({ mode: 'middleware' }),
-});
diff --git a/packages/astro/test/benchmark/simple/package.json b/packages/astro/test/benchmark/simple/package.json
deleted file mode 100644
index bec29ab55e62..000000000000
--- a/packages/astro/test/benchmark/simple/package.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "name": "@benchmark/simple",
- "scripts": {
- "start": "node server.mjs",
- "build": "astro build",
- "dev": "astro dev"
- },
- "dependencies": {
- "astro": "workspace:*",
- "@astrojs/node": "workspace:*"
- }
-}
diff --git a/packages/astro/test/benchmark/simple/server.mjs b/packages/astro/test/benchmark/simple/server.mjs
deleted file mode 100644
index 3fde151e1480..000000000000
--- a/packages/astro/test/benchmark/simple/server.mjs
+++ /dev/null
@@ -1,19 +0,0 @@
-import http from 'http';
-import { handler } from './dist/server/entry.mjs';
-
-const listener = (req, res) => {
- handler(req, res, (err) => {
- if (err) {
- res.writeHead(500);
- res.end(err.toString());
- } else {
- res.writeHead(404);
- res.end('Not found');
- }
- });
-};
-
-const server = http.createServer(listener);
-server.listen(3002);
-// eslint-disable-next-line no-console
-console.log(`Listening at http://localhost:3002`);
diff --git a/packages/astro/test/benchmark/simple/src/components/Layout.astro b/packages/astro/test/benchmark/simple/src/components/Layout.astro
deleted file mode 100644
index 7224ba686867..000000000000
--- a/packages/astro/test/benchmark/simple/src/components/Layout.astro
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
- {Astro.props.title}
-
-
-
-
-
diff --git a/packages/astro/test/benchmark/simple/src/pages/index.astro b/packages/astro/test/benchmark/simple/src/pages/index.astro
deleted file mode 100644
index 80f7e2eaa40e..000000000000
--- a/packages/astro/test/benchmark/simple/src/pages/index.astro
+++ /dev/null
@@ -1,79 +0,0 @@
----
-const content =
- "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
----
-
-
-
-
-
-
-
- Astro
-
-
-
Astro
-
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
{content}
-
-
-
diff --git a/scripts/memory/index.js b/scripts/memory/index.js
deleted file mode 100644
index 8eaffb56b581..000000000000
--- a/scripts/memory/index.js
+++ /dev/null
@@ -1,85 +0,0 @@
-import { fileURLToPath } from 'url';
-import v8 from 'v8';
-import dev from '../../packages/astro/dist/core/dev/index.js';
-import { openConfig } from '../../packages/astro/dist/core/config.js';
-import { nodeLogDestination } from '../../packages/astro/dist/core/logger/node.js';
-import prettyBytes from 'pretty-bytes';
-
-if (!global.gc) {
- console.error('ERROR: Node must be run with --expose-gc');
- process.exit(1);
-}
-
-const isCI = process.argv.includes('--ci');
-
-/** URL directory containing the entire project. */
-const projDir = new URL('./project/', import.meta.url);
-
-let { astroConfig: config } = await openConfig({
- cwd: fileURLToPath(projDir),
- logging: {
- dest: nodeLogDestination,
- level: 'error',
- },
- cmd: 'dev',
-});
-
-const telemetry = {
- record() {
- return Promise.resolve();
- },
-};
-const server = await dev(config, { logging: { level: 'error' }, telemetry });
-
-// Prime the server so initial memory is created
-await fetch(`http://localhost:3000/page-0`);
-
-async function run() {
- for (let i = 0; i < 100; i++) {
- let path = `/page-${i}`;
- await fetch(`http://localhost:3000${path}`);
- }
-}
-
-global.gc();
-const startSize = v8.getHeapStatistics().used_heap_size;
-
-// HUMAN mode: Runs forever. Optimized for accurate results on each snapshot Slower than CI.
-if (!isCI) {
- console.log(
- `Greetings, human. This test will run forever. Run with the "--ci" flag to finish with a result.`
- );
- let i = 1;
- while (i++) {
- await run();
- global.gc();
- const checkpoint = v8.getHeapStatistics().used_heap_size;
- console.log(`Snapshot ${String(i).padStart(3, '0')}: ${(checkpoint / startSize) * 100}%`);
- }
-}
-
-// CI mode: Runs 100 times. Optimized for speed with an accurate final result.
-for (let i = 0; i < 100; i++) {
- await run();
- const checkpoint = v8.getHeapStatistics().used_heap_size;
- console.log(`Estimate ${String(i).padStart(3, '0')}/100: ${(checkpoint / startSize) * 100}%`);
-}
-
-console.log(`Test complete. Running final garbage collection...`);
-global.gc();
-const endSize = v8.getHeapStatistics().used_heap_size;
-
-// If the trailing average is higher than the median, see if it's more than 5% higher
-let percentage = endSize / startSize;
-const TEST_THRESHOLD = 1.5;
-const isPass = percentage < TEST_THRESHOLD;
-console.log(``);
-console.log(`Result: ${isPass ? 'PASS' : 'FAIL'} (${percentage * 100}%)`);
-console.log(
- `Memory usage began at ${prettyBytes(startSize)} and finished at ${prettyBytes(endSize)}.`
-);
-console.log(`The threshold for a probable memory leak is ${TEST_THRESHOLD * 100}%`);
-console.log(``);
-console.log(`Exiting...`);
-await server.stop();
-process.exit(isPass ? 0 : 1);
diff --git a/scripts/memory/mk.js b/scripts/memory/mk.js
deleted file mode 100644
index ae14e1191587..000000000000
--- a/scripts/memory/mk.js
+++ /dev/null
@@ -1,11 +0,0 @@
-import fs from 'fs';
-
-const pages = new URL('./project/src/pages/', import.meta.url);
-
-for (let i = 0; i < 100; i++) {
- let content = `---
-const i = ${i};
----
-{i}`;
- await fs.promises.writeFile(new URL(`./page-${i}.astro`, pages), content, 'utf-8');
-}
diff --git a/scripts/memory/project/.gitkeep b/scripts/memory/project/.gitkeep
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/scripts/memory/project/src/pages/.gitkeep b/scripts/memory/project/src/pages/.gitkeep
deleted file mode 100644
index e69de29bb2d1..000000000000