Add benchmark for error stacks

.vscode/settings.json

@@ -34,5 +34,6 @@
   },
   "editor.codeActionsOnSave": {
     "source.organizeImports.biome": "explicit"
-  }
+  },
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

bench/bench.d.json.ts

@@ -1,4 +1,4 @@
-import type { BenchResults } from ".";
+import type { BenchResults } from "./src/index.ts";
 
 declare const results: BenchResults;
 

bench/download.d.json.ts

@@ -1,4 +1,4 @@
-import type { DownloadResults } from ".";
+import type { DownloadResults } from "./src/index.ts";
 
 declare const results: DownloadResults;
 

bench/package.json

@@ -16,13 +16,15 @@
   "exports": {
     ".": "./src/index.ts",
     "./bench.json": "./bench.json",
-    "./download.json": "./download.json"
+    "./download.json": "./download.json",
+    "./stack.json": "./stack.json"
   },
   "main": "./src/index.ts",
   "name": "@schema-benchmarks/bench",
   "private": true,
   "scripts": {
     "bench": "node --expose-gc --max-old-space-size=8192 ./src/scripts/bench.ts",
+    "stack": "node --expose-gc --max-old-space-size=8192 ./src/scripts/stack.ts",
     "download": "node ./src/scripts/download.ts",
     "typecheck": "tsgo"
   },

bench/src/results/types.ts

@@ -76,3 +76,39 @@ export const downloadResultsSchema = v.object({
   unminified: v.array(downloadResultSchema),
 });
 export type DownloadResults = v.InferOutput<typeof downloadResultsSchema>;
+
+const serializedErrorSchema = v.object({
+  message: v.string(),
+  name: v.string(),
+  stack: v.optional(v.string()),
+});
+export type SerializedError = v.InferOutput<typeof serializedErrorSchema>;
+
+const baseStackResultSchema = v.object({
+  libraryName: v.string(),
+  version: v.string(),
+  snippet: v.string(),
+});
+
+const successfulStackResultSchema = v.object({
+  ...baseStackResultSchema.entries,
+  line: v.number(),
+  error: serializedErrorSchema,
+});
+export const stackResultSchema = v.variant("line", [
+  successfulStackResultSchema,
+  ...(["no throw", "not an error"] as const).map((error) =>
+    v.object({
+      ...baseStackResultSchema.entries,
+      line: v.literal(error),
+    }),
+  ),
+  ...(["no stack", "no external stack", "not found"] as const).map((error) =>
+    v.object({
+      ...baseStackResultSchema.entries,
+      line: v.literal(error),
+      error: serializedErrorSchema,
+    }),
+  ),
+]);
+export type StackResult = v.InferOutput<typeof stackResultSchema>;

bench/src/scripts/stack.ts

@@ -0,0 +1,126 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as process from "node:process";
+import * as url from "node:url";
+import { assertNotReached, errorData } from "@schema-benchmarks/schemas";
+import { libraries } from "@schema-benchmarks/schemas/libraries";
+import type { SerializedError, StackResult } from "../results/types.ts";
+
+// this is probably quite fragile, worth keeping an eye on when we update node
+
+const libDist = import.meta
+  .resolve("@schema-benchmarks/schemas/libraries")
+  .replace("/index.js", "");
+
+const search = `    at Object.throw (${libDist}/`;
+
+const cwdRegex = new RegExp(
+  RegExp.escape(
+    url
+      .pathToFileURL(path.resolve(process.cwd(), ".."))
+      .href.replace(/^file:\/\//, ""),
+  ),
+  "g",
+);
+
+function serializeError(e: Error, customStack?: string): SerializedError {
+  return {
+    message: e.message,
+    name: e.name,
+    stack: customStack,
+    // cause?
+  };
+}
+
+const results: Array<StackResult> = [];
+
+function getScriptLineNumber(stack?: string) {
+  if (!stack) return "no stack";
+  const lines = stack.split("\n");
+  let i = 0;
+  while (i < lines.length) {
+    if (lines[i]?.startsWith(search)) return i + 1;
+    i++;
+  }
+  return "not found";
+}
+
+for (const getConfig of Object.values(libraries)) {
+  const {
+    library: { name: libraryName, version },
+    createContext,
+    stack,
+  } = await getConfig();
+  if (stack) {
+    const { snippet } = stack;
+    try {
+      const context = await createContext();
+      await stack.throw(context, errorData);
+      assertNotReached();
+    } catch (e) {
+      if (Error.isError(e)) {
+        if (e.name === "ShouldHaveThrownError") {
+          results.push({
+            libraryName,
+            version,
+            snippet,
+            line: "no throw",
+          });
+          continue;
+        }
+        const hasFrames = e.stack?.includes("    at ");
+        if (!hasFrames) {
+          results.push({
+            libraryName,
+            version,
+            snippet,
+            line: "no stack",
+            error: serializeError(e),
+          });
+          continue;
+        }
+        const frames = e.stack?.slice(e.stack.indexOf("    at "));
+        const line = getScriptLineNumber(frames);
+        if (frames && typeof line === "number" && line !== 1) {
+          results.push({
+            libraryName,
+            version,
+            snippet,
+            line,
+            error: serializeError(e, frames.replace(cwdRegex, "")),
+          });
+        } else {
+          results.push({
+            libraryName,
+            version,
+            snippet,
+            line: "no external stack",
+            error: serializeError(e),
+          });
+        }
+      } else {
+        results.push({
+          libraryName,
+          version,
+          snippet,
+          line: "not an error",
+        });
+      }
+    }
+  }
+  global.gc?.();
+}
+
+await fs.writeFile(
+  path.join(process.cwd(), "stack.json"),
+  JSON.stringify(
+    results.sort((a, b) => {
+      if (typeof a.line === "number" && typeof b.line === "number") {
+        return a.line - b.line;
+      }
+      if (typeof a.line === "number") return -1;
+      if (typeof b.line === "number") return 1;
+      return 0;
+    }),
+  ),
+);

bench/stack.d.json.ts

@@ -0,0 +1,5 @@
+import type { StackResult } from "./src/index.ts";
+
+declare const results: Array<StackResult>;
+
+export default results;

package.json

@@ -14,8 +14,9 @@
     "website:serve": "pnpm run --filter website serve",
     "website:storybook": "pnpm run --filter website storybook",
     "bench:bench": "pnpm run --filter '*/bench' bench",
+    "bench:stack": "pnpm run --filter '*/bench' stack",
     "bench:download": "pnpm run --filter '*/bench' download",
-    "bench:all": "pnpm bench:download && pnpm bench:bench",
+    "bench:all": "pnpm bench:download && pnpm bench:bench && pnpm bench:stack",
     "postinstall": "lefthook install"
   },
   "devDependencies": {

schemas/libraries/@vinejs/vine/benchmarks.ts

@@ -3,7 +3,7 @@ import { getVersion } from "@schema-benchmarks/utils/node" with {
 };
 import vine from "@vinejs/vine";
 import ts from "dedent" with { type: "macro" };
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 import { getVineSchema } from ".";
 
 export default defineBenchmarks({
@@ -37,4 +37,11 @@ export default defineBenchmarks({
       snippet: ts`vine.tryValidate({ schema: schema.bail(false), data })`,
     },
   },
+  stack: {
+    throw: async ({ schema }, data) => {
+      await vine.validate({ schema, data });
+      assertNotReached();
+    },
+    snippet: ts`await vine.validate({ schema, data })`,
+  },
 });

schemas/libraries/ajv/benchmarks.ts

@@ -1,6 +1,7 @@
 import { getVersion } from "@schema-benchmarks/utils/node" with {
   type: "macro",
 };
+import { ValidationError } from "ajv";
 import ts from "dedent" with { type: "macro" };
 import { defineBenchmarks } from "#src";
 import { getAjv, getAjvSchema } from ".";
@@ -43,4 +44,15 @@ export default defineBenchmarks({
       `,
     },
   ],
+  stack: {
+    throw({ validate }, data) {
+      validate(data);
+      throw new ValidationError(validate.errors ?? []);
+    },
+    snippet: ts`
+      // const validate = ajv.compile(schema);
+      validate(data);
+      throw new ValidationError(validate.errors ?? []);
+    `,
+  },
 });

schemas/libraries/arktype/benchmarks.ts

@@ -2,7 +2,7 @@ import { getVersion } from "@schema-benchmarks/utils/node" with {
   type: "macro",
 };
 import ts from "dedent" with { type: "macro" };
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 import { getArkTypeSchema } from ".";
 
 export default defineBenchmarks({
@@ -33,4 +33,11 @@ export default defineBenchmarks({
       snippet: ts`schema(data)`,
     },
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      schema.assert(data);
+      assertNotReached();
+    },
+    snippet: ts`schema.assert(data)`,
+  },
 });

schemas/libraries/effect/benchmarks.ts

@@ -2,8 +2,9 @@ import { getVersion } from "@schema-benchmarks/utils/node" with {
   type: "macro",
 };
 import ts from "dedent" with { type: "macro" };
+import { Effect } from "effect";
 import * as Schema from "effect/Schema";
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 import { getEffectSchema } from ".";
 
 export default defineBenchmarks({
@@ -74,4 +75,17 @@ export default defineBenchmarks({
       `,
     },
   },
+  stack: {
+    throw: ({ decodeAll }, data) => {
+      Effect.runSync(decodeAll(data));
+      assertNotReached();
+    },
+    snippet: ts`
+      // const decodeAll = Schema.decodeUnknownEither(
+      //  schema, 
+      //  { errors: "all" }
+      // );
+      Effect.runSync(decodeAll(data));
+    `,
+  },
 });

schemas/libraries/joi/benchmarks.ts

@@ -39,4 +39,10 @@ export default defineBenchmarks({
       snippet: ts`schema.validate(data, { abortEarly: true })`,
     },
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      throw schema.validate(data).error;
+    },
+    snippet: ts`throw schema.validate(data).error`,
+  },
 });

schemas/libraries/rod-js/benchmarks.ts

@@ -1,4 +1,4 @@
-import { defineBenchmarks } from "@schema-benchmarks/schemas";
+import { assertNotReached, defineBenchmarks } from "@schema-benchmarks/schemas";
 import { getVersion } from "@schema-benchmarks/utils/node" with {
   type: "macro",
 };
@@ -54,4 +54,11 @@ export default defineBenchmarks({
       },
     ],
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      schema.parse(data);
+      assertNotReached();
+    },
+    snippet: ts`schema.parse(data)`,
+  },
 });

schemas/libraries/sury/benchmarks.ts

@@ -3,7 +3,7 @@ import { getVersion } from "@schema-benchmarks/utils/node" with {
 };
 import ts from "dedent" with { type: "macro" };
 import * as S from "sury";
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 import { getSurySchema } from ".";
 
 export default defineBenchmarks({
@@ -78,4 +78,11 @@ export default defineBenchmarks({
       },
     ],
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      S.parseOrThrow(data, schema);
+      assertNotReached();
+    },
+    snippet: ts`S.parseOrThrow(data, schema)`,
+  },
 });

schemas/libraries/typebox/benchmarks.ts

@@ -4,7 +4,7 @@ import { getVersion } from "@schema-benchmarks/utils/node" with {
 import ts from "dedent" with { type: "macro" };
 import { Compile } from "typebox/compile";
 import Value from "typebox/value";
-import { defineBenchmarks } from "#src";
+import { assertNotReached, defineBenchmarks } from "#src";
 import { getTypeboxSchema } from ".";
 
 export default defineBenchmarks({
@@ -81,4 +81,11 @@ export default defineBenchmarks({
       },
     ],
   },
+  stack: {
+    throw: ({ schema }, data) => {
+      Value.Parse(schema, data);
+      assertNotReached();
+    },
+    snippet: ts`Value.Parse(schema, data)`,
+  },
 });