Evals

Author: bracesproul

evals/highlights.ts

@@ -0,0 +1,52 @@
+import { type Example, Run } from "langsmith";
+import { graph } from "../src/agent/index";
+import { evaluate, EvaluationResult } from "langsmith/evaluation";
+import "dotenv/config";
+
+const runGraph = async (input: Record<string, any>): Promise<Record<string, any>> => {
+  // Interrupt after updating the artifact
+  graph.interruptAfter = ["updateArtifact"];
+  return await graph.invoke(input);
+}
+
+const evaluateHighlights = (run: Run, example?: Example): EvaluationResult => {
+  if (!example) {
+    throw new Error("No example provided");
+  }
+  if (!example.outputs) {
+    throw new Error("No example outputs provided");
+  }
+  if (!run.outputs) {
+    throw new Error("No run outputs provided");
+  }
+
+  const { expectedGeneration } = example.outputs;
+  const { highlighted, artifacts } = example.inputs;
+  const expectedGenerationStart = artifacts[0].content.slice(0, highlighted.startCharIndex);
+  const expectedGenerationEnd = artifacts[0].content.slice(highlighted.endCharIndex);
+  const fullExpectedArtifact = `${expectedGenerationStart}${expectedGeneration}${expectedGenerationEnd}`;
+
+  
+  const generatedArtifact = run.outputs.artifacts[0].content;
+  if (generatedArtifact !== fullExpectedArtifact) {
+    return {
+      key: "correct_generation",
+      score: false,
+    }
+  }
+  return {
+    key: "correct_generation",
+    score: true,
+  }
+};
+
+async function runHighlightEval() {
+  const datasetName = "open-canvas-deterministic-highlights";
+  await evaluate(runGraph, {
+    data: datasetName,
+    evaluators: [evaluateHighlights],
+    experimentPrefix: "Highlight generation",
+  });
+}
+
+runHighlightEval();

package.json

@@ -40,6 +40,7 @@
     "@vercel/kv": "^2.0.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
+    "dotenv": "^16.4.5",
     "js-cookie": "^3.0.5",
     "langsmith": "^0.1.61",
     "lucide-react": "^0.441.0",

yarn.lock

@@ -2066,6 +2066,11 @@ doctrine@^3.0.0:
   dependencies:
     esutils "^2.0.2"
 
+dotenv@^16.4.5:
+  version "16.4.5"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f"
+  integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==
+
 double-ended-queue@^2.1.0-0:
   version "2.1.0-0"
   resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c"