[H-5747] Set up dev harness for Agentic Workflows

.cursor/mcp.json

@@ -3,6 +3,10 @@
     "filesystem": {
       "command": "npx",
       "args": ["-y", "@modelcontextprotocol/server-filesystem", "."]
+    },
+    "mastra": {
+      "command": "npx",
+      "args": ["-y", "@mastra/mcp-docs-server@beta"]
     }
   }
 }

apps/hash-ai-agent/.gitignore

@@ -0,0 +1,8 @@
+output.txt
+node_modules
+dist
+.mastra
+.env.development
+.env
+*.db
+*.db-*

apps/hash-ai-agent/README.md

@@ -0,0 +1,152 @@
+# HASH AI Agent (Mastra-based)
+
+Mastra-based implementation of HASH's AI inference capabilities, focusing on Named Entity Recognition (NER) and entity extraction optimization.
+
+## Project Structure
+
+```
+src/mastra/
+├── agents/
+│   └── entity-summary-agent.ts      # NER agent (baseline)
+├── evals/
+│   ├── test-data/
+│   │   └── ner-test-cases.ts        # Test dataset with scoring logic
+│   ├── test-entity-extraction.ts    # Simple smoke test
+│   └── run-entity-extraction-eval.ts # Full evaluation runner
+├── scorers/
+│   └── entity-recall-scorer.ts      # Entity recall/precision scorer
+├── shared/
+│   ├── dereference-entity-type.ts   # Entity type schema utilities
+│   └── generate-simplified-type-id.ts # Helper for type dereferencing
+├── types/
+│   └── entities.ts                  # Core entity type definitions
+└── index.ts                         # Mastra instance registration
+```
+
+## Quick Start
+
+### 1. Simple Test
+
+Run a single test case to verify the agent works:
+
+```bash
+pnpm tsx src/mastra/evals/test-entity-extraction.ts
+```
+
+This will:
+
+- Load the first test case from `ner-test-cases.ts`
+- Run the entity summary agent
+- Display extracted entities vs. expected entities
+
+### 2. Full Evaluation
+
+Run all test cases with scoring:
+
+```bash
+pnpm tsx src/mastra/evals/run-entity-extraction-eval.ts
+```
+
+This will:
+
+- Run all NER test cases
+- Calculate entity recall, precision, and type accuracy
+- Display average scores and detailed breakdowns
+
+## Architecture
+
+### Agents
+
+**Entity Summary Agent** (`agents/entity-summary-agent.ts`)
+
+- **Purpose**: Named Entity Recognition (NER) - identify entities in text
+- **Model**: `google/gemini-2.0-flash-exp:free` (via OpenRouter)
+- **Tool**: `registerEntitySummaries` for reporting found entities
+- **Input**: Text + research goal + entity types
+- **Output**: Array of entity summaries (name, summary, type)
+
+### Scorers
+
+**Entity Recall Scorer** (`scorers/entity-recall-scorer.ts`)
+
+- **Purpose**: Measure extraction quality (recall, precision, type accuracy)
+- **Methodology**: Ported from hash-ai-worker-ts optimization tests
+- **Metrics**:
+  - Gold entities found vs. missed (recall)
+  - Irrelevant entities identified (false positives)
+  - Wrong-type entities (type confusion)
+- **Score**: 0-1 (1 = perfect extraction)
+
+### Test Data
+
+**NER Test Cases** (`evals/test-data/ner-test-cases.ts`)
+
+- **Source**: Ported from hash-ai-worker-ts
+- **Format**: Stable fixtures (no external dependencies)
+- **Coverage**: AI companies, people, organizations, edge cases
+- **Ground Truth**: Gold entities, irrelevant entities, wrong-type entities
+
+### Type Definitions
+
+**Entity Types** (`types/entities.ts`)
+
+- `LocalEntitySummary` - Step 1: Entity name + summary + types
+- `Claim` - Step 2: Facts about entities (for future phases)
+- `ProposedEntity` - Step 3: Full entity with properties (for future phases)
+- `SourceProvenance` - Provenance tracking
+
+## Migration from hash-ai-worker-ts
+
+This project migrates AI inference capabilities from `hash-ai-worker-ts` (Temporal-based) to Mastra architecture:
+
+- **Workflows** (Temporal) → **Mastra Workflows** (structured steps)
+- **Activities** (LLM-based) → **Mastra Agents** (with tools)
+- **Activities** (deterministic) → **Mastra Tools**
+- **AI Tests** → **Mastra Evals** (scorers)
+
+See `docs/mastra-migration-plan.md` for complete architectural mapping.
+
+## Next Steps (Future Phases)
+
+- [ ] Create claim extraction agent
+- [ ] Create entity proposal agent
+- [ ] Build three-step workflow (summaries → claims → entities)
+- [ ] Create additional scorers (property accuracy, claim accuracy)
+- [ ] Add entity matching/deduplication agent
+- [ ] Integrate with HASH Graph API (currently using fixtures)
+
+## Development
+
+### Prerequisites
+
+- Node.js 22+
+- pnpm
+- OpenRouter API key (for Google models)
+
+### Environment Variables
+
+```bash
+OPENROUTER_API_KEY=your_api_key_here
+```
+
+### Running Tests
+
+```bash
+# Quick smoke test
+pnpm tsx src/mastra/evals/test-entity-extraction.ts
+
+# Full evaluation suite
+pnpm tsx src/mastra/evals/run-entity-extraction-eval.ts
+```
+
+### Type Checking
+
+```bash
+pnpm tsc --noEmit
+```
+
+## References
+
+- [Mastra Documentation](https://mastra.ai/docs)
+- [Original Plan](docs/ner-optimization-plan.md) - Original NER optimization plan
+- [Migration Plan](docs/mastra-migration-plan.md) - Detailed migration architecture

apps/hash-ai-agent/eslint.config.js

@@ -0,0 +1,18 @@
+import { createBase, defineConfig } from "@local/eslint/deprecated";
+
+export default [
+  {
+    ignores: ["_temp/**", ".mastra/**", "node_modules/**"],
+  },
+  ...createBase(import.meta.dirname),
+  ...defineConfig([
+    {
+      rules: {
+        /**
+         * @todo we should have separate browser/node configs
+         */
+        "react-hooks/rules-of-hooks": "off",
+      },
+    },
+  ]),
+];

apps/hash-ai-agent/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@apps/hash-ai-agent",
+  "version": "1.0.0",
+  "description": "",
+  "scripts": {
+    "test": "vitest",
+    "dev": "mastra dev",
+    "build": "mastra build",
+    "start": "mastra start",
+    "fix:eslint": "eslint --fix .",
+    "lint:eslint": "eslint --report-unused-disable-directives .",
+    "lint:tsc": "tsc --noEmit",
+    "ugi": "yarn upgrade-interactive"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "packageManager": "pnpm@10.24.0",
+  "type": "module",
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "2.0.80",
+    "@blockprotocol/graph": "0.4.0-canary.2",
+    "@blockprotocol/type-system": "0.1.2-canary.1",
+    "@local/advanced-types": "0.0.0-private",
+    "@local/hash-backend-utils": "0.0.0-private",
+    "@local/hash-graph-client": "0.0.0-private",
+    "@local/hash-graph-sdk": "0.0.0-private",
+    "@local/hash-isomorphic-utils": "0.0.0-private",
+    "@local/status": "0.0.0-private",
+    "@mastra/core": "1.0.0-beta.10",
+    "@mastra/evals": "1.0.0-beta.2",
+    "@mastra/libsql": "1.0.0-beta.7",
+    "@mastra/loggers": "1.0.0-beta.3",
+    "@mastra/mcp": "1.0.0-beta.6",
+    "@mastra/memory": "1.0.0-beta.4",
+    "dedent": "1.7.0",
+    "es-toolkit": "1.42.0",
+    "zod": "^4.1.13"
+  },
+  "devDependencies": {
+    "@local/eslint": "0.0.0-private",
+    "@local/tsconfig": "0.0.0-private",
+    "@types/dedent": "0.7.2",
+    "@types/dotenv-flow": "3.3.3",
+    "@types/jsdom": "21.1.7",
+    "@types/lodash": "4.17.20",
+    "@types/md5": "2.3.6",
+    "@types/mime-types": "2.1.4",
+    "@types/node": "22.18.13",
+    "@types/papaparse": "5.3.16",
+    "@types/sanitize-html": "2.16.0",
+    "@vitest/coverage-istanbul": "3.2.4",
+    "eslint": "9.38.0",
+    "jsdom": "27.3.0",
+    "mastra": "1.0.0-beta.5",
+    "rimraf": "6.1.2",
+    "sanitize-html": "2.17.0",
+    "tsx": "4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "3.2.4",
+    "wait-on": "9.0.1"
+  }
+}