Fix compatibility issues and improve error handling in Navie

README.md

@@ -14,7 +14,7 @@ projects.
 - [@appland/validate](https://github.com/getappmap/appmap-js/tree/main/packages/validate)
 
 Looking for the AppMap client for JavaScript? Find it here:
-https://github.com/getappmap/appmap-agent-js
+https://github.com/getappmap/appmap-node
 
 ## Development
 

packages/cli/package.json

@@ -38,7 +38,6 @@
     "@eslint/js": "~8.57.0",
     "@octokit/types": "^11.1.0",
     "@swc/core": "^1.3.76",
-    "@types/better-sqlite3": "^7.6.11",
     "@types/cors": "^2.8.17",
     "@types/fs-extra": "^9.0.13",
     "@types/gitconfiglocal": "^2.0.1",
@@ -103,7 +102,6 @@
     "applicationinsights": "^2.1.4",
     "async": "^3.2.4",
     "axios": "^0.27.2",
-    "better-sqlite3": "^11.5.0",
     "body-parser": "^1.20.2",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
@@ -131,6 +129,7 @@
     "lunr": "^2.3.9",
     "minimatch": "^5.1.2",
     "moo": "^0.5.1",
+    "node-sqlite3-wasm": "^0.8.34",
     "open": "^8.2.1",
     "openapi-diff": "^0.23.6",
     "openapi-types": "^12.1.3",
@@ -167,9 +166,7 @@
       "resources",
       "built/appmap.html",
       "built/sequenceDiagram.html",
-      "built/docs",
-      "../../node_modules/better-sqlite3/build/Release/better_sqlite3.node",
-      "node_modules/better-sqlite3/build/Release/better_sqlite3.node"
+      "built/docs"
     ],
     "outputPath": "dist"
   },

packages/cli/src/cmds/index/rpc.ts

@@ -81,6 +81,13 @@ export function rpcMethods(navie: INavieProvider, codeEditor?: string): RpcHandl
 
 export const handler = async (argv: HandlerArguments) => {
   verbose(argv.verbose);
+  const observer = new PerformanceObserver((list) => {
+    for (const entry of list.getEntries()) {
+      const duration = entry.duration;
+      console.log(`${entry.name}: ${duration.toFixed(0)} ms`);
+    }
+  });
+  observer.observe({ type: 'measure' });
 
   const navie = buildNavieProvider(argv);
   let codeEditor: string | undefined = argv.codeEditor;

packages/cli/src/cmds/record/action/guessTestCommands.ts

@@ -34,14 +34,14 @@ const TestCommands: TestCommandGuess[] = [
   {
     paths: ['package.json', 'node_modules/mocha'],
     command: {
-      command: 'npx @appland/appmap-agent-js --recorder=mocha -- npx mocha',
+      command: 'npx appmap-node npx mocha',
       env: {},
     },
   },
   {
     paths: ['package.json', 'node_modules/@jest'],
     command: {
-      command: 'npx @appland/appmap-agent-js --recorder=jest -- npx jest',
+      command: 'npx appmap-node npx jest',
       env: {},
     },
   },

packages/cli/src/cmds/record/state/agentProcessNotRunning.ts

@@ -24,7 +24,7 @@ export default async function agentProcessNotRunning(context: RecordContext): Pr
   Django:     https://appland.com/docs/reference/appmap-python.html#django
   Flask:      https://appland.com/docs/reference/appmap-python.html#flask
   Java:       https://appland.com/docs/reference/appmap-java.html#remote-recording
-  JavaScript: https://appland.com/docs/reference/appmap-agent-js.html#remote-recording
+  JavaScript: https://appmap.io/docs/reference/appmap-node.html#remote-recording
   `);
 
   await configureHostAndPort(context);

packages/cli/src/cmds/search/search.ts

@@ -1,5 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
 import yargs from 'yargs';
-import sqlite3 from 'better-sqlite3';
 import assert from 'assert';
 import { readFileSync } from 'fs';
 import { writeFile } from 'fs/promises';
@@ -186,7 +186,7 @@ export const handler = async (argv: ArgumentTypes) => {
     };
 
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(fileIndex, [process.cwd()]);
       return fileIndex;

packages/cli/src/rpc/explain/collect-location-context.ts

@@ -140,14 +140,22 @@ async function directoryContextItem(
   };
 }
 
-async function* listDirectory(path: string, depth: number): AsyncGenerator<string> {
+const MAX_SUBENTRIES = 100;
+
+export async function* listDirectory(path: string, depth: number): AsyncGenerator<string> {
   const entries = await readdir(path, { withFileTypes: true });
   for (const entry of entries) {
     const entryPath = join(path, entry.name);
     if (entry.isDirectory()) {
       if (depth > 0) {
-        yield `${entry.name}/`;
-        for await (const subentry of listDirectory(entryPath, depth - 1)) yield `\t${subentry}`;
+        const subentries: string[] = [];
+        for await (const subentry of listDirectory(entryPath, depth - 1)) subentries.push(subentry);
+        if (subentries.length <= MAX_SUBENTRIES) {
+          yield `${entry.name}/`;
+          for (const subentry of subentries) yield `\t${subentry}`;
+        } else {
+          yield `${entry.name}/ (${subentries.length} entries)`;
+        }
       } else yield `${entry.name}/ (${(await readdir(entryPath)).length} entries)`;
     } else if (entry.isFile()) {
       yield entry.name;

packages/cli/src/rpc/explain/collect-search-context.ts

@@ -1,4 +1,4 @@
-import { log } from 'console';
+import makeDebug from 'debug';
 
 import { ContextV2, applyContext } from '@appland/navie';
 import { SearchRpc } from '@appland/rpc';
@@ -10,6 +10,8 @@ import { searchProjectFiles } from './index/project-file-index';
 import { buildProjectFileSnippetIndex } from './index/project-file-snippet-index';
 import { generateSessionId } from '@appland/search';
 
+const debug = makeDebug('appmap:cli:search-context');
+
 export type SearchContextRequest = {
   appmaps?: string[];
   excludePatterns?: RegExp[];
@@ -69,7 +71,7 @@ export default async function collectSearchContext(
       type: 'appmap',
     };
 
-    log(`[search-context] Matched ${selectedAppMaps.results.length} AppMaps.`);
+    debug(`Matched ${selectedAppMaps.results.length} AppMaps.`);
   }
 
   const fileSearchResults = await searchProjectFiles(
@@ -95,23 +97,23 @@ export default async function collectSearchContext(
   try {
     let charCount = 0;
     let maxEventsPerDiagram = 5;
-    log(`[search-context] Requested char limit: ${charLimit}`);
+    debug(`Requested char limit: ${charLimit}`);
     for (;;) {
-      log(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`);
+      debug(`Collecting context with ${maxEventsPerDiagram} events per diagram.`);
 
       contextCandidate = await snippetIndex.search(maxEventsPerDiagram, charLimit, request);
       const appliedContext = applyContext(contextCandidate.context, charLimit);
       const appliedContextSize = appliedContext.reduce((acc, item) => acc + item.content.length, 0);
       contextCandidate.context = appliedContext;
       contextCandidate.contextSize = appliedContextSize;
-      log(`[search-context] Collected an estimated ${appliedContextSize} characters.`);
+      debug(`Collected an estimated ${appliedContextSize} characters.`);
 
       if (appliedContextSize === charCount || appliedContextSize > charLimit) {
         break;
       }
       charCount = appliedContextSize;
       maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5);
-      log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`);
+      debug(`Increasing max events per diagram to ${maxEventsPerDiagram}.`);
     }
   } finally {
     snippetIndex.close();

packages/cli/src/rpc/explain/index-files.ts

@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,

packages/cli/src/rpc/explain/index-snippets.ts

@@ -1,3 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
+
 import {
   buildSnippetIndex,
   FileSearchResult,
@@ -6,7 +8,6 @@ import {
   readFileSafe,
   SnippetIndex,
 } from '@appland/search';
-import sqlite3 from 'better-sqlite3';
 
 export default async function indexSnippets(
   db: sqlite3.Database,

packages/cli/src/rpc/explain/index/appmap-file-index.ts

@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { FileIndex, SessionId } from '@appland/search';
 
@@ -10,7 +10,7 @@ export async function buildAppMapFileIndex(
   appmapDirectories: string[]
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir<FileIndex>('appmaps', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     const fileIndex = new FileIndex(db);
     await buildAppMapIndex(fileIndex, appmapDirectories);
     return fileIndex;

packages/cli/src/rpc/explain/index/project-file-index.ts

@@ -1,5 +1,5 @@
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
@@ -71,7 +71,7 @@ export async function buildProjectFileIndex(
   excludePatterns: RegExp[] | undefined
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir('files', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexFiles(db, sourceDirectories, includePatterns, excludePatterns);
   });
 }

packages/cli/src/rpc/explain/index/project-file-snippet-index.ts

@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileSearchResult, SessionId, SnippetIndex } from '@appland/search';
 
 import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir';
@@ -76,7 +77,7 @@ export async function buildProjectFileSnippetIndex(
   fileSearchResults: FileSearchResult[]
 ): Promise<ProjectFileSnippetIndex> {
   const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexSnippets(db, fileSearchResults);
   });
 

packages/cli/src/rpc/search/search.ts

@@ -1,5 +1,7 @@
 import { isAbsolute, join } from 'path';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileIndex, generateSessionId } from '@appland/search';
 import { SearchRpc } from '@appland/rpc';
 
@@ -62,7 +64,7 @@ export async function handler(
     // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
     const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS;
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(
         fileIndex,

packages/cli/tests/unit/rpc/explain/collect-location-context.spec.ts

@@ -5,7 +5,9 @@ import { isBinaryFile } from '@appland/search';
 
 import Location from '../../../../src/rpc/explain/location';
 import ContentRestrictions from '../../../../src/rpc/explain/ContentRestrictions';
-import collectLocationContext from '../../../../src/rpc/explain/collect-location-context';
+import collectLocationContext, {
+  listDirectory,
+} from '../../../../src/rpc/explain/collect-location-context';
 
 jest.mock('node:fs/promises');
 jest.mock('@appland/search');
@@ -18,6 +20,39 @@ describe('collectLocationContext', () => {
     ContentRestrictions.instance.reset();
   });
 
+  describe('listDirectory', () => {
+    const mockReaddir = jest.spyOn(fs, 'readdir');
+
+    afterEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it('lists files and directories up to the specified depth', async () => {
+      mockReaddir.mockResolvedValueOnce([mockDirent('file1.js'), mockDirent('dir1', true)]);
+      mockReaddir.mockResolvedValueOnce([mockDirent('file2.js'), mockDirent('file3.js')]);
+
+      const result: string[] = [];
+      for await (const entry of listDirectory('/test', 1)) {
+        result.push(entry);
+      }
+
+      expect(result).toEqual(['file1.js', 'dir1/', '\tfile2.js', '\tfile3.js']);
+    });
+
+    it('limits the number of subentries', async () => {
+      const subentries = Array.from({ length: 101 }, (_, i) => mockDirent(`file${i}.js`));
+      mockReaddir.mockResolvedValueOnce([mockDirent('dir1', true)]);
+      mockReaddir.mockResolvedValueOnce(subentries);
+
+      const result: string[] = [];
+      for await (const entry of listDirectory('/test', 1)) {
+        result.push(entry);
+      }
+
+      expect(result).toEqual(['dir1/ (101 entries)']);
+    });
+  });
+
   describe('with empty locations', () => {
     it('handles empty locations', async () => {
       const result = await collectLocationContext(sourceDirectories, []);
@@ -151,7 +186,9 @@ describe('collectLocationContext', () => {
 
     it('handles directory listings', async () => {
       stat.mockResolvedValue({ isDirectory: () => true, isFile: () => false } as Stats);
-      jest.spyOn(fs, 'readdir').mockResolvedValue(['file1.js', 'file2.js'].map(mockDirent));
+      jest
+        .spyOn(fs, 'readdir')
+        .mockResolvedValue(['file1.js', 'file2.js'].map((p) => mockDirent(p)));
 
       const result = await collectLocationContext(sourceDirectories, [Location.parse('/src:0')]);
       expect(result).toEqual([
@@ -208,11 +245,11 @@ describe('collectLocationContext', () => {
   });
 });
 
-function mockDirent(name: string): Dirent {
+function mockDirent(name: string, isDirectory = false): Dirent {
   return {
     name,
     isFile: () => true,
-    isDirectory: () => false,
+    isDirectory: () => isDirectory,
     isBlockDevice: () => false,
     isCharacterDevice: () => false,
     isSymbolicLink: () => false,

packages/components/src/lib/documentation.js

@@ -2,7 +2,7 @@ const documentationUrls = {
   ruby: 'https://appmap.io/docs/reference/appmap-ruby.html',
   python: 'https://appmap.io/docs/reference/appmap-python.html',
   java: 'https://appmap.io/docs/reference/appmap-java.html',
-  javascript: 'https://appmap.io/docs/reference/appmap-agent-js.html',
+  javascript: 'https://appmap.io/docs/reference/appmap-node.html',
 };
 
 export function getAgentDocumentationUrl(language) {

packages/navie/package.json

@@ -33,6 +33,7 @@
     "eslint-plugin-promise": "^6.0.1",
     "eslint-plugin-unicorn": "^39.0.0",
     "jest": "^29.7.0",
+    "prettier": "^3.3.3",
     "ts-jest": "^29.2.5",
     "ts-node": "^10.4.0",
     "tsc": "^2.0.3",
@@ -44,6 +45,7 @@
     "@langchain/google-vertexai-web": "^0.1.0",
     "@langchain/openai": "^0.2.7",
     "fast-xml-parser": "^4.4.0",
+    "js-tiktoken": "^1.0.18",
     "js-yaml": "^4.1.0",
     "jsdom": "^16.6.0",
     "langchain": "^0.2.16",