refactor: code clean up, prep for review and release

Author: dustinbyrne

packages/cli/package.json

@@ -141,9 +141,11 @@
     "pretty-bytes": "^5.6.0",
     "ps-node": "^0.1.6",
     "read-pkg-up": "^7.0.1",
+    "reflect-metadata": "^0.2.2",
     "semver": "^7.3.5",
     "sql-formatter": "^4.0.2",
     "strip-ansi": "^6.0.0",
+    "tsyringe": "^4.8.0",
     "validator": "^13.7.0",
     "w3c-xmlserializer": "^2.0.0",
     "yargs": "^17.1.1"

packages/cli/src/cli.ts

@@ -6,6 +6,8 @@
 /* eslint-disable func-names */
 /* eslint-disable max-classes-per-file */
 
+import 'reflect-metadata';
+
 const yargs = require('yargs');
 const { promises: fsp, readFileSync } = require('fs');
 const { queue } = require('async');

packages/cli/src/cmds/index/rpc.ts

@@ -29,13 +29,16 @@ import { navieMetadataV1, navieMetadataV2 } from '../../rpc/navie/metadata';
 import { navieSuggestHandlerV1 } from '../../rpc/navie/suggest';
 import { navieWelcomeV2 } from '../../rpc/navie/welcome';
 import { navieRegisterV1 } from '../../rpc/navie/register';
-import { navieThreadSendMessageHandler } from '../../rpc/navie/thread/sendMessage';
+import { navieThreadSendMessageHandler } from '../../rpc/navie/thread/handlers/sendMessage';
 import {
   navieThreadPinItemHandler,
   navieThreadUnpinItemHandler,
-} from '../../rpc/navie/thread/pinItem';
-import { navieThreadQueryHandler } from '../../rpc/navie/thread/query';
+} from '../../rpc/navie/thread/handlers/pinItem';
+import { navieThreadQueryHandler } from '../../rpc/navie/thread/handlers/query';
 import NavieService from '../../rpc/navie/services/navieService';
+import { ThreadIndexService } from '../../rpc/navie/services/threadIndexService';
+import { container } from 'tsyringe';
+import ThreadService from '../../rpc/navie/services/threadService';
 
 export const command = 'rpc';
 export const describe = 'Run AppMap JSON-RPC server';
@@ -58,6 +61,8 @@ type HandlerArguments = yargs.ArgumentsCamelCase<
 >;
 
 export function rpcMethods(navie: INavieProvider, codeEditor?: string): RpcHandler<any, any>[] {
+  const threadService = container.resolve(ThreadService);
+  const threadIndexService = container.resolve(ThreadIndexService);
   return [
     search(),
     appmapStatsV1(),
@@ -77,19 +82,21 @@ export function rpcMethods(navie: INavieProvider, codeEditor?: string): RpcHandl
     navieMetadataV2(),
     navieSuggestHandlerV1(navie),
     navieWelcomeV2(navie),
-    navieRegisterV1(codeEditor),
-    navieThreadSendMessageHandler(),
-    navieThreadPinItemHandler(),
-    navieThreadUnpinItemHandler(),
-    navieThreadQueryHandler(),
+    navieRegisterV1(threadService, codeEditor),
+    navieThreadSendMessageHandler(threadService),
+    navieThreadPinItemHandler(threadService),
+    navieThreadUnpinItemHandler(threadService),
+    navieThreadQueryHandler(threadIndexService),
   ];
 }
 
 export const handler = async (argv: HandlerArguments) => {
   verbose(argv.verbose);
 
   const navie = buildNavieProvider(argv);
-  NavieService.registerNavieProvider(navie);
+
+  ThreadIndexService.useDefault();
+  NavieService.bindNavieProvider(navie);
 
   let codeEditor: string | undefined = argv.codeEditor;
   if (!codeEditor) {

packages/cli/src/cmds/index/rpcServer.ts

@@ -11,7 +11,7 @@ import { Server } from 'http';
 
 import shadowLocalhost from '../../lib/shadowLocalhost';
 import { RpcCallback, RpcHandler, toJaysonRpcError } from '../../rpc/rpc';
-import { sseMiddleware } from '../../rpc/navie/thread/subscribe';
+import { sseMiddleware } from '../../rpc/navie/thread/middleware';
 
 const debug = makeDebug('appmap:rpcServer');
 
@@ -64,7 +64,7 @@ export default class RPCServer {
     const app = connect();
     app.use(cors({ methods: ['POST'] }));
     app.use(jsonParser());
-    app.use(sseMiddleware);
+    app.use(sseMiddleware());
     app.use(server.middleware());
     const listener = app.listen(this.bindPort, 'localhost');
 

packages/cli/src/rpc/explain/navie/navie-local.ts

@@ -16,8 +16,9 @@ import {
 import reportFetchError from './report-fetch-error';
 import assert from 'assert';
 import Trajectory from './trajectory';
-import { getThread } from '../../navie/thread';
 import { verbose } from '../../../utils';
+import { container } from 'tsyringe';
+import ThreadService from '../../navie/services/threadService';
 
 const OPTION_SETTERS: Record<
   string,
@@ -135,7 +136,8 @@ export default class LocalNavie extends EventEmitter implements INavie {
       const startTime = Date.now();
       let chatHistory: Navie.ChatHistory = [];
       try {
-        const thread = await getThread(threadId);
+        const threadService = container.resolve(ThreadService);
+        const thread = await threadService.getThread(threadId);
         chatHistory = thread.getChatHistory();
       } catch (e) {
         if (verbose()) {

packages/cli/src/rpc/navie/register.ts

@@ -12,10 +12,10 @@ import { RpcHandler } from '../rpc';
 import { getLLMConfiguration } from '../llmConfiguration';
 import detectAIEnvVar from '../../cmds/index/aiEnvVar';
 import configuration from '../configuration';
-import { INavieProvider } from '../explain/navie/inavie';
-import { registerThread } from './thread';
+import ThreadService from './services/threadService';
 
 export async function register(
+  threadService: ThreadService,
   codeEditor: string | undefined
 ): Promise<NavieRpc.V1.Register.Response> {
   const modelParameters = {
@@ -42,18 +42,19 @@ export async function register(
   if (codeEditor) projectParameters.codeEditor = codeEditor;
 
   const thread = await AI.createConversationThread({ modelParameters, projectParameters });
-  registerThread(thread);
+  threadService.registerThread(thread);
 
   return { thread };
 }
 
 export function navieRegisterV1(
+  threadService: ThreadService,
   codeEditor?: string
 ): RpcHandler<NavieRpc.V1.Register.Params, NavieRpc.V1.Register.Response> {
   return {
     name: NavieRpc.V1.Register.Method,
     handler: async () => {
-      return register(codeEditor);
+      return register(threadService, codeEditor);
     },
   };
 }

packages/cli/src/rpc/navie/services/contextService.ts

@@ -5,7 +5,9 @@ import collectProjectInfos from '../../../cmds/navie/projectInfo';
 import collectHelp from '../../../cmds/navie/help';
 import detectCodeEditor from '../../../lib/detectCodeEditor';
 import configuration from '../../configuration';
+import { autoInjectable } from 'tsyringe';
 
+@autoInjectable()
 export class ContextService {
   private readonly codeEditor?: string;
 

packages/cli/src/rpc/navie/services/navieService.ts

@@ -3,6 +3,7 @@ import INavie, { INavieProvider } from '../../explain/navie/inavie';
 import { ContextService } from './contextService';
 import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
+import { autoInjectable, container, inject } from 'tsyringe';
 
 interface ContextEvent<Type, Req, Res> {
   type: Type;
@@ -44,22 +45,27 @@ function contextEvent<
   };
 }
 
+@autoInjectable()
 export default class NavieService {
-  private static readonly contextService = new ContextService();
-  private static navieProvider?: INavieProvider;
+  private static NAVIE_PROVIDER = 'INavieProvider';
 
-  static registerNavieProvider(navieProvider: INavieProvider): void {
-    NavieService.navieProvider = navieProvider;
+  constructor(
+    @inject(ContextService) private readonly contextService: ContextService,
+    @inject(NavieService.NAVIE_PROVIDER) private readonly navieProvider?: INavieProvider
+  ) {}
+
+  static bindNavieProvider(navieProvider?: INavieProvider) {
+    container.registerInstance(this.NAVIE_PROVIDER, navieProvider);
   }
 
-  static getNavieProvider(): INavieProvider {
+  getNavieProvider(): INavieProvider {
     if (!this.navieProvider) {
       throw new Error('No navie provider available');
     }
     return this.navieProvider;
   }
 
-  static getNavie(_navieProvider?: INavieProvider): [INavie, ContextEmitter] {
+  getNavie(_navieProvider?: INavieProvider): [INavie, ContextEmitter] {
     const navieProvider = _navieProvider ?? this.getNavieProvider();
     const contextEmitter = new EventEmitter();
     const navie = navieProvider(

packages/cli/src/rpc/navie/services/threadIndexService.ts

@@ -0,0 +1,165 @@
+import sqlite3 from 'better-sqlite3';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import configuration from '../../configuration';
+import { container, inject, injectable, singleton } from 'tsyringe';
+
+const INITIALIZE_SQL = `CREATE TABLE IF NOT EXISTS threads (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  uuid TEXT NOT NULL UNIQUE,
+  path TEXT NOT NULL,
+  title TEXT,
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  CONSTRAINT uuid_format CHECK (length(uuid) = 36)
+);
+
+CREATE INDEX IF NOT EXISTS idx_created_at ON threads (created_at);
+CREATE INDEX IF NOT EXISTS idx_uuid ON threads (uuid);
+
+CREATE TABLE IF NOT EXISTS project_directories (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  path TEXT NOT NULL,
+  thread_id TEXT NOT NULL,
+  FOREIGN KEY(thread_id) REFERENCES threads(uuid),
+  UNIQUE (thread_id, path)
+);
+
+CREATE INDEX IF NOT EXISTS idx_thread_id ON project_directories (thread_id);
+`;
+
+const QUERY_INSERT_THREAD_SQL = `INSERT INTO threads (uuid, path, title) VALUES (?, ?, ?)
+ON CONFLICT (uuid) DO UPDATE SET updated_at = CURRENT_TIMESTAMP, title = ?`;
+const QUERY_DELETE_THREAD_SQL = `DELETE FROM threads WHERE uuid = ?`;
+const QUERY_INSERT_PROJECT_DIRECTORY_SQL = `INSERT INTO project_directories (thread_id, path) VALUES (?, ?)
+ON CONFLICT (thread_id, path) DO NOTHING`;
+
+interface QueryOptions {
+  uuid?: string;
+  maxCreatedAt?: Date;
+  orderBy?: 'created_at' | 'updated_at';
+  limit?: number;
+  offset?: number;
+  projectDirectories?: string[];
+}
+
+interface ThreadIndexItem {
+  id: string;
+  path: string;
+  title: string;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+/**
+ * A service for managing the conversation thread index. The thread index is a SQLite database that
+ * stores information about threads, such as their path and title.
+ */
+@singleton()
+@injectable()
+export class ThreadIndexService {
+  private queryInsert: sqlite3.Statement;
+  private queryDelete: sqlite3.Statement;
+  private queryInsertProjectDirectory: sqlite3.Statement;
+
+  static readonly DEFAULT_DATABASE_PATH = join(homedir(), '.appmap', 'navie', 'thread-index.db');
+  static readonly DATABASE = 'ThreadIndexDatabase';
+
+  constructor(@inject(ThreadIndexService.DATABASE) private readonly db: sqlite3.Database) {
+    this.db.exec(INITIALIZE_SQL);
+
+    this.queryInsert = this.db.prepare(QUERY_INSERT_THREAD_SQL);
+    this.queryDelete = this.db.prepare(QUERY_DELETE_THREAD_SQL);
+    this.queryInsertProjectDirectory = this.db.prepare(QUERY_INSERT_PROJECT_DIRECTORY_SQL);
+  }
+
+  /**
+   * Binds the database to a sqlite3 instance on disk at the default database path
+   */
+  static useDefault() {
+    const db = new sqlite3(this.DEFAULT_DATABASE_PATH);
+    container.registerInstance(this.DATABASE, db);
+  }
+
+  /**
+   * Indexes a thread with the given path and title. If the thread is already indexed, it will be
+   * updated.
+   *
+   * @param threadId The thread ID
+   * @param path The path to the thread
+   * @param title The title of the thread
+   */
+  index(threadId: string, path: string, title?: string) {
+    const { projectDirectories } = configuration();
+    this.db.transaction(() => {
+      this.queryInsert.run(threadId, path, title, title);
+
+      // Project directories are written on every index update.
+      //
+      // This is likely not necessary but it'll handle the edge case where an additional project
+      // directory is added mid-way through a conversation.
+      projectDirectories.forEach((projectDirectory) => {
+        this.queryInsertProjectDirectory.run(threadId, projectDirectory);
+      });
+    })();
+    return;
+  }
+
+  /**
+   * Deletes a thread from the index.
+   *
+   * @param threadId The thread ID
+   */
+  delete(threadId: string) {
+    return this.queryDelete.run(threadId);
+  }
+
+  /**
+   * Queries the index for threads.
+   *
+   * @param options The options to query with
+   * @returns The threads that match the query
+   */
+  query(options: QueryOptions): ThreadIndexItem[] {
+    let queryString = `SELECT uuid as id, threads.path, title, created_at, updated_at FROM threads`;
+    const params: unknown[] = [];
+    if (options.uuid) {
+      queryString += ` WHERE uuid = ?`;
+      params.push(options.uuid);
+    }
+    if (options.maxCreatedAt) {
+      queryString += ` AND created_at < ?`;
+      params.push(options.maxCreatedAt);
+    }
+    if (options.orderBy) {
+      queryString += ` ORDER BY ? DESC`;
+      params.push(options.orderBy);
+    }
+    if (options.limit) {
+      queryString += ` LIMIT ?`;
+      params.push(options.limit);
+    }
+    if (options.offset) {
+      queryString += ` OFFSET ?`;
+      params.push(options.offset);
+    }
+    if (options.projectDirectories) {
+      if (options.projectDirectories.length === 0) {
+        // If `projectDirectories` is an empty array, we want to return all threads that have no
+        // project directories associated with them. This is edge-casey, but this does occur in
+        // development.
+        //
+        // If you're looking to query for threads with any project directory, leave `projectDirectories`
+        // as `undefined`.
+        queryString += ` LEFT JOIN project_directories ON uuid = thread_id WHERE project_directories.path IS NULL`;
+      } else {
+        queryString += ` INNER JOIN project_directories ON uuid = thread_id WHERE project_directories.path IN (${options.projectDirectories
+          .map(() => '?')
+          .join(',')})`;
+        params.push(...options.projectDirectories);
+      }
+    }
+    const query = this.db.prepare(queryString);
+    return query.all(...params) as ThreadIndexItem[];
+  }
+}