add typescript tokenizer and better chat trimmer

app/craco.config.js

@@ -1,36 +1,37 @@
 const cracoWasm = require("craco-wasm");
-
-/*
-{
-  "plugins": [
-    [
-      "formatjs",
-      {
-        "idInterpolationPattern": "[sha512:contenthash:base64:6]",
-        "ast": true
-      }
-    ]
-  ]
-}
-*/
+const webpack = require("webpack");
 
 module.exports = {
+  plugins: [
+    cracoWasm(),
+  ],
+  eslint: {
+    enable: false
+  },
+  babel: {
     plugins: [
-        cracoWasm(),
-    ],
-    eslint: {
-        enable: false
+      [
+        'formatjs',
+        {
+          removeDefaultMessage: false,
+          idInterpolationPattern: '[sha512:contenthash:base64:6]',
+          ast: true
+        }
+      ]
+    ]
+  },
+  webpack: {
+    configure: {
+      resolve: {
+        fallback: {
+          buffer: require.resolve("buffer"),
+        },
+      },
+      plugins: [
+        new webpack.ProvidePlugin({
+          Buffer: ["buffer", "Buffer"],
+        }),
+      ],
     },
-    babel: {
-        plugins: [
-            [
-                'formatjs',
-                {
-                    removeDefaultMessage: false,
-                    idInterpolationPattern: '[sha512:contenthash:base64:6]',
-                    ast: true
-                }
-            ]
-        ]
-    }
+  },
 }

app/package.json

@@ -13,6 +13,7 @@
     "@reduxjs/toolkit": "^1.9.3",
     "@svgr/webpack": "^6.5.1",
     "broadcast-channel": "^4.20.2",
+    "buffer": "^6.0.3",
     "csv": "^6.2.8",
     "expiry-set": "^1.0.0",
     "idb-keyval": "^6.2.0",

app/src/chat-manager.ts

@@ -8,7 +8,6 @@ import { createStreamingChatCompletion } from './openai';
 import { createTitle } from './titles';
 import { ellipsize, sleep } from './utils';
 import * as idb from './idb';
-import { selectMessagesToSendSafely } from './tokenizer';
 
 export const channel = new BroadcastChannel('chats');
 
@@ -145,7 +144,7 @@ export class ChatManager extends EventEmitter {
         this.emit(chat.id);
         channel.postMessage({ type: 'chat-update', data: serializeChat(chat) });
 
-        const messagesToSend = selectMessagesToSendSafely(messages.map(getOpenAIMessageFromMessage));
+        const messagesToSend = messages.map(getOpenAIMessageFromMessage)
 
         const { emitter, cancel } = await createStreamingChatCompletion(messagesToSend, requestedParameters);
 

app/src/components/pages/chat.tsx

@@ -9,7 +9,7 @@ import { useAppContext } from '../../context';
 import { backend } from '../../backend';
 import { Page } from '../page';
 
-const Message = React.lazy(() => import(/* webpackChunkName: "message" */ '../message'));
+const Message = React.lazy(() => import(/* webpackPreload: true */ '../message'));
 
 const Messages = styled.div`
     max-height: 100%;

app/src/openai.ts

@@ -70,7 +70,7 @@ export async function createStreamingChatCompletion(messages: OpenAIMessage[], p
 
     const emitter = new EventEmitter();
 
-    const messagesToSend = [...messages].filter(m => m.role !== 'app');
+    let messagesToSend = [...messages].filter(m => m.role !== 'app');
 
     for (let i = messagesToSend.length - 1; i >= 0; i--) {
         const m = messagesToSend[i];
@@ -87,6 +87,8 @@ export async function createStreamingChatCompletion(messages: OpenAIMessage[], p
         content: (parameters.initialSystemPrompt || defaultSystemPrompt).replace('{{ datetime }}', new Date().toLocaleString()),
     });
 
+    messagesToSend = await selectMessagesToSendSafely(messagesToSend, 2048);
+
     const eventSource = new SSE('https://api.openai.com/v1/chat/completions', {
         method: "POST",
         headers: {
@@ -151,4 +153,14 @@ export async function createStreamingChatCompletion(messages: OpenAIMessage[], p
         emitter,
         cancel: () => eventSource.close(),
     };
+}
+
+async function selectMessagesToSendSafely(messages: OpenAIMessage[], maxTokens: number) {
+    const { ChatHistoryTrimmer } = await import(/* webpackPreload: true */ './tokenizer/chat-history-trimmer');
+    const compressor = new ChatHistoryTrimmer(messages, {
+        maxTokens,
+        preserveFirstUserMessage: true,
+        preserveSystemPrompt: true,
+    });
+    return compressor.process();
 }