just talk

SUPERMASSlVE · Jul 2, 2024 · ba108a2 · ba108a2
1 parent e9177af
commit ba108a2
Show file tree

Hide file tree

Showing 4 changed files with 208 additions and 67 deletions.
diff --git a/app/page.tsx b/app/page.tsx
@@ -4,45 +4,44 @@ import clsx from "clsx";
 import React, { useCallback, useRef, useState, useTransition } from "react";
 import { toast } from "sonner";
 import { EnterIcon, LoadingIcon, MicrophoneIcon, StopIcon } from "@/lib/icons";
-import { useRecorder } from "@/lib/useRecorder";
 import { usePlayer } from "@/lib/usePlayer";
 import { useHotkeys } from "@/lib/useHotkeys";
 import { track } from "@vercel/analytics";
+import { useMicVAD, utils } from "@ricky0123/vad-react";
 
 export default function Home() {
 	const [isPending, startTransition] = useTransition();
-	const { isRecording, startRecording, stopRecording, volume } = useRecorder({
-		onUnsupportedMimeType() {
-			toast.error("Your browser does not support audio recording.");
-			track("Unsupported MIME type");
-		},
-		onMicrophoneDenied() {
-			toast.error("Access to microphone was denied.");
-		},
-		onRecordingStop(blob, duration) {
-			if (duration < 500) {
-				toast.info("Hold the button or spacebar to record.");
-				return track("Recording too short");
-			}
-
-			submit(blob);
-		},
-	});
+	// const { isRecording, startRecording, stopRecording, volume } = useRecorder({
+	// 	onUnsupportedMimeType() {
+	// 		toast.error("Your browser does not support audio recording.");
+	// 		track("Unsupported MIME type");
+	// 	},
+	// 	onMicrophoneDenied() {
+	// 		toast.error("Access to microphone was denied.");
+	// 	},
+	// });
 	const [input, setInput] = useState("");
 	const [latency, setLatency] = useState<number | null>(null);
 	const [response, setResponse] = useState<string | null>(null);
 	const messages = useRef<Array<object>>([]);
 	const inputRef = useRef<HTMLInputElement>(null);
 	const player = usePlayer();
 
+	const { userSpeaking } = useMicVAD({
+		startOnLoad: true,
+		onSpeechEnd: (audio) => {
+			player.stop();
+			const wav = utils.encodeWAV(audio);
+			const blob = new Blob([wav], { type: "audio/wav" });
+			submit(blob);
+		},
+		workletURL: "/_next/static/chunks/vad.worklet.bundle.min.js",
+		modelURL: "/_next/static/chunks/silero_vad.onnx",
+	});
+
 	useHotkeys({
 		enter: () => inputRef.current?.focus(),
 		escape: () => setInput(""),
-		blankDown: () => {
-			player.stop();
-			startRecording();
-		},
-		blankUp: stopRecording,
 	});
 
 	const submit = useCallback(
@@ -54,7 +53,7 @@ export default function Home() {
 					formData.append("input", data);
 					track("Text input");
 				} else {
-					formData.append("input", data, "audio.webm");
+					formData.append("input", data, "audio.wav");
 					track("Speech input");
 				}
 
@@ -105,7 +104,7 @@ export default function Home() {
 
 	function handleFormSubmit(e: React.FormEvent) {
 		e.preventDefault();
-		if (isRecording) return stopRecording();
+		// if (isRecording) return stopRecording();
 		submit(input);
 	}
 
@@ -117,47 +116,11 @@ export default function Home() {
 				className="rounded-full bg-neutral-200/80 dark:bg-neutral-800/80 flex items-center w-full max-w-3xl border border-transparent hover:border-neutral-300 focus-within:border-neutral-400 hover:focus-within:border-neutral-400 dark:hover:border-neutral-700 dark:focus-within:border-neutral-600 dark:hover:focus-within:border-neutral-600"
 				onSubmit={handleFormSubmit}
 			>
-				{player.isPlaying ? (
-					<button
-						className="p-3 box-border group"
-						type="button"
-						onClick={player.stop}
-						aria-label="Stop responding"
-					>
-						<div className="rounded-full bg-white dark:bg-black border border-neutral-300 dark:border-neutral-700 drop-shadow group-hover:scale-110 group-active:scale-90 transition-transform ease-in-out p-1 text-red-500">
-							<StopIcon />
-						</div>
-					</button>
-				) : (
-					<button
-						className="p-3 box-border group"
-						type="button"
-						aria-label="Hold to record"
-						onMouseDown={startRecording}
-						onMouseUp={stopRecording}
-						onKeyDown={(e) => !e.repeat && startRecording()}
-						onKeyUp={stopRecording}
-						onTouchStart={startRecording}
-						onTouchEnd={stopRecording}
-					>
-						<div
-							className={clsx(
-								"rounded-full bg-white dark:bg-black border border-neutral-300 dark:border-neutral-700 drop-shadow group-hover:scale-110 group-active:scale-90 transition ease-in-out p-1",
-								{
-									"text-red-500": isRecording,
-								}
-							)}
-						>
-							<MicrophoneIcon />
-						</div>
-					</button>
-				)}
-
 				<input
 					type="text"
-					className="bg-transparent focus:outline-none py-3 w-full placeholder:text-neutral-600 dark:placeholder:text-neutral-400"
+					className="bg-transparent focus:outline-none p-4 w-full placeholder:text-neutral-600 dark:placeholder:text-neutral-400"
 					required
-					disabled={isRecording || isPending}
+					// disabled={isRecording || isPending}
 					placeholder="Ask me anything"
 					value={input}
 					onChange={(e) => setInput(e.target.value)}
@@ -205,10 +168,13 @@ export default function Home() {
 			)}
 
 			<div
-				className="absolute size-36 blur-3xl rounded-full bg-gradient-to-b from-red-200 to-red-400 dark:from-red-600 dark:to-red-800 -z-50 transition-opacity ease-in-out"
-				style={{
-					opacity: isRecording ? Math.max(volume * 150, 0.4) : 0,
-				}}
+				className={clsx(
+					"absolute size-36 blur-3xl rounded-full bg-gradient-to-b from-red-200 to-red-400 dark:from-red-600 dark:to-red-800 -z-50 transition ease-in-out",
+					{
+						"opacity-30": !userSpeaking,
+						"opacity-100 scale-110": userSpeaking,
+					}
+				)}
 			/>
 		</>
 	);

diff --git a/next.config.mjs b/next.config.mjs
@@ -1,3 +1,6 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
 /** @type {import('next').NextConfig} */
 const nextConfig = {
 	experimental: {
@@ -6,3 +9,34 @@ const nextConfig = {
 };
 
 export default nextConfig;
+
+async function copyFiles() {
+	try {
+		await fs.access(".next/static/chunks/");
+	} catch {
+		await fs.mkdir(".next/static/chunks/", { recursive: true });
+	}
+
+	const wasmFiles = (
+		await fs.readdir("node_modules/onnxruntime-web/dist/")
+	).filter((file) => path.extname(file) === ".wasm");
+
+	await Promise.all([
+		fs.copyFile(
+			"node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js",
+			".next/static/chunks/vad.worklet.bundle.min.js"
+		),
+		fs.copyFile(
+			"node_modules/@ricky0123/vad-web/dist/silero_vad.onnx",
+			".next/static/chunks/silero_vad.onnx"
+		),
+		...wasmFiles.map((file) =>
+			fs.copyFile(
+				`node_modules/onnxruntime-web/dist/${file}`,
+				`.next/static/chunks/${file}`
+			)
+		),
+	]);
+}
+
+copyFiles();
diff --git a/package.json b/package.json
@@ -9,11 +9,14 @@
 		"lint": "next lint"
 	},
 	"dependencies": {
+		"@ricky0123/vad-react": "^0.0.24",
+		"@ricky0123/vad-web": "^0.0.18",
 		"@vercel/analytics": "^1.2.2",
 		"clsx": "^2.1.1",
 		"geist": "^1.3.0",
 		"groq-sdk": "^0.3.3",
 		"next": "15.0.0-rc.0",
+		"onnxruntime-web": "^1.18.0",
 		"react": "19.0.0-rc-100dfd7dab-20240701",
 		"react-dom": "19.0.0-rc-100dfd7dab-20240701",
 		"sonner": "^1.4.41",