[draft] Add NVIDIA Riva plugin with STT and TTS support

.changeset/nvidia-plugin-initial.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-nvidia": patch
+---
+
+Add NVIDIA Riva plugin with STT and TTS support

examples/package.json

@@ -34,6 +34,7 @@
     "@livekit/agents-plugin-inworld": "workspace:*",
     "@livekit/agents-plugin-livekit": "workspace:*",
     "@livekit/agents-plugin-neuphonic": "workspace:*",
+    "@livekit/agents-plugin-nvidia": "workspace:*",
     "@livekit/agents-plugin-openai": "workspace:*",
     "@livekit/agents-plugin-resemble": "workspace:*",
     "@livekit/agents-plugin-silero": "workspace:*",

examples/src/nvidia_agent.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type JobContext,
+  type JobProcess,
+  WorkerOptions,
+  cli,
+  defineAgent,
+  metrics,
+  voice,
+} from '@livekit/agents';
+import * as livekit from '@livekit/agents-plugin-livekit';
+import * as nvidia from '@livekit/agents-plugin-nvidia';
+import * as openai from '@livekit/agents-plugin-openai';
+import * as silero from '@livekit/agents-plugin-silero';
+import { fileURLToPath } from 'node:url';
+
+export default defineAgent({
+  prewarm: async (proc: JobProcess) => {
+    proc.userData.vad = await silero.VAD.load();
+  },
+  entry: async (ctx: JobContext) => {
+    const agent = new voice.Agent({
+      instructions:
+        "You are a helpful assistant using NVIDIA Riva for speech recognition and synthesis. You can hear the user's message and respond to it.",
+    });
+
+    const vad = ctx.proc.userData.vad! as silero.VAD;
+
+    const session = new voice.AgentSession({
+      vad,
+      stt: new nvidia.STT({
+        model: 'parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer',
+        languageCode: 'en-US',
+      }),
+      tts: new nvidia.TTS({
+        voice: 'Magpie-Multilingual.EN-US.Leo',
+        languageCode: 'en-US',
+      }),
+      llm: new openai.LLM(),
+      turnDetection: new livekit.turnDetector.MultilingualModel(),
+    });
+
+    const usageCollector = new metrics.UsageCollector();
+
+    session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
+      metrics.logMetrics(ev.metrics);
+      usageCollector.collect(ev.metrics);
+    });
+
+    await session.start({
+      agent,
+      room: ctx.room,
+    });
+
+    session.say('Hello, how can I help you today?');
+  },
+});
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/nvidia/README.md

@@ -0,0 +1,75 @@
+<!--
+SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+# NVIDIA Riva Plugin for LiveKit Agents
+
+This plugin provides NVIDIA Riva Speech-to-Text (STT) and Text-to-Speech (TTS) capabilities for LiveKit Agents.
+
+## Installation
+
+```bash
+npm install @livekit/agents-plugin-nvidia
+# or
+pnpm add @livekit/agents-plugin-nvidia
+```
+
+## Configuration
+
+Set your NVIDIA API key as an environment variable:
+
+```bash
+export NVIDIA_API_KEY=your_api_key_here
+```
+
+## Usage
+
+### Speech-to-Text (STT)
+
+```typescript
+import * as nvidia from '@livekit/agents-plugin-nvidia';
+
+const stt = new nvidia.STT({
+  model: 'parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer',
+  languageCode: 'en-US',
+});
+```
+
+### Text-to-Speech (TTS)
+
+```typescript
+import * as nvidia from '@livekit/agents-plugin-nvidia';
+
+const tts = new nvidia.TTS({
+  voice: 'Magpie-Multilingual.EN-US.Leo',
+  languageCode: 'en-US',
+});
+```
+
+## Options
+
+### STT Options
+
+- `model`: The ASR model to use (default: `'parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer'`)
+- `functionId`: NVIDIA function ID (default: `'1598d209-5e27-4d3c-8079-4751568b1081'`)
+- `punctuate`: Enable automatic punctuation (default: `true`)
+- `languageCode`: Language code (default: `'en-US'`)
+- `sampleRate`: Audio sample rate (default: `16000`)
+- `server`: NVIDIA Riva server address (default: `'grpc.nvcf.nvidia.com:443'`)
+- `useSsl`: Use SSL for connection (default: `true`)
+- `apiKey`: NVIDIA API key (defaults to `NVIDIA_API_KEY` environment variable)
+
+### TTS Options
+
+- `voice`: Voice name (default: `'Magpie-Multilingual.EN-US.Leo'`)
+- `functionId`: NVIDIA function ID (default: `'877104f7-e885-42b9-8de8-f6e4c6303969'`)
+- `languageCode`: Language code (default: `'en-US'`)
+- `sampleRate`: Audio sample rate (default: `16000`)
+- `server`: NVIDIA Riva server address (default: `'grpc.nvcf.nvidia.com:443'`)
+- `useSsl`: Use SSL for connection (default: `true`)
+- `apiKey`: NVIDIA API key (defaults to `NVIDIA_API_KEY` environment variable)
+
+## License
+
+Apache-2.0

plugins/nvidia/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/nvidia/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@livekit/agents-plugin-nvidia",
+  "version": "1.0.0",
+  "description": "NVIDIA Riva plugin for LiveKit Agents for Node.js",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"pnpm build:types\"",
+    "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugins-test": "workspace:*",
+    "@livekit/rtc-node": "^0.13.22",
+    "@microsoft/api-extractor": "^7.35.0",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@grpc/grpc-js": "^1.12.6",
+    "@grpc/proto-loader": "^0.7.15"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "^0.13.22"
+  }
+}

plugins/nvidia/src/index.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { Plugin } from '@livekit/agents';
+
+export * from './stt.js';
+export * from './tts.js';
+export * from './models.js';
+
+class NVIDIAPlugin extends Plugin {
+  constructor() {
+    super({
+      title: 'nvidia',
+      version: '0.1.0',
+      package: '@livekit/agents-plugin-nvidia',
+    });
+  }
+}
+
+Plugin.registerPlugin(new NVIDIAPlugin());

plugins/nvidia/src/models.ts

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type STTModels =
+  | 'parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer'
+  | 'parakeet-ctc-1.1b-asr'
+  | 'parakeet-ctc-0.6b-asr'
+  | string;
+
+export type STTLanguages =
+  | 'en-US'
+  | 'es-US'
+  | 'de-DE'
+  | 'fr-FR'
+  | 'it-IT'
+  | 'pt-BR'
+  | 'zh-CN'
+  | 'ja-JP'
+  | 'ko-KR'
+  | string;
+
+export type TTSVoices =
+  | 'Magpie-Multilingual.EN-US.Leo'
+  | 'Magpie-Multilingual.EN-US.Luna'
+  | 'Magpie-Multilingual.EN-US.Aria'
+  | 'Magpie-Multilingual.EN-US.Asteria'
+  | string;
+
+export type TTSLanguages =
+  | 'en-US'
+  | 'es-US'
+  | 'de-DE'
+  | 'fr-FR'
+  | 'it-IT'
+  | 'pt-BR'
+  | 'zh-CN'
+  | 'ja-JP'
+  | 'ko-KR'
+  | string;