✨ feat: support TTS & STT (lobehub#443)

* ✨ feat(tts): Add tts and stt basic features * ✨ feat(tts): Handle error * 💄 style(tts): Add alert to error handler * 🐛 fix(tts): Error display * ♻️ refactor: refactor the openai initial code to the createBizOpenAI * ♻️ refactor(tts): Refactor header config * ✨ feat: Add TTS voice preview * 🐛 fix(tts): Fix header * 🐛 fix: Fix api --------- Co-authored-by: Arvin Xu <arvinx@foxmail.com>
bryan31 · Nov 19, 2023 · 4fa2ef4 · 4fa2ef4
1 parent 26ef087
commit 4fa2ef4
Show file tree

Hide file tree

Showing 87 changed files with 1,937 additions and 230 deletions.
diff --git a/.i18nrc.js b/.i18nrc.js
@@ -8,5 +8,5 @@ module.exports = defineConfig({
   output: 'locales',
   outputLocales: ['zh_TW', 'en_US', 'ru_RU', 'ja_JP', 'ko_KR'],
   temperature: 0,
-  modelName: 'gpt-3.5-turbo',
+  modelName: 'gpt-3.5-turbo-1106',
 });
diff --git a/README.md b/README.md
@@ -317,6 +317,7 @@ This project provides some additional configuration items set with environment v
 | NPM                             | Repository                            | Description                                                                                                             | Version                                 |
 | ------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------- |
 | [@lobehub/ui][lobe-ui-link]     | [lobehub/lobe-ui][lobe-ui-github]     | Lobe UI is an open-source UI component library dedicated to building AIGC web applications.                             | [![][lobe-ui-shield]][lobe-ui-link]     |
+| [@lobehub/tts][lobe-tts-link]   | [lobehub/lobe-tts][lobe-tts-github]   | Lobe TTS is a high-quality & reliable TTS/STT React Hooks library                                                       | [![][lobe-tts-shield]][lobe-tts-link]   |
 | [@lobehub/lint][lobe-lint-link] | [lobehub/lobe-lint][lobe-lint-github] | LobeLint provides configurations for ESlint, Stylelint, Commitlint, Prettier, Remark, and Semantic Release for LobeHub. | [![][lobe-lint-shield]][lobe-lint-link] |
 | @lobehub/assets                 | [lobehub/assets][lobe-assets-github]  | Logo assets, favicons, webfonts for LobeHub.                                                                            |                                         |
 
@@ -483,6 +484,9 @@ This project is [MIT](./LICENSE) licensed.
 [lobe-lint-link]: https://www.npmjs.com/package/@lobehub/lint
 [lobe-lint-shield]: https://img.shields.io/npm/v/@lobehub/lint?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square
 [lobe-theme]: https://github.com/lobehub/sd-webui-lobe-theme
+[lobe-tts-github]: https://github.com/lobehub/lobe-tts
+[lobe-tts-link]: https://www.npmjs.com/package/@lobehub/tts
+[lobe-tts-shield]: https://img.shields.io/npm/v/@lobehub/tts?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square
 [lobe-ui-github]: https://github.com/lobehub/lobe-ui
 [lobe-ui-link]: https://www.npmjs.com/package/@lobehub/ui
 [lobe-ui-shield]: https://img.shields.io/npm/v/@lobehub/ui?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square

diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -317,6 +317,7 @@ $ docker run -d -p 3210:3210 \
 | NPM                             | 仓库                                  | 描述                                                                                                  | 版本                                    |
 | ------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------------- |
 | [@lobehub/ui][lobe-ui-link]     | [lobehub/lobe-ui][lobe-ui-github]     | Lobe UI 是一个专为构建 AIGC 网页应用程序而设计的开源 UI 组件库。                                      | [![][lobe-ui-shield]][lobe-ui-link]     |
+| [@lobehub/tts][lobe-tts-link]   | [lobehub/lobe-tts][lobe-tts-github]   | Lobe TTS 是一个专为 TTS/STT 建设的语音合成 / 识别 React Hooks 库                                      | [![][lobe-tts-shield]][lobe-tts-link]   |
 | [@lobehub/lint][lobe-lint-link] | [lobehub/lobe-lint][lobe-lint-github] | LobeLint 为 LobeHub 提供 ESlint，Stylelint，Commitlint，Prettier，Remark 和 Semantic Release 的配置。 | [![][lobe-lint-shield]][lobe-lint-link] |
 | @lobehub/assets                 | [lobehub/assets][lobe-assets-github]  | LobeHub 的 Logo 资源、favicon、网页字体。                                                             |                                         |
 
@@ -483,6 +484,9 @@ This project is [MIT](./LICENSE) licensed.
 [lobe-lint-link]: https://www.npmjs.com/package/@lobehub/lint
 [lobe-lint-shield]: https://img.shields.io/npm/v/@lobehub/lint?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square
 [lobe-theme]: https://github.com/lobehub/sd-webui-lobe-theme
+[lobe-tts-github]: https://github.com/lobehub/lobe-tts
+[lobe-tts-link]: https://www.npmjs.com/package/@lobehub/tts
+[lobe-tts-shield]: https://img.shields.io/npm/v/@lobehub/tts?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square
 [lobe-ui-github]: https://github.com/lobehub/lobe-ui
 [lobe-ui-link]: https://www.npmjs.com/package/@lobehub/ui
 [lobe-ui-shield]: https://img.shields.io/npm/v/@lobehub/ui?color=369eff&labelColor=black&logo=npm&logoColor=white&style=flat-square

diff --git a/__mocks__/zustand/traditional.ts b/__mocks__/zustand/traditional.ts
@@ -15,11 +15,11 @@ const createImpl = (createState: any) => {
 
 // Reset all stores after each test run
 beforeEach(() => {
-  act(() =>
-    { for (const resetFn of storeResetFns) {
+  act(() => {
+    for (const resetFn of storeResetFns) {
       resetFn();
-    } },
-  );
+    }
+  });
 });
 
 export const createWithEqualityFn = (f: any) => (f === undefined ? createImpl : createImpl(f));
diff --git a/locales/en_US/chat.json b/locales/en_US/chat.json
@@ -35,6 +35,11 @@
     "withSystemRole": "Include Agent Role Setting"
   },
   "stop": "Stop",
+  "stt": {
+    "action": "Voice Input",
+    "loading": "Recognizing...",
+    "prettifying": "Prettifying..."
+  },
   "temp": "Temporary",
   "tokenDetail": "Role Setting: {{systemRoleToken}} · Chat History: {{chatsToken}}",
   "tokenTag": {
@@ -57,9 +62,14 @@
     "openNewTopic": "Open a new topic"
   },
   "translate": {
-    "clear": "Clear Translation"
+    "clear": "Clear Translation",
+    "action": "Translate"
   },
   "translateTo": "Translate",
+  "tts": {
+    "action": "Text to Speech",
+    "clear": "Clear Speech"
+  },
   "updateAgent": "Update Agent Information",
   "upload": {
     "actionTooltip": "Upload Image",

diff --git a/locales/en_US/error.json b/locales/en_US/error.json
@@ -28,6 +28,12 @@
     "PluginServerError": "Plugin server request returned an error. Please check your plugin manifest file, plugin configuration, or server implementation based on the error information below",
     "NoAPIKey": "OpenAI API Key is empty, please add a custom OpenAI API Key"
   },
+  "stt": {
+    "responseError": "Service request failed, please check the configuration or try again"
+  },
+  "tts": {
+    "responseError": "Service request failed, please check the configuration or try again"
+  },
   "unlock": {
     "apikey": {
       "title": "Use Custom API Key",

diff --git a/locales/en_US/setting.json b/locales/en_US/setting.json
@@ -207,6 +207,44 @@
     },
     "title": "System Settings"
   },
+  "settingTTS": {
+    "showAllLocaleVoice": {
+      "desc": "If disabled, only voices for the current language will be displayed",
+      "title": "Show all locale voices"
+    },
+    "sttService": {
+      "desc": "The 'browser' option refers to the native speech recognition service in the browser",
+      "title": "Speech-to-Text Service"
+    },
+    "title": "Speech Services",
+    "ttsService": {
+      "desc": "If using the OpenAI text-to-speech service, ensure that the OpenAI model service is enabled",
+      "title": "Text-to-Speech Service"
+    },
+    "voice": {
+      "title": "Text-to-Speech Voices",
+      "desc": "Select a voice for the current assistant, different TTS services support different voices",
+      "preview": "Preview Voice"
+    },
+    "openai": {
+      "sttModel": "OpenAI Speech Recognition Model",
+      "ttsModel": "OpenAI Text-to-Speech Model"
+    },
+    "stt": "Speech Recognition Settings",
+    "sttLocale": {
+      "desc": "The language of the speech input, this option can improve the accuracy of speech recognition",
+      "title": "Speech Recognition Language"
+    },
+    "sttPersisted": {
+      "desc": "When enabled, speech recognition will not automatically end and requires manual click on the end button",
+      "title": "Manually End Speech Recognition"
+    },
+    "tts": "Text-to-Speech Settings",
+    "sttAutoStop": {
+      "desc": "When disabled, speech recognition will not automatically stop and will require manual intervention to end the process.",
+      "title": "Automatic Speech Recognition Termination"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "Avatar"
@@ -245,6 +283,7 @@
   "tab": {
     "agent": "Default Agent",
     "common": "Common Settings",
-    "llm": "Custom LLM API"
+    "llm": "Custom LLM API",
+    "tts": "Speech Services"
   }
 }
diff --git a/locales/ja_JP/chat.json b/locales/ja_JP/chat.json
@@ -35,6 +35,11 @@
     "withSystemRole": "エージェントの役割設定を含む"
   },
   "stop": "停止",
+  "stt": {
+    "action": "音声入力",
+    "loading": "読み取り中...",
+    "prettifying": "美化中..."
+  },
   "temp": "一時",
   "tokenDetail": "役割設定: {{systemRoleToken}} · チャット履歴: {{chatsToken}}",
   "tokenTag": {
@@ -57,9 +62,14 @@
     "openNewTopic": "新しいトピックを開く"
   },
   "translate": {
-    "clear": "翻訳をクリア"
+    "clear": "翻訳をクリア",
+    "action": "翻訳"
   },
   "translateTo": "翻訳",
+  "tts": {
+    "action": "音声読み上げ",
+    "clear": "音声削除"
+  },
   "updateAgent": "エージェント情報を更新",
   "upload": {
     "actionTooltip": "画像をアップロード",

diff --git a/locales/ja_JP/error.json b/locales/ja_JP/error.json
@@ -28,6 +28,12 @@
     "PluginApiNotFound": "申し訳ありませんが、プラグインのマニフェストに指定されたAPIが見つかりませんでした。リクエストメソッドとプラグインのマニフェストのAPIが一致しているかどうかを確認してください",
     "NoAPIKey": "OpenAI APIキーが空です。カスタムOpenAI APIキーを追加してください。"
   },
+  "stt": {
+    "responseError": "サービスリクエストが失敗しました。設定を確認するか、もう一度お試しください"
+  },
+  "tts": {
+    "responseError": "サービスリクエストが失敗しました。設定を確認するか、もう一度お試しください"
+  },
   "unlock": {
     "apikey": {
       "title": "カスタムAPIキーの使用",

diff --git a/locales/ja_JP/setting.json b/locales/ja_JP/setting.json
@@ -194,6 +194,44 @@
     },
     "title": "システム設定"
   },
+  "settingTTS": {
+    "showAllLocaleVoice": {
+      "desc": "关闭すると、現在の言語の音声のみが表示されます",
+      "title": "すべての言語の音声を表示"
+    },
+    "sttService": {
+      "desc": "ブラウザはブラウザのネイティブ音声認識サービスです",
+      "title": "音声認識サービス"
+    },
+    "title": "音声サービス",
+    "ttsService": {
+      "desc": "OpenAIの音声合成サービスを使用する場合、OpenAIモデルサービスが有効になっていることを確認する必要があります",
+      "title": "音声合成サービス"
+    },
+    "voice": {
+      "title": "音声合成音声源",
+      "desc": "現在のアシスタントに適した音声を選択します。異なるTTSサービスは異なる音声をサポートしています",
+      "preview": "プレビュー"
+    },
+    "openai": {
+      "sttModel": "OpenAI 音声認識モデル",
+      "ttsModel": "OpenAI 音声合成モデル"
+    },
+    "stt": "音声認識設定",
+    "sttLocale": {
+      "desc": "音声入力の言語、このオプションを選択すると音声認識の精度が向上します",
+      "title": "音声認識言語"
+    },
+    "sttPersisted": {
+      "desc": "有効にすると、音声認識が自動的に終了せず、手動で終了ボタンをクリックする必要があります",
+      "title": "音声認識の手動終了"
+    },
+    "tts": "音声合成設定",
+    "sttAutoStop": {
+      "desc": "オフにすると、音声認識が自動的に終了せず、手動で終了ボタンをクリックする必要があります",
+      "title": "音声認識の自動終了"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "アバター"
@@ -232,6 +270,7 @@
   "tab": {
     "agent": "デフォルトのアシスタント",
     "common": "一般設定",
-    "llm": "言語モデル"
+    "llm": "言語モデル",
+    "tts": "音声サービス"
   }
 }
diff --git a/locales/ko_KR/chat.json b/locales/ko_KR/chat.json
@@ -35,6 +35,11 @@
     "withSystemRole": "도우미 역할 포함"
   },
   "stop": "정지",
+  "stt": {
+    "action": "음성 입력",
+    "loading": "인식 중...",
+    "prettifying": "미화 중..."
+  },
   "temp": "임시",
   "tokenDetail": "역할 설정: {{systemRoleToken}} · 대화 기록: {{chatsToken}}",
   "tokenTag": {
@@ -57,9 +62,14 @@
     "openNewTopic": "새로운 주제 열기"
   },
   "translate": {
-    "clear": "번역 지우기"
+    "clear": "번역 지우기",
+    "action": "번역"
   },
   "translateTo": "번역",
+  "tts": {
+    "action": "음성 읽기",
+    "clear": "음성 삭제"
+  },
   "updateAgent": "도우미 정보 업데이트",
   "upload": {
     "actionTooltip": "이미지 업로드",

diff --git a/locales/ko_KR/error.json b/locales/ko_KR/error.json
@@ -28,6 +28,12 @@
     "OpenAIBizError": "OpenAI 서비스 요청 중 오류가 발생했습니다. 아래 정보를 확인하고 문제를 해결하거나 다시 시도해주세요.",
     "NoAPIKey": "OpenAI API 키가 비어 있습니다. 사용자 정의 OpenAI API 키를 추가해주세요."
   },
+  "stt": {
+    "responseError": "서비스 요청이 실패했습니다. 구성을 확인하거나 다시 시도해주세요."
+  },
+  "tts": {
+    "responseError": "서비스 요청이 실패했습니다. 구성을 확인하거나 다시 시도해주세요."
+  },
   "unlock": {
     "apikey": {
       "title": "사용자 정의 API 키 사용",

diff --git a/locales/ko_KR/setting.json b/locales/ko_KR/setting.json
@@ -194,6 +194,44 @@
     },
     "title": "시스템 설정"
   },
+  "settingTTS": {
+    "showAllLocaleVoice": {
+      "desc": "현재 언어의 음성만 표시하려면 닫으십시오",
+      "title": "모든 언어 음성 표시"
+    },
+    "sttService": {
+      "desc": "브라우저는 브라우저 기본 음성 인식 서비스입니다",
+      "title": "음성 인식 서비스"
+    },
+    "title": "음성 서비스",
+    "ttsService": {
+      "desc": "OpenAI 음성 합성 서비스를 사용하는 경우 OpenAI 모델 서비스가 열려 있어야 합니다",
+      "title": "음성 합성 서비스"
+    },
+    "voice": {
+      "title": "음성 합성 음성",
+      "desc": "현재 어시스턴트에 대한 음성을 선택하십시오. 각기 다른 TTS 서비스는 다른 음성을 지원합니다.",
+      "preview": "프리뷰 음성"
+    },
+    "openai": {
+      "sttModel": "OpenAI 음성 인식 모델",
+      "ttsModel": "OpenAI 음성 합성 모델"
+    },
+    "stt": "음성 인식 설정",
+    "sttLocale": {
+      "desc": "음성 입력의 언어, 이 옵션을 통해 음성 인식 정확도를 높일 수 있습니다.",
+      "title": "음성 인식 언어"
+    },
+    "sttPersisted": {
+      "desc": "활성화하면 음성 인식이 자동으로 종료되지 않고, 수동으로 종료 버튼을 클릭해야 합니다.",
+      "title": "음성 인식 수동 종료"
+    },
+    "tts": "음성 합성 설정",
+    "sttAutoStop": {
+      "desc": "자동으로 종료되지 않고 수동으로 종료 버튼을 클릭해야 하는 음성 인식을 사용하지 않습니다.",
+      "title": "음성 인식 자동 종료"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "아바타"
@@ -232,6 +270,7 @@
   "tab": {
     "agent": "기본 도우미",
     "common": "일반 설정",
-    "llm": "언어 모델"
+    "llm": "언어 모델",
+    "tts": "음성 서비스"
   }
 }
diff --git a/locales/ru_RU/chat.json b/locales/ru_RU/chat.json
@@ -35,6 +35,11 @@
     "withSystemRole": "С указанием роли помощника"
   },
   "stop": "Остановить",
+  "stt": {
+    "action": "Голосовой ввод",
+    "loading": "Идет распознавание...",
+    "prettifying": "Форматирование..."
+  },
   "temp": "Временный",
   "tokenDetail": "Роль помощника: {{systemRoleToken}} · История сообщений: {{chatsToken}}",
   "tokenTag": {
@@ -57,9 +62,14 @@
     "openNewTopic": "Открыть новую тему"
   },
   "translate": {
-    "clear": "Очистить перевод"
+    "clear": "Очистить перевод",
+    "action": "перевести"
   },
   "translateTo": "Перевести на",
+  "tts": {
+    "action": "воспроизвести речь",
+    "clear": "очистить речь"
+  },
   "updateAgent": "Обновить информацию о помощнике",
   "upload": {
     "actionTooltip": "Загрузить изображение",

diff --git a/locales/ru_RU/error.json b/locales/ru_RU/error.json
@@ -28,6 +28,12 @@
     "PluginServerError": "Запрос сервера плагина возвратил ошибку. Проверьте файл манифеста плагина, конфигурацию плагина или реализацию сервера на основе информации об ошибке ниже",
     "NoAPIKey": "Ключ OpenAI API пуст, пожалуйста, добавьте свой собственный ключ OpenAI API"
   },
+  "stt": {
+    "responseError": "Ошибка запроса сервиса. Пожалуйста, проверьте конфигурацию или повторите попытку"
+  },
+  "tts": {
+    "responseError": "Ошибка запроса сервиса. Пожалуйста, проверьте конфигурацию или повторите попытку"
+  },
   "unlock": {
     "apikey": {
       "title": "Использовать собственный ключ API",