Skip to content

Commit

Permalink
提供funasr集成
Browse files Browse the repository at this point in the history
1、修正语音和文字的交互逻辑;
2、提供funasr的可选集成。
  • Loading branch information
xszyou committed May 17, 2023
1 parent d90d699 commit 524e4d0
Show file tree
Hide file tree
Showing 42 changed files with 506 additions and 33,449 deletions.
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ Fay数字人助理版是fay开源项目的重要分支,专注于构建智能

## **推荐集成**

给Fay加上本地免费语音识别(达摩院funaar): https://www.bilibili.com/video/BV1qs4y1g74e/?share_source=copy_web&vd_source=64cd9062f5046acba398177b62bea9ad

消费级pc大模型(ChatGLM-6B的基础上前置Rasa会话管理):https://m.bilibili.com/video/BV1D14y1f7pr

UE5工程:https://github.com/xszyou/fay-ue5
Expand Down Expand Up @@ -153,7 +155,8 @@ python main.py

| 代码模块 | 描述 | 链接 |
| ------------------------- | -------------------------- | ------------------------------------------------------------ |
| ./ai_module/ali_nls.py | 阿里云 实时语音识别 | https://ai.aliyun.com/nls/trans |
| ./ai_module/ali_nls.py | 实时语音识别(免费3个月,asr二选一) | https://ai.aliyun.com/nls/trans |
| ./ai_module/funasr.py | 达摩院开源免费本地asr (asr二选一) | fay/test/funasr/README.MD |
| ./ai_module/ms_tts_sdk.py | 微软 文本转情绪语音(可选) | https://azure.microsoft.com/zh-cn/services/cognitive-services/text-to-speech/ |
| ./ai_module/xf_ltp.py | 讯飞 情感分析 | https://www.xfyun.cn/service/emotion-analysis |
| ./utils/ngrok_util.py | ngrok.cc 外网穿透(可选) | http://ngrok.cc |
Expand Down
120 changes: 120 additions & 0 deletions ai_module/funasr.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""
感谢北京中科大脑神经算法工程师张聪聪提供funasr集成代码
"""
from threading import Thread
import websocket
import json
import time
import ssl
import _thread as thread

from core import wsa_server, song_player
from utils import config_util as cfg

class FunASR:
# 初始化
def __init__(self):
self.__URL = "ws://{}:{}".format(cfg.local_asr_ip, cfg.local_asr_port)
self.__ws = None
self.__connected = False
self.__frames = []
self.__state = 0
self.__closing = False
self.__task_id = ''
self.done = False
self.finalResults = ""


def __on_msg(self):
if "暂停" in self.finalResults or "不想听了" in self.finalResults or "别唱了" in self.finalResults:
song_player.stop()

# 收到websocket消息的处理
def on_message(self, ws, message):
try:
self.done = True
self.finalResults = message
wsa_server.get_web_instance().add_cmd({"panelMsg": self.finalResults})
self.__on_msg()

except Exception as e:
print(e)

if self.__closing:
try:
self.__ws.close()
except Exception as e:
print(e)

# 收到websocket错误的处理
def on_close(self, ws, code, msg):
self.__connected = False
print("### CLOSE:", msg)

# 收到websocket错误的处理
def on_error(self, ws, error):
print("### error:", error)

# 收到websocket连接建立的处理
def on_open(self, ws):
self.__connected = True

def run(*args):
while self.__connected:
try:
if len(self.__frames) > 0:
frame = self.__frames[0]

self.__frames.pop(0)
if type(frame) == dict:
ws.send(json.dumps(frame))
elif type(frame) == bytes:
ws.send(frame, websocket.ABNF.OPCODE_BINARY)
# print('发送 ------> ' + str(type(frame)))
except Exception as e:
print(e)
time.sleep(0.04)

thread.start_new_thread(run, ())

def __connect(self):
self.finalResults = ""
self.done = False
self.__frames.clear()
websocket.enableTrace(False)
self.__ws = websocket.WebSocketApp(self.__URL, on_message=self.on_message,on_close=self.on_close,on_error=self.on_error,subprotocols=["binary"])
self.__ws.on_open = self.on_open

self.__ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})

def add_frame(self, frame):
self.__frames.append(frame)

def send(self, buf):
self.__frames.append(buf)

def start(self):
Thread(target=self.__connect, args=[]).start()
data = {
'vad_need':False,
'state':'StartTranscription'
}
self.add_frame(data)

def end(self):
if self.__connected:
try:
for frame in self.__frames:
self.__frames.pop(0)
if type(frame) == dict:
self.__ws.send(json.dumps(frame))
elif type(frame) == bytes:
self.__ws.send(frame, websocket.ABNF.OPCODE_BINARY)
time.sleep(0.4)
self.__frames.clear()
frame = {'vad_need':False,'state':'StopTranscription'}
self.__ws.send(json.dumps(frame))
except Exception as e:
print(e)
self.__closing = True
self.__connected = False
Binary file removed bin/Release_2.85/BCMakeCert.dll
Binary file not shown.
Binary file removed bin/Release_2.85/BasicFormatsForCore.dll
Binary file not shown.
Binary file removed bin/Release_2.85/CertMaker.dll
Binary file not shown.
Binary file removed bin/Release_2.85/ColorConsole.dll
Binary file not shown.
Binary file removed bin/Release_2.85/FiddlerCore4.dll
Binary file not shown.
Loading

0 comments on commit 524e4d0

Please sign in to comment.