Skip to content

Commit

Permalink
修复麦克风&文字回复问题
Browse files Browse the repository at this point in the history 
1、修复助理版文字输入不读取人设回复问题；
2、修复助理版文字输入不读取qa回复问题；
3、增强麦克风接入稳定性。
  • Loading branch information
@xszyou
xszyou committed Jul 12, 2023
1 parent 5c5e9d0 commit b53bc6e
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 71 deletions.
6 changes: 6 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,12 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127


## **三、升级日志**
**2023.07.12：**

+ 修复助理版文字输入不读取人设回复问题；
+ 修复助理版文字输入不读取qa回复问题；
+ 增强麦克风接入稳定性。

**2023.07.05：**

+ 修复无法运行唇型算法而导致的不播放声音问题。
Expand Down
83 changes: 24 additions & 59 deletions core/fay_core.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import difflib
import imp
import math
import os
import random
Expand All @@ -21,19 +22,21 @@
from core.tts_voice import EnumVoice
from scheduler.thread_manager import MyThread
from utils import util, storer, config_util

from core import qa_service

import pygame
from utils import config_util as cfg
from core.content_db import Content_Db
from datetime import datetime

from ai_module import nlp_rasa
from ai_module import nlp_chatgpt
from ai_module import nlp_gpt
from ai_module import nlp_yuan
from ai_module import yolov8
from ai_module import nlp_VisualGLM


import platform
if platform.system() == "Windows":
import sys
Expand Down Expand Up @@ -89,8 +92,21 @@ def determine_nlp_strategy(sendto,msg):
#文本消息处理
def send_for_answer(msg,sendto):
contentdb = Content_Db()
contentdb.add_content('member','send',msg)
text,textlist = determine_nlp_strategy(sendto,msg)
contentdb.add_content('member','send',msg)
textlist = []
text = None
# 人设问答
keyword = qa_service.question('Persona',msg)
if keyword is not None:
text = config_util.config["attribute"][keyword]

# 全局问答
if text is None:
answer = qa_service.question('qa',msg)
if answer is not None:
text = answer
else:
text,textlist = determine_nlp_strategy(sendto,msg)

contentdb.add_content('fay','send',text)
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}})
Expand Down Expand Up @@ -121,27 +137,6 @@ def __init__(self):
# self.W = np.array([0.01577594,1.16119452,0.75828,0.207746,1.25017864,0.1044121,0.4294899,0.2770932]).reshape(-1,1) #适应模型变量矩阵
self.W = np.array([0.0, 0.6, 0.1, 0.7, 0.3, 0.0, 0.0, 0.0]).reshape(-1, 1) # 适应模型变量矩阵

self.command_keyword = [
[['播放歌曲', '播放音乐', '唱首歌', '放首歌', '听音乐', '你会唱歌吗', '我想首听歌'], 'playSong'],
[['关闭', '再见', '你走吧'], 'stop'],
[['静音', '闭嘴', '我想静静'], 'mute'],
[['取消静音', '你在哪呢', '你可以说话了'], 'unmute'],
[['换个性别', '换个声音'], 'changeVoice']
]

# 人设提问关键字
self.attribute_keyword = [
[['你叫什么名字', '你的名字是什么'], 'name'],
[['你是男的还是女的', '你是男生还是女生', '你的性别是什么', '你是男生吗', '你是女生吗', '你是男的吗', '你是女的吗', '你是男孩子吗', '你是女孩子吗', ], 'gender', ],
[['你今年多大了', '你多大了', '你今年多少岁', '你几岁了', '你今年几岁了', '你今年几岁了', '你什么时候出生', '你的生日是什么', '你的年龄'], 'age', ],
[['你的家乡在哪', '你的家乡是什么', '你家在哪', '你住在哪', '你出生在哪', '你的出生地在哪', '你的出生地是什么', ], 'birth', ],
[['你的生肖是什么', '你属什么', ], 'zodiac', ],
[['你是什么座', '你是什么星座', '你的星座是什么', ], 'constellation', ],
[['你是做什么的', '你的职业是什么', '你是干什么的', '你的职位是什么', '你的工作是什么', '你是做什么工作的'], 'job', ],
[['你的爱好是什么', '你有爱好吗', '你喜欢什么', '你喜欢做什么'], 'hobby'],
[['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact']
]

self.wsParam = None
self.wss = None
self.sp = Speech()
Expand All @@ -156,36 +151,6 @@ def __init__(self):
self.playing = False
self.muting = False

def __string_similar(self, s1, s2):
return difflib.SequenceMatcher(None, s1, s2).quick_ratio()

def __read_qna(self, filename) -> list:
qna = []
try:
wb = load_workbook(filename)
sheets = wb.worksheets # 获取当前所有的sheet
sheet = sheets[0]
for row in sheet.rows:
if len(row) >= 2:
qna.append([row[0].value.split(";"), row[1].value])
except BaseException as e:
print("无法读取Q&A文件 {} -> ".format(filename) + str(e))
return qna

def __get_keyword(self, keyword_dict, text):
last_similar = 0
last_answer = ''
for qa in keyword_dict:
for quest in qa[0]:
similar = self.__string_similar(text, quest)
if quest in text:
similar += 0.3
if similar > last_similar:
last_similar = similar
last_answer = qa[1]
if last_similar >= 0.6:
return last_answer
return None

def __play_song(self):
self.playing = True
Expand All @@ -197,7 +162,7 @@ def __play_song(self):
def __get_answer(self, interleaver, text):
if interleaver == "mic":
#指令
keyword = self.__get_keyword(self.command_keyword, text)
keyword = qa_service.question('command',text)
if keyword is not None:
if keyword == "playSong":
MyThread(target=self.__play_song).start()
Expand Down Expand Up @@ -225,14 +190,14 @@ def __get_answer(self, interleaver, text):
config_util.save_config(config_util.config)
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
return "NO_ANSWER"

# 人设问答
keyword = self.__get_keyword(self.attribute_keyword, text)
keyword = qa_service.question('Persona',text)
if keyword is not None:
return config_util.config["attribute"][keyword]

answer = None
# 全局问答
answer = self.__get_keyword(self.__read_qna(config_util.config['interact']['QnA']), text)
answer = qa_service.question('qa',text)
if answer is not None:
return answer

Expand Down
83 changes: 83 additions & 0 deletions core/qa_service.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@

from winreg import QueryInfoKey
from openpyxl import load_workbook
import difflib
import shlex
import subprocess
from utils import config_util as cfg

def question(query_type,text):
qa = QAService()
answer = qa.question(query_type,text)
return answer

class QAService:

def __init__(self):
# 人设提问关键字
self.attribute_keyword = [
[['你叫什么名字', '你的名字是什么'], 'name'],
[['你是男的还是女的', '你是男生还是女生', '你的性别是什么', '你是男生吗', '你是女生吗', '你是男的吗', '你是女的吗', '你是男孩子吗', '你是女孩子吗', ], 'gender', ],
[['你今年多大了', '你多大了', '你今年多少岁', '你几岁了', '你今年几岁了', '你今年几岁了', '你什么时候出生', '你的生日是什么', '你的年龄'], 'age', ],
[['你的家乡在哪', '你的家乡是什么', '你家在哪', '你住在哪', '你出生在哪', '你的出生地在哪', '你的出生地是什么', ], 'birth', ],
[['你的生肖是什么', '你属什么', ], 'zodiac', ],
[['你是什么座', '你是什么星座', '你的星座是什么', ], 'constellation', ],
[['你是做什么的', '你的职业是什么', '你是干什么的', '你的职位是什么', '你的工作是什么', '你是做什么工作的'], 'job', ],
[['你的爱好是什么', '你有爱好吗', '你喜欢什么', '你喜欢做什么'], 'hobby'],
[['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact']
]

self.command_keyword = [
[['播放歌曲', '播放音乐', '唱首歌', '放首歌', '听音乐', '你会唱歌吗', '我想首听歌'], 'playSong'],
[['关闭', '再见', '你走吧'], 'stop'],
[['静音', '闭嘴', '我想静静'], 'mute'],
[['取消静音', '你在哪呢', '你可以说话了'], 'unmute'],
[['换个性别', '换个声音'], 'changeVoice']
]

def question(self, query_type, text):
if query_type == 'qa':
answer_dict = self.__read_qna(cfg.config['interact']['QnA'])
answer = self.__get_keyword(answer_dict, text)
elif query_type == 'Persona':
answer_dict = self.attribute_keyword
answer = self.__get_keyword(answer_dict, text)
elif query_type == 'command':
answer = self.__get_keyword(self.command_keyword, text)
print(answer)
return answer


def __read_qna(self, filename) -> list:
qna = []
try:
wb = load_workbook(filename)
sheets = wb.worksheets
sheet = sheets[0]
for row in sheet.rows:
if len(row) >= 2:
qna.append([row[0].value.split(";"), row[1].value])
except BaseException as e:
print("无法读取Q&A文件 {} -> ".format(filename) + str(e))
return qna

def __get_keyword(self, keyword_dict, text):
last_similar = 0
last_answer = ''
for qa in keyword_dict:
for quest in qa[0]:
similar = self.__string_similar(text, quest)
if quest in text:
similar += 0.3
if similar > last_similar:
last_similar = similar
last_answer = qa[1]
if last_similar >= 0.6:
return last_answer
return None


def __string_similar(self, s1, s2):
return difflib.SequenceMatcher(None, s1, s2).quick_ratio()


33 changes: 25 additions & 8 deletions core/recorder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def __print_level(self, level):
text += "-"
print(text + " [" + str(int(per * 100)) + "%]")

def __waitingResult(self, iat):
def __waitingResult(self, iat:asrclient):
if self.__fay.playing:
return
self.processing = True
Expand All @@ -95,20 +95,37 @@ def __waitingResult(self, iat):


def __record(self):
stream = self.get_stream() #把get stream的方式封装出来方便实现麦克风录制及网络流等不同的流录制子类

try:
stream = self.get_stream() #把get stream的方式封装出来方便实现麦克风录制及网络流等不同的流录制子类
except Exception as e:
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")
return
isSpeaking = False
last_mute_time = time.time()
last_speaking_time = time.time()
data = None
while self.__running:
data = stream.read(1024, exception_on_overflow=False)
if not data:
try:
data = stream.read(1024, exception_on_overflow=False)
except Exception as e:
data = None
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")
return

if data is None:
continue

if cfg.config['source']['record'].get("channels"):

if cfg.config['source']['record']['enabled']:
if len(cfg.config['source']['record'])<3:
channels = 1
else:
channels = int(cfg.config['source']['record']['channels'])

#只获取第一声道
data = np.frombuffer(data, dtype=np.int16)
data = np.reshape(data, (-1, cfg.config['source']['record']['channels'])) # reshaping the array to split the channels
data = np.reshape(data, (-1, channels)) # reshaping the array to split the channels
mono = data[:, 0] # taking the first channel
data = mono.tobytes()

Expand Down
16 changes: 12 additions & 4 deletions fay_booter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@ def get_stream(self):
device_id,devInfo = self.__findInternalRecordingDevice(self.paudio)
if device_id < 0:
return
rate = int(devInfo['defaultSampleRate'])
channels = int(devInfo['maxInputChannels'])
if channels == 0:
util.log(1, '请检查设备是否有误，再重新启动!')
return
self.stream = self.paudio.open(input_device_index=device_id, rate=self.__RATE, format=self.__FORMAT, channels=channels, input=True)
return self.stream

Expand All @@ -51,9 +53,15 @@ def __findInternalRecordingDevice(self, p):

def stop(self):
super().stop()
self.stream.stop_stream()
self.stream.close()
self.paudio.terminate()
try:
self.stream.stop_stream()
self.stream.close()
self.paudio.terminate()
except Exception as e:
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")




#Edit by xszyou on 20230113:录制远程设备音频输入并传给aliyun
Expand Down

0 comments on commit b53bc6e

Please sign in to comment.