Skip to content

Commit

Permalink
集成灵聚NLP api及UI修正
Browse files Browse the repository at this point in the history
1、集成灵聚NLP api(支持GPT3.5及多应用);
 2、ui修正。
  • Loading branch information
xszyou committed Jun 21, 2023
1 parent ec482dd commit 750e77e
Show file tree
Hide file tree
Showing 10 changed files with 390 additions and 234 deletions.
19 changes: 16 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ Fay数字人助理版是fay开源项目的重要分支,专注于构建智能

## **推荐玩法**



灵聚NLP api(支持GPT3.5及多应用):b站"Fay数字人集成灵聚NLP api(支持GPT3.5及多应用)"

集成本地唇型算法:https://www.bilibili.com/video/BV1Zh4y1g7o7/?buvid=XXDD0B5DD6C43C070DF9E7E67930FC48B24DF&is_story_h5=false&mid=Pvwl%2Ft1ahPM726k1L4%2FnRA%3D%3D&plat_id=202&share_from=ugc&share_medium=android&share_plat=android&share_source=WEIXIN&share_tag=s_i&timestamp=1686926382&unique_k=Jdqazy3&up_id=2111554564

给数字人加上眼睛(集成yolo+VisualGLM):B站视频
Expand Down Expand Up @@ -85,7 +89,7 @@ Fay(服务端)与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127

## **二、Fay控制器核心逻辑**

![](images/luoji.png)
![](images/luoji.jpg)

**注:**

Expand All @@ -103,9 +107,9 @@ Fay(服务端)与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127
├── ai_module
│   ├── ali_nls.py # 阿里云 实时语音
│   ├── ms_tts_sdk.py # 微软 文本转语音
│   ├── nlp_lingju.py # 灵聚 人机交互-自然语言处理
│   ├── xf_aiui.py # 讯飞 人机交互-自然语言处理
│   ├── chatgpt.py # gpt3.5对接
│   ├── nlp_gpt.py # 对接chat.openai.com(免key)
│   ├── yuan_1_0.py # 浪潮.源大模型对接
│   ├── nlp_rasa.py # ChatGLM-6B的基础上前置Rasa会话管理(强烈推荐)
│   ├── nlp_VisualGLM.py # 对接多模态大语言模型VisualGLM-6B
Expand All @@ -116,6 +120,10 @@ Fay(服务端)与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127
│   ├── fay_core.py # 数字人核心模块
│   ├── recorder.py # 录音器
│   ├── tts_voice.py # 语音生源枚举
│   ├── authorize_tb.py # fay.db认证表管理
│   ├── content_db.py # fay.db内容表管理
│   ├── interact.py # 互动(消息)对象
│   ├── song_player.py # 音乐播放(暂不可用)
│   └── wsa_server.py # WebSocket 服务端
├── gui # 图形界面
│   ├── flask_server.py # Flask 服务端
Expand All @@ -134,6 +142,11 @@ Fay(服务端)与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127

## **三、升级日志**

**2023.06.21:**

+ 集成灵聚NLP api(支持GPT3.5及多应用);
+ ui修正。

**2023.06.17:**

+ 集成本地唇型算法。
Expand Down Expand Up @@ -195,9 +208,9 @@ python main.py
| ./ai_module/ms_tts_sdk.py | 微软 文本转情绪语音(非必须,不配置时使用免费的edge-tts) | https://azure.microsoft.com/zh-cn/services/cognitive-services/text-to-speech/ |
| ./ai_module/xf_ltp.py | 讯飞 情感分析 | https://www.xfyun.cn/service/emotion-analysis |
| ./utils/ngrok_util.py | ngrok.cc 外网穿透(可选) | http://ngrok.cc |
| ./ai_module/nlp_lingju.py | 灵聚NLP api(支持GPT3.5及多应用)(NLP多选1) | https://open.lingju.ai 需联系客服务开通gpt3.5权限|
| ./ai_module/yuan_1_0.py | 浪潮源大模型(NLP 多选1) | https://air.inspur.com/ |
| ./ai_module/chatgpt.py | ChatGPT(NLP多选1) | ******* |
| ./ai_module/xf_aiui.py | 讯飞自然语言处理(NLP多选1) | https://aiui.xfyun.cn/solution/webapi |
| ./ai_module/nlp_rasa.py | ChatGLM-6B的基础上前置Rasa会话管理(NLP 多选1) | https://m.bilibili.com/video/BV1D14y1f7pr |
| ./ai_module/nlp_VisualGLM.py | 对接VisualGLM-6B多模态单机离线大语言模型(NLP 多选1) | B站视频 |

Expand Down
100 changes: 100 additions & 0 deletions ai_module/nlp_lingju.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import json
import requests
import uuid
from datetime import datetime, timedelta
import time
from utils import util
from utils import config_util as cfg
from core.authorize_tb import Authorize_Tb

def question(cont):
lingju = Lingju()
answer = lingju.question(cont)
return answer

class Lingju:

def __init__(self):
self.userid = str(uuid.getnode())
self.authorize_tb = Authorize_Tb()

def question(self, cont):
token = self.__check_token()
if token is None or token == 'expired':
token_info = self.__get_token()
if token_info is not None:
#转换过期时间
updated_in_seconds = time.time()
updated_datetime = datetime.fromtimestamp(updated_in_seconds)
expires_timedelta = timedelta(days=token_info['data']['expires'])
expiry_datetime = updated_datetime + expires_timedelta
expiry_timestamp_in_seconds = expiry_datetime.timestamp()
expiry_timestamp_in_milliseconds = int(expiry_timestamp_in_seconds) * 1000
token = token_info['data']['accessToken']
if token == 'expired':
self.authorize_tb.update_by_userid(self.userid, token_info['data']['accessToken'], expiry_timestamp_in_milliseconds)
else:
self.authorize_tb.add(self.userid, token_info['data']['accessToken'], expiry_timestamp_in_milliseconds)
else:
token = None

if token is not None:
try:
lat,lng,city = self.__get_location()
url="https://dev.lingju.ai/httpapi/ljchat.do"
req = json.dumps({"accessToken": token, "lat": lat, "lng": lng, "input": cont})
headers = {'Content-Type':'application/json;charset=UTF-8'}
r = requests.post(url, headers=headers, data=req)
if r.status_code != 200:
util.log(1, f"灵聚api对接有误: {r.text}")
return "哎呀,出错了!请重新发一下"
info = json.loads(r.text)
if info['status'] != 0:
return info['description']
else:
answer = json.loads(info['answer'])
return answer['rtext']
except Exception as e:
util.log(1, f"灵聚api对接有误: {str(e)}")
return "哎呀,出错了!请重新发一下"

def __check_token(self):
self.authorize_tb.init_tb()
info = self.authorize_tb.find_by_userid(self.userid)
if info is not None:
if info[1] >= int(time.time())*1000:
return info[0]
else:
return 'expired'
else:
return None

def __get_token(self):
try:
cfg.load_config()
url=f"https://dev.lingju.ai/httpapi/authorize.do?appkey={cfg.key_lingju_api_key}&userid={self.userid}&authcode={cfg.key_lingju_api_authcode}"
headers = {'Content-Type':'application/json;charset=UTF-8'}
r = requests.post(url, headers=headers)
if r.status_code != 200:
util.log(1, f"灵聚api对接有误: {r.text}")
return None
info = json.loads(r.text)
if info['status'] != 0:
util.log(1, f"灵聚api对接有误:{info['description']}")
return None
else:
return info
except Exception as e:
util.log(1, f"灵聚api对接有误: {str(e)}")
return None

def __get_location(self):
try:
response = requests.get('http://ip-api.com/json/')
data = response.json()
return data['lat'], data['lon'], data['city']
except requests.exceptions.RequestException as e:
util.log(1, f"获取位置失败: {str(e)}")
return 0, 0, "北京"


66 changes: 66 additions & 0 deletions core/authorize_tb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import sqlite3
import time
import threading
import functools
def synchronized(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
with self.lock:
return func(self, *args, **kwargs)
return wrapper
class Authorize_Tb:

def __init__(self) -> None:
self.lock = threading.Lock()



#初始化
def init_tb(self):
conn = sqlite3.connect('fay.db')
c = conn.cursor()
try:
c.execute('SELECT * FROM T_Authorize')
except sqlite3.OperationalError:
c.execute('''
CREATE TABLE T_Authorize
(id INTEGER PRIMARY KEY autoincrement,
userid char(100),
accesstoken TEXT,
expirestime BigInt,
createtime Int);
''')
conn.commit()
finally:
conn.close()

#添加
@synchronized
def add(self,userid,accesstoken,expirestime):
conn = sqlite3.connect("fay.db")
cur = conn.cursor()
cur.execute("insert into T_Authorize (userid,accesstoken,expirestime,createtime) values (?,?,?,?)",(userid,accesstoken,expirestime,int(time.time())))

conn.commit()
conn.close()
return cur.lastrowid

#查询
@synchronized
def find_by_userid(self,userid):
conn = sqlite3.connect("fay.db")
cur = conn.cursor()
cur.execute("select accesstoken,expirestime from T_Authorize where userid = ? order by id desc limit 1",(userid,))
info = cur.fetchone()
conn.close()
return info

# 更新token
@synchronized
def update_by_userid(self, userid, new_accesstoken, new_expirestime):
conn = sqlite3.connect("fay.db")
cur = conn.cursor()
cur.execute("UPDATE T_Authorize SET accesstoken = ?, expirestime = ? WHERE userid = ?",
(new_accesstoken, new_expirestime, userid))
conn.commit()
conn.close()
48 changes: 11 additions & 37 deletions core/fay_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import numpy as np
# import tensorflow as tf
import fay_booter
from ai_module import xf_aiui
from ai_module import xf_ltp
from ai_module.ms_tts_sdk import Speech
from core import wsa_server, tts_voice, song_player
Expand All @@ -38,6 +37,7 @@
import sys
sys.path.append("test/ovr_lipsync")
from test_olipsync import LipSyncGenerator
from ai_module import nlp_lingju

#文本消息处理
def send_for_answer(msg,sendto):
Expand All @@ -52,9 +52,8 @@ def send_for_answer(msg,sendto):
if sendto == 2:
text = nlp_gpt.question(msg)
else:
if cfg.key_chat_module == 'xfaiui':
text = xf_aiui.question(msg)
elif cfg.key_chat_module == 'yuan':

if cfg.key_chat_module == 'yuan':
text = yuan_1_0.question(msg)
elif cfg.key_chat_module == 'chatgpt':
text = chatgpt.question(msg)
Expand All @@ -63,10 +62,12 @@ def send_for_answer(msg,sendto):
text = textlist[0]['text']
elif cfg.key_chat_module == "VisualGLM":
text = VisualGLM.question(msg)
elif cfg.key_chat_module == "lingju":
text = nlp_lingju.question(msg)


else:
raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置!')
raise RuntimeError('灵聚key、yuan key、gpt key都没有配置!')
util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
if text == '哎呀,你这么说我也不懂,详细点呗' or text == '':
util.log(1, '[!] 自然语言无语了!')
Expand Down Expand Up @@ -126,16 +127,6 @@ def __init__(self):
[['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact']
]

# 商品提问关键字
self.explain_keyword = [
[['是什么'], 'intro'],
[['怎么用', '使用场景', '有什么作用'], 'usage'],
[['怎么卖', '多少钱', '售价'], 'price'],
[['便宜点', '优惠', '折扣', '促销'], 'discount'],
[['质量', '保证', '担保'], 'promise'],
[['特点', '优点'], 'character'],
]

self.wsParam = None
self.wss = None
self.sp = Speech()
Expand Down Expand Up @@ -229,24 +220,6 @@ def __get_answer(self, interleaver, text):
answer = self.__get_keyword(self.__read_qna(config_util.config['interact']['QnA']), text)
if answer is not None:
return answer


def __get_list_answer(self, answers, text):
last_similar = 0
last_answer = ''
for mlist in answers:
for quest in mlist[0]:
similar = self.__string_similar(text, quest)
if quest in text:
similar += 0.3
if similar > last_similar:
last_similar = similar
answer_list = mlist[1]
last_answer = answer_list[random.randint(0, len(answer_list) - 1)]
# print("相似度: {}, 回答: {}".format(last_similar, last_answer))
if last_similar >= 0.6:
return last_answer
return None

def __auto_speak(self):
while self.__running:
Expand Down Expand Up @@ -285,9 +258,7 @@ def __auto_speak(self):
util.log(1, '自然语言处理...')
tm = time.time()
cfg.load_config()
if cfg.key_chat_module == 'xfaiui':
text = xf_aiui.question(self.q_msg)
elif cfg.key_chat_module == 'yuan':
if cfg.key_chat_module == 'yuan':
text = yuan_1_0.question(self.q_msg)
elif cfg.key_chat_module == 'chatgpt':
text = chatgpt.question(self.q_msg)
Expand All @@ -296,8 +267,10 @@ def __auto_speak(self):
text = textlist[0]['text']
elif cfg.key_chat_module == "VisualGLM":
text = VisualGLM.question(self.q_msg)
elif cfg.key_chat_module == "lingju":
text = nlp_lingju.question(self.q_msg)
else:
raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置!')
raise RuntimeError('灵聚key、yuan key、gpt key都没有配置!')
util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
if text == '哎呀,你这么说我也不懂,详细点呗' or text == '':
util.log(1, '[!] 自然语言无语了!')
Expand Down Expand Up @@ -435,6 +408,7 @@ def __send_or_play_audio(self, file_url, say_type):
audio_length = eyed3.load(file_url).info.time_secs #mp3音频长度
# with wave.open(file_url, 'rb') as wav_file: #wav音频长度
# audio_length = wav_file.getnframes() / float(wav_file.getframerate())
# print(audio_length)
# if audio_length <= config_util.config["interact"]["maxInteractTime"] or say_type == "script":
if config_util.config["interact"]["playSound"]: # 展板播放
self.__play_sound(file_url)
Expand Down
Loading

0 comments on commit 750e77e

Please sign in to comment.