重构nlp模块逻辑

+ 重构NLP模块管理逻辑，便于自由扩展； + gpt：拆分为ChatGPT及GPT、更换新的GPT接口、可单独配置代理服务器； + 指定yolov8包版本，解决yolo不兼容问题； + 修复：自言自语bug、接收多个待处理消息bug。
xiaojiayang · Jun 28, 2023 · 8a4c9d7 · 8a4c9d7
1 parent e2859e9
commit 8a4c9d7
Show file tree

Hide file tree

Showing 10 changed files with 117 additions and 99 deletions.
diff --git a/README.md b/README.md
@@ -109,8 +109,9 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127
 │   ├── ms_tts_sdk.py       # 微软 文本转语音
 │   ├── nlp_lingju.py       # 灵聚 人机交互-自然语言处理
 │   ├── xf_aiui.py          # 讯飞 人机交互-自然语言处理
-│   ├── chatgpt.py          # gpt3.5对接
-│   ├── yuan_1_0.py         # 浪潮.源大模型对接
+│   ├── nlp_gpt.py          # gpt api对接
+│   ├── nlp_chatgpt.py      # chat.openai.com逆向对接
+│   ├── nlp_yuan.py         # 浪潮.源大模型对接
 │   ├── nlp_rasa.py         # ChatGLM-6B的基础上前置Rasa会话管理(强烈推荐)
 │   ├── nlp_VisualGLM.py    # 对接多模态大语言模型VisualGLM-6B
 │   ├── yolov8.py           # yolov8资态识别
@@ -141,6 +142,12 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127
 
 
 ## **三、升级日志**
+**2023.06.28：**
+
++ 重构NLP模块管理逻辑，便于自由扩展；
++ gpt：拆分为ChatGPT及GPT、更换新的GPT接口、可单独配置代理服务器；
++ 指定yolov8包版本，解决yolo不兼容问题；
++ 修复：自言自语bug、接收多个待处理消息bug。
 
 **2023.06.21：**
 

diff --git a/ai_module/nlp_chatgpt.py b/ai_module/nlp_chatgpt.py
@@ -0,0 +1,30 @@
+from revChatGPT.V1 import Chatbot
+from utils import config_util as cfg
+import time 
+
+count = 0
+def question(cont):
+    global count
+    try:
+        chatbot = Chatbot(config={
+            "access_token": cfg.key_gpt_access_token,
+            "paid": False,
+            "collect_analytics": True,
+            "proxy": cfg.proxy_config,
+            "model": "gpt-4",
+            "conversation_id":cfg.key_gpt_conversation_id
+            },conversation_id=cfg.key_gpt_conversation_id,
+            parent_id=None)
+
+        prompt = cont
+        response = ""
+        for data in chatbot.ask(prompt):
+            response = data["message"]
+        count = 0
+        return response
+    except Exception as e:
+        count += 1
+        if count < 3:
+            time.sleep(15)
+            return question(cont)
+        return 'gpt当前繁忙，请稍后重试' + e
diff --git a/ai_module/nlp_gpt.py b/ai_module/nlp_gpt.py
@@ -1,25 +1,13 @@
-from revChatGPT.V1 import Chatbot
-from core.content_db import Content_Db
+from revChatGPT.V3 import Chatbot
 from utils import config_util as cfg
 import time 
 
 count = 0
 def question(cont):
     global count
     try:
-        chatbot = Chatbot(config={
-            "access_token": cfg.key_gpt_access_token,
-            "paid": False,
-            "collect_analytics": True,
-            "model": "gpt-4",
-            "conversation_id":cfg.key_gpt_conversation_id
-            },conversation_id=cfg.key_gpt_conversation_id,
-            parent_id=None)
-
-        prompt = cont
-        response = ""
-        for data in chatbot.ask(prompt):
-            response = data["message"]
+        chatbot = Chatbot(proxy = cfg.proxy_config, api_key = cfg.key_chatgpt_api_key)
+        response = chatbot.ask(cont)
         count = 0
         return response
     except Exception as e:

diff --git a/ai_module/yuan_1_0.py → ai_module/nlp_yuan.py b/ai_module/yuan_1_0.py → ai_module/nlp_yuan.py
diff --git a/ai_module/yolov8.py b/ai_module/yolov8.py
@@ -100,10 +100,6 @@ def run(self, cap):
                     cv2.putText(operated_frame, f"{res.names[int(cls.item())]}", (int(x1.item()), int(y1.item()) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)        
                 if res.keypoints is not None and res.keypoints.size(0) > 0:  # check if keypoints exist
                     keypoints = res.keypoints[0]
-
-                    #TODO人脸相似性的比较，待优化
-                    keypoints_np = keypoints[0:5].cpu().numpy()
-                    mtx1, mtx2, disparity = procrustes(keypoints_np[:, :2], self.my_face)
                     #总人数
                     person_count += 1
                     #坐着的人数
@@ -142,5 +138,6 @@ def new_instance():
     return __fei_eyes
 
 
+
 
 
diff --git a/core/fay_core.py b/core/fay_core.py
@@ -21,64 +21,77 @@
 from core.tts_voice import EnumVoice
 from scheduler.thread_manager import MyThread
 from utils import util, storer, config_util
-from ai_module import yuan_1_0
-from ai_module import chatgpt
+
+
 import pygame
 from utils import config_util as cfg
 from core.content_db import Content_Db
 from datetime import datetime
 from ai_module import nlp_rasa
+from ai_module import nlp_chatgpt
 from ai_module import nlp_gpt
+from ai_module import nlp_yuan
 from ai_module import yolov8
-from ai_module import nlp_VisualGLM as VisualGLM
+from ai_module import nlp_VisualGLM
 
 import platform
 if platform.system() == "Windows":
     import sys
     sys.path.append("test/ovr_lipsync")
     from test_olipsync import LipSyncGenerator
+
 from ai_module import nlp_lingju
 
+modules = {
+    "nlp_yuan": nlp_yuan, 
+    "nlp_gpt": nlp_gpt,
+    "nlp_chatgpt": nlp_chatgpt,
+    "nlp_rasa": nlp_rasa,
+    "nlp_VisualGLM": nlp_VisualGLM,
+    "nlp_lingju": nlp_lingju
+}
+
+
+def determine_nlp_strategy(sendto,msg):
+    text = ''
+    textlist = []
+    try:
+        util.log(1, '自然语言处理...')
+        tm = time.time()
+        cfg.load_config()
+        if sendto == 2:
+            text = nlp_chatgpt.question(msg)
+        else:
+            module_name = "nlp_" + cfg.key_chat_module
+            selected_module = modules.get(module_name)
+            if selected_module is None:
+                raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！')   
+            if cfg.key_chat_module == 'rasa':
+                textlist = selected_module.question(msg)
+                text = textlist[0]['text'] 
+            else:
+                text = selected_module.question(msg)  
+            util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
+            if text == '哎呀，你这么说我也不懂，详细点呗' or text == '':
+                util.log(1, '[!] 自然语言无语了！')
+                text = '哎呀，你这么说我也不懂，详细点呗'  
+    except BaseException as e:
+        print(e)
+        util.log(1, '自然语言处理错误！')
+        text = '哎呀，你这么说我也不懂，详细点呗'   
+
+    return text,textlist
+
+
+
+
+
 #文本消息处理
 def send_for_answer(msg,sendto):
         contentdb = Content_Db()
         contentdb.add_content('member','send',msg)       
-        text = ''
-        textlist = []
-        try:
-            util.log(1, '自然语言处理...')
-            tm = time.time()
-            cfg.load_config()
-            if sendto == 2:
-                text = nlp_gpt.question(msg)
-            else:
-
-                if cfg.key_chat_module == 'yuan':
-                    text = yuan_1_0.question(msg)
-                elif cfg.key_chat_module == 'chatgpt':
-                    text = chatgpt.question(msg)
-                elif cfg.key_chat_module == 'rasa':
-                    textlist = nlp_rasa.question(msg)
-                    text = textlist[0]['text']  
-                elif cfg.key_chat_module == "VisualGLM":
-                    text = VisualGLM.question(msg)  
-                elif cfg.key_chat_module == "lingju":
-                    text = nlp_lingju.question(msg)  
-
-
-                else:
-                    raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！')    
-                util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
-                if text == '哎呀，你这么说我也不懂，详细点呗' or text == '':
-                    util.log(1, '[!] 自然语言无语了！')
-                    text = '哎呀，你这么说我也不懂，详细点呗'                    
-        except BaseException as e:
-            print(e)
-            util.log(1, '自然语言处理错误！')
-            text = '哎呀，你这么说我也不懂，详细点呗'
+        text,textlist = determine_nlp_strategy(sendto,msg)
 
-        now = datetime.now()
-        timetext = str(now.strftime("%Y-%m-%d %H:%M:%S"))
         contentdb.add_content('fay','send',text)
         wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}})
         if len(textlist) > 1:
@@ -88,6 +101,8 @@ def send_for_answer(msg,sendto):
                   wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":textlist[i]['text']}})
                   i+= 1
         return text
+
+
 class FeiFei:
     def __init__(self):
         pygame.mixer.init()
@@ -238,8 +253,8 @@ def __auto_speak(self):
                         fay_eyes = yolov8.new_instance()            
                         if fay_eyes.get_status():#YOLO正在运行
                             person_count, stand_count, sit_count = fay_eyes.get_counts()
-                            if person_count != 1: #不是有且只有一个人，不互动
-                                 wsa_server.get_web_instance().add_cmd({"panelMsg": "不是有且只有一个人，不互动"})
+                            if person_count < 1: #看不到人，不互动
+                                 wsa_server.get_web_instance().add_cmd({"panelMsg": "看不到人，不互动"})
                                  continue
 
                         answer = self.__get_answer(interact.interleaver, self.q_msg)#确定是否命中指令或q&a
@@ -252,35 +267,10 @@ def __auto_speak(self):
                         wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":self.q_msg}})
                         text = ''
                         textlist = []
+                        self.speaking = True
                         if answer is None:
-                            try:
-                                wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."})
-                                util.log(1, '自然语言处理...')
-                                tm = time.time()
-                                cfg.load_config()
-                                if cfg.key_chat_module == 'yuan':
-                                    text = yuan_1_0.question(self.q_msg)
-                                elif cfg.key_chat_module == 'chatgpt':
-                                    text = chatgpt.question(self.q_msg)
-                                elif cfg.key_chat_module == 'rasa':
-                                    textlist = nlp_rasa.question(self.q_msg)
-                                    text = textlist[0]['text']
-                                elif cfg.key_chat_module == "VisualGLM":
-                                    text = VisualGLM.question(self.q_msg)
-                                elif cfg.key_chat_module == "lingju":
-                                    text = nlp_lingju.question(self.q_msg) 
-                                else:
-                                    raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！')    
-                                util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
-                                if text == '哎呀，你这么说我也不懂，详细点呗' or text == '':
-                                    util.log(1, '[!] 自然语言无语了！')
-                                    wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
-                                    continue
-                            except BaseException as e:
-                                print(e)
-                                util.log(1, '自然语言处理错误！')
-                                wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
-                                continue
+                            wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."})
+                            text,textlist = determine_nlp_strategy(1,self.q_msg)
                         elif answer != 'NO_ANSWER': #语音内容没有命中指令,回复q&a内容
                             text = answer
                         self.a_msg = text
@@ -293,8 +283,7 @@ def __auto_speak(self):
                                 wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":textlist[i]['text']}})
                                 i+= 1
                     wsa_server.get_web_instance().add_cmd({"panelMsg": self.a_msg})
-                    self.last_speak_data = self.a_msg
-                    self.speaking = True
+                    self.last_speak_data = self.a_msg               
                     MyThread(target=self.__say, args=['interact']).start()
 
             except BaseException as e:

diff --git a/core/recorder.py b/core/recorder.py
@@ -28,7 +28,7 @@ def __init__(self, fay):
         self.__processing = False
         self.__history_level = []
         self.__history_data = []
-        self.__dynamic_threshold = 0.35 # 声音识别的音量阈值
+        self.__dynamic_threshold = 0.7 # 声音识别的音量阈值
 
         self.__MAX_LEVEL = 25000
         self.__MAX_BLOCK = 100

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 requests~=2.26.0    
-numpy~=1.22.0
+numpy
 pyaudio~=0.2.11
 websockets~=10.2
 ws4py~=0.5.1

diff --git a/system.conf b/system.conf
@@ -20,20 +20,20 @@ ms_tts_region=
 xf_ltp_app_id=
 xf_ltp_api_key=
 
-#NLP五选一:lingju、yuan、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、VisualGLM
+#NLP五选一:yuan、gpt、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、lingju
 chat_module=lingju
 
-#灵聚 服务密钥(NLP多选1) https://open.lingju.ai
-lingju_api_key=
-lingju_api_authcode=
 
-#浪.潮源大模型 服务密钥(NLP5多1) https://air.inspur.com/
+#浪.潮源大模型 服务密钥(NLP5选1) https://air.inspur.com/
 yuan_1_0_account=  
 yuan_1_0_phone=
 
-#gpt 服务密钥(NLP多选1) https://openai.com/
+#gpt 服务密钥(NLP5选1) https://openai.com/
 chatgpt_api_key=
 
+#灵聚 服务密钥 https://open.lingju.ai
+lingju_api_key=
+lingju_api_authcode=
 
 #ngrok内网穿透id，远程设备可以通过互联网连接Fay（非必须）http://ngrok.cc
 ngrok_cc_id=
@@ -42,3 +42,6 @@ ngrok_cc_id=
 gpt_access_token=
 gpt_conversation_id=
 
+#gpt代理
+proxy_config=http://192.168.1.13:7890
+
diff --git a/utils/config_util.py b/utils/config_util.py
@@ -21,6 +21,7 @@
 key_chat_module = None
 key_gpt_access_token = None
 key_gpt_conversation_id = None
+proxy_config = None
 
 ASR_mode = None
 local_asr_ip = None 
@@ -45,6 +46,7 @@ def load_config():
     global key_gpt_conversation_id
     global key_lingju_api_key
     global key_lingju_api_authcode
+    global proxy_config
 
     global ASR_mode
     global local_asr_ip 
@@ -73,6 +75,8 @@ def load_config():
     local_asr_ip = system_config.get('key', 'local_asr_ip')
     local_asr_port = system_config.get('key', 'local_asr_port')
 
+    proxy_config = system_config.get('key', 'proxy_config')
+
     config = json.load(codecs.open('config.json', encoding='utf-8'))