From 161f562171ef32b2e0c47069822e49202a22fa83 Mon Sep 17 00:00:00 2001 From: shanshi Date: Sat, 15 Jun 2024 22:14:58 +0800 Subject: [PATCH 001/128] =?UTF-8?q?=E5=8D=87=E7=BA=A7langchain=E7=AD=89?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=E5=90=8C=E6=97=B6=E8=B0=83=E6=95=B4llm=5Fcon?= =?UTF-8?q?fig=E7=9B=B8=E5=85=B3=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- muagent/chat/search_chat.py | 2 +- muagent/codechat/code_search/code_search.py | 2 +- .../codechat/code_search/cypher_generator.py | 5 +- .../codebase_handler/code_importer.py | 2 +- muagent/connector/memory_manager.py | 6 +- .../{ => llm_models}/embeddings/__init__.py | 0 .../get_embedding.py | 6 +- .../huggingface_embedding.py | 2 +- muagent/llm_models/llm_config.py | 8 +- .../openai_embedding.py | 26 +++- muagent/llm_models/openai_model.py | 129 ++++++++++++------ .../commands/__init__.py | 0 .../commands/default_vs_cds.py | 0 .../retrieval/document_loaders/json_loader.py | 2 +- .../document_loaders/jsonl_loader.py | 2 +- muagent/{embeddings => retrieval}/faiss_m.py | 12 +- .../{embeddings => retrieval}/in_memory.py | 4 +- muagent/{embeddings => retrieval}/utils.py | 2 +- muagent/service/base_service.py | 4 +- muagent/service/cb_api.py | 2 +- muagent/service/faiss_db_service.py | 10 +- muagent/service/kb_api.py | 2 +- muagent/utils/path_utils.py | 2 +- requirements.txt | 13 +- tests/codechat/codebasehander_test.py | 10 +- tests/connector/agent_test.py | 11 +- tests/connector/chain_test.py | 1 - tests/connector/memory_manager_test.py | 2 +- tests/connector/phase_test.py | 1 - tests/llm_models/openai_test.py | 43 ++---- 31 files changed, 187 insertions(+), 127 deletions(-) rename muagent/{ => llm_models}/embeddings/__init__.py (100%) rename muagent/{embeddings => llm_models}/get_embedding.py (96%) rename muagent/{embeddings => llm_models}/huggingface_embedding.py (98%) rename muagent/{embeddings => llm_models}/openai_embedding.py (82%) rename muagent/{embeddings => retrieval}/commands/__init__.py (100%) rename muagent/{embeddings => retrieval}/commands/default_vs_cds.py (100%) rename muagent/{embeddings => retrieval}/faiss_m.py (96%) rename muagent/{embeddings => retrieval}/in_memory.py (89%) rename muagent/{embeddings => retrieval}/utils.py (92%) diff --git a/.gitignore b/.gitignore index ed2377e..73e0fc3 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ setup_test.py build *egg-info dist -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +zdatafront* \ No newline at end of file diff --git a/muagent/chat/search_chat.py b/muagent/chat/search_chat.py index 3d21798..cfc2b23 100644 --- a/muagent/chat/search_chat.py +++ b/muagent/chat/search_chat.py @@ -5,7 +5,7 @@ from langchain.callbacks import AsyncIteratorCallbackHandler from langchain.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper from langchain.prompts.chat import ChatPromptTemplate -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document # from configs.model_config import ( # PROMPT_TEMPLATE, SEARCH_ENGINE_TOP_K, BING_SUBSCRIPTION_KEY, BING_SEARCH_URL, diff --git a/muagent/codechat/code_search/code_search.py b/muagent/codechat/code_search/code_search.py index 554d228..d3f5c22 100644 --- a/muagent/codechat/code_search/code_search.py +++ b/muagent/codechat/code_search/code_search.py @@ -15,7 +15,7 @@ from muagent.codechat.code_search.cypher_generator import CypherGenerator from muagent.codechat.code_search.tagger import Tagger -from muagent.embeddings.get_embedding import get_embedding +from muagent.llm_models.get_embedding import get_embedding from muagent.llm_models.llm_config import LLMConfig, EmbedConfig diff --git a/muagent/codechat/code_search/cypher_generator.py b/muagent/codechat/code_search/cypher_generator.py index 19929d3..893dc73 100644 --- a/muagent/codechat/code_search/cypher_generator.py +++ b/muagent/codechat/code_search/cypher_generator.py @@ -5,7 +5,7 @@ @time: 2023/11/24 上午10:17 @desc: ''' -from langchain import PromptTemplate +from langchain.prompts import PromptTemplate from loguru import logger from muagent.llm_models.openai_model import getChatModelFromConfig @@ -14,7 +14,8 @@ from langchain.schema import ( HumanMessage, ) -from langchain.chains.graph_qa.prompts import NGQL_GENERATION_PROMPT, CYPHER_GENERATION_TEMPLATE +# from langchain.chains.graph_qa.prompts import NGQL_GENERATION_PROMPT, CYPHER_GENERATION_TEMPLATE +from langchain_community.chains.graph_qa.prompts import CYPHER_GENERATION_TEMPLATE schema = ''' Node properties: [{'tag': 'package', 'properties': []}, {'tag': 'class', 'properties': []}, {'tag': 'method', 'properties': []}] diff --git a/muagent/codechat/codebase_handler/code_importer.py b/muagent/codechat/codebase_handler/code_importer.py index aad74c5..588c7dc 100644 --- a/muagent/codechat/codebase_handler/code_importer.py +++ b/muagent/codechat/codebase_handler/code_importer.py @@ -12,7 +12,7 @@ from muagent.db_handler.graph_db_handler.nebula_handler import NebulaHandler from muagent.db_handler.vector_db_handler.chroma_handler import ChromaHandler -from muagent.embeddings.get_embedding import get_embedding +from muagent.llm_models.get_embedding import get_embedding from muagent.llm_models.llm_config import EmbedConfig diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index 5ce1486..6096464 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -6,14 +6,14 @@ from loguru import logger import numpy as np -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document from .schema import Memory, Message from muagent.service.service_factory import KBServiceFactory from muagent.llm_models import getChatModelFromConfig from muagent.llm_models.llm_config import EmbedConfig, LLMConfig -from muagent.embeddings.utils import load_embeddings_from_path +from muagent.retrieval.utils import load_embeddings_from_path from muagent.utils.common_utils import * from muagent.connector.configs.prompts import CONV_SUMMARY_PROMPT_SPEC from muagent.orm import table_init @@ -489,7 +489,7 @@ def check_chat_index(self, chat_index: str): from muagent.utils.tbase_util import TbaseHandler -from muagent.embeddings.get_embedding import get_embedding +from muagent.llm_models.get_embedding import get_embedding from redis.commands.search.field import ( TextField, NumericField, diff --git a/muagent/embeddings/__init__.py b/muagent/llm_models/embeddings/__init__.py similarity index 100% rename from muagent/embeddings/__init__.py rename to muagent/llm_models/embeddings/__init__.py diff --git a/muagent/embeddings/get_embedding.py b/muagent/llm_models/get_embedding.py similarity index 96% rename from muagent/embeddings/get_embedding.py rename to muagent/llm_models/get_embedding.py index 8619e25..307b0f6 100644 --- a/muagent/embeddings/get_embedding.py +++ b/muagent/llm_models/get_embedding.py @@ -8,8 +8,8 @@ from loguru import logger # from configs.model_config import EMBEDDING_MODEL -from muagent.embeddings.openai_embedding import OpenAIEmbedding -from muagent.embeddings.huggingface_embedding import HFEmbedding +from muagent.llm_models.openai_embedding import OpenAIEmbedding +from muagent.llm_models.huggingface_embedding import HFEmbedding from muagent.llm_models.llm_config import EmbedConfig def get_embedding( @@ -35,7 +35,7 @@ def get_embedding( oae = OpenAIEmbedding() emb_res = oae.get_emb(text_list) elif engine == 'model': - hfe = HFEmbedding(model_path, embedding_device) + hfe = HFEmbedding(model_path, embed_config.model_device) emb_res = hfe.get_emb(text_list) return emb_res diff --git a/muagent/embeddings/huggingface_embedding.py b/muagent/llm_models/huggingface_embedding.py similarity index 98% rename from muagent/embeddings/huggingface_embedding.py rename to muagent/llm_models/huggingface_embedding.py index 1b6d5d0..5568241 100644 --- a/muagent/embeddings/huggingface_embedding.py +++ b/muagent/llm_models/huggingface_embedding.py @@ -8,7 +8,7 @@ from loguru import logger # from configs.model_config import EMBEDDING_DEVICE # from configs.model_config import embedding_model_dict -from muagent.embeddings.utils import load_embeddings, load_embeddings_from_path +from muagent.retrieval.utils import load_embeddings, load_embeddings_from_path class HFEmbedding: diff --git a/muagent/llm_models/llm_config.py b/muagent/llm_models/llm_config.py index 9dac682..504639d 100644 --- a/muagent/llm_models/llm_config.py +++ b/muagent/llm_models/llm_config.py @@ -11,6 +11,7 @@ class LLMConfig: def __init__( self, model_name: str = "gpt-3.5-turbo", + model_engine: str = "openai", temperature: float = 0.25, stop: Union[List[str], str] = None, api_key: str = "", @@ -19,12 +20,15 @@ def __init__( llm: LLM = None, **kwargs ): - + # only support http connection with others + # llm_model init config self.model_name: str = model_name + self.model_engine: str = model_engine self.temperature: float = temperature self.stop: Union[List[str], str] = stop self.api_key: str = api_key self.api_base_url: str = api_base_url + # custom llm self.llm: LLM = llm # self.check_config() @@ -55,7 +59,7 @@ def __init__( self.model_device: str = model_device self.api_key: str = api_key self.api_base_url: str = api_base_url - # + # custom embeddings self.langchain_embeddings = langchain_embeddings # self.check_config() diff --git a/muagent/embeddings/openai_embedding.py b/muagent/llm_models/openai_embedding.py similarity index 82% rename from muagent/embeddings/openai_embedding.py rename to muagent/llm_models/openai_embedding.py index 180f654..17c70da 100644 --- a/muagent/embeddings/openai_embedding.py +++ b/muagent/llm_models/openai_embedding.py @@ -5,7 +5,9 @@ @time: 2023/11/22 上午10:45 @desc: ''' + import openai +from openai import OpenAI import base64 import json import os @@ -14,18 +16,34 @@ class OpenAIEmbedding: def __init__(self): - pass + + try: + from zdatafront import ZDataFrontClient + from zdatafront.openai import SyncProxyHttpClient + # zdatafront 分配的业务标记 + VISIT_DOMAIN = os.environ.get("visit_domain") + VISIT_BIZ = os.environ.get("visit_biz") + VISIT_BIZ_LINE = os.environ.get("visit_biz_line") + # zdatafront 提供的统一加密密钥 + aes_secret_key = os.environ.get("aes_secret_key") + + zdatafront_client = ZDataFrontClient(visit_domain=VISIT_DOMAIN, visit_biz=VISIT_BIZ, visit_biz_line=VISIT_BIZ_LINE, aes_secret_key=aes_secret_key) + self.http_client = SyncProxyHttpClient(zdatafront_client=zdatafront_client, prefer_async=True) + except Exception as e: + logger.warning("There is no zdatafront, just as openai config") + self.http_client = None def get_emb(self, text_list): openai.api_key = os.environ["OPENAI_API_KEY"] - openai.api_base = os.environ["API_BASE_URL"] + openai.base_url = os.environ["API_BASE_URL"] # change , to ，to avoid bug modified_text_list = [i.replace(',', '，') for i in text_list] + client = OpenAI(http_client=self.http_client) - emb_all_result = openai.Embedding.create( + emb_all_result = client.embeddings.create( + input=modified_text_list, model="text-embedding-ada-002", - input=modified_text_list ) res = {} diff --git a/muagent/llm_models/openai_model.py b/muagent/llm_models/openai_model.py index fa19255..ead3b1c 100644 --- a/muagent/llm_models/openai_model.py +++ b/muagent/llm_models/openai_model.py @@ -3,7 +3,8 @@ from loguru import logger from langchain.callbacks import AsyncIteratorCallbackHandler -from langchain.chat_models import ChatOpenAI +# from langchain.chat_models import ChatOpenAI +from langchain_openai import ChatOpenAI from langchain.llms.base import LLM from .llm_config import LLMConfig @@ -21,57 +22,107 @@ def __call__(self, prompt: str, def _call(self, prompt: str, stop: Optional[List[str]] = None): - return self.llm(prompt, stop) + return self(prompt, stop) def predict(self, prompt: str, stop: Optional[List[str]] = None): - return self.llm(prompt, stop) + return self(prompt, stop) def batch(self, prompts: str, stop: Optional[List[str]] = None): - return [self.llm(prompt, stop) for prompt in prompts] + return [self(prompt, stop) for prompt in prompts] + + + +class OpenAILLMModel(CustomLLMModel): + + def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None,): + # logger.debug(f"llm type is {type(llm_config.llm)}") + try: + from zdatafront import ZDataFrontClient + from zdatafront.openai import SyncProxyHttpClient + # zdatafront 分配的业务标记 + VISIT_DOMAIN = os.environ.get("visit_domain") + VISIT_BIZ = os.environ.get("visit_biz") + VISIT_BIZ_LINE = os.environ.get("visit_biz_line") + # zdatafront 提供的统一加密密钥 + aes_secret_key = os.environ.get("aes_secret_key") + + zdatafront_client = ZDataFrontClient(visit_domain=VISIT_DOMAIN, visit_biz=VISIT_BIZ, visit_biz_line=VISIT_BIZ_LINE, aes_secret_key=aes_secret_key) + http_client = SyncProxyHttpClient(zdatafront_client=zdatafront_client, prefer_async=True) + except Exception as e: + logger.warning("There is no zdatafront, you just do as openai config") + http_client = None + + if llm_config is None: + self.llm = ChatOpenAI( + streaming=True, + verbose=True, + api_key=os.environ.get("api_key"), + base_url=os.environ.get("api_base_url"), + model_name=os.environ.get("LLM_MODEL", "gpt-3.5-turbo"), + temperature=os.environ.get("temperature", 0.5), + model_kwargs={"stop": os.environ.get("stop", "")}, + http_client=http_client + ) + else: + self.llm = ChatOpenAI( + streaming=True, + verbose=True, + model_name=llm_config.model_name, + temperature=llm_config.temperature, + model_kwargs={"stop": llm_config.stop}, + http_client=http_client, + # callbacks=[callBack], + ) + if callBack is not None: + self.llm.callBacks = [callBack] + + def __call__(self, prompt: str, + stop: Optional[List[str]] = None): + return self.llm.predict(prompt, stop=stop) + + +class LYWWLLMModel(OpenAILLMModel): + + def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None,): + if llm_config is None: + api_key=os.environ.get("api_key") + base_url=os.environ.get("api_base_url") + model_name=os.environ.get("LLM_MODEL", "yi-34b-chat-0205") + temperature=os.environ.get("temperature", 0.5) + model_kwargs={"stop": os.environ.get("stop", "")} + else: + api_key=llm_config.api_key + base_url=llm_config.api_base_url + model_name=llm_config.model_name + temperature=llm_config.temperature + model_kwargs={"stop": llm_config.stop} + + self.llm = ChatOpenAI( + streaming=True, + verbose=True, + api_key=api_key, + base_url=base_url, + model_name=model_name, + temperature=temperature, + model_kwargs=model_kwargs, + ) + def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None, ) -> Union[ChatOpenAI, LLM]: - # logger.debug(f"llm type is {type(llm_config.llm)}") - if llm_config is None: - model = ChatOpenAI( - streaming=True, - verbose=True, - openai_api_key=os.environ.get("api_key"), - openai_api_base=os.environ.get("api_base_url"), - model_name=os.environ.get("LLM_MODEL", "gpt-3.5-turbo"), - temperature=os.environ.get("temperature", 0.5), - stop=os.environ.get("stop", ""), - ) - return model if llm_config and llm_config.llm and isinstance(llm_config.llm, LLM): return CustomLLMModel(llm=llm_config.llm) - - if callBack is None: - model = ChatOpenAI( - streaming=True, - verbose=True, - openai_api_key=llm_config.api_key, - openai_api_base=llm_config.api_base_url, - model_name=llm_config.model_name, - temperature=llm_config.temperature, - stop=llm_config.stop - ) + elif llm_config: + model_class_dict = {"openai": OpenAILLMModel, "lingyiwanwu": LYWWLLMModel} + model_class = model_class_dict[llm_config.model_engine] + model = model_class(llm_config, callBack) + logger.debug(f"{model}") + return model else: - model = ChatOpenAI( - streaming=True, - verbose=True, - callBack=[callBack], - openai_api_key=llm_config.api_key, - openai_api_base=llm_config.api_base_url, - model_name=llm_config.model_name, - temperature=llm_config.temperature, - stop=llm_config.stop - ) - - return model + return OpenAILLMModel(llm_config, callBack) import json, requests diff --git a/muagent/embeddings/commands/__init__.py b/muagent/retrieval/commands/__init__.py similarity index 100% rename from muagent/embeddings/commands/__init__.py rename to muagent/retrieval/commands/__init__.py diff --git a/muagent/embeddings/commands/default_vs_cds.py b/muagent/retrieval/commands/default_vs_cds.py similarity index 100% rename from muagent/embeddings/commands/default_vs_cds.py rename to muagent/retrieval/commands/default_vs_cds.py diff --git a/muagent/retrieval/document_loaders/json_loader.py b/muagent/retrieval/document_loaders/json_loader.py index 574e078..7f6ab95 100644 --- a/muagent/retrieval/document_loaders/json_loader.py +++ b/muagent/retrieval/document_loaders/json_loader.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import AnyStr, Callable, Dict, List, Optional, Union -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document from langchain.document_loaders.base import BaseLoader from langchain.text_splitter import RecursiveCharacterTextSplitter, TextSplitter diff --git a/muagent/retrieval/document_loaders/jsonl_loader.py b/muagent/retrieval/document_loaders/jsonl_loader.py index bec8033..e9a9135 100644 --- a/muagent/retrieval/document_loaders/jsonl_loader.py +++ b/muagent/retrieval/document_loaders/jsonl_loader.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import AnyStr, Callable, Dict, List, Optional, Union -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document from langchain.document_loaders.base import BaseLoader from langchain.text_splitter import RecursiveCharacterTextSplitter, TextSplitter diff --git a/muagent/embeddings/faiss_m.py b/muagent/retrieval/faiss_m.py similarity index 96% rename from muagent/embeddings/faiss_m.py rename to muagent/retrieval/faiss_m.py index d7b6721..7f3fc69 100644 --- a/muagent/embeddings/faiss_m.py +++ b/muagent/retrieval/faiss_m.py @@ -21,13 +21,13 @@ import numpy as np -from langchain.docstore.base import AddableMixin, Docstore -from langchain.docstore.document import Document -# from langchain.docstore.in_memory import InMemoryDocstore +from langchain_community.docstore.base import AddableMixin, Docstore +from langchain_community.docstore.document import Document +# from langchain_community.docstore.in_memory import InMemoryDocstore from .in_memory import InMemoryDocstore from langchain.embeddings.base import Embeddings -from langchain.vectorstores.base import VectorStore -from langchain.vectorstores.utils import maximal_marginal_relevance +from langchain_community.vectorstores import VectorStore +from langchain_community.vectorstores.utils import maximal_marginal_relevance class DistanceStrategy(str, Enum): @@ -86,7 +86,7 @@ class FAISS(VectorStore): .. code-block:: python from langchain.embeddings.openai import OpenAIEmbeddings - from langchain.vectorstores import FAISS + from langchain_community.vectorstores import FAISS embeddings = OpenAIEmbeddings() texts = ["FAISS is an important library", "LangChain supports FAISS"] diff --git a/muagent/embeddings/in_memory.py b/muagent/retrieval/in_memory.py similarity index 89% rename from muagent/embeddings/in_memory.py rename to muagent/retrieval/in_memory.py index f92484d..342bd5e 100644 --- a/muagent/embeddings/in_memory.py +++ b/muagent/retrieval/in_memory.py @@ -1,8 +1,8 @@ """Simple in memory docstore in the form of a dict.""" from typing import Dict, List, Optional, Union -from langchain.docstore.base import AddableMixin, Docstore -from langchain.docstore.document import Document +from langchain_community.docstore.base import AddableMixin, Docstore +from langchain_community.docstore.document import Document class InMemoryDocstore(Docstore, AddableMixin): diff --git a/muagent/embeddings/utils.py b/muagent/retrieval/utils.py similarity index 92% rename from muagent/embeddings/utils.py rename to muagent/retrieval/utils.py index 25088b1..9a42dbf 100644 --- a/muagent/embeddings/utils.py +++ b/muagent/retrieval/utils.py @@ -1,6 +1,6 @@ import os from functools import lru_cache -from langchain.embeddings.huggingface import HuggingFaceEmbeddings +from langchain_huggingface import HuggingFaceEmbeddings from langchain.embeddings.base import Embeddings # from configs.model_config import embedding_model_dict diff --git a/muagent/service/base_service.py b/muagent/service/base_service.py index 76248fb..b180918 100644 --- a/muagent/service/base_service.py +++ b/muagent/service/base_service.py @@ -3,7 +3,7 @@ import os from langchain.embeddings.base import Embeddings -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document # from configs.model_config import ( # kbs_config, VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD, @@ -16,7 +16,7 @@ from muagent.orm.commands import * from muagent.utils.path_utils import * from muagent.orm.utils import DocumentFile -from muagent.embeddings.utils import load_embeddings, load_embeddings_from_path +from muagent.retrieval.utils import load_embeddings, load_embeddings_from_path from muagent.retrieval.text_splitter import LCTextSplitter from muagent.llm_models.llm_config import EmbedConfig diff --git a/muagent/service/cb_api.py b/muagent/service/cb_api.py index 392d7dd..0c60abb 100644 --- a/muagent/service/cb_api.py +++ b/muagent/service/cb_api.py @@ -12,7 +12,7 @@ from fastapi.responses import StreamingResponse, FileResponse from fastapi import File, Form, Body, Query, UploadFile -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document from .service_factory import KBServiceFactory from muagent.utils.server_utils import BaseResponse, ListResponse diff --git a/muagent/service/faiss_db_service.py b/muagent/service/faiss_db_service.py index 499b793..77720f8 100644 --- a/muagent/service/faiss_db_service.py +++ b/muagent/service/faiss_db_service.py @@ -4,10 +4,10 @@ from functools import lru_cache from loguru import logger -# from langchain.vectorstores import FAISS +# from langchain_community.vectorstores import FAISS from langchain.embeddings.base import Embeddings -from langchain.docstore.document import Document -from langchain.embeddings.huggingface import HuggingFaceEmbeddings +from langchain_community.docstore.document import Document +from langchain_huggingface import HuggingFaceEmbeddings from muagent.base_configs.env_config import ( KB_ROOT_PATH, @@ -18,8 +18,8 @@ from muagent.utils.path_utils import * from muagent.orm.utils import DocumentFile from muagent.utils.server_utils import torch_gc -from muagent.embeddings.utils import load_embeddings, load_embeddings_from_path -from muagent.embeddings.faiss_m import FAISS +from muagent.retrieval.utils import load_embeddings, load_embeddings_from_path +from muagent.retrieval.faiss_m import FAISS from muagent.llm_models.llm_config import EmbedConfig diff --git a/muagent/service/kb_api.py b/muagent/service/kb_api.py index 002c16b..6462c07 100644 --- a/muagent/service/kb_api.py +++ b/muagent/service/kb_api.py @@ -8,7 +8,7 @@ from fastapi.responses import StreamingResponse, FileResponse from fastapi import Body, File, Form, Body, Query, UploadFile -from langchain.docstore.document import Document +from langchain_community.docstore.document import Document from .service_factory import KBServiceFactory from muagent.utils.server_utils import BaseResponse, ListResponse diff --git a/muagent/utils/path_utils.py b/muagent/utils/path_utils.py index 4df121a..8d48cfc 100644 --- a/muagent/utils/path_utils.py +++ b/muagent/utils/path_utils.py @@ -1,5 +1,5 @@ import os -from langchain.document_loaders import CSVLoader, PyPDFLoader, UnstructuredFileLoader, TextLoader, PythonLoader +from langchain_community.document_loaders import CSVLoader, PyPDFLoader, UnstructuredFileLoader, TextLoader, PythonLoader from muagent.retrieval.document_loaders import JSONLLoader, JSONLoader # from configs.model_config import ( diff --git a/requirements.txt b/requirements.txt index ef0698c..f129643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ -openai==0.28.1 -langchain<=0.0.266 +openai +langchain==0.2.3 +langchain_community==0.2.4 +langchain_openai==0.1.8 +langchain_huggingface==1.3.0 sentence_transformers loguru -fastapi~=0.99.1 +# fastapi~=0.99.1 +fastapi pandas jieba psutil @@ -17,4 +21,7 @@ SQLAlchemy==2.0.19 docker redis==5.0.1 pydantic<=1.10.14 +# pydantic # duckduckgo-search + +sseclient \ No newline at end of file diff --git a/tests/codechat/codebasehander_test.py b/tests/codechat/codebasehander_test.py index b6f4d2c..b885247 100644 --- a/tests/codechat/codebasehander_test.py +++ b/tests/codechat/codebasehander_test.py @@ -23,8 +23,11 @@ embed_model_path = "" logger.error(f"{e}") - - +# local debug +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler from muagent.base_configs.env_config import CB_ROOT_PATH @@ -63,4 +66,5 @@ # search_type = [tag, cypher, description] if you have llm and nebula-api -code_text, related_vertex = cbh.search_code(query="remove函数做什么", search_type="tag", limit = 3) \ No newline at end of file +code_text, related_vertex = cbh.search_code(query="remove函数做什么", search_type="tag", limit = 3) +print(code_text) \ No newline at end of file diff --git a/tests/connector/agent_test.py b/tests/connector/agent_test.py index 137758e..f4b4c36 100644 --- a/tests/connector/agent_test.py +++ b/tests/connector/agent_test.py @@ -24,11 +24,10 @@ embed_model_path = "" logger.error(f"{e}") -# src_dir = os.path.join( -# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# ) - -# sys.path.append(src_dir) +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) from muagent.connector.agents import BaseAgent, ReactAgent, ExecutorAgent, SelectorAgent from muagent.connector.schema import Role, Message from muagent.llm_models.llm_config import EmbedConfig, LLMConfig @@ -191,4 +190,4 @@ # base_agent.pre_print(query) output_message = base_agent.step(query) print(output_message.input_query) -print(output_message.role_content) +print(output_message.parsed_output_list) diff --git a/tests/connector/chain_test.py b/tests/connector/chain_test.py index 427a1f1..649b50f 100644 --- a/tests/connector/chain_test.py +++ b/tests/connector/chain_test.py @@ -27,7 +27,6 @@ src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) - sys.path.append(src_dir) from muagent.base_configs.env_config import JUPYTER_WORK_PATH diff --git a/tests/connector/memory_manager_test.py b/tests/connector/memory_manager_test.py index 24a6656..5518ea7 100644 --- a/tests/connector/memory_manager_test.py +++ b/tests/connector/memory_manager_test.py @@ -33,7 +33,7 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_type="openai", api_key=api_key, api_base_url=api_base_url, temperature=0.3, ) embed_config = EmbedConfig( diff --git a/tests/connector/phase_test.py b/tests/connector/phase_test.py index 4ea3cfe..158dd55 100644 --- a/tests/connector/phase_test.py +++ b/tests/connector/phase_test.py @@ -27,7 +27,6 @@ src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) - sys.path.append(src_dir) from muagent.base_configs.env_config import JUPYTER_WORK_PATH diff --git a/tests/llm_models/openai_test.py b/tests/llm_models/openai_test.py index 795ff85..d8c9ea8 100644 --- a/tests/llm_models/openai_test.py +++ b/tests/llm_models/openai_test.py @@ -24,9 +24,8 @@ - -from langchain.chat_models import ChatOpenAI -from langchain import PromptTemplate, LLMChain +# test 1 +from langchain_openai import ChatOpenAI from langchain.prompts.chat import ChatPromptTemplate model = ChatOpenAI( streaming=True, @@ -39,34 +38,12 @@ # test 1 print(model.predict("please output 123!")) -# # test 2 -# chat_prompt = ChatPromptTemplate.from_messages([("human", "{input}")]) -# chain = LLMChain(prompt=chat_prompt, llm=model) -# content = chain({"input": "who are you!"}) -# print(content) -# test 3 -# import openai -# # openai.api_key = "EMPTY" # Not support yet -# openai.api_base = api_base_url -# # create a chat completion -# completion = openai.ChatCompletion.create( -# model=model_name, -# messages=[{"role": "user", "content": "Hello! What is your name? "}], -# max_tokens=100, -# ) -# # print the completion -# print(completion.choices[0].message.content) - -# import openai -# # openai.api_key = "EMPTY" # Not support yet -# openai.api_base = "http://127.0.0.1:8888/v1" -# model = "example" -# # create a chat completion -# completion = openai.ChatCompletion.create( -# model=model, -# messages=[{"role": "user", "content": "Hello! What is your name? "}], -# max_tokens=100, -# ) -# # print the completion -# print(completion.choices[0].message.content) \ No newline at end of file +# # test 2 +# from openai import OpenAI +# http_client = None +# client = OpenAI(api_key=os.environ.get("api_key"), http_client=http_client) +# model = 'gpt-3.5-turbo' +# messages=[{'role': 'user', 'content': 'Hello World'}] +# result = client.chat.completions.create(model=model, messages=messages) +# print(result) From e19e3bb8e08f6504de37dc70efd2f4d6bb435911 Mon Sep 17 00:00:00 2001 From: shanshi Date: Sun, 16 Jun 2024 14:35:21 +0800 Subject: [PATCH 002/128] update open interface params --- Dockerfile | 20 ++++ docker_build.sh | 3 + .../muagent_examples/baseGroup_example.py | 8 +- .../muagent_examples/baseTask_examples.py | 16 +++- .../muagent_examples/codeGenDoc_example.py | 22 +++-- ...example_copy.py => codeGenTest_example.py} | 18 +++- .../muagent_examples/codeReact_example.py | 20 +++- .../muagent_examples/codeRetrieval_example.py | 20 +++- .../muagent_examples/codeToolReact_example.py | 20 +++- examples/muagent_examples/codechat_example.py | 52 ++++++---- examples/muagent_examples/docchat_example.py | 20 +++- examples/muagent_examples/load_codebase.py | 21 ++-- examples/muagent_examples/metagpt_example.py | 19 +++- examples/muagent_examples/search_example.py | 21 +++- .../muagent_examples/toolReact_example.py | 12 ++- examples/start.py | 5 +- examples/test_config.py.example | 1 + muagent/chat/agent_chat.py | 44 +++++---- muagent/chat/base_chat.py | 44 +++++---- muagent/chat/code_chat.py | 50 ++++++---- muagent/chat/knowledge_chat.py | 12 ++- muagent/chat/search_chat.py | 2 +- muagent/llm_models/openai_model.py | 8 +- muagent/service/cb_api.py | 63 ++++++------ muagent/service/kb_api.py | 96 ++++++++++--------- muagent/tools/cb_query_tool.py | 12 ++- muagent/tools/codechat_tools.py | 15 ++- muagent/tools/docs_retrieval.py | 9 +- requirements.txt | 4 +- tests/connector/agent_test.py | 11 ++- tests/connector/chain_test.py | 5 +- tests/connector/flow_test.py | 10 +- tests/connector/memory_manager_test.py | 6 +- tests/connector/phase_test.py | 6 +- tests/test_config.py.example | 2 +- 35 files changed, 468 insertions(+), 229 deletions(-) create mode 100644 Dockerfile create mode 100644 docker_build.sh rename examples/muagent_examples/{codeGenTest_example_copy.py => codeGenTest_example.py} (93%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2ac6041 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +From python:3.9.18-bookworm + +WORKDIR /home/user + +COPY ./requirements.txt /home/user/docker_requirements.txt + + +# RUN apt-get update +# RUN apt-get install -y iputils-ping telnetd net-tools vim tcpdump +# RUN echo telnet stream tcp nowait telnetd /usr/sbin/tcpd /usr/sbin/in.telnetd /etc/inetd.conf +# RUN service inetutils-inetd start +# service inetutils-inetd status + +RUN wget https://oss-cdn.nebula-graph.com.cn/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb +RUN dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb + +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install -r /home/user/docker_requirements.txt + +CMD ["bash"] \ No newline at end of file diff --git a/docker_build.sh b/docker_build.sh new file mode 100644 index 0000000..ac6dfc1 --- /dev/null +++ b/docker_build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t muagent:0.0.1 . \ No newline at end of file diff --git a/examples/muagent_examples/baseGroup_example.py b/examples/muagent_examples/baseGroup_example.py index 84973e4..315f525 100644 --- a/examples/muagent_examples/baseGroup_example.py +++ b/examples/muagent_examples/baseGroup_example.py @@ -13,6 +13,7 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] try: from test_config import BgeBaseChineseEmbeddings @@ -24,12 +25,17 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" embeddings = None logger.error(f"{e}") - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import JUPYTER_WORK_PATH from muagent.tools import toLangchainTools, TOOL_DICT, TOOL_SETS from muagent.llm_models.llm_config import EmbedConfig, LLMConfig diff --git a/examples/muagent_examples/baseTask_examples.py b/examples/muagent_examples/baseTask_examples.py index c88ffd9..1e367c7 100644 --- a/examples/muagent_examples/baseTask_examples.py +++ b/examples/muagent_examples/baseTask_examples.py @@ -13,16 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase from muagent.connector.schema import Message @@ -32,7 +46,7 @@ os.environ["log_verbose"] = "0" llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/codeGenDoc_example.py b/examples/muagent_examples/codeGenDoc_example.py index 63113e8..28be9d4 100644 --- a/examples/muagent_examples/codeGenDoc_example.py +++ b/examples/muagent_examples/codeGenDoc_example.py @@ -1,5 +1,4 @@ import os -import json from loguru import logger try: @@ -14,22 +13,31 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") import sys, os -src_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -) -print(src_dir) -sys.path.append(src_dir) +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import CB_ROOT_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -151,7 +159,7 @@ def start_action_step(self, message: Message) -> Message: llm_config = LLMConfig( - model_name="gpt-4", api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path diff --git a/examples/muagent_examples/codeGenTest_example_copy.py b/examples/muagent_examples/codeGenTest_example.py similarity index 93% rename from examples/muagent_examples/codeGenTest_example_copy.py rename to examples/muagent_examples/codeGenTest_example.py index ecfd27f..4a245cc 100644 --- a/examples/muagent_examples/codeGenTest_example_copy.py +++ b/examples/muagent_examples/codeGenTest_example.py @@ -1,5 +1,4 @@ import os -import json from loguru import logger try: @@ -14,15 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") + +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import CB_ROOT_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -162,7 +176,7 @@ def start_action_step(self, message: Message) -> Message: llm_config = LLMConfig( - model_name="gpt-4", api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path diff --git a/examples/muagent_examples/codeReact_example.py b/examples/muagent_examples/codeReact_example.py index 19868fd..5a0e155 100644 --- a/examples/muagent_examples/codeReact_example.py +++ b/examples/muagent_examples/codeReact_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,16 +13,29 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import JUPYTER_WORK_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -31,7 +45,7 @@ os.environ["log_verbose"] = "0" llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/codeRetrieval_example.py b/examples/muagent_examples/codeRetrieval_example.py index 1a7a232..6174f0e 100644 --- a/examples/muagent_examples/codeRetrieval_example.py +++ b/examples/muagent_examples/codeRetrieval_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,16 +13,29 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.agents import BaseAgent, ReactAgent, ExecutorAgent, SelectorAgent from muagent.connector.chains import BaseChain @@ -132,7 +146,7 @@ def end_action_step(self, message: Message) -> Message: # llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/codeToolReact_example.py b/examples/muagent_examples/codeToolReact_example.py index 20395cf..effa33b 100644 --- a/examples/muagent_examples/codeToolReact_example.py +++ b/examples/muagent_examples/codeToolReact_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,16 +13,29 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.tools import toLangchainTools, TOOL_DICT, TOOL_SETS from muagent.llm_models.llm_config import EmbedConfig, LLMConfig @@ -38,7 +52,7 @@ os.environ["log_verbose"] = "0" llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/codechat_example.py b/examples/muagent_examples/codechat_example.py index 241aae3..9fefc95 100644 --- a/examples/muagent_examples/codechat_example.py +++ b/examples/muagent_examples/codechat_example.py @@ -13,15 +13,25 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") +# # test local code # src_dir = os.path.join( # os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ) @@ -38,7 +48,7 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( @@ -70,26 +80,26 @@ ) -# round-1 -query_content = "代码一共有多少类" -query = Message( - chat_index="codechat_test", role_name="human", role_type="user", input_query=query_content, - code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="cypher", - local_graph_path=CB_ROOT_PATH, use_nh=use_nh - ) - -output_message1, output_memory1 = phase.step(query) -print(output_memory1.to_str_messages(return_all=True, content_key="parsed_output_list")) - -# round-2 -query_content = "代码库里有哪些函数，返回5个就行" -query = Message( - chat_index="codechat_test", role_name="human", role_type="user", input_query=query_content, - code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="cypher", - local_graph_path=CB_ROOT_PATH, use_nh=use_nh - ) -output_message2, output_memory2 = phase.step(query) -print(output_memory2.to_str_messages(return_all=True, content_key="parsed_output_list")) +# # round-1 +# query_content = "代码一共有多少类" +# query = Message( +# chat_index="codechat_test", role_name="human", role_type="user", input_query=query_content, +# code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="cypher", +# local_graph_path=CB_ROOT_PATH, use_nh=use_nh +# ) + +# output_message1, output_memory1 = phase.step(query) +# print(output_memory1.to_str_messages(return_all=True, content_key="parsed_output_list")) + +# # round-2 +# query_content = "代码库里有哪些函数，返回5个就行" +# query = Message( +# chat_index="codechat_test", role_name="human", role_type="user", input_query=query_content, +# code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="cypher", +# local_graph_path=CB_ROOT_PATH, use_nh=use_nh +# ) +# output_message2, output_memory2 = phase.step(query) +# print(output_memory2.to_str_messages(return_all=True, content_key="parsed_output_list")) # # round-3 diff --git a/examples/muagent_examples/docchat_example.py b/examples/muagent_examples/docchat_example.py index 8c7f982..1ba7ed4 100644 --- a/examples/muagent_examples/docchat_example.py +++ b/examples/muagent_examples/docchat_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,17 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import KB_ROOT_PATH from muagent.tools import toLangchainTools, TOOL_DICT, TOOL_SETS from muagent.llm_models.llm_config import EmbedConfig, LLMConfig @@ -35,7 +49,7 @@ # llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/load_codebase.py b/examples/muagent_examples/load_codebase.py index e8074a5..1267c80 100644 --- a/examples/muagent_examples/load_codebase.py +++ b/examples/muagent_examples/load_codebase.py @@ -13,21 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") -src_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -) -print(src_dir) -sys.path.append(src_dir) +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.base_configs.env_config import CB_ROOT_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -40,7 +49,7 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/metagpt_example.py b/examples/muagent_examples/metagpt_example.py index f9336cd..455549b 100644 --- a/examples/muagent_examples/metagpt_example.py +++ b/examples/muagent_examples/metagpt_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,16 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -32,7 +47,7 @@ # llm_config = LLMConfig( - model_name="gpt-4", api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/search_example.py b/examples/muagent_examples/search_example.py index 7a686b0..068bd0d 100644 --- a/examples/muagent_examples/search_example.py +++ b/examples/muagent_examples/search_example.py @@ -1,7 +1,8 @@ -import os, sys, json +import os from loguru import logger try: + import os, sys src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) @@ -12,18 +13,30 @@ model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] + + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" + embeddings = None logger.error(f"{e}") - - +# # test local code +# src_dir = os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ) +# sys.path.append(src_dir) from muagent.tools import toLangchainTools, TOOL_DICT, TOOL_SETS from muagent.llm_models.llm_config import EmbedConfig, LLMConfig @@ -37,7 +50,7 @@ # llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) embed_config = EmbedConfig( diff --git a/examples/muagent_examples/toolReact_example.py b/examples/muagent_examples/toolReact_example.py index 3632fc6..872ceeb 100644 --- a/examples/muagent_examples/toolReact_example.py +++ b/examples/muagent_examples/toolReact_example.py @@ -8,24 +8,30 @@ ) sys.path.append(src_dir) import test_config - from test_config import BgeBaseChineseEmbeddings api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] + model_engine = os.environ["model_engine"] - embeddings = BgeBaseChineseEmbeddings() + try: + from test_config import BgeBaseChineseEmbeddings + embeddings = BgeBaseChineseEmbeddings() + except: + embeddings = None except Exception as e: # set your config api_key = "" api_base_url= "" model_name = "" + model_engine = "" embed_model = "" embed_model_path = "" embeddings = None logger.error(f"{e}") +# test local code src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -38,7 +44,7 @@ # llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) if embeddings: diff --git a/examples/start.py b/examples/start.py index e3bb358..524ea16 100644 --- a/examples/start.py +++ b/examples/start.py @@ -14,7 +14,7 @@ DEFAULT_BIND_HOST = "127.0.0.1" os.environ["DEFAULT_BIND_HOST"] = DEFAULT_BIND_HOST CONTRAINER_NAME = "muagent" -IMAGE_NAME = "devopsgpt:py39" +IMAGE_NAME = "muagent:latest" SANDBOX_CONTRAINER_NAME = "devopsgpt_sandbox" SANDBOX_IMAGE_NAME = "devopsgpt:py39" SANDBOX_HOST = os.environ.get("SANDBOX_HOST") or DEFAULT_BIND_HOST # "172.25.0.3" @@ -213,8 +213,7 @@ def start_api_service(sandbox_host=DEFAULT_BIND_HOST): '''curl -X PUT -H "Content-Type: application/json" -d'{"heartbeat_interval_secs":"2"}' -s "http://127.0.0.1:19669/flags"''', '''curl -X PUT -H "Content-Type: application/json" -d'{"heartbeat_interval_secs":"2"}' -s "http://127.0.0.1:19779/flags"''', - "pip install zdatafront-sdk-python==0.1.2 -i https://artifacts.antgroup-inc.cn/simple", - "pip install jieba", + "pip install zdatafront-sdk-python -i https://artifacts.antgroup-inc.cn/simple", "pip install duckduckgo-search", f"export DUCKDUCKGO_PROXY=socks5://host.docker.internal:13659 && export SANDBOX_HOST={sandbox_host}", diff --git a/examples/test_config.py.example b/examples/test_config.py.example index 01e99ca..1b96b3f 100644 --- a/examples/test_config.py.example +++ b/examples/test_config.py.example @@ -7,6 +7,7 @@ os.environ["API_BASE_URL"] = OPENAI_API_BASE os.environ["OPENAI_API_KEY"] = "sk-xxx" openai.api_key = "sk-xxx" os.environ["model_name"] = "gpt-3.5-turbo" +os.environ["model_engine] = "openai" # os.environ["embed_model"] = "{{embed_model_name}}" diff --git a/muagent/chat/agent_chat.py b/muagent/chat/agent_chat.py index 79628ee..618b404 100644 --- a/muagent/chat/agent_chat.py +++ b/muagent/chat/agent_chat.py @@ -70,14 +70,16 @@ def chat( kb_root_path: str = Body("", description="知识库存储路径"), jupyter_work_path: str = Body("", description="sandbox执行环境"), sandbox_server: str = Body({}, description="代码历史相关节点"), - api_key: str = Body(os.environ.get("OPENAI_API_KEY"), description=""), - api_base_url: str = Body(os.environ.get("API_BASE_URL"),), - embed_model: str = Body("", description="向量模型"), - embed_model_path: str = Body("", description="向量模型路径"), - model_device: str = Body("", description="模型加载设备"), - embed_engine: str = Body("", description="向量模型类型"), - model_name: str = Body("", description="llm模型名称"), - temperature: float = Body(0.2, description=""), + # api_key: str = Body(os.environ.get("OPENAI_API_KEY"), description=""), + # api_base_url: str = Body(os.environ.get("API_BASE_URL"),), + # embed_model: str = Body("", description="向量模型"), + # embed_model_path: str = Body("", description="向量模型路径"), + # model_device: str = Body("", description="模型加载设备"), + # embed_engine: str = Body("", description="向量模型类型"), + # model_name: str = Body("", description="llm模型名称"), + # temperature: float = Body(0.2, description=""), + llm_config: LLMConfig = Body({}, description="llm_model config"), + embed_config: EmbedConfig = Body({}, description="llm_model config"), chat_index: str = "", local_graph_path: str = "", **kargs @@ -88,8 +90,8 @@ def chat( custom_phase_configs, custom_chain_configs, custom_role_configs) params = locals() params.pop("self") - embed_config: EmbedConfig = EmbedConfig(**params) - llm_config: LLMConfig = LLMConfig(**params) + # embed_config: EmbedConfig = EmbedConfig(**params) + # llm_config: LLMConfig = LLMConfig(**params) logger.info('phase_configs={}'.format(phase_configs)) logger.info('chain_configs={}'.format(chain_configs)) @@ -216,14 +218,16 @@ def achat( kb_root_path: str = Body("", description="知识库存储路径"), jupyter_work_path: str = Body("", description="sandbox执行环境"), sandbox_server: str = Body({}, description="代码历史相关节点"), - api_key: str = Body(os.environ["OPENAI_API_KEY"], description=""), - api_base_url: str = Body(os.environ.get("API_BASE_URL"),), - embed_model: str = Body("", description="向量模型"), - embed_model_path: str = Body("", description="向量模型路径"), - model_device: str = Body("", description="模型加载设备"), - embed_engine: str = Body("", description="向量模型类型"), - model_name: str = Body("", description="llm模型名称"), - temperature: float = Body(0.2, description=""), + # api_key: str = Body(os.environ["OPENAI_API_KEY"], description=""), + # api_base_url: str = Body(os.environ.get("API_BASE_URL"),), + # embed_model: str = Body("", description="向量模型"), + # embed_model_path: str = Body("", description="向量模型路径"), + # model_device: str = Body("", description="模型加载设备"), + # embed_engine: str = Body("", description="向量模型类型"), + # model_name: str = Body("", description="llm模型名称"), + # temperature: float = Body(0.2, description=""), + llm_config: LLMConfig = Body({}, description="llm_model config"), + embed_config: EmbedConfig = Body({}, description="llm_model config"), chat_index: str = "", local_graph_path: str = "", **kargs @@ -236,8 +240,8 @@ def achat( # params = locals() params.pop("self") - embed_config: EmbedConfig = EmbedConfig(**params) - llm_config: LLMConfig = LLMConfig(**params) + # embed_config: EmbedConfig = EmbedConfig(**params) + # llm_config: LLMConfig = LLMConfig(**params) # choose tools tools = toLangchainTools([TOOL_DICT[i] for i in choose_tools if i in TOOL_DICT]) diff --git a/muagent/chat/base_chat.py b/muagent/chat/base_chat.py index 5771b40..1df3938 100644 --- a/muagent/chat/base_chat.py +++ b/muagent/chat/base_chat.py @@ -43,20 +43,22 @@ def chat( stream: bool = Body(False, description="流式输出"), local_doc_url: bool = Body(False, description="知识文件返回本地路径(true)或URL(false)"), request: Request = None, - api_key: str = Body(os.environ.get("OPENAI_API_KEY")), - api_base_url: str = Body(os.environ.get("API_BASE_URL")), - embed_model: str = Body("", ), - embed_model_path: str = Body("", ), - embed_engine: str = Body("", ), - model_name: str = Body("", ), - temperature: float = Body(0.5, ), - model_device: str = Body("", ), + # api_key: str = Body(os.environ.get("OPENAI_API_KEY")), + # api_base_url: str = Body(os.environ.get("API_BASE_URL")), + # embed_model: str = Body("", ), + # embed_model_path: str = Body("", ), + # embed_engine: str = Body("", ), + # model_name: str = Body("", ), + # temperature: float = Body(0.5, ), + # model_device: str = Body("", ), + llm_config: LLMConfig = Body({}, description="llm_model config"), + embed_config: EmbedConfig = Body({}, description="embedding_model config"), **kargs ): params = locals() params.pop("self", None) - llm_config: LLMConfig = LLMConfig(**params) - embed_config: EmbedConfig = EmbedConfig(**params) + # llm_config: LLMConfig = LLMConfig(**params) + # embed_config: EmbedConfig = EmbedConfig(**params) self.engine_name = engine_name if isinstance(engine_name, str) else engine_name.default self.top_k = top_k if isinstance(top_k, int) else top_k.default self.score_threshold = score_threshold if isinstance(score_threshold, float) else score_threshold.default @@ -106,20 +108,22 @@ def achat( stream: bool = Body(False, description="流式输出"), local_doc_url: bool = Body(False, description="知识文件返回本地路径(true)或URL(false)"), request: Request = None, - api_key: str = Body(os.environ.get("OPENAI_API_KEY")), - api_base_url: str = Body(os.environ.get("API_BASE_URL")), - embed_model: str = Body("", ), - embed_model_path: str = Body("", ), - embed_engine: str = Body("", ), - model_name: str = Body("", ), - temperature: float = Body(0.5, ), - model_device: str = Body("", ), + # api_key: str = Body(os.environ.get("OPENAI_API_KEY")), + # api_base_url: str = Body(os.environ.get("API_BASE_URL")), + # embed_model: str = Body("", ), + # embed_model_path: str = Body("", ), + # embed_engine: str = Body("", ), + # model_name: str = Body("", ), + # temperature: float = Body(0.5, ), + # model_device: str = Body("", ), + llm_config: LLMConfig = Body({}, description="llm_model config"), + embed_config: EmbedConfig = Body({}, description="llm_model config"), ): # params = locals() params.pop("self", None) - llm_config: LLMConfig = LLMConfig(**params) - embed_config: EmbedConfig = EmbedConfig(**params) + # llm_config: LLMConfig = LLMConfig(**params) + # embed_config: EmbedConfig = EmbedConfig(**params) self.engine_name = engine_name if isinstance(engine_name, str) else engine_name.default self.top_k = top_k if isinstance(top_k, int) else top_k.default self.score_threshold = score_threshold if isinstance(score_threshold, float) else score_threshold.default diff --git a/muagent/chat/code_chat.py b/muagent/chat/code_chat.py index 32fa8d1..12ffa6b 100644 --- a/muagent/chat/code_chat.py +++ b/muagent/chat/code_chat.py @@ -53,24 +53,31 @@ def check_service_status(self) -> BaseResponse: return BaseResponse(code=404, msg=f"未找到代码库 {self.engine_name}") return BaseResponse(code=200, msg=f"找到代码库 {self.engine_name}") - def _process(self, query: str, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, local_graph_path=""): + def _process(self, query: str, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, local_graph_path="", use_nh=True): '''process''' + # codes_res = search_code(query=query, cb_name=self.engine_name, code_limit=self.code_limit, + # search_type=self.cb_search_type, + # history_node_list=self.history_node_list, + # api_key=llm_config.api_key, + # api_base_url=llm_config.api_base_url, + # model_name=llm_config.model_name, + # temperature=llm_config.temperature, + # embed_model=embed_config.embed_model, + # embed_model_path=embed_config.embed_model_path, + # embed_engine=embed_config.embed_engine, + # model_device=embed_config.model_device, + # embed_config=embed_config, + # local_graph_path=local_graph_path + # ) codes_res = search_code(query=query, cb_name=self.engine_name, code_limit=self.code_limit, search_type=self.cb_search_type, history_node_list=self.history_node_list, - api_key=llm_config.api_key, - api_base_url=llm_config.api_base_url, - model_name=llm_config.model_name, - temperature=llm_config.temperature, - embed_model=embed_config.embed_model, - embed_model_path=embed_config.embed_model_path, - embed_engine=embed_config.embed_engine, - model_device=embed_config.model_device, + llm_config=llm_config, embed_config=embed_config, + use_nh=use_nh, local_graph_path=local_graph_path ) - context = codes_res['context'] related_vertices = codes_res['related_vertices'] @@ -108,21 +115,24 @@ def chat( local_doc_url: bool = Body(False, description="知识文件返回本地路径(true)或URL(false)"), request: Request = None, - api_key: str = Body(os.environ.get("OPENAI_API_KEY")), - api_base_url: str = Body(os.environ.get("API_BASE_URL")), - embed_model: str = Body("", ), - embed_model_path: str = Body("", ), - embed_engine: str = Body("", ), - model_name: str = Body("", ), - temperature: float = Body(0.5, ), - model_device: str = Body("", ), + # api_key: str = Body(os.environ.get("OPENAI_API_KEY")), + # api_base_url: str = Body(os.environ.get("API_BASE_URL")), + # embed_model: str = Body("", ), + # embed_model_path: str = Body("", ), + # embed_engine: str = Body("", ), + # model_name: str = Body("", ), + # temperature: float = Body(0.5, ), + # model_device: str = Body("", ), + llm_config: LLMConfig = Body({}, description="llm_model config"), + embed_config: EmbedConfig = Body({}, description="llm_model config"), local_graph_path: str=Body(", "), + use_nh: bool =Body(True, description=""), **kargs ): params = locals() params.pop("self") - llm_config: LLMConfig = LLMConfig(**params) - embed_config: EmbedConfig = EmbedConfig(**params) + # llm_config: LLMConfig = LLMConfig(**params) + # embed_config: EmbedConfig = EmbedConfig(**params) self.engine_name = engine_name if isinstance(engine_name, str) else engine_name.default self.code_limit = code_limit self.stream = stream if isinstance(stream, bool) else stream.default diff --git a/muagent/chat/knowledge_chat.py b/muagent/chat/knowledge_chat.py index 1558f05..4d198a3 100644 --- a/muagent/chat/knowledge_chat.py +++ b/muagent/chat/knowledge_chat.py @@ -47,11 +47,15 @@ def check_service_status(self) -> BaseResponse: def _process(self, query: str, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, ): '''process''' + # docs = search_docs( + # query, self.engine_name, self.top_k, self.score_threshold, self.kb_root_path, + # api_key=embed_config.api_key, api_base_url=embed_config.api_base_url, embed_model=embed_config.embed_model, + # embed_model_path=embed_config.embed_model_path, embed_engine=embed_config.embed_engine, + # model_device=embed_config.model_device, + # ) docs = search_docs( - query, self.engine_name, self.top_k, self.score_threshold, self.kb_root_path, - api_key=embed_config.api_key, api_base_url=embed_config.api_base_url, embed_model=embed_config.embed_model, - embed_model_path=embed_config.embed_model_path, embed_engine=embed_config.embed_engine, - model_device=embed_config.model_device, + query, self.engine_name, self.top_k, self.score_threshold, self.kb_root_path, + llm_config=llm_config, embed_config=embed_config ) context = "\n".join([doc.page_content for doc in docs]) source_documents = [] diff --git a/muagent/chat/search_chat.py b/muagent/chat/search_chat.py index cfc2b23..3854b88 100644 --- a/muagent/chat/search_chat.py +++ b/muagent/chat/search_chat.py @@ -3,7 +3,7 @@ from langchain import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler -from langchain.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper +from langchain_community.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper from langchain.prompts.chat import ChatPromptTemplate from langchain_community.docstore.document import Document diff --git a/muagent/llm_models/openai_model.py b/muagent/llm_models/openai_model.py index ead3b1c..2f32877 100644 --- a/muagent/llm_models/openai_model.py +++ b/muagent/llm_models/openai_model.py @@ -91,13 +91,15 @@ def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler base_url=os.environ.get("api_base_url") model_name=os.environ.get("LLM_MODEL", "yi-34b-chat-0205") temperature=os.environ.get("temperature", 0.5) - model_kwargs={"stop": os.environ.get("stop", "")} + stop = [os.environ.get("stop", "")] if os.environ.get("stop", "") else None + model_kwargs={"stop": stop} else: api_key=llm_config.api_key base_url=llm_config.api_base_url model_name=llm_config.model_name temperature=llm_config.temperature - model_kwargs={"stop": llm_config.stop} + stop = [llm_config.stop] if llm_config.stop else None + model_kwargs={"stop": stop} self.llm = ChatOpenAI( streaming=True, @@ -110,7 +112,6 @@ def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler ) - def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None, ) -> Union[ChatOpenAI, LLM]: if llm_config and llm_config.llm and isinstance(llm_config.llm, LLM): @@ -119,7 +120,6 @@ def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbac model_class_dict = {"openai": OpenAILLMModel, "lingyiwanwu": LYWWLLMModel} model_class = model_class_dict[llm_config.model_engine] model = model_class(llm_config, callBack) - logger.debug(f"{model}") return model else: return OpenAILLMModel(llm_config, callBack) diff --git a/muagent/service/cb_api.py b/muagent/service/cb_api.py index 0c60abb..974a32f 100644 --- a/muagent/service/cb_api.py +++ b/muagent/service/cb_api.py @@ -47,21 +47,22 @@ async def create_cb(zip_file, cb_name: str = Body(..., examples=["samples"]), code_path: str = Body(..., examples=["samples"]), do_interpret: bool = Body(..., examples=["samples"]), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), - model_name: bool = Body(..., examples=["samples"]), - temperature: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + # model_name: bool = Body(..., examples=["samples"]), + # temperature: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, embed_config: EmbedConfig = None, local_graph_path: str = '', ) -> BaseResponse: logger.info('cb_name={}, zip_path={}, do_interpret={}'.format(cb_name, code_path, do_interpret)) - embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config - llm_config: LLMConfig = LLMConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config + # llm_config: LLMConfig = LLMConfig(**locals()) # Create selected knowledge base if not validate_kb_name(cb_name): @@ -92,20 +93,21 @@ async def create_cb(zip_file, async def delete_cb( cb_name: str = Body(..., examples=["samples"]), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), - model_name: bool = Body(..., examples=["samples"]), - temperature: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + # model_name: bool = Body(..., examples=["samples"]), + # temperature: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, embed_config: EmbedConfig = None, local_graph_path: str="", ) -> BaseResponse: logger.info('cb_name={}'.format(cb_name)) - embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config - llm_config: LLMConfig = LLMConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config + # llm_config: LLMConfig = LLMConfig(**locals()) # Create selected knowledge base if not validate_kb_name(cb_name): return BaseResponse(code=403, msg="Don't attack me") @@ -136,16 +138,17 @@ def search_code(cb_name: str = Body(..., examples=["sofaboot"]), code_limit: int = Body(..., examples=['1']), search_type: str = Body(..., examples=['你好']), history_node_list: list = Body(...), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), - model_name: bool = Body(..., examples=["samples"]), - temperature: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + # model_name: bool = Body(..., examples=["samples"]), + # temperature: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), use_nh: bool = True, local_graph_path: str = CB_ROOT_PATH, + llm_config: LLMConfig = None, embed_config: EmbedConfig = None, ) -> dict: @@ -156,8 +159,8 @@ def search_code(cb_name: str = Body(..., examples=["sofaboot"]), logger.info('code_limit={}'.format(code_limit)) logger.info('search_type={}'.format(search_type)) logger.info('history_node_list={}'.format(history_node_list)) - embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config - llm_config: LLMConfig = LLMConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) if embed_config is None else embed_config + # llm_config: LLMConfig = LLMConfig(**locals()) try: # load codebase cbh = CodeBaseHandler(codebase_name=cb_name, embed_config=embed_config, llm_config=llm_config, diff --git a/muagent/service/kb_api.py b/muagent/service/kb_api.py index 6462c07..68da570 100644 --- a/muagent/service/kb_api.py +++ b/muagent/service/kb_api.py @@ -16,7 +16,7 @@ from muagent.orm.commands import * from muagent.orm.utils import DocumentFile from muagent.base_configs.env_config import KB_ROOT_PATH -from muagent.llm_models.llm_config import EmbedConfig +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.utils.server_utils import run_async async def list_kbs(): @@ -27,16 +27,17 @@ async def list_kbs(): async def create_kb(knowledge_base_name: str = Body(..., examples=["samples"]), vector_store_type: str = Body("faiss"), kb_root_path: str =Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, embed_config: EmbedConfig = None, ) -> BaseResponse: - embed_config: EmbedConfig = embed_config if embed_config else EmbedConfig(**locals()) + # embed_config: EmbedConfig = embed_config if embed_config else EmbedConfig(**locals()) # Create selected knowledge base if not validate_kb_name(knowledge_base_name): return BaseResponse(code=403, msg="Don't attack me") @@ -93,15 +94,17 @@ def search_docs(query: str = Body(..., description="用户输入", examples=[" top_k: int = Body(5, description="匹配向量数"), score_threshold: float = Body(1.0, description="知识库匹配相关度阈值，取值范围在0-1之间，SCORE越小，相关度越高，取到1相当于不筛选，建议设置在0.5左右", ge=0, le=1), kb_root_path: str =Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, + embed_config: EmbedConfig = None, ) -> List[DocumentWithScore]: - embed_config: EmbedConfig = EmbedConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) kb = KBServiceFactory.get_service_by_name(knowledge_base_name, embed_config, kb_root_path) if kb is None: return [] @@ -132,18 +135,19 @@ async def upload_doc(file: UploadFile = File(..., description="上传文件"), override: bool = Form(False, description="覆盖已有文件"), not_refresh_vs_cache: bool = Form(False, description="暂不保存向量库（用于FAISS）"), kb_root_path: str =Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, embed_config: EmbedConfig = None, ) -> BaseResponse: if not validate_kb_name(knowledge_base_name): return BaseResponse(code=403, msg="Don't attack me") - embed_config: EmbedConfig = embed_config if embed_config else EmbedConfig(**locals()) + # embed_config: EmbedConfig = embed_config if embed_config else EmbedConfig(**locals()) kb = KBServiceFactory.get_service_by_name(knowledge_base_name, embed_config, kb_root_path) if kb is None: return BaseResponse(code=404, msg=f"未找到知识库 {knowledge_base_name}") @@ -184,17 +188,19 @@ async def delete_doc(knowledge_base_name: str = Body(..., examples=["samples"]), delete_content: bool = Body(False), not_refresh_vs_cache: bool = Body(False, description="暂不保存向量库（用于FAISS）"), kb_root_path: str =Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, + embed_config: EmbedConfig = None, ) -> BaseResponse: if not validate_kb_name(knowledge_base_name): return BaseResponse(code=403, msg="Don't attack me") - embed_config: EmbedConfig = EmbedConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) knowledge_base_name = urllib.parse.unquote(knowledge_base_name) kb = KBServiceFactory.get_service_by_name(knowledge_base_name, embed_config, kb_root_path) if kb is None: @@ -220,17 +226,19 @@ async def update_doc( file_name: str = Body(..., examples=["file_name"]), not_refresh_vs_cache: bool = Body(False, description="暂不保存向量库（用于FAISS）"), kb_root_path: str =Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, + embed_config: EmbedConfig = None, ) -> BaseResponse: ''' 更新知识库文档 ''' - embed_config: EmbedConfig = EmbedConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) if not validate_kb_name(knowledge_base_name): return BaseResponse(code=403, msg="Don't attack me") @@ -289,12 +297,14 @@ async def recreate_vector_store( allow_empty_kb: bool = Body(True), vs_type: str = Body("faiss"), kb_root_path: str = Body(""), - api_key: bool = Body(..., examples=["samples"]), - api_base_url: bool = Body(..., examples=["samples"]), - embed_model: bool = Body(..., examples=["samples"]), - embed_model_path: bool = Body(..., examples=["samples"]), - model_device: bool = Body(..., examples=["samples"]), - embed_engine: bool = Body(..., examples=["samples"]), + # api_key: bool = Body(..., examples=["samples"]), + # api_base_url: bool = Body(..., examples=["samples"]), + # embed_model: bool = Body(..., examples=["samples"]), + # embed_model_path: bool = Body(..., examples=["samples"]), + # model_device: bool = Body(..., examples=["samples"]), + # embed_engine: bool = Body(..., examples=["samples"]), + llm_config: LLMConfig = None, + embed_config: EmbedConfig = None, ): ''' recreate vector store from the content. @@ -302,7 +312,7 @@ async def recreate_vector_store( by default, get_service_by_name only return knowledge base in the info.db and having document files in it. set allow_empty_kb to True make it applied on empty knowledge base which it not in the info.db or having no documents. ''' - embed_config: EmbedConfig = EmbedConfig(**locals()) + # embed_config: EmbedConfig = EmbedConfig(**locals()) async def output(): kb = KBServiceFactory.get_service(knowledge_base_name, vs_type, embed_config, kb_root_path) if not kb.exists() and not allow_empty_kb: diff --git a/muagent/tools/cb_query_tool.py b/muagent/tools/cb_query_tool.py index 6790e5c..a4ebc1e 100644 --- a/muagent/tools/cb_query_tool.py +++ b/muagent/tools/cb_query_tool.py @@ -50,11 +50,15 @@ def run(cls, }.get(search_type, 'tag') # default + # codes = search_code(code_base_name, query, code_limit, search_type=search_type, history_node_list=history_node_list, + # embed_engine=embed_config.embed_engine, embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, + # model_device=embed_config.model_device, model_name=llm_config.model_name, temperature=llm_config.temperature, + # api_base_url=llm_config.api_base_url, api_key=llm_config.api_key, use_nh=use_nh, + # local_graph_path=local_graph_path, embed_config=embed_config + # ) codes = search_code(code_base_name, query, code_limit, search_type=search_type, history_node_list=history_node_list, - embed_engine=embed_config.embed_engine, embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, - model_device=embed_config.model_device, model_name=llm_config.model_name, temperature=llm_config.temperature, - api_base_url=llm_config.api_base_url, api_key=llm_config.api_key, use_nh=use_nh, - local_graph_path=local_graph_path, embed_config=embed_config + use_nh=use_nh, local_graph_path=local_graph_path, + llm_config=llm_config, embed_config=embed_config ) return_codes = [] context = codes['context'] diff --git a/muagent/tools/codechat_tools.py b/muagent/tools/codechat_tools.py index 0695c69..9ed77b5 100644 --- a/muagent/tools/codechat_tools.py +++ b/muagent/tools/codechat_tools.py @@ -47,12 +47,17 @@ def run(cls, code_base_name, query, embed_config: EmbedConfig, llm_config: LLMCo code_limit = 1 # default + # search_result = search_code(code_base_name, query, code_limit, search_type=search_type, + # history_node_list=[], + # embed_engine=embed_config.embed_engine, embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, + # model_device=embed_config.model_device, model_name=llm_config.model_name, temperature=llm_config.temperature, + # api_base_url=llm_config.api_base_url, api_key=llm_config.api_key, embed_config=embed_config, use_nh=kargs.get("use_nh", True), + # local_graph_path=kargs.get("local_graph_path", "") + # ) search_result = search_code(code_base_name, query, code_limit, search_type=search_type, - history_node_list=[], - embed_engine=embed_config.embed_engine, embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, - model_device=embed_config.model_device, model_name=llm_config.model_name, temperature=llm_config.temperature, - api_base_url=llm_config.api_base_url, api_key=llm_config.api_key, embed_config=embed_config, use_nh=kargs.get("use_nh", True), - local_graph_path=kargs.get("local_graph_path", "") + history_node_list=[], use_nh=kargs.get("use_nh", True), + local_graph_path=kargs.get("local_graph_path", ""), + llm_config=llm_config, embed_config=embed_config ) if os.environ.get("log_verbose", "0") >= "3": logger.debug(search_result) diff --git a/muagent/tools/docs_retrieval.py b/muagent/tools/docs_retrieval.py index b06572d..db2bfcc 100644 --- a/muagent/tools/docs_retrieval.py +++ b/muagent/tools/docs_retrieval.py @@ -26,10 +26,13 @@ class ToolOutputArgs(BaseModel): def run(cls, query, knowledge_base_name, search_top=5, score_threshold=1.0, embed_config: EmbedConfig=EmbedConfig(), kb_root_path: str=""): """excute your tool!""" try: + # docs = search_docs(query, knowledge_base_name, search_top, score_threshold, + # kb_root_path=kb_root_path, embed_engine=embed_config.embed_engine, + # embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, + # model_device=embed_config.model_device + # ) docs = search_docs(query, knowledge_base_name, search_top, score_threshold, - kb_root_path=kb_root_path, embed_engine=embed_config.embed_engine, - embed_model=embed_config.embed_model, embed_model_path=embed_config.embed_model_path, - model_device=embed_config.model_device + kb_root_path=kb_root_path, llm_config=None, embed_config=embed_config ) except Exception as e: logger.exception(e) diff --git a/requirements.txt b/requirements.txt index f129643..f8be989 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -openai +openai==1.34.0 langchain==0.2.3 langchain_community==0.2.4 langchain_openai==0.1.8 -langchain_huggingface==1.3.0 +langchain_huggingface==0.0.3 sentence_transformers loguru # fastapi~=0.99.1 diff --git a/tests/connector/agent_test.py b/tests/connector/agent_test.py index f4b4c36..8f2f96c 100644 --- a/tests/connector/agent_test.py +++ b/tests/connector/agent_test.py @@ -13,6 +13,7 @@ api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] except Exception as e: @@ -20,10 +21,12 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = os.environ["model_engine"] embed_model = "" embed_model_path = "" logger.error(f"{e}") +# test local code src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -35,7 +38,7 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3, + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, stop="**Observation:**" ) @@ -188,6 +191,6 @@ tools=tools, ) # base_agent.pre_print(query) -output_message = base_agent.step(query) -print(output_message.input_query) -print(output_message.parsed_output_list) +# output_message = base_agent.step(query) +# print(output_message.input_query) +# print(output_message.parsed_output_list) diff --git a/tests/connector/chain_test.py b/tests/connector/chain_test.py index 649b50f..5624ccf 100644 --- a/tests/connector/chain_test.py +++ b/tests/connector/chain_test.py @@ -13,6 +13,7 @@ api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] except Exception as e: @@ -20,10 +21,12 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = os.environ["model_engine"] embed_model = "" embed_model_path = "" logger.error(f"{e}") +# test local code src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -38,7 +41,7 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3, + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, stop="**Observation:**" ) diff --git a/tests/connector/flow_test.py b/tests/connector/flow_test.py index b8070d9..d2b4b06 100644 --- a/tests/connector/flow_test.py +++ b/tests/connector/flow_test.py @@ -12,6 +12,7 @@ api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] except Exception as e: @@ -19,17 +20,22 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = os.environ["model_engine"] embed_model = "" embed_model_path = "" logger.error(f"{e}") - +# test local code +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler from muagent.base_configs.env_config import CB_ROOT_PATH llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3 ) # define your customized llm # llm_config = LLMConfig(llm=ReadingModel()) diff --git a/tests/connector/memory_manager_test.py b/tests/connector/memory_manager_test.py index 5518ea7..5a759dd 100644 --- a/tests/connector/memory_manager_test.py +++ b/tests/connector/memory_manager_test.py @@ -12,6 +12,7 @@ api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] except Exception as e: @@ -19,21 +20,22 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = os.environ["model_engine"] embed_model = "" embed_model_path = "" logger.error(f"{e}") +# test local code src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) -print(src_dir) sys.path.append(src_dir) from muagent.connector.memory_manager import LocalMemoryManager, Message from muagent.llm_models.llm_config import EmbedConfig, LLMConfig llm_config = LLMConfig( - model_name=model_name, model_type="openai", api_key=api_key, api_base_url=api_base_url, temperature=0.3, + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, ) embed_config = EmbedConfig( diff --git a/tests/connector/phase_test.py b/tests/connector/phase_test.py index 158dd55..62d8d64 100644 --- a/tests/connector/phase_test.py +++ b/tests/connector/phase_test.py @@ -13,6 +13,7 @@ api_key = os.environ["OPENAI_API_KEY"] api_base_url= os.environ["API_BASE_URL"] model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] embed_model = os.environ["embed_model"] embed_model_path = os.environ["embed_model_path"] except Exception as e: @@ -20,10 +21,12 @@ api_key = "" api_base_url= "" model_name = "" + model_engine = os.environ["model_engine"] embed_model = "" embed_model_path = "" logger.error(f"{e}") +# test local code src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -39,10 +42,11 @@ llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3, + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, stop="**Observation:**" ) + embed_config = EmbedConfig( embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path ) diff --git a/tests/test_config.py.example b/tests/test_config.py.example index 01e99ca..6fe823f 100644 --- a/tests/test_config.py.example +++ b/tests/test_config.py.example @@ -7,7 +7,7 @@ os.environ["API_BASE_URL"] = OPENAI_API_BASE os.environ["OPENAI_API_KEY"] = "sk-xxx" openai.api_key = "sk-xxx" os.environ["model_name"] = "gpt-3.5-turbo" - +os.environ["model_engine] = "openai" # os.environ["embed_model"] = "{{embed_model_name}}" os.environ["embed_model_path"] = "{{embed_model_path}}" From 32bd3e6a852887f2630822866bbf0d512caed359 Mon Sep 17 00:00:00 2001 From: shanshi Date: Sun, 16 Jun 2024 16:55:44 +0800 Subject: [PATCH 003/128] debug chatbot' service --- muagent/chat/agent_chat.py | 10 +++++----- muagent/chat/base_chat.py | 10 ++++++---- muagent/chat/code_chat.py | 7 ++++--- muagent/chat/knowledge_chat.py | 4 ++-- muagent/chat/llm_chat.py | 4 ++-- muagent/chat/search_chat.py | 4 ++-- .../configs/prompts/qa_template_prompt.py | 14 ++++++-------- muagent/llm_models/openai_model.py | 5 ++--- muagent/service/service_factory.py | 2 +- 9 files changed, 30 insertions(+), 30 deletions(-) diff --git a/muagent/chat/agent_chat.py b/muagent/chat/agent_chat.py index 618b404..fe0dc0c 100644 --- a/muagent/chat/agent_chat.py +++ b/muagent/chat/agent_chat.py @@ -174,7 +174,7 @@ def chat_iterator(message: Message, local_memory: Memory, isDetailed=False): result["related_nodes"] = related_nodes # logger.debug(f"{result['figures'].keys()}, isDetailed: {isDetailed}") - message_str = step_content + message_str = final_content if self.stream: for token in message_str: result["answer"] = token @@ -238,8 +238,8 @@ def achat( custom_phase_configs, custom_chain_configs, custom_role_configs) # - params = locals() - params.pop("self") + # params = locals() + # params.pop("self") # embed_config: EmbedConfig = EmbedConfig(**params) # llm_config: LLMConfig = LLMConfig(**params) @@ -302,7 +302,7 @@ def chat_iterator(message: Message, local_memory: Memory, isDetailed=False): step_content = local_memory.to_str_messages(content_key='step_content', filter_roles=["human"]) step_content = "\n\n".join([f"{v}" for parsed_output in local_memory.get_parserd_output_list() for k, v in parsed_output.items() if k not in ["Action Status", "human", "user"]]) # logger.debug(f"{local_memory.get_parserd_output_list()}") - final_content = message.role_content + final_content = step_content or message.role_content result = { "answer": "", "db_docs": [str(doc) for doc in message.db_docs], @@ -322,7 +322,7 @@ def chat_iterator(message: Message, local_memory: Memory, isDetailed=False): result["related_nodes"] = related_nodes # logger.debug(f"{result['figures'].keys()}, isDetailed: {isDetailed}") - message_str = step_content + message_str = final_content if self.stream: for token in message_str: result["answer"] = token diff --git a/muagent/chat/base_chat.py b/muagent/chat/base_chat.py index 1df3938..6f89a25 100644 --- a/muagent/chat/base_chat.py +++ b/muagent/chat/base_chat.py @@ -3,7 +3,7 @@ import asyncio, json, os from typing import List, AsyncIterable -from langchain import LLMChain +from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler from langchain.prompts.chat import ChatPromptTemplate @@ -55,8 +55,8 @@ def chat( embed_config: EmbedConfig = Body({}, description="embedding_model config"), **kargs ): - params = locals() - params.pop("self", None) + # params = locals() + # params.pop("self", None) # llm_config: LLMConfig = LLMConfig(**params) # embed_config: EmbedConfig = EmbedConfig(**params) self.engine_name = engine_name if isinstance(engine_name, str) else engine_name.default @@ -78,6 +78,7 @@ def _chat(self, query: str, history: List[History], llm_config: LLMConfig, embed def chat_iterator(query: str, history: List[History]): # model = getChatModel() model = getChatModelFromConfig(llm_config) + model = model.llm result, content = self.create_task(query, history, model, llm_config, embed_config, **kargs) logger.info('result={}'.format(result)) @@ -142,6 +143,7 @@ async def chat_iterator(query, history): callback = AsyncIteratorCallbackHandler() # model = getChatModel() model = getChatModelFromConfig(llm_config) + model = model.llm task, result = self.create_atask(query, history, model, llm_config, embed_config, callback) if self.stream: @@ -166,7 +168,7 @@ def create_task(self, query: str, history: List[History], model, llm_config: LLM content = chain({"input": query}) return {"answer": "", "docs": ""}, content - def create_atask(self, query, history, model, llm_config: LLMConfig, embed_config: EmbedConfig, callback: AsyncIteratorCallbackHandler): + def create_atask(self, query, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, callback: AsyncIteratorCallbackHandler): chat_prompt = ChatPromptTemplate.from_messages( [i.to_msg_tuple() for i in history] + [("human", "{input}")] ) diff --git a/muagent/chat/code_chat.py b/muagent/chat/code_chat.py index 12ffa6b..2a7dba1 100644 --- a/muagent/chat/code_chat.py +++ b/muagent/chat/code_chat.py @@ -11,7 +11,7 @@ from typing import List from fastapi.responses import StreamingResponse -from langchain import LLMChain +from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler from langchain.prompts.chat import ChatPromptTemplate @@ -129,8 +129,8 @@ def chat( use_nh: bool =Body(True, description=""), **kargs ): - params = locals() - params.pop("self") + # params = locals() + # params.pop("self") # llm_config: LLMConfig = LLMConfig(**params) # embed_config: EmbedConfig = EmbedConfig(**params) self.engine_name = engine_name if isinstance(engine_name, str) else engine_name.default @@ -151,6 +151,7 @@ def _chat(self, query: str, history: List[History], llm_config: LLMConfig, embed def chat_iterator(query: str, history: List[History]): # model = getChatModel() model = getChatModelFromConfig(llm_config) + model = model.llm result, content = self.create_task(query, history, model, llm_config, embed_config, local_graph_path, **kargs) # logger.info('result={}'.format(result)) diff --git a/muagent/chat/knowledge_chat.py b/muagent/chat/knowledge_chat.py index 4d198a3..981dd04 100644 --- a/muagent/chat/knowledge_chat.py +++ b/muagent/chat/knowledge_chat.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode from typing import List -from langchain import LLMChain +from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler from langchain.prompts.chat import ChatPromptTemplate @@ -75,7 +75,7 @@ def _process(self, query: str, history: List[History], model, llm_config: LLMCon result = {"answer": "", "docs": source_documents} return chain, context, result - def create_task(self, query: str, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, ): + def create_task(self, query: str, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, **kargs): '''构建 llm 生成任务''' logger.debug(f"query: {query}, history: {history}") chain, context, result = self._process(query, history, model, llm_config, embed_config) diff --git a/muagent/chat/llm_chat.py b/muagent/chat/llm_chat.py index 7d8887d..bf265b1 100644 --- a/muagent/chat/llm_chat.py +++ b/muagent/chat/llm_chat.py @@ -1,7 +1,7 @@ import asyncio from typing import List -from langchain import LLMChain +from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler from langchain.prompts.chat import ChatPromptTemplate @@ -31,7 +31,7 @@ def create_task(self, query: str, history: List[History], model, llm_config: LLM content = chain({"input": query}) return {"answer": "", "docs": ""}, content - def create_atask(self, query, history, model, llm_config: LLMConfig, embed_config: EmbedConfig, callback: AsyncIteratorCallbackHandler): + def create_atask(self, query, history: List[History], model, llm_config: LLMConfig, embed_config: EmbedConfig, callback: AsyncIteratorCallbackHandler): chat_prompt = ChatPromptTemplate.from_messages( [i.to_msg_tuple() for i in history] + [("human", "{input}")] ) diff --git a/muagent/chat/search_chat.py b/muagent/chat/search_chat.py index 3854b88..9e1351e 100644 --- a/muagent/chat/search_chat.py +++ b/muagent/chat/search_chat.py @@ -1,9 +1,9 @@ import os, asyncio from typing import List, Optional, Dict -from langchain import LLMChain +from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler -from langchain_community.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper +from langchain.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper from langchain.prompts.chat import ChatPromptTemplate from langchain_community.docstore.document import Document diff --git a/muagent/connector/configs/prompts/qa_template_prompt.py b/muagent/connector/configs/prompts/qa_template_prompt.py index 0eeb487..63f35fe 100644 --- a/muagent/connector/configs/prompts/qa_template_prompt.py +++ b/muagent/connector/configs/prompts/qa_template_prompt.py @@ -5,20 +5,18 @@ Based on the information provided, please answer the origin query concisely and professionally. Attention: Follow the input format and response output format -#### Input Format - -**Origin Query:** the initial question or objective that the user wanted to achieve - -**Context:** the current status and history of the tasks to determine if Origin Query has been achieved. - -**DocInfos:**: the relevant doc information or code information, if this is empty, don't refer to this. - #### Response Output Format **Action Status:** Set to 'Continued' or 'Stopped'. **Answer:** Response to the user's origin query based on Context and DocInfos. If DocInfos is empty, you can ignore it. If the answer cannot be derived from the given Context and DocInfos, please say 'The question cannot be answered based on the information provided' and do not add any fabricated elements to the answer. """ +# **Origin Query:** the initial question or objective that the user wanted to achieve + +# **Context:** the current status and history of the tasks to determine if Origin Query has been achieved. + +# **DocInfos:**: the relevant doc information or code information, if this is empty, don't refer to this. + CODE_QA_PROMPT = """#### Agent Profile diff --git a/muagent/llm_models/openai_model.py b/muagent/llm_models/openai_model.py index 2f32877..4d7c1a7 100644 --- a/muagent/llm_models/openai_model.py +++ b/muagent/llm_models/openai_model.py @@ -47,7 +47,7 @@ def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler VISIT_BIZ_LINE = os.environ.get("visit_biz_line") # zdatafront 提供的统一加密密钥 aes_secret_key = os.environ.get("aes_secret_key") - + # logger.debug(f"{VISIT_DOMAIN}, {VISIT_BIZ}, {VISIT_BIZ_LINE}, {aes_secret_key}") zdatafront_client = ZDataFrontClient(visit_domain=VISIT_DOMAIN, visit_biz=VISIT_BIZ, visit_biz_line=VISIT_BIZ_LINE, aes_secret_key=aes_secret_key) http_client = SyncProxyHttpClient(zdatafront_client=zdatafront_client, prefer_async=True) except Exception as e: @@ -112,8 +112,7 @@ def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler ) -def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None, ) -> Union[ChatOpenAI, LLM]: - +def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None, ) -> Union[ChatOpenAI, LLM, CustomLLMModel]: if llm_config and llm_config.llm and isinstance(llm_config.llm, LLM): return CustomLLMModel(llm=llm_config.llm) elif llm_config: diff --git a/muagent/service/service_factory.py b/muagent/service/service_factory.py index f59c7b9..14a3253 100644 --- a/muagent/service/service_factory.py +++ b/muagent/service/service_factory.py @@ -145,5 +145,5 @@ def get_kb_doc_details(kb_name: str, kb_root_path) -> List[Dict]: for i, v in enumerate(result.values()): v['No'] = i + 1 data.append(v) - + return data From b317fd74efb38538f3d5394dfb80e7315ed8cb35 Mon Sep 17 00:00:00 2001 From: shanshi Date: Sun, 16 Jun 2024 20:24:43 +0800 Subject: [PATCH 004/128] update langchain search wrapper use --- muagent/chat/search_chat.py | 2 +- muagent/llm_models/openai_model.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/muagent/chat/search_chat.py b/muagent/chat/search_chat.py index 9e1351e..6f3e5ec 100644 --- a/muagent/chat/search_chat.py +++ b/muagent/chat/search_chat.py @@ -3,7 +3,7 @@ from langchain.chains.llm import LLMChain from langchain.callbacks import AsyncIteratorCallbackHandler -from langchain.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper +from langchain_community.utilities import BingSearchAPIWrapper, DuckDuckGoSearchAPIWrapper from langchain.prompts.chat import ChatPromptTemplate from langchain_community.docstore.document import Document diff --git a/muagent/llm_models/openai_model.py b/muagent/llm_models/openai_model.py index 4d7c1a7..ce492ed 100644 --- a/muagent/llm_models/openai_model.py +++ b/muagent/llm_models/openai_model.py @@ -113,12 +113,14 @@ def __init__(self, llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler def getChatModelFromConfig(llm_config: LLMConfig, callBack: AsyncIteratorCallbackHandler = None, ) -> Union[ChatOpenAI, LLM, CustomLLMModel]: + # logger.debug(f"{llm_config}") if llm_config and llm_config.llm and isinstance(llm_config.llm, LLM): return CustomLLMModel(llm=llm_config.llm) elif llm_config: model_class_dict = {"openai": OpenAILLMModel, "lingyiwanwu": LYWWLLMModel} model_class = model_class_dict[llm_config.model_engine] model = model_class(llm_config, callBack) + # logger.debug(f"{model.llm}") return model else: return OpenAILLMModel(llm_config, callBack) From 9f219a0ee59ed9a7f67b736f844095da739c819d Mon Sep 17 00:00:00 2001 From: shanshi Date: Tue, 18 Jun 2024 11:44:13 +0800 Subject: [PATCH 005/128] update muagent to 0.0.5 --- README.md | 6 +++--- README_zh.md | 6 +++--- setup.py | 14 ++++++++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1f1c601..929f475 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Developed by the Ant CodeFuse Team, CodeFuse-muAgent is a Multi-Agent framework ![](docs/resources/agent_runtime.png) ## 🚀 快速使用 -For complete documentation, see: [CodeFuse-muAgent](docs/overview/o1.muagent.md) -For more [demos](docs/overview/o3.quick-start.md) +For complete documentation, see: [CodeFuse-muAgent](https://codefuse-ai.github.io/docs/api-docs/MuAgent/overview/multi-agent) +For more [demos](https://codefuse-ai.github.io/docs/api-docs/MuAgent/connector/customed_examples) 1. Installation ``` @@ -115,7 +115,7 @@ We are deeply grateful for your interest in the Codefuse project and warmly welc Feel free to raise your suggestions, opinions, and comments directly through GitHub Issues. There are numerous ways to participate in and contribute to the Codefuse project: code implementation, writing tests, process tool improvements, documentation enhancements, etc. -We welcome any contribution and will add you to the list of contributors. See [Contribution Guide...](docs/contribution/contribute_guide.md) +We welcome any contribution and will add you to the list of contributors. See [Contribution Guide...](https://codefuse-ai.github.io/contribution/contribution) ## 🗂 Miscellaneous diff --git a/README_zh.md b/README_zh.md index 9981b58..498902e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -34,8 +34,8 @@ CodeFuse-muAgent 是蚂蚁CodeFuse团队开发的Mulit Agent框架，其核心 ## 🚀 快速使用 -完整文档见：[CodeFuse-muAgent](docs/overview/o1.muagent.md) -更多[demo](docs/overview/o3.quick-start.md) +完整文档见：[CodeFuse-muAgent](https://codefuse-ai.github.io/zh-CN/docs/api-docs/MuAgent/overview/multi-agent) +更多[demo](https://codefuse-ai.github.io/zh-CN/docs/api-docs/MuAgent/connector/customed_examples) 1. 安装 ``` @@ -120,7 +120,7 @@ print(output_memory3.to_str_messages(return_all=True, content_key="parsed_output 您对 Codefuse 的各种建议、意见、评论可以直接通过 GitHub 的 Issues 提出。 -参与 Codefuse 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表。详见[Contribution Guide...](docs/contribution/contribute_guide.md) +参与 Codefuse 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表。详见[Contribution Guide...](https://codefuse-ai.github.io/zh-CN/contribution/issue) ## 🗂 其他 diff --git a/setup.py b/setup.py index 1db8fe2..e6c8321 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="codefuse-muagent", - version="0.0.4", + version="0.0.5", author="shanshi", author_email="wyp311395@antgroup.com", description="A multi-agent framework that facilitates the rapid construction of collaborative teams of agents.", @@ -19,21 +19,27 @@ "Operating System :: OS Independent", ], install_requires=[ - "openai==0.28.1", - "langchain<=0.0.266", + "openai==1.34.0", + "langchain==0.2.3", + "langchain_community==0.2.4", + "langchain_openai==0.1.8", + "langchain_huggingface==0.0.3", "sentence_transformers", "loguru", - "fastapi~=0.99.1", + "fastapi", "pandas", "Pyarrow", "jieba", "psutil", "faiss-cpu", "notebook", + "docker", + "sseclient", # "chromadb==0.4.17", "javalang==0.13.0", "nebula3-python==3.1.0", + "SQLAlchemy==2.0.19", "redis==5.0.1", "pydantic<=1.10.14" ], From 4f792d5a96fc6729817485dc70f37dda3408f8e2 Mon Sep 17 00:00:00 2001 From: lightislost <31849436+lightislost@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:17:34 +0800 Subject: [PATCH 006/128] Delete docs/resources/wechat.png --- docs/resources/wechat.png | Bin 24858 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/resources/wechat.png diff --git a/docs/resources/wechat.png b/docs/resources/wechat.png deleted file mode 100644 index 773c7f1be5fe72eb8469005357acba313294764b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24858 zcmdqIbySt_w)edNY3Wjsln%i_q!H-`>29Q@n+1Y`Al=d>-4csV>6R`fBo^JUi1*^R z_a4tV@7eEr#(wsB{&|Kn7;C}ezVGXr@xA8!%+E>+k~mo8SP%#VM_Niu1p+}jzWc&J z1Ml=tE`@+^_bgt^zlK06BCxNFP{C^&6DbvW2*i^S0`UukKrX>sej5;o3l{{kWe9=r zB|;!%?^EiP1;7u`jAbRoAh&n_GMe&Yz&n`lrL>&DZxGylA@wmH2}2-;zS3f^Ro!QI z7Ti*ZCN72dhjVPo*xAi2$CN{8LU0IP6ZjVe)6kqEP%D(g&OORLFk?(f5Qx7fz|RTJ z`XHsIniHCT{Djg~tETq*5yIqU6~DAA&y2j(vx`Q9vfyMn#$mZ=567XDqc8M<_%w8M zSsCAs+#q6NVyZ538{ivxYoLOdSPs(voi7ac-5~xMBc?w#u^+y2U#4^27W~|w|>?KFAGPO5k&%he*;yjpVIz7l^Hq0{yxhv@R)x#hW42bK3N zTnHNEIH6hmGDWe!{o*_OmCx#<=G$D;xuDYqN0OCT^&gKD0+^Nd>(ri7)r#6E>AsFiGQi>~xisTkOllcjGQ$5T zKvg?Ww$)t{DYKDYI^E9s5ncVdlC0r0{m|`!7;RWL$8g}G1Dh$~-NOi+sbX?AA92f> zhqiRkU`Y$usVU@xKg^g3tu~5id4r+tg`0Zfh9O4#xx1tf8$5~>{qYAnCn@=Z{Tdis z%_aZ6tjLZP^@+e`2R3@b%cRhUH@LL%NhiH_ubQVR};Ver9`KG-6WF?j7D2ime8p zhf*$Gf?8*9SI92ExN*!mV3%G})#_q@4&2?H2(q)EnqJC8D^^0}%C6dj zmDbCrtQDopY>>4^kq(wZxX;t^Gdk1J(cu`1GSShELH@fJ7@x<=#Ex3#cK7fh5RJ+s z6m$NnBY~oU1h5qGy!@Dc;-jURV@vkJ<~4G~Nqw^cTVHW1|Bv ztBW$-)+gUBu%kx(mVnw<8n~`t0pZf(EEgr ze-fQ7ePP;dnb}!G+2ljgz>d(FZ4KOI5XC57aD>DN>#dJZ@j5)AWE#p{Szn4khL|@U za%uC(ADsOmQpR6)kkxU=U63vPkoMD=7g^A0^rw1!KgM>n|s z!5=Wd?Q{h0qUH@59JpcB#ip(7;RiDZ@7LuD%5CWX+F_xJbKreGwM2`rjQ`s~)+=%3 zb8zqrKg`tNU@4TpzldAzVPYK84>*~_z2yHQgPEg}mKINoGiiIW<~nq1*C1(TjbfIL z?-9cBuWZ==&%ScF9e#Fr<7GHMLS@IfXUSo&xMdsRduF{o@7A}}a1XLSLlJRlvjDH5 zYR%!b7W53~w9fZBq^j`cQ=X{!_|GD}{s=Qe?-Ff>YnVGUNJQ*42rfHDqGun#mS6DS z`&mp1|BuTu(%@tb4f1p4*aAQNUrYPlE`E7x%6^<3RT^{jW)Vk>SPbkE71*WV`#i>z zYxupib;1Bozw(txk$=?hY#g<5L9CCrj6?@-4r2ip);dS(x8RMWXm!p_? zU0i-^qP6Tu;rm+j^m%+og0_tSb(({D~t?a?$i4 zbO(EK6J1kcp=Lxh&)=`=T0TAOKzBKC>xnzPS(X>lsbxZ}ZEUJS_d@)-SK?oCWybtz zI#V%#24zfF?@>PzO#O3um_}c^TdN-mzTkWE_aNR)4tU=bt-$#O*l9sEK~>DSZ~jiqJ7z zapG}OeK~e7HormJ&UcU=stiA-D%W`NoW1cgult1#^s^u;=8Bo@K(3AQ;!mm5mTCyf* z&X$(kcpp97VBf7rC?VM(&?qL}aOvq;O|?-fJv2y;SjBzb@yCk#g=p5t#qX~x=lZ`6 zsNYYm>s+Emf27}#xUseTkLThzj$H+2hyQc8X=gT)f~@$J;;qmT2$gO0Y=3Qz`rO|H?~DKg54e&^2qS`!otmyrIvn52aW6?i&LW(Xy<86y)M1*O8qy!Rz-Pm3EL zWF*qS-~O@NZ(_$~lE6W|F%8{y{?{p<9p1)q5cv7Zu0U2p-Col2k0#!z1t197TiQlfv4tMJmoVscAU41 zvOWv?uD2D~LSYLrKECJas_7+jQ;4pFihI$$>7{Qx%}ZN4Fg)cpo&?Ga8XMTH*JTQF z0UZXlr-NiDFKmH~unr>jb^?c2_{!SDr3)`2wzZ7{)f@|0i-CZHUd-%t`E}&5@tK*e zuR$rIp0V2DS7(=BFhgW>_GS_c$*Jk1;g2V7|AL_UXZbnawU(F>|g7nH><9{Nz%IE~395F&6X}rw|O9V7UYT zCziY3Y}g?pzNJLZ1O#OAI1R%?Hw>iSgr(8%Umij27u8mJedSKxx42(YT|9GjlqVF$ zfTI7lZZ2kdB=rvim~ll@xn4E&i4Z7R+bu9$;B3AMM;kjK6}S$+)Moyvh&Q<>W>((+ zujIgg5=v<3tzhWUc|SOhXX?kFIifJeiyUsD7rhta9^Y9L_F7R+?{YkiMmVT`V%po> z41Jhyz&5(G<^wbEhNW(5T_6$lgExi3K8osq6NgCQ8oZxxG0m>1%=$UH3_I#a#`HZm zw~3OZU&`!h+Z`@Cc!pys)l^+X%!sbHX+L)QO+;vRIz;#;M<=@fZ4;2jpuM6L%Q!H5 zsG=zQ+>=~3Bi1+Nl^4#WJ^uq|2O^KYge7Hz1dCI(`BiFK<^I&~k{BY0r!2L8^7~T_pA_z}Q+4>**)?qGQxiJskS;#{OaEDJm%36IF3&(`w8O7eZW0qVwO9Y|AG8A4F-`@B2aSL5xtGJ!zh(i|%|S{K`F+=~{VNy#|%_ zwcxz!`&4X$lub~sjr0a}Njv6YFb2A#Bg6fTY{?z+mk>VhmuDv1^CwJSBp$PqV$*kr zxy^PXOJoy>i;LIoH6z~NUZ0^re6LS58c!y5fTP^ilE?N3}O3=2e;6)mG=*Rl^zMuw5Dfr$6`q45vr9z$btIY73}yT*Lax zV;^2)JC_B$z1f(nEjwzi)y|xPAFQWCm$M^;L^U-N=jPsVJus}$R4UQtmP>h6F;Ewb zOi9Ft%Fw-X++J3Ee!#;v_S~ji`(w)SJ;MrQBMD3wtId&H1GIUM&>>68+?YCgBpqn2 z(Pk}a3a|Z}DCyS4&kWLp&3bH^V97m~Y(<9cwoqEirTuVUIqCl@nlS%afN5%mqO9yE zsH=QMU4#Rn@9dU&wb&p6V7E6O&zkyZ^i6mG?PU^VOlO^Y%0WHT3FfY0Hi0nnUs+T9bZ6 z43nbqPzrC{><_DnLZh)9={V0P$~1qiI;mG&VU@rDG7*f%#arfE2`51RK zcSu~dtjjcusyO)K;h{tqp-d1l1U^K~qLW>&z(m6I%ZV|p#vmrf+P>-kv~*i({1g9i z{ruK6U#t?xYf+=jz#a@t^H7mQWtAFvjM&wiF{;Ra`GA(jHZtgNh9Qp-&r z)2bO7X~IDKLiLi-cY!lD65ig8js~Wt^mLCO$8~R8{mjn{5fBh~6crVvP^M93E}u{~ z(E^3C8#F(H;FFM`7EOd?Wzj+W6!PA>{IH>3}qk8&4RKA8UedZn>69Ekvwk^K{!5nV>x+ApP7K6J_m258GL#fjejE7>qKQP0)FGAM zkUftk^oQa_w@6$r>)O;nW_U&9Vnr0v=1J`Lb~&G+;QI|S;u9b}I3_eUfdsUDx*dE$ z`&8)ulmSy&&*Dj10%v3llVPi*aXeqXu$V4;IjvI#M;L9`8L5sCO# zrck+jEflGy`^2PJViJ%GR1upmxR?i?tAF~}zuBt$SCQS@`gt$Dtn$~YC9J^SzIe-7 zB$0rNT;_(Ge9dXIVpK$zU5TUO9NmvTGL9oM0k2aGdmzQyUw*n)8JVWdFjA4?0p;XF z5|%PWSQl}b@V8Z$TH`lNsI{TZ5s4llJ!Z zSW-U7_FOIAbBm!sg*>NA0i16;i8=`xSH!4s(YvgQHzAX?yc1#)gtirV|z`JA*<`_Y`vw0(!&QAAWA%0)K4CNR#8#$?q|Mo5`4Dag-s!qFZXIc z_{*2$S5#h8-sa|vuQr1|JZzX}Vb1-Aozg;&PxYiWc&*JC85LC|^c&i|S02^o!`2Y$ zfpRPgo0q(qo6df1H05WJIMF$h?sfC2zQ2NT_)_N=3VC|;;C#|S7F5q^7`1%|^UyXb z9I9~5t8gT;=MTBb@8MF%5JmZve=SvA88Jnpd+gUPn&BIQ+mZgV-4snBja2hI?jdRD zm2RqnT?L%paL$g+ZZF?Srt|b@g@6D#Nvkym4=i|gxC0cjWOG$@L(;moI5qW-f)hm1576)m8xMBUkAyLT$> zJCE&bj;C(k_-hRRW*@|I)v(YTUEN2%7i(nmUVBZdKVT)qX<b%r3QDOCuY`Z5_#)Ca}cBLzdNRQ{0PR(y)rAQCV>H z^ovPI^$d`9tv9T#Ip>mRyrhVohGZBWNS&I8(I?8GdB0i;ob4s7bVAL_rWcdI(5<6C zBc4t^tyf|DKD*i=Z%&h2lT%zKU|y9nqP9M#JjS5k;_LI>cCM!V1SID+L#}+ZcC}AU z{JI$a$H&gPjqb`Pb>O&?_!Py#!2w@1!)bl!0hb{Jr0X(rC8ogEwg7atZEg%~>^DKU zl!e4R${8ki$);LrMEbDDF;vZtk}Szjn|n2CjWqP0ln6ZjaN~WET95P~)&`DaGYdBAIxAs$(k-dG-fp9AYSyQdUi4xM&Nj1f*dA&LQSqGh5R@DFc!Y;ywIXxbQ8 zsYyvgMcf}m#@Vt>c&grtl;4&qFxA$HAN*__J6ewO-|Kpg)VJkW+adv?p7&*DzSKkM zU8(4tXliz~lsvt9xDY4j@TLO-E-5L=){jXW8+Cfxy`y)Wi@qJ#WnNWKf7~A$wm!MqZCxVAfi@y6$w0<<}OGn>Mz*h^bq9S9bM&OUIT4O!! zlKin;9IM@~;U`XL;mE)v+1|Kdm$Q4 z4;nA|yKbI4%Ot8N0#2JJ31=&<2PySU{0qK?B0w)$RRfz9oiOHvb8B&Z`W}v{XEJFk z`0K8KyFOm&UGLl5MJXCPfXd0yQl9fGqz$s+jO>|u{ix70BVnvkTv~Q^h*3sR-8=mn zT|~TX77Bq3aaiPaE!fa_&PEnnb&Tr-EXy$Ib;Ew=ii?RcukZLrvpAo=wmDrVB_UCJ zK%ujsw!T@Bizu0@Z`vMi2+!!Taqt2)B1)OW5xxB!^Mk9s8k1cfoomWUV<4z+a9{r( zpB)azRL+n2mO98trd5Hb`J8;!X-qjvZCkrU^zp<^rFya_L+(~bEJ7C%5!^0{6e_;3 zrmtclvHQc#f?@}A%=8%x%A{)Y)*5Ukm&g-HrSTQMRNR-zIBe?0tw1HlO{KrEBuw+% zVgY`fIB5RhKH8l(=PSQqxz?Wa69jrFrdQP*TR(5S?cIw{^8(HWlW}MCo28)V;kwJe z7UL2`$monEvrFYzqD~`-68t4{-LoH)l1nR)^+MzFfxa?Y=?09-E~Pv!B_bj;PLZRI zYI?_4>G$v7kYCS|`Vl`GFiVd!>T_~Hkx>KBVkkvv&YsVSDLCjYi*NCL_x2YH`ja6 zqBQuck&lC7^k*8K?V*MpM0+`S$gLeWopuwYG-WD zTW%4;$6FJ!NwxZX2Myz1hTv@Rg4~GfLF_Fa9$wJsJ24rVkm1^>o~gtUX|G2^VQ}Z@ z{^XU6dPADp9}F8V$XYPRg^Qog7fZ4U{<4;bw+3=0o@D_(vzox#mN9`c=WMwm#3p4| z`j*M7?lTTQ(QNYM{`%53gh4hwUwdp%6fK5Y#gWLW;pRq=}l&Kocqhx>O~|Gf~OB~ zsfE5g(W|d(yea_V*Us(<8V-4&MVg0%$3SXe=@eXz2P`+R1>iP6OZ6xq!|89|n}{;) z5QuDEta%FRF3M0h1mqg2*MCflVWSm%(;j!9M(~ZOC)#^n6sh5RPx3UHKH3R)?-qVd z=A71<2dS@HQTRI0a{L=h#0*hQ(^RV=m*MI5%mZ#KV!!Cnp4f!Q67`}j^9oJFXG!4L z_I`6Lx2Q5qthw{GLk@5Bm}o*|>f>^UDsU!f=KI>DTdaWY-17FrC=C|m~QBzsdsrLN)4jh>iZkR?#$o~F&Ab7HY zG4&lTD)DX3O8%UivWE^>RU5DNL~Tr#P*6dy9Ig(!sS91EO;Uw?_$Nyy4Su!6DY%?US%$@K!Yp`QvH2s<^$ZceMVHC&fjtM?D5{Bg>Rnt zk591nd9)fwK_iJtkZj9hMw(r^F!PHgirNhQdQIB{@PvXt6EUgRQN2+^?1%EkJG*cf z>i+FCtX8Bx-Q>k15k~%Tb3Cu8uVEIw%jckr%zZOAfks*-3FRD|h`0<}Av~vQzr1mS z1-b$izP`Qevp_>pYWU<`a&U#rn<-0(PqZ_wa_sgqmnUO8C}%moP62&&HSP=gd+zZ* zU*74;vzRMUApU`wnW={W8?>v|U6B8j3w=OLN@~15lp0Ad&7B8sX=PH!UM)WPr7iiL zt5EX0G7Dm#e%%EjMp?h1__eKiQT2RL=Ztk%kaq4X(0w~*y*%1~7!eOO?6iRfd4nQX z$VZ}VG^)8y=JjIvX_<4Ms9eUE?b?nk8WIG z3%OeZl7R_d(3$z`97Q=5bit{Pi#*IuQITyAnK?IZBCFQm9vb9{Zk^vyDu09S?)FSo zpYtLY3;CPYfiCj|yB2HMO`C#g{FuRB0lA&Rc?}>mz8zxPZ!FZDzH^EXWbn z-IQ@SnIT)VWclVBSf!xo zE^oQ!jz*wU^QyIjR9{2@U5o(6P(gpwDi%kfY=inS%8!K{4fV5{CN>~yNzVN0n&+)H zKXB#dedB#%q4>+yOmA@_RlOR&-I6>7WT`a0UE=SFBP`6D^sVL}qgi&QSZfnRIZ}p+k#$FXAY%iRf(aZmy=bPO0b0WJZ zXAWy_?4mQ|lIrQNrG2WPcl_;`L08rwg@I3I(cA|$uU6KerkZsdv$nW?bA32nqq>jN zpjo%EginpImh*9-h_0?~QkjKf>*>)fuu!%|>X#Fax$C_NabH~cQm4>P2vAvl+gX^! zCHkaCt)dzxLPJ*l?JPH6+Fa}Xn#3h!CpGLM$99g6B&vzb%UdJq(O|{KBkKD6R@9kT z$&?fWkJIvlY;+)3b`t}K!His9Q00>*T&^U)95FB1y4A)e(MpY?7@bjG81l>HKqoTP zcyDWQ?_%%K2ieL`9=ugz&UBUc!mgwDn0t8NQ3aGjaH)t!D8zcdS1;pllf8SI#5s^{hd9sDXoP@yX!F&#Kb_}C@`?#WT7&h4x!}&S~;C|8rrvzK4EauVl zcNH=z1qBH!t3oS1QH;j@cI-r9s1P9$QH8^b1iOBtX0;Hag+u#${(CMRkoeWh^h>r- z%g^Yhy+}c@7^$|_s2O2-cc)-w3Q*}gHzVUr#Kr2slPW#YJ{*!1ultsUi~6zkqbda< z(P}3JDI-(w2hohzf*vobd)Wk;?#5C@0c ztHko(kb<40>*_HLBXRCNsg0LuHR{R>&ZYzT@1?*QH4zQc3=GuW{Da@aCL8HV$Ua2* z>G1a?)#jYFEKe8F#9#M4i@+*^nLn^zi-Ji@_Hl>whr=a&`2NRA!X!Jv}YVGvay2U+Q)+uFMX6=EF9f6~cK&h-m-I|P# zOxBoR;IK)o9SEGk4vkE(_}B(nT}7^rW=I?m3Gnm;#e$UJ_KuK-K3hlYIpesP#x7R3 zx>eQ1FKQS$=K%R2f)<|F$k>2Ne&iFtQ9*KE%ApuZzyDESckwrn%9ZhD{@!iAf_Z zEg_YyjGv#ZXCNxw+8}+xiseQ_DBU)FSMBo373S5bN|4C%_FD#UpSyHMzHo3=qi18o z9*Cd#%nWO5vZP2ibl$2u3rNie&&}@_tFwU$7~7Y}baco-HG`CMz8uO4iA_nUtsT*` zjWOBOK$PU?V*pvTxw+Z+3LBN%6$TE4$nVCK+OzQ<9`urd!Y~VyT&jX--Mz#CM_E1r zIibANSenTAC!p9FF*-age0qAc;P`{$`U}$IZtR>~ov1`5-8lK^Si(Zam=x2M6~8NQ zf4VAMqHc1d^i4Cx6i*a<5(e65qWY4i_hOaL9Y2utU2M10Wt1z?JlAROxyA0lDUsC< zRypv#2!0RBOoVig=^qRr8+8dkqH9l;oa!HM@=Ij)XiaO%DZv*J6QjQ?d3<0uGBV;y zk~0gZa_d?|R+N%zy*zMdlZTEu<$0{wm3Y}t+P)4*Xw>p9e z!qxTJJm;B+-xEdBi8m4tz74!s8F`A`{>AULeL$}%^#sRV4OGXEkNf1oQ89sh8H+UJ z)M>-~d1Mj;{zKY?G;ZFXf&^JPsMHT>Gkdu`1$bNi;^k9fKGRG0^pf=S5ppH1QnUVE zlOw53e^aaF5BeXWO$Y-sl>3u@|Zp+*|;f@=lxG`I^mA^Ma*T zOmEJW;R9jc#<6O#*n4=5c;{)|+luzT=$& zw|apB`rhpM7(#}5?*IZ!heHo9FWFiBHTf~jPBH0snu_h!gY+-#i=9Hy@nx`^)UXAu z82ZVgD>O5Irn`0dA|-4v(=cg>n!5M)ASO%7x;LLE5IlVN`RdBMv$Ks{cV*3-Oj<-fjgm_BQ_1X=70 zIX4yxI&S9&RP1tKF)6%GXsD>DQZh24b2YZ3pmKi&H@I~S>8J7vBjr zFAfoVmh7wAQuNM>!og|Y<;WbFh-fB++445GX3LJDNGIWdw+}uJaawbF^Ob^Eh_JIB zcI>bn*WMdNhuD9!`48b76178kW+S-s&nWr#_TG~eK^KBVyJgejvh8bK?Kr)Toml~y zGnDNvpasp}U#(h$pP-K+R{F0UH&z+G*M0jo*N;BtOFO06zwptp1o3E~0`Y8jbk8>| z6`^oJAODZuwvA4}$}eReUpLx9^mhMEq!;uh1tVoh zJ;_5kWuce##rd2jn6YQaHLhWTA#0$69pKu6a7;&Bdj0NqZ1DeX;LQ?wDx2YLhl2Az zh%X3|6uV>rPNO%EJ3OEo&4Z?7?rn4No9ou#8L8pEquzRit94NG!Ym9l+INl&|MaiO z(oBfqQ#=F zde)c_&~~!(mG{IEm9JW(#SD zE`wW!Yw#UYc^eD=W3A&81U2hYw$i_S)jL3kfo=-m4ZTcoL03Kwnq`{XgH=I6m){}7 zlY-t#WB>^)Vx%Xm=Y*>XJKPSsFW)3zH+@F3_*(?YXz8e96}a#gGh)@PI^?%v9jP(v=eKltd44&k?*gcZB5Ae(M#XpMg#}5O8<@N z;OrjG`CWwLwSQfPxT2I1c`&=@3dlLE&ko8O*VR5+mrj7_ta4dKj zn1|?0NGCI}%PTWEFkQ%0N#oUQv5zB&<^W9hb0NJ7RKU@6EM?w{o#}^mY_GN4k#=`D z-Lx?iU_xmVMyigm4)zmaJ%gNw0F1L9OW{6;?`wVW$fAhhBT$aSD0uCFDsIl`^dEtA zf~aIs)=X%?Y=%zdblP~UsqI&pv$8uVIBj07Vo@MhwZ zARx)#?eIEJ0#cVi85MAr0HNpQF2Mq*-4@Phuxf@MIGexsB7GQXUlRJ%>qiLbm33#vkR8mgn%Oa&5Ls|O z2TU1idE|y;3m}yy9c{nnzT->cw{no@81RO*==7VflV0-cg6EV3jOBP~om*7gr2O}Z zrrmIBiYZrXSn)h^#3aRlOO}ro;sV5A!NTDkY4*P(zCJZ2AY3#TYwfZaOvT~F?%e;Q z*;9>gnc0vqV*fXioV^B+%gBI_adxw`g*+l4_=x-8A>;pqaJqxIJh{#OteUJZzMZaH z!BZrLPCgj-~%uriatSUn>DOX92|TrB{p|pDW))9z6Y(SJ+%Xm9W!{1su)K z;f;^R?NW>;?yXa=g@e1*^<=I$I@X)V^El7N3>xmomPlHX#wd%rf}%;nk!fUKMKg1m;_ zr$4fV7KTKkVhlEvIc}8-iKvRFUSZG!3`j|4W83j8x+rQu&*OB^?3F9y;8L=pn3(Sm zfP*LTUieD?aa9c0!2tZE8wvrz1p$D^0TvHv(_c1L-&)>)F&bfGfJle&;{pnLo8NQs z&h({T(XTyo~cU24pu? zobLTe!aRS{22R<0G!@u_5(^2ue+RHc@+0w?IB{o7Z*Bg!ImQ1RiLGHVis$tT&iOwb zPuE<9xQt(pRULZQ_$qEydDW-3X=~hEcO{p5UsFW@KKOdM-vzzT1yyuS6RG?!m>lyT zV?(>BVJgw!!L6o(&%hXn*s3I8=#^SrxdANxb;20X%zt0U=O9U3^I3FNKfSbQ z)7Q`W8qY_UGC}mfFcM|LG6czYHhXQ!tDtNSI48-a*U^i%n6r1c2iP~bJT&q*&Xi+K z^T`iG5=VNOe&Fo*10eq#(EkTHr$_OE-3NzX_JBed^2zhA*{$2ah_HfrRT?xe)}fD+ z*&b?b`*$YnMU|n-q#hj)=ti8HWOP!#d&M07AEpZaKa)ERNKUW!7R}B75TEn^^viW~ zq9!DWf(CXf^UVxef3%ZP(q3QW&^aC9D2%4U-u#6>@uL{W**Bc%1^r3W8sm*lh8tYo zU;AI-Vq*Rr^!4*^qgSpeMU@%=E6v-N8qD1Ku~S*QY~1Kw3(#m1BU?*NO?M8Ng%v0! zitd+|s(ry9nDLXD0TYZ4?1MojF;#y6djkV;caeIppFWq)h}S|LO@G0EpN_6jq=()a zQnwyYy0HMO29PN^(WetJnZbL1r58Jlg1{%zgg*pE!=x8*?tU^AK6p)uMnC{#$sy#U zVLGW$TnsBPxxD0$^}D;G?2>0lZ%Qz8Phh8jvxZ$`DU$D%>8sO6e4$_hYS}PYE!fF< z&vq&cdUgtcb+q9mkQOo{uKf8vB#b{IxKISnXZ%4vDHFmU>|RA zN8!hwlwc@;fSSlZ=ltD^;+VkBI_9D0Adfi$!5Rl_2R_XXDjNU&Fwj4V?Emcm(f^Up z1hk-Fs8aCQb*8#4Vg|H%kYUntx$WvgM4X*ppy58+aHM2X1fr)uVYJdr7Mw#>`QpLB z`&au%>~~AeO*;KL?4-q7RcL_Z@VPub=(~BqWA_Mj3No$)!E09Ax8;?0EFv<(Z6EYt zM#_yk{6gMh_=B^kVShZwG@xHpR8;o=B->vykcM`B03@Q(&DDkb&BgjC%*mpq(_#PD zYj&;5djOaLRiiH&Q>m$`VTp-}GCpXBM@JPt2-mCAIs3Y`c%AoW3vLLr+i9DZHZzr| zwTqr?$DU2&D@7V*KW?rOqXlo7?z&U}m^hM6K@yLQ5f3HBgk+fyelrN6LW)SWSh&~b1KqI>CcFfmDP&sO*E%q=!~3HpYCQiQUm z=Eh_t;%twML#Kw=>Cb?J^+klxrPSRAVS_2WZ%@tvFeeT=D{*t6xu?436CV{tyERqD zx!bTO6crb@_R+14AKZHG<<)~Q2?s!Y7EyKQNXLkkXjjL+va8?xRz%eS8sOsKbRYKs zw5B_w9!oaX_*TuZ+qGP~jy8HY9bBxZV;4={vKqeu14Rx~ET- zUG^OW{2G!+(DA5>*_ouHdjhFJ>2qYjli^5*{TyOWsxO=S&JI`5}ANQ~GZO zeE|VmL}k$I?Gbl21Zv% zK!4(aqoZRi^}?yn-eOZz%?+7>w{rC^ysSmvX0dUmWSx|hG-PX{s3@%kv@m752&kxb zN6*}5zgvo8&|+a>iGgQ{MZm#UfP#qC!o|e}9Uq^z7Nd|*^H{eAhf#tT%goHo#zF(n zvuDp@5uV2z{(#h3$xYBlhRiz+3)aC)8&c95Y-XMTo4|%}c^sMse*E|wl-B!J05%XD z9)9}jTNnioU{8tqmYO_IL~U$rlKK?^)o@quQdcL17=jM&U4If-Gtas65{F(rak}rN zW!z+iaTgY_LN(s_#>Pe&!EUSSdT(%-cMb*iL2cJ-ElvXz+Jbo&-Lv}+Jo~f6L(82v zRo35{= zzdW||234tzsj~FSfwLuO*=3qA_;?&&vlR}J=hLd0FF{nE7|`46T1NvJ87v{h_H#g0 z{cbv2M4^E`)MGg2ww^R>`+y2r%^@cx{nQbJ#};AWwNo7z0>3_6(zw=}yzp!QLe!*+Xhgy#!d)vP5Qrq3>3{0LDk(_dJ-1~`lX^qQN>d;j&-eg|IPHCTke z39FMin4u&lCMG~XR2gOg>X&6bReF9hWdPbC+tpKi?)x-j}Nr)ps)TTg4$ry zXT1-^U@prsdQEmxhSM=jA7CAQw{T}ZfwOrheV=RK&A7Zt!iPPNr^}5391XxWG++Fo z_*tlios^VBRdxgn6M*6&hiBl)_73>&c!!B14L0ahu@(`q;Byd;`+f8~KH)F%Mk8sN6*A$j2iE?s!vzg9!}--i0NQg20UsJvSd{Wh-m^@Zke8il41Br`zL z!OYBz1&3V2YO;9qSth4erP$_)CwSh-uV1ks?hjAs&QvYdMYZ>RtuoLK^3O=e*N3{rfS*25dU7j@hi{ce8pyLk_FJdNIjMG z_P$b!$hKcRTgxUzI z{>Xk{jpGveTng*FFDiR_dkMNDsL4QUm(XqAAqW5r%OLENfR2Jd&=YrDN?^TxF#86r zD)Y>@c?vy!eWKpp-no+q#9m9$O&V~le!IPHYUsQf&n1v2s_So%U}g^7Lo;c7E?5TM z2jZ~)Si6TH4gLc317M95%-o)^0!~{SCwb0J#KNs}~RNtQ6$_!!5otFqCiw zTvI=SJNAX;*u9qPQwl)Tgnzd-ieEe(0h=%TKn$2bl;Gk z9;y3O0-7)=Z~=GFTsQzNxq9!WyMC{qcl{_up4kVj9;Rb}G#LWT@j$?x%bAbTd?3+3 z^G(oS83bS}BMol$0{g8feTR2cL25GKM9x8sO&wy2-z7x$15ity_jM)VIPlffpGR_eDD z>v+69>vNj3t%WVD?Yb9cW^HN$Pp`DPSc2XzSSB^>*2`AQsjZ8n%OrSN0wL|}*#gFX z)YndcI(s|sI%nIrGizOT;;FFa> z>#Z|dqQg@GlDx8#lG5+Pj~_qYfm+L7txF4zW1lKE-rgVzeTXS3IY6W@7O<<(R6Y8G z6nuMg)?!vVECg+=tOe3FUu9TUM&|IM8OvF2qieg14Cs#WAUDwbr}8OZ^hMa_p^SIn z3A1~i?UvNv*38)v0h&~)E9&c4R?wwnliPs7V8%d@sny5DwaPKZ^z`&7Er2vu7_xRJ zjZES=443Uhf%6wSy)V(JEq%YBD0GoywNQ$+9h;_8HU*DsCtX~Q@Yo0k*6{0?cMN_K z$J+ijEltqNQ!vxZ1lI>xv>Lqe_a}s8!7$3xPL4 zLp3b$m9q(sDCdpCTLmWD49UY=9WScGT1Y6h+gew*S`=N}|``F`DWZ*0gGD zlHTHhl5OC0jjc}7a1w}}wP8jF zrIbDpOt~DE#U_0An$DE4H*xAuV2B0)jB|6bc}vREr$y@VWc7l8Y&K_qo7MDS>ys2=J6(NU9X>P@woKKl*MJdoABqSQ7+5x~pbTpcOqtO@--9RehJ^=_a$f3^s`T=c9Yi$k$ z1O!4e35(7kaZDIewtD!bEPyqJKg(<=jQ*o3klh&A?0w;=CJ2#K6gUX%7c+X9tOhmM z@wkzVXJ5 zQwcEcMarrF1!UEhb{A*(Bo6fI039qtHo2eI)=!qZ_;fdasC$#aO&<1c_B%^Bm0)50 zE%5TYLlZcG1Tm$#BZHX+6%r}dl18lurH}ZWF&;~XI~Njq++2Ai2nzsX)KOy#!lC#^ zZn{w?Z{Tx60`?)N-X1{cLS?}ZHxfr^Yjf@}(>zOJSDjirwK&V;pJ1W{BBF*>1J3=0 zW}u-BqH*3AiLkJ+yUi_Tb$vSv!sX^kfPvW~UdMY4ZhJq#DvvgKxkVIq!Jc!#LHIhZ z`=D!QS6Yay;pkX7-t1^Y`RWM~3CY?)&uxV8jpj-1%D^YEnFaA6Z9r8c?5E3LPq2nY zM3j|3Gt>fGzH;1=2fU`R!1Hd%D~Q@RnBcL5>K1pON!3hgA92AU0O{2nm>^)I@%G!$ zsofh(%gf6hV^Sxqxzy}m*p5m zp@K13>z!D|pjGV>O7r;hFz>G#0+8mh{w!cRh2!Xvr-DK_m_}8Mvjaz}h&e>h!6VNy z{iaxKIWR+@tg1?1@4BO%d%fx&z7O<$A(Hs14#3F7K~X?WV-Q(JqF`^SrKNA(9*;^O zzf*bi!U$2R<|E&od~N(Z?)NLwdw!bV-$jkreeCQMa{v_sR%j6N!J|vSyf=}`0$J4O zO&QZmZ`X7(`E049<@QGB?$~Aw&Ns_9h&@i8KY*}ME>h=k0K=XkcPo3RpO$AV(p_M? zZc9G3^Cyj`v(|L%>~au%;j8_CLrowvfW8zhn+gSCJ_JPg&maPZlIgp@2FKOc-eB*5 zaB9W}f`f$LZI>9xS`v*?U1gkss^K(Hfg%FovmOvZg;T;Hpmrnbj(MHCoe;b8;G~tz z3Jnj?10oAg=scsH{yFrof4w9XnEe_B`fLOuGie=yXf@{fCa&l%G zguQkEoaS9##W(r!g#Z-H)=J^vWT>yW0k*;sHVVR`_%`M>FpB)cW)?l(J&>Y| zfpF;?toVOAJM(|2*Z+;v`E=U!Q8^-~bVLgd4zi3bXF{h$S%$JNIZD~rsZb|QvQ@HV zuOyi{#*%exrKF4!lY~LZVC-9z^?SX~_dodj;5QG$`+eW9`?{Xj>vh{`eQ)nkQ$ckNA1PYCzKWOwN7OF+K}pYr`)rD)`KRQCkXIU`RuH$QqHzB(uY^3 zuU$IWUm9*^QD+9Y3|TbN zK}AJGMIDBz^ad@Fy+%5gSw=U>`Voi2h|ew)6BCd6G1l+iEiEm^E+(wbKp3TU8iC`{ zs_vZ*Z8VB_b%=hpJ%*wk%hpLIegLCMx+cU@c6)z-yX0?UlV&Qx63RZJvRf5EJ zmOX)F-M&}&4$6Fy<)eDp@|yL@{tWLT;n)TA@?{ys z$oN(nxCme@Fdg!vRUTGd8UI+A)s_8s@8ubzs)b>CV=vdXynzAX{|(~; zx6{+}G%U1WfA4BhimeU0nIuv+BNBQ!eH!M6`Dt3eiHpVHE!lKb8ll z@%6LFs5PDEv|hqF#CvOJyE>$41{x>xJLRt=rKZxwe&~N2ezxXY-=?-xiLg3Zbsc0p zjBA)g)hH$X_rT^&n|#GipU{0(_AdNZR3ir^p%4424`9Q$!_9Ayyd_q|{jSRSL>89- z<`y>Gl6S%9Ai0D+M!;a{OB%v&4#QfKCHOi?uv=TAm%OgOB^8^j+RUhrHr!c4XD#kh*k-x|RkPC4 z&9xV8i!{SqYkX{n#Z=s$;ywW&?;%cwau0QI7iHvGw?-?)1Y+yp`40l!m4WkTk7ee7 znwm@AIS=`k{nZ5&gV8R)_phFq7yji$TPj}Wu+Ft-pQg7ZfSJ7nKFoGI7bB{;27b=O zHl(Zwo^l}J+-<_%UHfsf%-4ljNzMdwM7}I@;^SdTTeDtm$nraQmFr#WI&#ztOb`}P4ptxbDjXtaz5uw`|jO1!|PiamRKxJqwQRD4rJ#6(1{uiJgO zpm8d8C#iL8{Q0y^xz{tEjB}jSXJ`UlOfMDLgVa;F?X+m9eF+k&V=S!Ze7iiQ-9aBm zl10w7a5{A)gXn8y7n3}fB4lsYJ3Rx@Nn;4=5bHuZZtgBYk|>s!iSAJBPBe!qP>f$@ zhOUrwaG=U~woh|evI*~SI33Oz8{1qzF@;X6muqrtM3W5^ex_z5pH$#Jpm_$-r z+g3%NWq0M}&m!->PlT@0X?as{6o=RIAB{85H1V{XCDuvn3G<}O{9|D5{lr~93l3d<}Lv!il_LFG^T z+k;Ovx^t`1i~ntweIg;!?#ksI@3Df5>K+`tIto}yV=bnyRVb$JS-? zgqYL`UN}Jc4>sdSF(-P5a%0}UK?V?nddxad>9Y9Uyd2|n)A9V_ptY6J(E_uh9c;Y}4`i-?@!~ffGHpq%k=8E0D|wZc^ryCl08pZbx}lcv2Y*$x}Y_ zRx_ct%(M4x*tq`jArGb>FK*a#@oJIZJOeioFbg42!?DpEVOVH`m<3#Yj!B8J506g= zejV!)ff@2Y-+>a?ey-TR00xrq@6{HYn0}jt4}@NR3K)A|%#cr)*E;&KP%&ofIpWl};bJ;w@_uYSw5o;myH{^;jqg#dvL2O9WOzl5GkY zP(8fhjU&5*-}7`+lap@%silZG=KeL!yhem30GUt6{7XZ}VPh6V=F4+=jI#JF# zO&qL;*L_F0M%J?0T-F`dy&5c8ntviA!XhIQ z<5_Jo-i#0D${?;~jHuEl0s=}+#f5Is7f9QefsMA{oXDO0Y7Nn*gL3E%&xxFpw$Z-w z-o1NGb)wMWTQQrGqwY6!i#~>Y?o#gKNqj z5x@1Zpa2?^=LPEMEaXMaHACdv0*iOpzARXg#u3d}A7r=d(4 zR=8JZib!Gj_}?h^r~M`P7D-1iK?>ve2@a*+zuopazWE-;YKPkBNyBxb?w!TY;43Fl zCuQf2kQ-ujTWH_$$J2fl*u2tgN148$wmpF)yFco?%$Br(!FWYKmn+#mj}lR^h>MZ|~{P8O3agUwQV_ewx~5>pPM z0tHri7^THs4djpz9VKODN*VE~-S_Dhd8U!^2<04+r4kB`y;c_jaRpdkrcKGCqi8P? zz<^71@glE@$lE;SRQRULZ?egNtK!F6oK=r)jGp*d%`YL@Q33nCS34xlJV!@_&p8#ucu5DLsuWHKNu@Fiv@P17-{r(Yxp4?hSDt8GiM@* zkg6ZK4bGnk&kJsuu+&jC)?{37i%_1z zvVXIkSzQpFgA}nIZ}>&n2B8=6(q=D^3YzC_st#P z59e2j#FpK?E1E4(u!UT;JlDl7odtNY;x@0sR;0~gM1yF7;OaDsWZRqkzDZj_;l$G5 zq1CMyt9y7*+2O1tz>Na~H$x~0%Sz-EG6T8x+OCi&>i zj&-%?`Mi#kx~DygDR;ZS3XMY>#AQtM()q#$ao-A9w;|!~kK}x@eSYg=H!Pz=;sL~GV=seN)lqqD z*ZDI?^zoRm;>9P2$z-VI3nm!69wa@mV5)@qBn>8(27`-O5IvTp`pSJ zvMzAA%EFDXwakxPyQ|;61zm;`Ai4>>hTcge)eDSL5(x|A(uk{?w1}S@1s%?63Fk`a5xCDRn2LIvr-YGRd@L_NGGbhH%dgs)~Bi$;x+H%d- zGWfV7I?DE!=6Hs?i5+XzfI~BHrzC2~I$cKL z3V59Zf)nFR9@yt@I2E4}?NoJdrwYlIYSQ9eBqopuP<)rS8^T z^aOn_@lUGnq1}->it7Y!$Hz;d-$Vj(z+!v`LDvglo}sA`+X^53o!S@8N4qO6(R5;n zVJchXg=WMIK9RZfLF)HG$?n0fIpgAz%lyciffA|1dJBB{8kx%7#f-#OEl2eCnk7TdM<2q z#4&P`vd+-hcDa-jE(<8fA;^<#dl>@Fiz^if7!>^DdXQ0@Ez4R)?<(z{`x;4!|$^I From e63eec2e75066cdd13d7543979961458df20afc5 Mon Sep 17 00:00:00 2001 From: lightislost <31849436+lightislost@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:18:06 +0800 Subject: [PATCH 007/128] Add files via upload --- docs/resources/wechat.png | Bin 0 -> 60564 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/resources/wechat.png diff --git a/docs/resources/wechat.png b/docs/resources/wechat.png new file mode 100644 index 0000000000000000000000000000000000000000..04fe244c4a8611efa7638c2f71e696e30560cdbe GIT binary patch literal 60564 zcmdSBWmHvB-!CePf`D{)vz6`!+0wbCk?!tJr5hAB-Q5kF4y9X4IwYlY)8JY7KJPou z{dVuT<9s;70Sg#wueIh}bN=VArr{7psW&LZD9@ffdm|$)uKes7!qBs4&q9Cz65@M?I5k?{OlQe_tO_bzkRXEvu6iy zWyHl)J@gLRkS>s)2Kcb1HCt`RereG;Ho@Z%CTOY&T*vtzxI?w@KOuj1BDD_f_EJj zU9(W*k=Qnla2?b&77AJDi^%pcQ3aveEvaf$?UXnZRF!O)vy^=3A1`~58ZT`R9WP!E zRV))OfEL<>DwX+4jggAGzVd=ZnmdCk(Lk&8duVOPgiqXkMHDKG4y6Qtdo6|2{C1<~H zl{LTP{VQ4Tb|lO}59O_wo;*2MKU998XVQ4ns-!lP8UwNS5r(uw${M{2B{AS5{le3A z1z(|u%}QRS8v7S2$K0f)BG$ZmuEZS>g-=adwo9JV$YL#GQg`_?gm8zgm|sFST1Rqux?&!1Dc<**$}y8yBRSf&mpg=*FhkQdjhBK zy>Y_WUhnvAu|jLCL>4gxfAdSLo~VXW$;gR9;Xk4<7de{t4v9{Ag0lunO2WFNV@P{H ziyFK|QC%$!O{3~F_j|Qst%aSwqtdoa68*c{POr`hHn>FS>(a&IUaN)m3ajDF4YQo& zH^{mm_XE)q`mxn~fj8RO#PH-o5nEx&cK9fziVQ~Q>__1XGeP*8b;XQm$7HSQdDc6a za;H~b#qDzIYH2?_NWVt$vs(MSv|8gi?M&&a4Y$SFy9Td)G1o0GuUBiq!?O)+R~oCV zvp-WQL_gmuJHxxpNm6)*cy!vFbfYs`#oBf4t6Mt3D)93pZO!2|2GhcWX)&4p`Un+?R`vYRnte-3mfLFbawHRKV0CLa3N$SQ#7G6V85WA5Pn^ z*j4$J-Bca>XwzRWHX3@v(;64j#7;1mBq*9uu`4M>nX~FCcv|nQ zCy>JJ0;$Zf>$ZB876M?q>|Qb#M+XR7U1ycQY=TNyZ=)*E{2t|%RuWw2vF77qB=#?p zIUc~Q^v1&}ERW}96;W!FCgG(B=@2oihYI^XcG^580k!~#=ngGt%5fdJi|1(YrBKIo<S7Q9z*Zy^7=H*8IyDyLS4v)mj2+3SacfMMQNrkkKAi^ zF5);vW-=mQHt!I{FT9AAu9VdSNN@W=FDK49hm&ryI}AQ+D)tJv6hB)ZrZ zvVM}wY-?zA_89j<(dN5TSLnDEhkrc^U6D3_(?)xIJ)UZEvQ78dN^SsbMEfG6Kr!>p zV~6tpbecLSR8hA=@3uVl)^`iNb~*KV`{U)=j_;`D@9e{`Oq29bABJ%5!5~KJ6mln* z)TBp2^z^7-wSwaR-4c_aII^6=cC%z9OHIVZNxWAy#_8iOF_|rx6$zsWB zL_k{NPjSUT=V$iCx=4y83y1w_G3hm?4~!;GPsgd zI-?v(XKdy?#h2TNI*=;sk?B@&PMQKa5rQaJO!|tj?la;uNMrrBBK=BbHw^c1eC;cE zjMXY&srQMmloLUscHE!Z^BC}nlj60fC30VfxDzS)&M8Ev)rkSajqcN;9r8OUnsi%$ z3CVSn@7{XFUiUW5aFU^V3h_ z%j?+OHu(~uL6F@ph>0=Bqm16(;P+%S3JHg415tKT)mG{ZLEu+{`hq~{R^eNm4t9cy z<-C>{O%r-wbfJ$LFoCmy+`#=N^Seib#i~JM(<&xh*&e&0T;@_QY;gGTNL-9O%2uDQ zRGw0Ftk81fDJ>m1h49+;P{gVROIYOZ4@h+ZYKc@2_V?@c#n*ibYnKlU(=(*Pyn6zy z70w$e=Ni*X34)`7Quw&B>bo}RhjV)aG%=qX4rkr(oi;FdpPt>(A?8?prHFkOztPGF zVhBt-VJX&++E?Vf-XnakSWyieeoK4x7QQG|41K+j8>5-n%gltSOkeG#TU1;?aw(zW z_<~WZ9Ny_&>t~`Zh<*dz>EB8t9QKZ?Y2zEG`8BybJC(4&-Bs|oTumbPxm8v3>Wc=2sMR;;vbEN zl_sZ+Okf+im7_MEiUf{tZ%~)@BVgRd=hz?ma5jc!g2~3ViT1Zd_RJP~6Ok;KWR}ei z_})zV2D2h#YKa#xt=$*w_q`S9>tU#nBpIHptG0#r<_69xx83QsJ~(hCoP5R4p+!MP zA3I$pw(nJvGHk_@!~VuJOvvjVSXvq!?q#Si0;{Bi2iG|&9*k3)+(>Y z$FJAZj3z6)zk_aq$Gyr3-rV)j?I!-MFS4{>I&Q!4r1-Gd5HitOcR2D38;pPDrSSN8 zU(_y7#+Ea`Hz0Y|)1#lGzHV+3!S{JPY1}u!AR`meB2Tu(_QbL2-J!jS z6r#IVQH*^_sIWu9oMDcEYrAN3PtISCEPvJd!}q!->Lyx;$xjq{P4?cw6(6o!@tCqP zr&2%3;afSrA+f}gML%1!r9PvEU5|BLVwJJOv^iZry(RJgU?L=EpjC5b?rKuTx)=ZY zFPZRrhLJ_W_Qf|sfrkg@UR%RCi5{B*>Xi=+OuU{sfz)O!v{~}Cu+I5ES^}_6y#Cdj zJz8X2%VkF8s(Ti_M&1Rmoq=@(&yRp`+eXc^z2~!`e2e|mbu%(TMiV{Ti;@dpyY-{J z6?wHc+N5Ax3|evYTbgZlv9~Q#w485WVfkrLD~`xK^s(|Teu;Xa?RRe?(1}~6-R`Mj ziEgr+pd&=Q3Qb+P47*x=T~f8>_2{1KZ6}&{{hFTq!E(tOo627wmZ)v zH&G-kXS{cNS!pj`CgNi8O)8165#LlQ5&+{XPl6f6S~J%#`&4hPH3H-a&Q zq6JZJwaF=eaX8cb!g4g!I)R!l{WKc9)4J87rg{Ftb}yHCLa_V8r=U1Zt2Ry3xCXo- z8t?mbJ5vdIh}BKsgBhOw((}OZa6F;KoHhy+k~U6S)^NRi)W9jhsP5nCY+}{gE2mAQ z>#~VVI@Ox5F$nN6QH83w{)li>Lk%^d$OU!;x4xPN`{^1;q{+2g2(@!2cAHGkL){RE z+N_MIx)DUQGl*!wsHVNFwRTc%7N^CAGT^NuSynjc#o9BbR?d~ z0Bz&M&0(6y%|Q#9q=njG(c{W4>jB6-a)r*Iaa0BbAn zvhMfNyfT7O;oONn+2gc!!ylNi@cnL@$g(=6lytdPN+t&ZQdRmyoQB?nlxC?~cxjMN zN{D239^cTHY8|IcWXAh#ZsH}M`pfZ7WdiS@C3fxLMbXv{i!P@~nJs*8=Qqwy^lbJI zFt0f#HS`};t3@`|0(riAL4_9QMu=n0R8wv<-uTUH>+0NXBWfQ^O_!ghE5)RG_-agT z+;>Uk!MHckeTz!hd6i=(&PIE!uB=bP56@`D`+ z%Du^GSwL<^6@O~2X#{vzzyJ68{&h-ZA2T6Ktk<6e!#lBa)qMnKdDWA!RGt4)s zwL^TXy?LKoAo#(PsIi=yvpeZ{g7U}}F5kWf3o_=W`b?SDW@ZbdeK#AHD$UPyP!;i8 zk>s|%akKWjPp@%FyLo6^_P`O|*&4qiMq1pZ9czCgJmIMUe~0IU@+0F3?9?~5?Of1@ z6VQ;++YLtT)b18f3*U5WUatwn^rC+gJ`uAmo6=m*&%o5IT0bK2j+PW?nL&ay&b_;i7Tk;`C#Z{=BOfTCRqWkUz{qcPew1gGk3YaFD9@Hb*jel% z&AJ2;AlzL$RJ&2~E@2f^;R@-;MWA$7i`rK*O1DXN@ z+2gNpI8S*!EKzx}523%xY)>{n;Z2We{>DCX9M~d{C|(S(*WKA5i^Z0mrWPM=cnvc+ z9AYPE>S381hpN)VdII^81{1$ie1><`=*litKjH_+AzVi2 zO`nK>m`M3uzn43Gz>^jJ?K5d5zt*8sHH_6gS6@9CdtWp|d#;9(kAwa22d?Zb zN0#BqvJoaFbjezdQ;ma&21?>a9xhU`Nj(MLm0e)b4z8`P{@Ei)m&xEv?o^#syq9xamYPvq8h zZLZm3EZ{>B2O#!DKtg(L@MRS@u~icJgB@+W)sw}!_-)4oujz%0%HqxAKbyY=r{qGM z975gWY~Hecu^E4z=3+zq6}`rDyypLT89dps(-9rmXdAEj*n1P3ag;P@@OUU6X+i`R z)zdX<{~nSpH75mz)=ol7TC6#}!(M@Xdjo!FLYLhbTdrI=)SHO$C+a$SHr!bS=8$U`Z! zf0=`9$__LS-4KN?aSR5qb9XiCMzRPw`uqy-*tC`cxrw7Q@9r7PF{d<-FHV zpLo~x^47qB!2a=)4xGf4?-zOIBx!yGv6>!VU#A3O!hHJUl;T$^M8%m`>-BBtokvpL zMQ<*XR%1U!DgQmV*23{LtUWm+Ogq+mT-yyPeDF}jOmB8jtM~;+d+=n5vF6&U@&| zoA+)DCIZS3Z-+@>CXdPYv%{~Rr4p+tAvK<{bhAIJ3Yq7L6{sEtSeq=j698#}aEF|U z&BjzQxSk(AbZxXQQEhX$R)lP71gw=Wu!c4<3nA;<7`*D6@#tINOw(a0!g}FV$x72M zM90YBD|#j?*m1S|b+nWZ(kc_v^m8`32c0X!*^b~|^I>P9mQUrFS*iN&sU~O0M(1R5 zLqor*XE7nDohG_n~Xk z+8Rkz0)(RyR7)5Jp$}yXj`P1aTNb0YqtuRIaY+G9iqVG*P)f`1r5v*7Mh8V-2JR3! zIyDsU23BsJ5cYg3T6)v4m57$ir|XY{!K8aj;`?;Z`O^ErbA<+OGQH#q`P}2{Ewdx+ z9C-`?5BMXSIw5O=h|8UsFEx_V%NLPsJj$;zJfROLdcp}p^eIQW4_E_?z5bHHF`{m<2bALRk!Jp`ba z6utj+GcaC~pQfN8P<7g$X3D>1oP+&I$mkA1HuZW7)5(U!B z^JkeTGUvxx*5y_WWJN5xqwJX`*9r`*`on6xfKXhQXsdzO`Ny1|LTGyKY=gnm`+cL# zM&{H^L%L5wyoQ0os#4HagdaKiSDi2JbVELHuS{-X5}NS=&dkIBpx^$d3z&v_n1=D` ze9`n9Y@#dYT)OTH)wNQ2kcG2LcPfeGC4 zUaucI%{O1H7Q91_l8-76l2vPpPo!jV-wYYciQ=(n)@dt!X`TKSPmoK8iXj#hclmig zP0T8$QlAvIc$WDNE0Ib?qpUPKv)ef`=u%~ z=2b&j(5f|`s@jC*d+i0k4lVcWniz~MK8lLNYlE0BEkWuKw)R0cf}cY!5N$sr8FN!>T>X@k_qAJ20Gz$w zjT{~Aq2ST%`Z?>ToKtf1CeX%~Gv!#w;oa?YPnV1^>dQW%`htCi#R0KQf0~PiMU0Xv z0R^t_VU`5iEq;%G<|WMn(24i!r;!WvXzx{nIBNrg@mR9;6xO!wCb~w^e~(+VKCLkQ<4yf6FAF3t3ifwIt*8KHKXwf~ zFZ>RQPHh%+7Fo*(g|{lOZ&m-#!ZcJ#gTGf#{URP6|1L`Y8weh&cE|*N3npq8{09SC zp5@SHU)txIHYEY*;zu1d)SV_C%x^LDPHjanXNT>5H+BD?6dj%;TtGZT!}zzJuEBj9 zzeUbWwZyze_ha3WT=-2?0r*Dj1NjgZomMH|dB@ycAJzOre#7&|R4z)>m9{%gF7xj| zT>x+=ZVvEH%zvR2s1KzK!^%F{_NVf9yP}1PDgNkGZmnAQd||`o{&X$?K5}h)qPU;r z?$P`hi}_4E{N3qJzq|~u+)o7Xlq0COiY9kB#EjsH@YwF&rc{7m13pR7t&&dV-Z>=^jIOdAuj-d^1BY#LZ2|wYr>yfF2)Wv=^=yDenPoX(oU*>0VQrG)fQ<6 zo_=pp`Gk^M0pkm7DK|R~;H0j%mWV+Y$t1SpsDV&_H$d;_R4eL6bao3=@`(dr7je+p zQm=LC)tqUmSKYV0UE*a0(=!_ZV^I0rP+R;azh+$6X3tHUs-oT*^=olZ9qLm*!dxCk z+;Z{{bVFFV&VYeMOv^ZQ=e& z4tk)Vf}ng_A`S=Xmg#Q~(JM-xB6!Xp>*nr`E&jJ$|$BT~r&zvgOxEwjG>q%`DI1|H`krX70uSni-+F?8P z*vT!@JXoxhW|$%Ltp^8EvDktax%Y*;(;ri1zXMWgRI$-pwmD!iP>`W{5f_M|f^Qp+E8h+DJc4$Dhq+PfLZV#@;Ua59-aNQs zLCZGGE@$EFun1`-5IZ(5Jk&h~CUhM>GsIIQ>QKk*`^T^mhVC8p$tD+iudL(;4A%T( zW1>2bQv6*noBXVsoB$H z)e4GZfd+A|zZ7BcuMyW#j5MPmib9j^l-PZ~@?h}2Hw}?4ota&X)dFbdARFkH0>%$`L*&#@@}VXOHlAM4 zB)|#&bG{*pDGk;THB>go1oI@vo;qjKg$k_Y@F58MBk?nHH1V9O~jF zZc2|D<1ZQvX_&#`W%(KnujRjMlFg&MA{scJCb2>M(U6xQdE$66v+rqjkPeTaEbCPu z_4s6X=2ds}-uE!%&+`0>e^v}sEvIKx+tXIOX(Srab1m%`qn9113^X0TQBcO;mqEYU zsB90rUANcVOK{a5(QIM%a`%vqgpdolDFQ&D_|yr@zSiVR8z2Vk2^f77a9Q`cII0a- z@Osv;$2&Z&)i!n#c7!JW9GN3rDgzo6V-hsEPh-Pd_u@MuR&06C&7T$1=T`%qO)GGs z>vz4p?-qr{s$GvARqTAeEljotPWV(`JHlUeyXqWwijof^-B9U^eUWRH)qlhG@W#KU z;E=&JB0iWB9VsTgE&Db~*Gh+cUAil=GAnG;_m-&X1ogRF*~w#|y_R;1`^Ti<(#K_g=nFZ_*4{wkV0hiO)~ z*;?XBRl0kH(9fQ+%xOn8O_Ox7zB6305CxjRmWq+7vUD9rH|yB`_tArvVl&hSLA&GO z0)2*kqMD{>&LqK3t~E@6LgzdiKPAyC$Gz?(an~AlLFGC~xLWu!FHMiqnJv0jYwEH% zODxTJ*ZZ3#3f$Rd_wKdkm(Gvw*+hcBfO@US9WI2cebvie3^@|=ko7HQaMmHi=N$t? zj!0yYEMu%UL%)^?-!lH9jm(?vB)8axuHz5Xvp&H072`*=VygjfP{{W&Vvoxwr?gl@ zoXf)zr||0d`c<$0DF7O$f!aSLA8OE=AofLC-HvVA##Lf(>!>9_9v7_|=C##fXl`=S z6kv|UgCaGZRZotS#MfB4VAABtiitRgOjqUsTSa~$UXOAghcBhmgC$!LnQ5?kZ}ydl zRB`;L`tvg9qh-gnd~RFO@ap%iE@aKpYSscjdQ$3V;^sc0t4Ctr06=F4*3K|u z4|sFf9m7`x|Jw-L90%xHIGLs#>kmHuFze=tx?d?_WUzW+BdVGK+Xd?ot&@dGc)a=a zbiC+#?+=IDNn5M_$7@?KlpIDGs-$n6o<1eYzxY&)7tnWhZRj@o#uH&#LZSP7;@wAq z!<7<{oE}&GG=Jxj=ie{%td!gKK*%GTelzbzH-@56He4QB3@wVZ&Ah& zd!Q)t@X>unYk%zW3kylS7$CKS&%g=7ZuD0k+GsA40%-BpGkx8vyu$nn+HEuCDAzi< zTyeFhdWnEU)W~5-34DL@??QbvP0>?YK4CPWPAs@}0}+ZD#)iAppC8bUJWTday|;*q zV?OLi9lL$t01x`J&~FcGIOmo0ASL!NF_((>WOuB?nPbx{&HNoWvxB3$d>Gy*GvgD8 zea|1^sbXHO*=#XU1jn(SnGZK+P2-G9za2?D5g<7fo>`@e0z5bD zC7z&AFWLI(z>h+}%^I1H&OViT>I)9us;(Yd8qyf+CgNDti22)hFQT9`dKf%LzF3dQ z`>3ew$biDvd44A( z`VaD3E&v`#2L!O$q+%0f3NvfcZ7fl7;(2l(fn1Z~sBZIzEfa%9ynHoKmCm@@x%%-W z4#TP}cOAYeVV3-)jV>pDJ%x})MyJaRsl{uQrE96MiQ` z&NfRf?{(j6zeKi=kLhw_jDJo&$g9U|M}I7&_XIE!3!?$Bg(WMZk-K{Cg)%Og9^mR} zrp4geQ)I}*B4WqCAYO)YyQ@(u-3q(`nxBf@_{C2l5In0(j{a_{qVzT##lH2ZPZ?;i zNqI%*kLpA6^C=hnhrP7D>S$yEPczv_rjR=BNuvFqn_*0Ih9JfeA}dLMF7UfA5;e7J z6Xq2;^OJIRKob7#ec|o@GidS!InVXiH1zNN=RWeoq5YYROgd_g#Ar%r1u5bP-v^&{ zX!8?A^^@%d9@A2kMVtr!-d;mLrkz4~btVW8O~x2Xx!(AOK17Q%jo?}s6vrmuk@}mH znm%UkzCVtGGesALFXeaZv>NCp>@i;5Sk+JLea-`BifCrIF8B4VvpG!=AjppPLs zWVr$HJ?9qglt4kAwUkg0%BpW^M+)NlsXswl$+74%#Fey#(yNgKP@f6v)%YZ@QW`sm z@{hbb!m+*pJ#luoMZDKSj>~ss2}H!A`IFBMVkm?XP4bUK;Fvw+nAO6bx z*N-EUP$Q~J)t@sUeussGk)+W(&%AOOf8a?2+-evD3U{)I>%B7Vkd z%e6K6W}F(Kf02)`-<{EJH^6Jj@g*v*`oljjJmNGg5820^1+A&#+v}Ws(x7*_=<_KS z%b#HlXqb_hey4|s?U-rhefu5U#=CARJEzjg0_BGlAKOZ{EIPhw(XUHP+@i9 zK11ce_0|LD)xu=d|Bh&yMP5*gm;Z5O;CJ6rv|b<|><3VTnyqk74Vk2GBbkLg#=z?0 zn{?|+^D}ctK1MO$6i>0F!21(2cIw?)kKY?UZj~W><|8r?bZOiNOF3AnA5J0(((&nS zq7)F^i115M#Km3WqWS?l5;hc%_lcf`N0oI=3b2BQLX`CH{gh^GIX=VfHoS>6LU$`S zRKI@A!G*GzfI;yH$Hjy}HioA{E8b0CujpVx1X4rRat!3U^gg1^D~XCiE%kxXD}uS= zCX7&bDdb2-bW-6muV^hJnErf7Ljj508BTo%AJy(Bbo42MV2+Qdh^GKIk|v&i2W1$y z%1Q57XxlL?!;c&c77LE!aHe25Gn$Gha93C}|Fi#a)o(n(83#msXq?VkJkHQ6e~Sy^ zCo+ahH3&B#-n!owop>h%8Z=1dEUOD@hct3Xu) zg})8Uv&&WV_~uyauZJ7O_Tq)PEU3LH8s=0OcwPM9cthVum%*q^qUi~cZo=ACZXhLKN6i@K1ayJP;SaF9i9fhVgpf55G9K9hzEf75KD zS?LICQ^_TTaG6yjg?ZZpr)LXd&YO2oXrMIqH=dfEUK-yQ{b!4pWE#GG%-qk(6X9Ub z(^%O*#pa?%mOfD{-j`eeCt-<9G)Z}zv&qkA-I&DfDx%h&CPN{bC-#0s_g3p`O|Qow z&zg6dXK?M@nN40wq|=Y9@3Ox0nV$(<{hS(Df}*i$pooEe^X%wVPeAr{9yF}#&ugNh zk;;T-cjkGBse>6xC&wg{8 zL5HzRc^D@T*e#S6$rafnL6BNsc|q)X8XRPf(_;aTp&rP+UUSH=#~lXkR?&%byl*zO zZ^RvMYd6H}UWk#~-o(xFTjqQV6$=bEf3BHlv)rP2c%ik;_`}PT z1k=YYBl97@XfTiYZ%w|w@06+2AFvEE*O~ak{24+!=kxYHmgp5ca&eF^;H+i=ekfqg z`aFR;Gv+5;2H0AFeENU#4^KpV0VA_W@4me8boJ(E+M&EeFAYV$Upd2@rOD=J<3d0- zSVb)j5l5CDeerziyZ*kDNcQkVOXpoaVx`Wo+VM1n&A(ovjzPg3!ZBx+awewGsN5+X z^oqO5-bIG5l~qL3SOYe}`b;oSO@mrX@+epZ^_oA6x$w*Q-wjG0+l%_ZzLicmxo`#dG&$BmDps5z zYE$vGWsvozl<@AqOD=Hu$UfFWM%v)sBCB z>D14gzQ%WY1~$81`D9o!ZMADg%tTF!n}Z`)ObZkhlY<_|wOgy>cdl8c9jP5A+80L# z$SSdJ2cnkZv9+|RDpa3=maW*e0WDjfOdxG=dC=o_S+J_(FfGs`E#dQX99IQG2^%zS znNr_DM9_8VaXy|R?EDWvf zoa|c9R9jlB+UsIG0rRu&eoL7efe4WS!yh>wKZPIS85OK*;FG)UE~~`wkhgR9+(C7ay)_ zByN}XIsB01b#k^V7EIHtLSO&b9Mx)`+GL(c-}~mrZ9VUXH8dH->>Xpt=#n^JX`B`` z=X^Sl$nvK9S4M@Q4XrMW^+#Vlo%ZNE68$1r+ue<^ZG-To7=i1b>zgNGjp-5y=KM_b zK%(J+-QV3fm!s(G}v=bQ{L2=`z3mB(wKYCAe^!K z+50`TMo%}>A1{3BAsdU)BkK-8hbhbGz<57Hb8FBUSBFZ5LCi4Aap!CEm3Om|&2+5x z1BWoq{36A4s`jCOaJpZRg39dbd|eXcTzHO1-qBg*JBY8L!86I4@$={>)R2{4CX z(cz!4rmeQ35EiT3u&V+8Zv48q7lb9NOF#`J>RPZ3je@L`WuaS@9jFclrG%>(q^HVF zI|@-Vr4Mbm=)nvJZ`(hC`bi>_&gRaJ8E}xwY+!M^j@O%>+&~xx zsu54XTc|d-hZD8KsA^}k>^v#L{;%_`{q$_1u2eLuBB_IO6`=NqGZ5sCkP5p_cv#kK z?Vw985}U*6$QoVDi$7_es!g?W6?jr znSPYBoaZY#0sonH>>w6iRPS+4b_irEUl6;TK-J8G2iI;&H^kwt41$h@Blf24XWo*i zudqaP1S~k@B|+g~6Xr|GaLcj@_pRl=Tiv4MT0jaBt)FLtqW(JI$r*2x32mdNHj<)+ z-s&=xwPaA+S~Gl3%V%z9FP%(!h?e6RSF&i4SpjP0{6(*qfd1wE()P|_U&^YjcdV3k z?&fpRBAjhoW#-!vt_`*o&7gWBTjln7WO#4mMr5D4ggn3a1~ z{f3;URV%oV#L-v6t?Ehf9_a#jaxtsrmbgZnr`$6Jsi z(+a|+cX`YdesK(b;qIR%{Y{nHv5zlmFa*l_U0Qn*uN2+5TLu1>_EXyqd|s?-RhY~l zE9E7zr2mnbNAuBT{o5sdZ9lLTZX&(Kuy^|f{X5CoZ;|D@w>K4juc{CN1SBwYLGb%2 zu_OeF@oD{B$1{_9h2+5Ntxnt7WiyRgK+&F+zk@~C+`!nw*35y7f`px)e`<$IT4`kk>I&~! z<&e6lpK?_KTsDO{ovp@p^|i5@@+vT0!_`KPa~r&K-IH{5#z%Q`vR7`+sx2{K14hFn za<{A-6PmBlO`c0>B9A7bJxA-$$J!)H!ITAOQJGEG4fCT5%XqWB?N7Y2le{ftNDL3V z#x{y}A1y#2!WuIC1Zt3wBxc#5wMa|eZ(3P0zH5zMfr}kYOkZ8^_ef&NqEdtT_1l5X z1D?C&;(R|$Cc!00&jIEwp8#B&pG%Bw#cEVC`6YIlf_BQoW(NK-AB*1WOY*lMqk_B>m!%M-p7Cm-Kc*rv<^?Cu`5Y z%Nz~HrBc8Ey^GC^7OB~!ew#CG`(uZy=>i335P2&OUeNXX3$m%ff9}5aBPQlcXrc|a zY)_@$tN|Fv+;XEgbmI>JaXyJGO{Z_=koDAkE5)YOkHGL6<~fR8(RBi96rdmOngfAE zU>|C7Y*je@x3Dr8DFkQ7URmEq@ViH5*EoF!3sy=OrkXZgQKV2Fkf@Wmi5m50(y$jU z-mY3PWiq~(TR0<=gJCpUCQsV$p;4x2;kFE1&i{<+`#+Fr>Ld#I$PTMqq2-C4ierO^LgKC zcV}oVa)d{)2h)c+W@$m2);=kAII@PFV=TPrCxW?22YYDzaA$0lHVpJr`fE-)x7V*k z+JN(j`gv6BOKbOWg0+G8Pjx-f&O24#Cvmt798M^jPr%H?rqd@QQ?yGn&ZU|W${O_x z?9pTn=69XIhnt3m_ua6gJHKvMCGFhf`slml1a;KES~vHNQVaiy+Yi(n&dx2k$*yPV zNgE_T%>(o8UrQyGSLv@7&yIJJ$QTIwd+mRbnG9j4U1=)+iqOB{9Luc9)gAILf=aC% z7X$7+$DYALCPA$Dc~`;81)X+YMZ9p66SKysUQM^=;S;^3H`f*x+d~n#l2L-!b{y5Y zZKN;ct1s5vF4R~7`|>7=T7Z?_OMTWmIs~8p0k*ZJcQW-WPa*S?+|j!z+PbTzj`vBF zWhIV>T}lV%hkv*LwjoJn5tIM9y7=7ZU*UIOK4AnsEs5warZn&m(qHpL3?0q>0f{2|x6=fKFgFGFtU9L162G=uR(*<{!K4ebnJ=fsbN{f z=My&D8WxlTpU#Dub{&tHE=DojvUS@~=>TaGP3ZVsTYsE)@-t)vw_U3-T_8`nMN3Pt zB?$$(BzwIrS!Rh&XEGoi|2dI1lWwj1Z|X3FHe06d+FOK3GG=h~HV}i<-1H2WPIAytFh>%{!RTl3cBVRF)2~ zg{t19mu77#13&0N{#N*5HPV|X*5^-PfQ%(j7ulX5C(1fN$-4nFf`S_Q9rT1o z!!Z8eU1T&rpJ5iDDN@e6Aw+VGq3@tg?R&y3cckdFFGv-mlAC#|A%QoPlUmI&s(pH+TOWgh(xuq;xYT-am+RU^U->PG;!j-5H>R+ z$L|e}gK#Td`FS7@%n%7$aJ)Ex?`@BQRd!$(K!_;OZq6Jrgr}jvn#MQSe13sSE!Ab! z#$4!dE>o9uSL0%^cYo=Ywg<^x(az5$-xOm&R4))nYOngOb7eiQIR1Ig*%|Z}$`|K8 zh?74a-cpV89mci69=5u$)b^aS&t`Xyh-gP#9_9Jc)%;ZCmTP=bwc~h@(^@`W zUjj*{+1#Z;bstSk(r!MWVMi2xB?RieDxivU;vjEzUKuik4jcOab6+oa$zCUiqU9jmI{GA_m6{)HLvOR&=d{!oy-m;<%VL043;YiY-1ajp&RKF=eY^LNai*_LZF{aC21h(t!7vK zG`BhY`c7*6lRZ8z-u?wDt}VSqR{m#*+dYOk$(UYSDM0Lva!2RRju;+xySMXHUoMAg zftnFXrPV>6GvXP|QOpbelyEU7;qq}qFH-M^*)9GA5_Im)N>uV7?`C`lCWp+A1*a#i zfHC6Q``l;tyUG&*2K(3i&CI;B*#&RE+kQzc7aZ+P4=tfwyr;kxK+Q4XwkOfyG1@Xx zFgEO=d9-~^GGo@2{%7h(H;6eA$i@Ar-`cX1gxz8y+^rrLHVO~4!#Yg4xeJd+%qhb>F|6?27a( zH2e8>UM_56B9q5jb;&$011`mJonG+b9eds5wlWaADbZKo?$57f!Q=_D!%%nDFpu-o z1_dUEAr&r3mEi`iSG$t3RT10%D%ot733f7-jsIOnYWWIf#ev($r=UOPm#WV6WaACj zX1Ugjl8YZT$sFUZ=aH^f2_%7l3KQ_HsO1!k8`ED~*C@|beFsGqD20&^hMoigYH8%d zk1OF$XnE}@;xIC;8^8uSsZTNlT);*AQ{zm1#_sozaelwzBYu7k^?FX|Wv1Xx|J)fk zYu=^(40y{cf67fz;SW*^&9zvhY}bZ!JyNU!lQ4>@Zo&spzum| zWZ2;Oo>zS`4HyrJ-F}queVWshSBY=)oPl!RBk~xJ77&_uhItqPI$7E|ddVPJI*H)X zFp=m!fa_wPBm6WXE$QB6&W|D^fQF;>yFoY|@*`8(^ zog|T=RwEwKJ}@AOxHFZp)3o&SR(r2_(u+5x$_JI`PTFV3OU_jh$DB9!Zc`$ZL@J?x zk=#Maq!#xsQU0x{{$?o|g&6EIow8v?)W0RsP*gor)da99<3Y%L_oog+u-rloxI0a~ z#)F=kNy_O-25)^}#Qdp!I9PEgUYH4N!pek6T{-vvz5qouInWpz-u!ee{_$X`FjI_B zo4w|!T`Fo4$0;M|nl$L0wA0~W_PEaay@-2&XjMmWF;)=0!T`?svj=K>w03=Gt<~)! zv*GKtME+HOv&wAjkk;U5g9SK#sz{zh72%f?crbZdmdMBB$T7J&+f~{PG}+E>S78;@ zY(Ei(rf<{{+~?Q!rfFaY7WYcspwPBzj&#m5NE2Ayv8ruAUrNJJS+2g3h<*;7*MG7V z3^=A&3a8ah64m38dyoJ!K{ee=$B)Nvfg972){*r{|I{(}NdGX{z&m?yKc{M*)5u)T zYCI{f3Gx)>GWzv*}5ee+2cEa5J17B{6@#}{XbeE|ErATRg3YrKmRCJJg z#X{E8Z*#2JycZL68Ytj6Bj$9wxN{|);Fsu@%Vaf0c0VGF)5}?L6!%u#^Wd}$^BFP! zka^aEt8*OFQbV2geKj(dgLYo3AHeaw=msSP%19N#wV! zPV{4yrCfI;Nd8uVv$z7BmQBVst$|uxhb7e@fhg_KXZ#n&SpZMLdRez7MK(R%m!%dXI20II@0Pu}(iq`ja{IL5W>U9ql)}#nR{Y-0{eD3uJs0?%}pjK+MeZFN1@#P#P zvmG00RHTt%=QYyIhGaRraFoP!{p=&F*G>2$Wglqea>ab+r?R+Br3v2l@TTJa&MjU! z@ww#ndx*oTI_R;S_ju2E>Lnq|`z4?+Fh2A$>1!+*c6K0wr13-+R}7cj0H z@i{Ke`QXj%2%0LcNKHOQ(m-fJ=*REBmxauoMXlb7|DkUs&i=Zb{p>kHo$`~W3}ErH zr;y+Q4G0oV{QGH)df<9v4j!KJq#Ia(3HkWePyID?)pT8AXnXg;G?!K(}04&Jz_8+QL zZd_K&VuKEbFmMZQi0rcNgXg0)xh3Pwf7ZFRcVGKQWw-&ZJc|=Ed>aEsq8DriepN%C zXZ4n?fgU#NZ8`DaR_$HDjmQ98U^EKslk&Gucg9;Obp4j(BtV(@fp3#}TV_xJ&s)Ih z&}NtUpY2?g{I)w2`aj|#v?|cR0@O?n`M@8uA-V01FCcRq08ujV22uOZZ}K0>#dw)t zjZ|4UTC*_f_K(TlHq{-50tx{TPA?f7M1=O2HZB8%FR?X&*v(W@mt_bTXq9G9r%Lwy z@M*OGP*H{2uM!vGlNt-3&iZ@bI`kCYz=yt)f1|rKivq_9D)&r_Z^qp_E7G$DK~sSc zjz{MOLOFhJ)E1mhT+LtiyA*3d1GnGEF@u?XVa{cB)&j#HV;$C8D<8k@t{t?kC{z~S zAV9DMsF@wlkaX~omU|6;sPl@;!Ba%eT)x+D`Q zbZBGaTBafGfVXf@MS8JXqAc7a)g6DVo5HxLMqX1IFcO@@9(}%CD!fmoTuDq*f?rVK z>fHU-2be43eJD#O@S^YCGy=z3_X2Z=}2F76DUJ(NyFsewjBKWOH|H~-^0snKK)R$z=qxK%Ulz_r&0 zXoY6CGRut4vTV5PasfLNKMa6em4%BOn`XwG;QkF!AeV!n{%1Vnr z)xFBTjJY2tz6CDoA62^OkmNXSS4a@~9(bAE4|kn#`eM{%>m!i1X3>X9wjEYAv*(rL z5ID(Zjp2X~`x#5$1NkBU+L&&SlnzCP+a}sOMOa5!1@|ff#ynj>c{}Wi{-F#GGRF?lvFF4`alHcbQ z*>a-><<7J5%w1d_#O}L-Si||)p$Kt{@^A);$$ld<>6hX#2%9E5Z_uq;`Z{kE@4~Qk zgqu1n0x)JR*Lw$W4_1l!0&%G=rY28P`3@T%!HgbaL4&@!OFw@Gk`Vicg`848(|AC=Fn30?s|_N7c?xm6<+CXjP~vgGl0mDG5Fq zB5U&I@P(m20>%%xsoz4COglxt&>Ni&hAd3VFW=7gk{JmT=#5NN?mdG6TQbgHReKEH zLnuxPZL{Y8@3}JQb~`)UyH#fndI3ET;JNS!7z6@Lt=p%KDlpXo9zOp+{ZifSj>$3a z9W|^ntE3}Ue&}3=Pm>`Ka%p+Or19$bJev((hNdRcZT7<*FZvS72y7wPA)#E}PQwySu9?S7s;(NeYVO)zR;JI?!jl}(_4rm}e!8BLQCx_osm;zW*qJ-zT>{<*&!GSHNIRO=_*ELg{uyT1?H|SB$2P z)q5(`5Y;-L0+wM_OrV|ZTIS}_% zIhYSoIgk!ip_BGjG3_Xw8o56bxV6CTiOBoWIA!6>1v*(7#|E8tb9=9!a(f^C=Jr1D zKkc=ww!%`OMuj(~w5Pw6*hy0ta0HUQG{2xuG{-D%WMkE{WA%7vXe*Yljb=kD%i=TR zHNipCN|G2?ZPe8)yhOTfTvHOMcn{UTpScXC^%zS=G)}m@dUqi0ufEh``27Q7xkAYG zZ}Ci(d6kgM!q1rs<02vOa=H39kvtC%kNko^H$EsK)I!mLH}y(!gxTf08^Fgkf?!y7 zF6Nj%%qtb7DqLje+qP#jXuq%*^n!DEAOH4i2w5>}@JBf|SPzgjxUGRshU(7#&Hv!} zCJoMUhhOUoI&`w4)Y1k@jp`B$U7YHlm6%mcPC^jq6!?mKU>Lq9fB*Q+NVpfua2sb;`9B`f zjq4cEYREFv&lPU2q=wF}mttK8;1b+#aN)KVKVz3~{4Q|6G-Ab*Y}L*3NSg=hz@;DB zA5Kw@Ag9?-C%Q{|xTSpwG>+rZAa4+%?Qv7Vj0=yBOTb^v5%H(R6xoD2WtF(41(>an z#@&2uBdY_N-d`>*H(0x3(Mv^I+`pAL@$E&bnCD?3wWOei;6CI>hx3P}YoT$QiWOGm zV*Q4T(8N~d>~@Re#2V0^GBG8_EtShP5%e~zq&Uig|L!?)Jo}3aM+vXIgN2-vqNXOt zO4)KaigJ8wzxg9eibfC{ncL{7qA4paf9%;+1onHTfJ-G2U6g#mJk-0aRIeh_|8!hw z8XSB=gPOC7FA2Fw3(?Y$L_y$-ja3Vc;M`^+pnQ7Me(IrsDl^9_-kgNObck=Z&8MX$ zj`&d~7Y`kW=Z^ig+*KQs56cR(y{?|f_fC=CAi5%m^$~3+UoZDGLP{}6UhKP@f!yn; z%UtY9NU1pnnFjA7me#TA`>N~b=eK=HKfTp%I*OnB zZYw8PQ4;;v6FzU3%e@}M+hLEXFlqKE^+7Dd|K!Q}Cqk}85d5wgRQv|5&|}EM_KUAI z`|uSJs=S{nWVpDLpUx+P>Xol+CG=99`t|Wp;wtpf%&m8T`aq{%WhZXZAjyYm|9ykH zQT6;Y5rb5yK(b8~x$;>{i5FCd1c#EzUv}BSFE1YO#?uEmzG8tbtH#QJ3|}MZ9j-9s zw@1MvZKx0dp7sBn1+CDXN57q)>gPV1qiao}mNF|Tq=S^sucrF%OlHATN`-ZMh+Rv4 zoG3~&QQ#8AgHu6r$U-xlzZI=XQ}NU=Zlg}!*Nf6#VLx~rbu+wxqZuBK_L$~m*evyN z+$u48e4^7Qh3nW^3zEF61di?e`$BkHT<~PNY}#YmiD5g_%M8n|uRDRgYJex!uVyjq zlscMkm-ttl=;p2oPc5j0b4W{PMk-oZ$g^v}z572VDo>wZruyrq9sHP;> zK13_zWEQJ`1A|>klVZG2g$H4nGHeCr602yBF~(vVv6n^gE=S#|5^~Z!la9b<|J}TZ zDeO4|R_>+`_%R)FQ-#&NSq!-X1247^tcIZT+#iEm=V~8u)=1B z#Qs_YgB}V#J(&u)8?${Zujj_$;vpyca9&Jb?vk$hISf0C6|gxv;l-@;s`_F6jbT)$ zOSJn8(u7YfISi}!H$@#A9|MGn$0+kh%(xm>(o0#Fg(Rugak$_=|y_vwB2`a`Y^nzg1$s$rksCcqg~yC}f} zt zT$!lRPo$1h&_QKT_@KRp0Nw7|REDwRDTD?-dFcm_QWk~3R67hYD!LjrzC^3tfLYu3 zfi2~!OXuRiV`^H=HfoO}HEiIf-CkVv65Z<^gz(nIEYYI=E{ekE*)Pz2-e%ZyWtOU+ zanGzAEaUu*Ye@5SY_racT|RY`D_o(UR_@n(PMsjkW-pp6a{`o#D&A6WCbvl=lXzN# zdndtLC4FYv@B zQp+kDX;Cw5qVStBRESfD8M9Kmk))C?Mm0v+hTFjCjo2xVW>#YW`ECW%^Yx%jRc_a( z4Y-jB?HBH0y!vsG#AC9%zP+str*&Q}_8PLQYqpbZS&jLS1@b<^PP~(qL6&d zrUs_&attf_Tl!{FS{J(oyQbv4ph36YoGrqS*4{)`q;Tq&uG}2@0Jk_ATZy>=e)L|h zXTmW1FugcxK|a0oh$wQl(%kD!HdeaIEkBd1U-yX9D<=8vT+z(;DL9Fo5}oE8Z#1xb z(ivA0og0)fNiaW$Aav`B?sS5sMg81jhTk*#KB(3%!!!GQR-UOO49e-y?j29!(k$*BGczt8YJvau-DpS~gq_se_^Ip^$e?J< zzIz}^20b08c5C2x(&wK@l1Ok=q(XBDHaNNQDW_h#PclBs?=6BEEU@CU z&g~u~yhRM7!Pvpc3J)PN%81hJ|Wxdp79+^XJ}Ri)L*W6rq`S+2$@^zdSj^ zq-WSe>@o7;=lpsg+iU{t1Z?1dzuancJ(uIM@OtH&P&-uvYmH_^?8`qC<}8o9pS}*P zefJf5h@7fE79Q4fM1R6%rR&E31X6vA2o>{ljDfllK77NQwEeB8jd^5p>?=M~>dSol zute;Q)WxL39?YT>8B-8F$LxzT+M=mJeN>pYn|q9cLh0vA72MwT4G~(GrJ+mkU7Ce` zs|HVy-lpuj4Wo~_>v1{s3AeU}?4~0Ve@y#yocQR;{#xitiViZztD*K&I-fBHKP7m9 zU>DWgAlL-9?hODkJ$ScFb-AMTl}32fAg~5OMcQrF?w`S}{c-Ed^x(?%=Z7^MZ?n)T z&WOwKudrhkvyMx6S-zcl>jyQ|?sykR^d==|5^{b}EvOp5*t8XjUjX!Kn*MhgbH-B_ z%fp37D43aa-x`0Ya$ycPCg8y5{^sv zTYaUG!pj6AsG%SggetY}wYg7t&o$9(OS0hwNxa@_->3syyxuPDv~rd2(WkONh>9>l z+5&=%u}hn}=2t+p!0Cidju4_SQ)0g`^ZVqNWZR`bSkq$G5uL(OAu!B=f|`=|oAwW- zc&!k$j}PGs5VDsBTs)dBUIvEUobf=xKuzH|t*n;8r(oLVJHF{MUuy%`hh6F>HhkIY zcB=d0d|#1MIx}LyMrBWpqbTix0w;OIbc{vKD(;g>}Siv!NxC{mhQLfAY3Z zNXkbUu;+a~1dQ05eRHf43bC>xK{=G9U6zx6741>7$THj};triipvu95 zkILg`esYA#6r5Zt+#JqYQhB&v(SD(Q%bG=FOMIbjeTT-1?@6WQ6XafzlzATyF6u2& z1>H0`2sg-VXLj*{L#RoWK%hOVs=`%We!6h)8q;&u4lxn0-lPX8j zytvXNPW5PKELYpAH@4y73kt*8O4I3qWVR7IIw6l=4`i(Zo^YAAOhqQ4F%xxglEDX$ zssGlXqBSu`5z<^_LSvVwVaR{;b&4NF?}yz%3Tvn~r*a!jNcbFndmG<2RjONNPcyGV z9D@R5&%klT_oViACyO)^Bd-R;HMy4V;SoLK%;59Tj50^S{D=@5q4LHZ~(jc=Z9YaWhdpAC(a9e7h^Q?#jlqS)seAT2b29lI4s1qSgXj@ zQ0?(KMMH;(F79X4xi_6#mqBi%l@0e9tvcwObc$P?+)(sd(&I5xS0`1LrsY%hOo0KD zorfCik}-UM*2}osxxH3Ud6{uVA$Ui8<)t!l&8AU@6U$>_{Yn!pe!Ec}4TZ{PU` z)FZCk;qq6TMXzyqPfxuUv$jRc?CczZ!G!Pkwrm+Wg)Fd@S*Or9Kq;65fznAKcB~`q1$)Bmf)fmne z#_xY2*fgM7h{p`N@R*L20f1PaJMHKqQ_R^L9XZQ9S8zOWAYg_Xig}J60vXC6?!(@37cK`+~Ky9QR_6N@}sM(T@9kmWj3fVeMpGR)r_%|a}ZYff}|^!15b5OAQEJj$6? zsS`OL`o;xpfDFJGr%m#e#QCHsw|+ioO!zf@)bB3_z#QEG(x@ZN4RLN@g6MWc;`f$a z!o*KI^3G_RBuk+O(X<`1A4T3qS04{uwVTO3j6JeQXM~sw4(V$+; z$1XY^eJ<==6z&o7Q^E(|W-9*?(0(RaLvAj;8$KKup?WCIi_NdsDSPEeVBdZRT_5+wiQrW7y6#WwvQOS6RrUeEnK*#s zS!Al<*xgIrKK&Nm?r*w&0yyk-U<5;P(i5-T^?benc@o9A3$>fV^l2`C!If8l2n?Wb z$vbr*zVonc75s&)^whW&h9~(z0yTC;AIPwis+M=F2Z4**rSiHxi92*dP{ZD+vZ5-g z=a*?7v$4PG5r*HtW_Pd2Sns3uDK2`WImwj4``+P8`PTc>yb#j{&BI!P0*ZlFs9EEa z(tR!IwF_LW7-ZM8yeYrfr-R~9GwB#-NzXeaa=ADSL1PDYi(18O-xdLEj|F`g86h`* zHT&^FW~`=v1o%3_kH@6xy;RSt7-l>1VD|P*)_9DDqt84qkTx4_f#dlKti0Et4#ep< zCO7;dkv#dI?(iG}K;+EtoM+wmm@@EwhXK?g*QjzwWl`cGW#!TjK;~OOL>oiPL`eQZ zkzLZ?0%~6^&my5aX8|}~j)VVrg|X^wo2| zW>AX)uL}?V)C&Fkm5U!E10;xh2ny!?mwd12Y+O(2w&*|?7MqKMia>h96erVH-C;ok zNJYzbnN1ntbhwJLJM!rco;=GMB@)rxd+4ieK32?5(9JwWh3vIgK*YHq_Sk{AN~H5 zAZ^^Cz13dLe(~=f;RG+FN@qmZrIqqY`{d-3A)TaIu`O>)iI}5Ga+kP4{xxz9(`FM6 zM?PgrvLEu~M6e^IhP922A;Ed_<%cB7E?WK|2le)rhg+=N5Um(KHgxLolduQ|m_pR_ zVsTLLxLz$A1Thus%lw<1bXE#INzrq9>eWr1I?Y2-MocdBOzjYQJUu;!{V+S1QKH?x#X+7lH4 z?8pvXBy(=T9WL1pmrbxx(%1?XgWeWb2ATWG3|syPV1$@x1JiQ(yLb=JBYR|Vatn`K zlgaDwJz75;3|oG5S<`tA#Ehuo7ZB*ItVpu^vTBxwPk4y{!(uTLR+->u_<(2#p_B0# z6ji*MSm?Kd)9}Oo>LfL17@UEQj(jZP%Yq#8Ne#4$6pSjOUZ%2?#HD@Z*C|nNJSKJuM=7 zqS^I}HS>0?ldubLZKW%v5iL-&j`yoak zQInO4DO$YEE6@oTjQu}8>v6Vn!X{mgCVV`c|I=rhMlU-1?WK1YmuU!!+quS3SnY?6 zKvaLsLjDx!B9MZx;8-9`p(H#BuyPnc8khtU-w3P-QM~w^S=7;;KQ-3Yl^@Fti6r>E zOS*Q_cIcM=o3rO#tSn<= zl0wcb-%%JsjMy2zKC}TqoQNwfW!gq^!TZwCYZ`=OZt;_-u7M~6DGQ*ZC868!^_|LW zg1QcSay7f8tenizC-}-2dG_2uTmvKl0b|9xfJ#g=H4OQ+LSM2aQgEP|$Rvpx zUU(ebenJl%aUxWH&Fp7`sE*xZs*H0^x}8Mf#749OH0l&{(Ltvv61o?j1*zp8l*;nM zb*$~0I@e&#mvi?f+?68wwu;$TT!RK~)w>O|X8fb~o5mC|dNr0ktz0U!B>`ioWTfXp z+QE`RlJppii&X91|9i1EUug{SVB#_05>9<^iJTI+ByN%EZ-+Rt)w>9TwsIbVU#`6x zgMv1~@d(TNXPtx|+F)r|y(rZmi40raf476`szMG!`AOwlh}d7Pcp)zc&7wpeiwAyw z7eJiU_y)scB2t9A57q3Aevs>UWXghPLmzk~D@TO9?vlO^6?GVFmFK0Ui1bO1nEWBj zrL4g&nrRU3LaVEgC66#@nh zmcHC&CWpr)SJ84IVbW?qJktNQ-W=f`Q`w-sAo|bd`iNG!UoZ8)r3*Xizwa-49@rfZ zo+|MKmlCGAz8np5{!4Z|9I9~kp%z57DL(ituWes^uvio6_vW7u1RO-`$BJNnOT#M} z2o_aK!_#^}>?}<#8NQMo04}+SNNMr9fNJK5o447~mn*fcyWu<~jE|&AKnM7aK0kk* zx&ZW))Qmok$pr%qCMv>MfoqTh;osqs2}a{8p`(t3^OUWjG+r_JO%3ACki+Jk=<69K zh)Mj`uIWCh))x5Nw;rv#5uX8z*@Yf?AMR$1!2kIj%6uTSsU}Jz0Adv`#0TvhPc;Mh z*&)GW5;>awM0gT}k5*?Y93pUl=Z5HA|0M%UVz{})QT=qj|Ec`c+Q%{eF*+hVK1K*F z7c~_ppnCaZ7zCT?YWqE&QGQ|s8I7=OIn=rN;33)C z`ky#7rb;9xDY*=v{436N>|@W{k={l+9xhUH3r) z>7B4*7YUn8K(ngE4T;y^x14Z0UQKxQHH-0=Zf{>b)kwls)d7tXAM1o10Dxci2lMNCx8CNQ@|4<)~BWw+AB=4E! zWCDdfzK`ofy0eQ^V3LL(oY&o0YFBX)o+S4Q;{)7D zEBU8Hv%Vc>^ULxi5sztXJK_V;OFp_lLw{t4!8wBH*{jvBKH=Swn@@NayD_IE@6{`i zfih;K%8TNIjN@|e6#DW`rqP3f0tHwHAepKOb=dR>bzZfENZcqWl5cW=!a{Ugo9 zBlG38O8ojX4r@PXITR5Gfmxs(#*0`Vn2l4EFINE8zl38?7nMm3BnZ>Zukqk**B=>A z{X-ofsH^&tP&MT?$8Qmx9XS(5b0oPK^4nu0W(K|@s1OW*bshhDwq!zm%Zxfoe9fci zwNV-N-}2)lMHG0+ly98s*rTrAKu9oO)BS(<2E19a|G_6cyedbt*E8CzlMQn2zqERS8eGfQN@k`gpp?9y*KOqd2iVqHlphq-$GtOw zAws&q#`E=#L*Uuq87<|p`Nnfe%~Vu~NsPX8y(cD4^C=s|xY1Q`_Ab=yhN6JYxv4(J@0?+MdsZd&09fTCHcS& zRy@7<^7}o|%MLS@=X+DKXaD%aWAr_yS|5K&2ejG;YzSKe(^}k=M{T~q$1pZKB~wgb~*HCaPV@ieH`Le7$NdTZ;q zUS#c!7?Vk|D3Hy@wqM>dD)e5~91p*H;Q;Wld*Lx)HtTH7?aimf!f>Lg3n%N5)?BPk zO-j`Mj7|6N-K9>=CryTg51CN1*^ve{Ph%2R1(xnn(5KSpDof-=84GiD>Ptl$d;}_M ziWiBXxf})d+`jN^^+^NBYvU@b9|-n#UkE;RhrXyq<||+Un>6-!O1O=Fot!Ywj;G?k zDEjb{T)uzEA*S|iM4%@l!O_3UdrY9H?<=H|KYc#U+kPd+&^NOtDu~(+Kl3zY@|}5# zXO7v+#gx2ZlpG?hCki{fj*Nnh>ew=nsiGSZ1W00}~>K!M^ z%D2{9{tg!dPz2DQ9Pyt20sv^z9_C3DbJqaLbj3S3_yti6sDu9;m_{iX zK72`0zz6CVSH2(a%JuA#Vk{bOzxQiUzGaXApk%v@5?()i*NvOsz zhcK^RXM0$aS!&th_2hHOKFQPBZtc(8V9JePB*WM2aTHZ#KMqx{CsBu(wUGIt^)Koo zhlImY_UXH7uLZ{1b|n11B!3UoH!OxX3<}Rs&Axx37zVH+%>0b6NA1TmC9VqH#W4LU z3zy|3lauSw?@?6#atHh?9E#L^_%_)RCm3Y-?hNN_(7AU`rWB%3)T`Sk`+gqgJgc7q zAlAM)3FO+8>Uwy3pW-dC7DDv=%lg$@3j9c^EhhSHeQsofuK%R-UZshCacV_W0=C`b zhP%Pi)Ox%c>y|e_x1v?Z0l|;o(N1wufyzrp+@r5qCgFriM==aMe@=V%yz03B1D76; zrMyVh_DEJ)F7|rI$25XuojvI0616Q>{&umh`Or@KDkI9BfW!gl2Yt9XrDb;e7+HIg z4crAs;*L}7Cop$CnUD5Re@PWFa=@_%P=!+&qcu6a&)VMAQBstm>+g8HVIzCN%*0ov zH>T^|uv$~G&}{w*n6%_vuroM476u!{Xy^@U3DRtq;fBq(P?x+ZZxC8yO5G_lViyh6 zHjAMSeivb{)!{2zt1!66*fG6n_%l>D!hqQ^uqvu&&th~vOMz&ZQ~=q}I|CH5S>D6K zMMlkEPOBwI-(!Tgej~r-E6h7NAtRq@n*oYq$|{M!5xyR8iq$6ryT+)uGA0}3{ZW@^+}i2plk{f&9piTmB9iISo(5vVvh4R8`!)dxM#8%K%@vI6D9xVFyZK z9{d?@X8g8UIhN&fb|yDPxBC9)z9gOeh@03*$LAC4pY7K>clv80>wrdQt)B%OImvD@2U!Eb84w}?HG zJHH=#6eBj`b9!8CtM^7ifNNk`RNUuEdCL(vcR1a4-6@e(@n%a-M&}W2V9g zErtL=jhjP-f~s~4Xx6|+%Fs^zmVlx$gw#UL`Wm#s1PP8wVrQtc9pTGgFIb}-GVmV@ zjhr&aY?+r1rf||YJx&m^91u>&=XQ`qELpibcvP^p1yX;;^}9TENM_wTUIEURW9a3? zmIQR6L2b?W+uwjY9>7o6-UoEnI3| zaOvt@`IGD}P_wJIhZGF&{e5NCv#;n|8d9R59it~7dlI{aE`&ZH6)hKon&t*%k=Hku z>dxY9Y_WarM||EAjx*QmE>B_|>&e_iA80TgGesSB7I6e7{f6mJdeXyka23R6u=YsD3dr>k*YCD`? zb#rckN9P;ylrCw=5`64pE-Q;90(>pZvv+RNjWvAuo-<{?Er8MCc&L5zWopyb2G~SS z#)*hE>J~9j0K}GR|HQYskKYfE3pe_XjB~SSc&>Q=wXOI>-EyoT<&AK;oO}g30jFVT zhj=Dycg!3JjpS}~SawKaMO#!@e6kd*eL4g?S;6Pidyx|;lF<^#ookkEEx@#Ap~bYx z+g22g4uaY-Up(hs7T2|s?4Y@5yU&@lqXSHt@z+a=e=slSecmWyitr=Rfd-6#WQw%& zJT}`oj9VcWf@%{A-JHR0rdZ<+|Fs)qcvltS)c4mHteW24`Lsb13VTsRplwSn|0AN) zI~{hLbwpu0y~@_$suu2yQWBi=bj$c{%4iW=~loE2ANZK$4#-K_jn zyB;EjZJV+&tSKGFGbTM-Y!H!M%~I`-y&^S3YCq3p3rh_ZjTD(h{>`%j7f;$T$J}+} z=YaI7|K&6La!vucC%+8(*d@(sYU;|R*8Q!m?%=YVVhD*?eIx5IR4ap`$^k>oc+JJi zD$2AJ5KnH5Gq0K0d4Su=BJtN<+Mg%Oe`2=m7Z)~HNLl4;foY*nHU2L;jc%2v9D@EmAM@_mn^AG}WS0?^xJ^)_-w7)IYi`CkljH@x@D za~v;_X}@`!qaI3LDcY;m$zU!U>7gs=g7RkQ7wbH=HG_hf`%(Z_K+qe}@f zGcko-yt1Kb(CZ4EF4z7fIuHZXBO3>jw{DT)Cu+eGMn|n2{f#~i9qF6BdqyId_RAR9xys2) zyu~CD{6>N}PG8D5_&hjtbuQ8G{XzF?&P7DD>ulBP8V1GXVY-f8K(QRRL%VD!b$!&O zNJ9F0LatxrZt}X4Ed$r@x3R1hW(33VknH z&0<^yME^|C0bz?A*W(=7(|}K7u1SQWB=BguAFo!H_TdJ9jcjAdcxFHSU(USKYO67B za!IRdXs`d{mzqltww<02>EF~>$7bYDP3S6Ti!-#(-EHGogld4of3JVPvwk^xwPJsU zri*$l-_%qy7NM>s3s46KjIaaLao!}xoqXd1vDTNs!rvhqP4BmdGkTpgnJhHeJ}^ie zNUi9`UaudHP76C1hLQCVs0EB^YUm*x1G|<}d{Ikm4>CFKn8X}SZHr_pb^r){i zTZM%o2D6S%cqIt^Qgdv_+o%aBBdxz-KbtHyPpW4NVJt#d%?ND(aVr*4*y5f%#^jnl z@XKiXnTN|KC^e9;X7H>4z4EhfMa{pZqiLCyVoyFV_|DkoycxYEwlNnPf<`_fbbSIB zD0!X@Wd-iDZE*9#DRg?oMGD;(#$1==BtnAy5z?87p z=U0P=Csyfz;?7S}*j)e8z2U=^MT5moB3b{B!bz^BYK>c?d{|g#ZM~p67ZsOt7YUlo zZUBgJVrZmwVTy}QuMq;`p8NIk{<^QG!-XxPE=xCjb$@KVamsMHu1DGHmK{7p8MeNL zpPcz5hg8km#x|d|f05G#q9vz*JO#i{aF2CZjIOtgIEz?#jJWrWUz-H^hYH&)clZgI zt&~?x^Ke!7H8>D~DMwt@D=!W(ai>eM&Zv@6eeN4ol6{$j*ZJVA+M?Ra8qztS*15T`rCIUpW8pMO(lW9jtGz%r;7rm;@fCh_o05k(H zxq7Jg^k(vfgsx`NHqe)OmB?00Ffi%^3H4}}-&<_~!o|Z034LZ?8Q+G=^sbe1S1TO9-FTvTch>`{0Cc~R~YPLyhTt}P?N5Yr*Kx-?le8s7}x=(7C zY(d6Kyseh!s)uv!;R{X|=#Schssv(y(f<3s_?vt6Hpd^@J8jpAncut$O6uYR0@J=q zHCTrp4~Y;>=HGg$fy^TP6P_uB1b41c-sg2DGC#9$ojw56FxHGK5}Oczz)80y`5Q2h zF+rXKs16ALrpe34384c4>5A5Fx%f6zK>zB0U2-h|Z}a9!UYpXb1?JKJq@Di{ZV3G! zknTT+9AKFlJk<_E(%&ZB^WoouIZXkFc5t-jGV`s~?td(NoPZ<_2=)sAJh%UA;PVRq z52fT6?{;BSnb8k0M7ky(<8Ot%dw$Dm7*Wn+F}Gl<15pL%4&MiirlWVHra>9zB*QMi zr}Je`1gO|AAFXndj*(Aieg$+*AFUVPkIaPne^=9qPs7Rk>$m#Tzub}1)MtKNzyCIB zW8imzDF7l3>f?=(5`6YMlQ~I&F)y&DxpZ2{w>}+!K9;eS#%O-|1W*V7;*#JDjd6C& z>_HTUVl2Gn_1ZvP;}05^-x-AZ)3>)RPR!TYGq* z2>030ZdKfB(tG&QfrW_xU_h~6$5n=fBS=){MrOrHWp*rgpCIZD{w;85`QQPqY1N85 zi^US?dY5OV$s9#pC$H-&((UDOeixdSP$g{C>rz4)f6F5Xw_`JwgF z+x7QRwiq{K48H;e{jf2Rj2pfE zN&;Z%bPu47S03qA15{0GxCPTh-pCEqD`->f+6T%Y!E3}JeAIyg5}rmjZ*#K% z$3<~)1zU~O#^;LVvHt;9I6gp?dD|-mwELF>8pm1Q#Y#xFrULZ(r!v5yH)RVZZtvq3 zBsw&`+}H==R|q-u5S7W&KdXFidQ;I%id6&?jsI<$+w=^Y+^Tt%_L~vLx?-=p6kDq= z%XCr{yHo-Bb#2OB#i34b`9E%t1#xSl2ZQF+bUnC+Jab9Dy^j?O!YB(A&EM8Gc)F?`C z046HH9PqIRV-rDOYa!b%`G^|@XWO!S%@=S%lq_u={O*Mfs|$1@^!w7UX4?YCLCb;G z+(+*Qot&&_+B-<=!9apbFl7>Nvb0i?kjt(={w|O+Am)CSv`~ZnxhMW7HRMfZmJfsW zZNhL|i-sk+~Bl--$I}7a9>+oXDd)?bwkZ)S!a~ekR89?tj%W}#X-MC`TCpYpLR<569<1hLv@VE5o*fI^g zAKCvtoP$%mgC!I{FgO(XK6d}}6z(`>KZ8w8N~cDh+awGs1O-SBPB@BZtNz#;*s8({n$t(;t%NbHWYp&zLOGplz^>MzvwF34|I;b*4x*nX|BXopPy4av5 z%d%EFT&lX(0lak`l?5!h!v`el(lD=XgHmVi6SCwoL@^cLrfLE1IDDI-;JV@HeRDI7 zwrDlzMm60hO@DMBGX>$|1Dez5;-B~OHs6nuPq(Dwei?F1{e<;6==fAItIF8_DdsyD z+<5;pin|h^DLiPG8sL!RI>I#e759{07fFs(e&%KDTJwLf z_LfmmhHd+=K?;%zBHbw64T5wIAmsqk2n-<7NQZ!wbf<)LcXx|)cL_){()hmypJ%`C z+H38#_F8*?@xu%bGxu@O++!rl$N>zdoyEqzZ-Q?LFuUX z<&z8&UMF7{uvX>G?JjdU<)C-SI$JqqPu+b!h`>II_Z$>%2o7?>%zN+#wae6QG@NQt z--XVThx3svzc%cAIt`3(;Lo`t?~%{0os4EH>T?XgVhHW<#ugxSU$$nfNK-In`uc~A z-)OXSqf7bNVZ$}`cBtZZL25taw-_mF<-X{ggmpSob>rMK!HXvdtHn2rl<6=Ba{2BT zMq~#z>c=?h_{Hp)F7 zy)NA2TRXw0Z*v|nd%zrqd5?~bE?9t+E`Mcg>iLu&pfYWr%w-NMz#W^!q>?{;L?Gs{ z>dAjjQ8>aIl-M_j_{z@BBx*->Gyx4esvCgU7GtcLJdfSf#x9cnBpU2j<$j{in=-kt z>CVTL)~H`$du%N~Hhhb(@g1CEFNH9X?~8Dwqxwmc%?3D@#}KKu1skBRG)J&J=-=6ECkb2Z@)JP zxW1gO7P(3F=;kX0QK1|7sJXf!-(QgyE`rRG^hi-tnASssI>_g#8pe zI#-J3yJ0TTj8jnBLm5&pIyKfJnZrsh*fBqHz1j8dp5Z*38~yX^hSW*hW}Ni>{B#v5 zUr%CnMm5bWX1vj{YQ>NiT(2X;6@p#Qs7q{HtM_f=S0-2C_Ej%;BkxV=g$%IFRTS+* zwXVC|4Z}s!EZ(qaOUu{0EDQ(4)TKf}nHpVDZr91nUklc?`OHnWc0Av0P=`59e5yHK z7jpM2y?U!Ntx_zpcW!+{S>J@3Zzi&Dz~t|A>N!^&Ol#7uEqEm9>t%H#C7jd693Y*n zgI{#N=jZRhX*w3fwJ(p9rqTXA8g|tE$Vx%*Bia23iEAyL>Eji80_?EMr99E;5YMx!Bn^X%et||I6mE^9r4!+yje3Tb4Rw0aohfQ? ztt_RP$E*B&>dIUMj}d&92+4v1VRtFkoS^s%(P&wB(uzkG z3g5ifA}mj&)T4?j!caqFJg!-UcS^7@hTxkBhMd2Lt z<_uL2mIJ&~_!E7It$}Nyk2f+%W~4Qi#-EDgBPkG|VcZ?FbTqRp_#^8sEcZ4tP{e`^mTlH6Snm)IP71d=JBqOUqVd&>rF3HfnpeR2%Df81<(@xPfo< zfXG+N;J*f!$NRZcH9lX;JfTam)=a*Wgf#A(f)A)_^?`FsHy?#z*nn2Zpwp1zFV3kH zGreKJ@S6x^ZwQ#QkD{4m+v~?i0ZnP!Jhskm<#`miomfw7qf2fB)8;RD}R0Je1?Vz18H zr6&z5Vt&x_IrM1~_IFl6i;Qu6&Hw?}MUY8^%-dU2>R`ftvTDf3<#x2}4)5U6m(i!n zm*GXE`{~xAgz}@`fW)`R|1rYm}lt}c_I7uev%K%wZW z$ScUhVlI> zJRwLI3w(?;v)SWS`n4e2r_-6^YjL*8A=$s=j{n#WjQlv*41%;f!Yk5F?a6}0wgiVW z*SXa=P@`5yl4@K{^x0{}w*i3Ue+n_c9YK(ih!Bs zk;53b>|T)3n!8(bNF$@OC>_njZC))St7D~O3>okUH9)z;w&>Z8CnxZC6n^z)f^J_t zXnKX40qwjPANy6Yc#!|N_3}IjHp^Gz{f`3@0hWE&4ESd?GS(q}uZi-hV81F=^tqA4 zeGK9G(K6~c;P={ayTn#N5}P;v+|;0gRGSY_R}PkvPLo;0x4kOSHk6t;+D6W>)69)E zOl|W7Y1TW`b)K)}hFhsWFXAH;v0e1@xd4a^>RuL>4vJ7fc>(o-K71EkmF8f z-qa)RPX**Vz-{vKf|Ge#Ct&MaX_@|zHN4CkUw|KgX49y4%GYuUODI{Q;-yiri?MVs zHOhr-HzO1IJfjly_ZgQuQ$5L>EH_IW&!W(47d<_75--0foU`1T2~)L)EcIdV7Ods6 zz8%<&5D80PZFuTrs&E~c4J^=Qh!~V184{h?zu1A1p)7EcTKE05NuZ;%x;7(unDb35 z6ZWH^JYC#n^*1fIVfXWj%eJd<<{>S-n5@ybhYBKBzfgjK14v!YT%hFU&5yN-dad1V zWu1p2Dn$kPi?FO6ZquL)5PPqFU5cOfPOV)cV$P~lT(yxPGpumltrd30)~Ct>_4x9g zP?E3CLkF6My?Z^gvm`}gULnWhZs~RnZqKW2`y_a(L28GD(NIgc7VH}3v3V!__G&Fc zlIF(08q=_j;?H@Z)wp~9j;KLhRU;WX0;VKyMPx>t__ zAUsLC+%2|7^IQq;{AM+96UVlTxLU`A?QBJ^aK^>+LG`g~mIzTvs;?c?bz2`?uL60!6rgp!JuqffBh!0Wd&>B%(ncjW1PX4@_#>F5 zpePZ!*4ZGVSoOT`qYYr88@i-%9FPVD9}V+m^m)og;l0=@dGbX&KGjB^0V zu?!iQ&*O$Vrum6O^sg_6sy9i|a~y3u>pM6-emYT#`Hwb!o5%T0N(-f4XN*+-bbCYd zgz#)c*4OJ<0&EUK2mW-*6?=AKun3>X@|eAE^%gP#jX(yYrUux_2%W-v9p$C636*?f z;-b$#)Hptna`uH4fg2(5r$cppkThMk1bhQ-OP9ySBpcrMh_;NuW#iCD7{%~}Ouqkt z?QiP=-%v^Sf%&A`yE{uS1gcpYXiVs>iCyhB4az)RI=80s!ytMY50t0<#_YgdG>U%b zXDqhXDpS8P8{m}yXp%bC8rwZ^0s7p?@S-=DC1QTiNkMTA_oKr)Cd0vPkf1-^+-kwN zz5MK!rm4Po66W{f)3bw)$c(9)kEB-`Isf={8x2`(+8OH$>>1zq3=k}ytYHpoq872% z>sBFLW3optqZ7`BXtWh{8@X59=>Oxh^;r^l+aS;pFU0fid~hDz6?%xxVgCtHm)2w;EjWGrtKe#BEcgNatUl05$GNeMG$Sj z(>2i2I3%(R$xQ(a`uo=wuJM59qMxzudlQK@Yh0dcoX<<1pe`Pirl&)=*r@$%$lTHE zx>jI**wj^g#!{hF_Hgimv76fzu&CE~(c;PD>HG5`KeQ5^n;C&ZKVT$K1-Rs2g!Z(z zi?035&}g+bGuniCViE#F_Rv2BqjP};qUK9S*kBz zgo#}eVinu+y-K?ejPl2biT(ZCOrIeN?r^XFK&XUJib^Oj1w85kWJ`f@RE`cWywLzu zeap*7b~`b%Kqb946>BZD&oKE2W` z7u2N?p*29KP4>sLs?Kz<@ew{cP6L>#aoiRDK6taIpzFUr&#gl9GB1aN1^ zyA0etT=-J+bWLVH5>@m_?zq88@VhJq55F+=)6va7=X)QKt>K68s~l_msXskc&+&=$ z6k%{R64|8*W@2-EI_rtphZYTbU#EAvhpXnOjeJE2kp36`Sg77PtcEDK+Eo_S7M;lW zdembbHP~#fdChf^&jL7m)fyyMdVx2$98|L}3U!|`@X^lbq$VYA{$t_^SKVvQU;ByN9mIP;^#ynorxJ^3 z9mU8=SE=hTfnHR>pNG=JL}#w{u(j#w$h0{fLpQw1J)s$);jV|Gp~H@xxLk>UQVoi} z2~Jz(=Px;Y$$OVdn8D0iBkR^wOih?@S~rzw{aj|&!9EGqVfZQ17OO5N$fmc~1x7g9 zmScloErC9D&16KHpSEFexY>u2U}}rw_CGw7lW6M31>~aI{wKua4wQE>`B==OSF;hF zE>feu+H7B}ENcIMu5hY`O84ID^XW&H6v&X$~DotY1 zaLMwxKiz$RxFX~oCjtEKFK-qn;3^Q=XeM=kfks)&Py%9mpRyDHT?~9E?Mcge6LZqm z1Iz|>su#2wj?zLM2Hh^zbAu+2FJ~Z8j-A?qCEBY)a9Sk>Cvu^p)SnL=Tl8W|g*U~E zY00v8cLJpLINS`rA2CEBlX}O{F#;oLm@%II&zYTWWu~83sWygGS*Z8tSS1+R>MOI8 zunc9FfN!pOT&ez|3d=k!lNIug<$%FOv?sH_%Eslr==;=}a(j2|eq*I~l@p3g+rihM zpclO4;{5TyI@te`HJxqpy}`!t+oDoM4eOlQtk~#~qI(VH-wSEP#>BIwW@u{e z$`n9*))HdWT_Jo>(-&U^U|6R=A|G$=rUJML z;qtoouSL=sLz$^EWowr^X9nmRm3ba*l!so0LQrOWOtRM}B!$X3OrbOY>sBH$zYNls zZHkmco*vi77;(iCN=+az&ZaEa7rPIopgsYT#AB-YC%A%O9AahAzX9o4P_*_h8W3P-hzu+RHcaZ2Mi)lo8XOY>nD4Cux}?8QFKmRO@TgN+le z3cpS?=)53Gpf2q^cV`^gQo%*HG=czU!|ml_}_!THWG16-4#3rcO$7J0Pp>O`$8Q*9_EHTTp|}*_-T(q zxGEphH#o6?SNyd;R41$7b5!kS{UXPd19bhi5Eg!JXYb&~Gh%Et#DK8hsDNHHk5$O~rw(gZnh^qcZmtD4IX6|VAMN*kX;4-g56v8{)tWF03#CO zEs>?DbNLBGJ)u8iE~$AVubVRIzdIX0@a5aLI-dMQy=Yyn(?clxVzETJ^=vk4*%ZfhPBj>{xt zakzBQTcf`HHZ@mF%|iSHPS}OAzbVToxHxesk7PW}bP`7|M)C3rW2#B87 zz^TmrkVBpjMCJaVB?~4)_rfv0?|M7xPz4G#b@}jjZtJuC0GdN17|nFK>XPshEYbRN zQiScOQ$HmOw@*~tDt=aDX8+vgp7R3>>Zj7Q_^B4az#G><*iv~(`PdH704fVpR{^18 z6`R232H|WR2hTDFG;w>SY8>>jjtBS4N+%-HYJ)Sa%kXqad+9sC9Qi$U(dDozg1Qmi z6ARP@pZwo8O&F;+JVPDFgedE;ML3%IJlc5>vd1+RpgcFG=&!(bjk2%+=CP;DIwO?M zKiuV~w*4-WLM(eF_9B3#v-l3B3=-0Z8IPA}@sp#MaYsn(4?Z_z#y6j+~9eEwd8;cqEvu!!USL9HfvjBY`SB+$A&aXP5%Iy=Ezdl;U z7`To5TaCLN9dK~YpOSxtiO%lyD(_(2H-Z?|fet+uNYDWCN|pO<7#KAby$A;umaF9W zVTpAtiQ;U2!ZS3!6h?G`pv@5Y!@CL}$r8SrJ*xRIBwJb+*jTDZ@{T&W|J{W%XD0ZC z-d++31^>7G(gkOMwufk5y8LV%B5VKM0;}=_D-Ny_7%c-@elQHr$w}csjdK069_?u7 zg9`a*2wgA$r6j3iGAhKykobmtNh*IY4_J%93((l0wJqrJeQYI&b^k%VRSY_K&+?N8 z>J9=Sy!ZkefX+v+lvNh3We_U~X5C{or&J;FfFb?2OThKKDNMA1;5uvXf4e@U&2z6Mv{agrJ zTMl3>{4X#fgmGYxj4JwHxC4*ZgTemeyO}-kNW6P|l=PoPI`G;3@8kOjGY~`&?Lfff z5Rcmfh{tn8_Wm!pA|4mqPk@BQmq}rm*sKYB371R238sG9P1EWyW{(WC6;|`)3Jh@L zYL~Mz(6>on6FRrBTws5zz005uP#yHLU9IV#Z>GYbwW(%PcFjBzY-w62g>& z7!WbS#tR*GkeQ=v88V|dhzR6$4V>GSJ%_at-AS!CHXlkc^raQqU0e*S3=4t$JrIo8 z1+W}m_0)c|9+lMXDfaCGrordnhUw;1o?x-~)Y>fhw~G_o`>~Gyu~Q;;)6D+9M>O4N ziHA;e(2kIs0asj5#P9h9r|a^ce#dbt848o%wBlQ(oyPPy%T*b)wP+F#8bu<5y578H z>aQRy&Ie5J=PlkaqfR56CRZcm0H^CI_;RQFc>K)nMCrEWmyP!~XMm}ZkDZ9g%a|^* z7Mqu9^n2SZQkeTnS;s&CD+O)IuZ)y2*1k7@(TNL62EdcNv1qu(jh}bxy#PPbpW3vL zn*|vR;(hwk3QV+T{XVYD3g@puE)Q25W+`I%>QVt?W|;v`3;>h!lsm)NS3zIy^-VV` z?@i6T^uqYTlEX>Imv0stCm|~UcakVsQuHocTyKai7imGNJ5SJv2JpW=eljK#r%BNK zE~t6tD%ShVKxo2RbaBvOhQmf>f79LcIqJc0@!GHp(A{CCAFkcaN-OZF{B0!HKLB*$e)#5wd|M_Lnd|*c)I*J5P=iM?7+VnFmQ3h0VJK^ibln{J z8KEE|&-f8wIZe?km;BB~5&>zvX7p&$c|B47u9OYhcdrc7P&<|<1D2^dwG{sOOGTA` zn%|l@Lp#Fnj)W2u;0@`-E-~pn9b2h**4{vGOg_9V1cxg_-<`Dt~=RDJWiD zo)gT(G7*5FK@GfFiqCiiX2*h=F>;wwM+(mTb4z=#?|cbcV;9J z4O=T#p*%2(dAN+m6SH=dD!S-$i07c|`^mIpAnlJ1;9!~k@%7!C7q0)q94>ZMf-5JS zVUnqgf_&0Bkaos>5?oZs^VvYQJJLUJZ{{qr^3%_9)TeTp=O2pwoR$Zl0^nQUv=OYT zzq~nyAY=ZUf22CeQ-cB>Vxsq*vG)Jd_{LxChDB+OGKFSQSM*bfyH4v1eq+KgS!Osb zk}}JNd{!(`E-&rn!Rh|;0RkisP6if$OPI?#lfMWI{Hyh$+GD2`?*Yy+1afDjO%*Mq z>^BO~*()FpAtPW3AUesV(u;3{!7FGZAiPC7zJaJhJB*)U4>|?4|5jaBR6yX0+zNu! zgU*2t@Y%ix0tCXnyYB>8Qh-fp7D)!gx~?-K3tV>rPTrt-5O-&kgAgixfgB3~e{S&m z{e5$MW2sc5Uu7 zC{OTO_vx(61{twiv!uGb)1HwNL2NCPfN%3OPyX zn%+;K8VMHS-#*%S8;fvnM$2YAJ{A9XyQ52|`eZQ@W`5Z>{hIhKumo3Zd))WKrC25y zLS=|R>F5=WEjiEGt}wbzOqE`}F&h>_w9a4W7W3T7AaL5;L!-S;l@mjch|)fbeW)n? zS&5Nh|A#~bKMo;UDsDOK3M^nIOsir8`*lAsT-w9gTxd$FInHGhUI;qiAc1^cF=cMH zeOME298$VJ?Kvtm5F7rcR`aBDM&4q4&>@HHLoq4hv!c;rP47CU7GjYI=3bYpZ1oY>#BSfK#YP9aR2#xF<9Hjs>1S|f8Dx3gW;Sg(s>-lcg#+=l!)e7UReg;$R zlTgJuBa_sO9!Y+y??3l{Q;~fcJJ@bH4ONF?<%>mW=D)S-&gb35s?2dmg>!2CVp}BI z%`%V|Z67l&Olxpv7VznK{njrMefxMq$3YAlKv=TQ7hzeVGd#w4zb5;Y!3uLsl%~Gl zM68}Uor8lgujpHR>vUnQM*VG0FTCTZRH;UvLK$ES{Z+_T*rjq)_;YkZ`Mdf=eD{3f z??zK1ZYQQ03DMdw@E>=B3nBX^fIi=eIp25MwJuJA34tF@JoqHJgQ1z&Sxf6#?UpOY zBr5>`^ccV$SkZE+m&@)W#_53rfYXq`Ysu)(iyU5vVn95C#%9y#ldz=#Nld&P!HBli4^!5F5}(T+Vx*Qi*p&XX@l&I@K@h{i8zVqk{e?&I1X?Z= zEcv2Wq*TC&(bwJc@3IU%CvCJ?-_`T#2p#M%v>4*A3=}!|=Z-0v+!2M}vv)H{A&;vW z%UXRNZ6E-Zd7Y>PD?Z!}cc2%>=lI{7<;;3HWzxvQFMbq z{lfyEuigDVTYpwNOXN`I_@`anfw^ZvPz8tW>(Y0@pKgcpX7Um`Sw}~WMXw2ZK=Ll8 zU9hLMChV_dJs_XocyK}2{pr!7M0+Rb#f1#8iG);de53j5*mLB@kqYxT(fDMUxJ`vm zJ6tM!rB}vv*Ez_{qv;0Iu@K1Zz^)bi+{!^SsS9tZ$hm(uYwLm{z7E!4{@1yfSUwvp zOKukEqAC$N$tn~A8uHJm8H%+Bzx#F;LKvxZ3}d8Jd6>eF(@x(YtgS_0>~d8^eA1yn zbWpVDDwERW%>7n=`{BcjA{&&vG@+$&dp^R^BN%BpA02HpaK9Oz4OI_soEj~LAyqUja!i&$6{Dkt8kuR3a7MTKV8082XI19FRlYta zF&$V^4}@RFVx3wSDrGK8kps3>Yz(z4_- z&*TH5OU8~V7~&%yM=E#IJ@lhqyBU&2Z?yUsG)Xiz8(d>QpydPq9KpVi7-UQ|1B6Vy|0Y9fN) z#B_Ird^HQ!2cDc-GJGRW+fh2aAG_Z=tmQPQ{RMthb7JiV{FInSPb~+qVS*ou(F-(+ zw$bo~jr-e~ZRgzc>Gt>}hHvO}d&+vN*0ZGsMshH*yJl1vo{GBwC?O}Msu-)eN^8R5 zFg&aOUI&b~KvEaofS>h;6i1uH^A=s>sw_e@v|l%@T%57!j8%QXJj5l}jo2~-ef#TD z?XU%cQweZxb9$e|mE_@q#ID(&Tx0BPbP(9Xzhk|Wl1KwvSpa$lztns<*uP{FA<;zj z%phQ97HDPl^p$y`&SLK{{JM<&M=@4Dd#XY`kE7Vs)So}*EXmi22!Rmz&y3n}Y?NBT0u zLd~WN8)uk4oC~KQg zy&cPocdpp;cjJi+^QP^gpgb1)MmF{!?TP$Tz!7=@zMlVJz2V02VkO2gW0|E}31!o| z`*ansu2l!wMGb2);#phZZj1V|B(1tIFqi5SZ|1ifUlR!AJDBmxHwg}S(g zKG!NLYkn}rv;_@0r+Bwe#K~M>R@v|R{~z})Q_XrIJW4&QpeHdwR2a%-bNs@6IiXG5#1dZuMj=8 zbXmp;?{8)kd3Ks|UB+uZj8iA}F04t)?{~I;eAZHB?Rrc`KneuSg*%R^Fgi|6nq6b} z`T5WsPSm5p4#91JJ(5=I#ad1<8~p9?mZ1R|PmTcH4Kft4qJVaEUZXs&rk;cuF2epy zXO14z?1HCIv7J+SS{&VR8`*wrk+TvKi#06dJij8iJ>C3d;NVv?+yh1U=f4TfLKYyIFp4u>jNH8FKVZJ=~swbAs*?)7|?OpnoMuY}py>B)VOpa9gsgeqaU^ z2Fh@Y)Ti!32>Y<3voF0s|1_USE;?l`3iI$)Dn+!Ci^d%!8?eWNH5{D>&mZ^QVH^m? z$Dc;n7hVKKdQ@5rh;ak_)mlckofCj_<@6b+hCqN8qQ*YDXENS(bsC!+IwmKAe);;;X<5rfvm7GMNqcu1>=3bTsv9Ofez|1G{!}a_<7h8s z%qm9rPO6hVwS8_KAi%6yb(#Y#JU{?>toBy1Vb)t&+nssFzVT33u^cSs}(dE{3W>zppU-tK}v7ZgMafqG#Wi9J`EJ5vcFh8 zwn`RBo8}%PzQ={E1#N{c=yEq`3e$AznF&Xn{79j=SKZhd>1HwzThIf*RN8#ct^3eT z>_6H0SD%+ZmZ&mNh|On%x;;Mz3KJvkaQed+{37wwvTytf7}4UtKnQrz zUBxdl3HlAFqBFow^D{&S+98$qRBpbA7Eexz%hvGauiH=N{tb{nxEYucU^sEdrsCQQWy4x22$IdlCsIV z*=X4)v2AeHrk352Ild956+NX`2Q&G>MSW2lQz=>ZVQ5c;D9JHr+aRAT*ia>hBKo$U ziHYe=fO4`XXPlLqdpEOM(bcd9yg^YN@XRVF`sggA!Ajhl;AT#BkYR-m)V)UV z%2Ng8d?BmjDe7rQmnD5CqmrXW;@x&CIGUykPcW(W9fH1iVbp_hVP^L*#-?%&kZ@WB zHi*v8(B`w2gyzjnwUDLPW4?ROL`1i}8t8jK2PD8}&oQ}g(>+x)wsp5J%~LbRuy-^@ z(EP|a2VWPDuu-%tk|K24_`G(yN%!p(ocWVPnq{^}_ zF3ZP_6m*@?`>?1xetBV=${dd$IjsuxcC=|<)7^9X@{v`n&C2{r8)TVwd&gc-NVX)y zwprkyAUuGkkagDYJ1?vI;*`XLb{`j=s3qslwhJ&yi2sEjlqx2A3Dhs};vd*2j|gD4 z8CTa?3%2=TqULdqTl~=hxALNnE+|uMrrYmecC-D7wo)fz&F|%*&1(??@>&EpXFUZD z>{DZ!jh+p=0;2FG@*+H2*f5{0a}%9=kmb9)g{eyAMt*-0anHnZwrM2@2iVwhSvhuK z)?$2P!i;2(_1p&xTRC-U+*3Qe=w}}ktS8N8Dk9VBGLVm67*URBL!Ww~p@aO=oLU7n zaEagX2MRvA|5t5=LO|?LO){7Wa7;HPJbkOpz|OAWS5+*6RElbkYvbU9>+D@q zR8$O$;zoaEvC`Eto>#I{4S2teuU3VaTTGle2>H$@_5UPHJ zK~ud@2X*jADE|Si)rCE!H)tQtXxWEv*f_dQH7s;gRZ>&_eowx64?5>!{M)pB3>t*r zpHzQ5X(I|VY+!#i6Kh04`I05-J%=#|g?oGR2ORH@Z>)soooneA-KiJUrE&tH``+Eu6s?5SOjpHrZncm**kB){Qo+xD;k3s|0o)1zp# zX+gDKl0Pm<1Jr%@)u_d{DGURT7{T&c&#=yyoK=*?pfXN82?qZa9IlrCvM~PTT>Gf*-{F>OS! zdJSw}DFz4r>OF_6FudG{h{j&3UHcQcZVgayP>FA!lbUo1+>BqyPi5xSW`K${hddav z_~{G80_;-;eegf7N9Z_qq=&hAwi|E)9#y+N=}|!6sC6M$-}wAm@o&Z=Vx5me?N2&+ zDr#5#UVx(T{JV@iZgP*~datF7deZQe=J&~d2Xe29*T4!Bx^`aGHm@@@$f{(uSJ=%O zAeKx|6rg%St8mk({gXYEAL^{0i&M^UTXTH3ZK`gN^?jMtb6bC35M#9kwmeZh}Xq(UA_+y|U$an!0d)%G`8dfFt*1lcT{Cc|{Wqe;JxqWU{SvZDiadVpH9g94xF z2D2qM=y($B-l(_iK7s$?^_@_gXxO%WR`vjhlx0QH4OR5&Vhf~(^-4tE3s9|I^oVt8 z{aW#b0}7)$(4kucPcxQbOt{p7_a+eUDB*sQG-nWZp$FIhn!I>|V0b=8P*H!J3in8uJcCUIE11=gq)y}V57 z$>)yO=pFpVE;yl0zt1e1cAZeQOL_@;MsY>_&{1 zIkHO>ZwORgIZr7-KXZ$YmK-(=p2XV7&C~lv9u1v$tqy}ZDP13opr7Ja?&cY<)3*dP z9bqyPaRxQXa)$`C#$0(6Eh9g8W*C0NZmkFdfS=`}ahh0GICtikZ&1f-{qcJxH%Xo= zzd(BuhDWtrym9!;u*lUIBRTM?1Y183&dXplQa(bb85h~lrv;xEawXHW96<&`Q?Zb# zfYOm$!X6-{v!ou2lrhLNzRS@S@{rCRn&P^C0KQByuU4GIK%kAz@eL8i`ce7F_7hWr zT1f&0ykCqxGs&1Za6smJKAV5!G5$z)As7|N#!6+70J*z>P)6m18B-n0cgW`R43fR; z5aZryBTT$bUMAxTh}5E|SscVL%S@G_V>Yi(1AKm2MD#EQ6*8DhfW0*Y&`%Z-0yv1k zko2E=vBMwyiJVcpvXC9wgXh$$v4DY>HhUqV??7DZ2jM zg#7B^1G&7yj86YBH=8#712LTO%xhpn z7bT68bfEQs?(jLYG;k5?jI(~&1nCtBF2BWbVA*kAn8fm+o2#i^Y?uOCV6R<-868>4 zWEU9qo=|U;%(+C?+jbNwf7bRmIPfQWRs${#$DIFYZRwn&%n(>(W7et=WU(c!G90_V zcE-`0625e@vAu77X-P)*@)R|*3Bn;`w+pm)CCO!tRFqXo1FAvtk7jBV*=^mcZ;3N+xoGr+QNdO*h}49 z#MFUfb*mn&Pa{JJ;C~a0Z9PB#l?OVJ_2+J1ibpVlOF^1l_8aW~4y;F98J_5pm@6+* z*L`*J;v?%xaPGt_+v*w~L^Ekl1iABe3s_;;73ik9wY9A^st&ES8jR8>F*o%TQ}4`!BxCU z8(fJwZVpABD3E+ir7x4^uq;{-h>U~XCz;w3&-`gQ`11xXS50SdY|nTc>a5j5{WwH$ z&JJ@F&7hw6zEXR-ImjM~;hm*SCwg$NY8#T7ax7lXU$psE()#x8lR5c zI6cqji%kr)hz&r#<#A?B)AZ>@EEff1cBo*5CIf#rnXxKXKc0aKfm)3R-s5kTT7Vsj zM2F@a@FtN-uD#qpNH}1aZ^(%a)b+*pFq%{iRCYen7^$e*++CHl)_lU?QLsyo-$imC z-Qs@sUAEBo>TWWMNE}|E(XO`38^07?ymg3QZ+-Y!=EF4er!Yx70unGfy%TXLfH(x; z!~qJA?wZ!|{JTRmGStSdS20`cANZ+$oJpXD(Ry-e`*D!m$vx2T_Hi%2%4Q%T_Nj-* ze;JHYAG>HRVb1i*uU|28aV|MoO->qH>xpq`wo%?~2!RsS@6HL?y|)rV^#G^`)T`#g zHHY0t$C5p$tWwDdi2PcJB|rMNHFO8w0Yr^#Y)~F!18U(LMOP(W zU*95#1Sh6Q!9@tm5@Xd>;+}>Gx~w!2_&;LWg#GELgOLRNw+0v+5rt3sn`F^8#?!^~ z2n9{F8FGUlIm!$rTbl(@W~>DddwwMRXgH9m?QB5uW#bjS*wYY|3x@n5sQ^jl@74%4 z!olrduqXZCPW9Uih>_i9XO4#cAqf{5dOX0h4{S)M4j3*>akt0~?u&LMIg)cP#)Lqc zx9OsvRzX0=i#Ab?Prn)cWi%iX2Lla=9&6bz(8{N}1=9BnrfoD$hLYVr z8?PfpSu;gJ#%hj~=)Sy#^g4vl%mFmz9X8w58Hd}0HHUg;nJmDD983hWM)wmTyt5~> z_vt6BG!2B40A;{*Y|TVb8rTfDwSR$+U%|a3qAGg58o9fLh0nD9$pESS?O%dhgnIC} zRg{iE8%Ad6qO?V-X$!M!(rGBU@Vee&t*Ok}qfe7MDifY0v+~%s2;+ADRV8SRto?p; z|GH-HZTLBIR|D3qkgfOUC5wVe76*#%SK_Ajsq`n6Jn0T9YkZ5zJKGg8!BWte<*74>deMy2V)vJea zshgQm|3TqO7cgjL+eSU~R&93HNNQOY04301Ml#yeE;Ft4a$G~T^8t9kL6gi-_R}D=Y78tC zDTTjFZCi)QHw0cjm+-l~tlfA&wpkG;-EX?Z!>uN2c7Dn&@J@c*?}8Y=TF@LpNrY`{ zfj>Mdu0SOxyNeE=W8SPwe=){8ld08>N(=+ za6e+o*BX8I>tRj_-)Y>>ypvbW&jLV=Z@V8gjt`f-$P=}kO}tL&hymTi@UULchIhfn z&4J)~nD%%_S->ZKH*a3m*SRmDj~(Hz5n8}CY}|x@25GVAgb!5m?#z}PolFSaOvtC5 zq0<;@vE0zH@DgMvl}xSsMWWHh5M#3M84Wa~@g;A?35^tkRb;$@y>A5H6x?kVK3x}s z0rJ$(#di+LzoO}7noHYLzfVerCRQmSEFcoAr3okpc=t6yV$Ig<_ioC(Q3=|eMGU+p z`_!^oNoRHcrW7qB8rk@C!}FE9-?>FStjFB(@jwLU zW6-kUarv0XsH6O`s0?SxahLJy?Xl7gWj8IQgTa6@o$QQuDMchX54riJS5c#fs%Uh| z+0r<%`Sj@L+Ae4X{HuV6z`PRxnH`f+F=Y%&=t#B_;2u3F6TdApl;?yTby$9@(QWTD zDAXx@c{}G=l-7uGCF`GKv78k%l1nF*v#7!WN$3SC{~+!I4_6Zt%W=E5_&^x{aTp6j ze01|)`;LZV9QhK(X{qeTxU!lbc9PLk_MEz#nBBv3790d?s|7vj6{{mZp}PIG2Tn8j zkcyhi70?4W&+RAwmgXgJ3*x-F*W; z9DgUk)zAPJ$v_#XQqFJ;-bN?fJ9yH(&o!b&jfN5+1ly4DZ{#)lNXlcSc$ya!k*8L& zX>2$FN@q4bK3ZZV1qC4Pu-dQ6HmANYV+>ZW$8dfpq`&$IY|FRhEa)lqiqcRN-+Ul#UR9H&%ddU0pgng#y;$b1lRZO6v6~6 z|4x<}ZaGqCeqfJg5{-5#4LzD)I1oaGRDF5u7pay>qSVKOIDx((@@6p5aa&gGeWAoO z^XMSr(f5&$(HCAi>ErF(4Qp?son==9sYLRY=SpG!_Jt5BG#L9de={3$QdZzOqHqH=f2dw}H zFsWFx^^+CVlWvul0|S^4dYt_i16mC?r53ZFK40X7$wL_-q5?Ix%ni;R#u~O@xCgSPD&np7to!UT)8Kr?0avRGTg$uZN>*^fxcdL4sY=4 z(n+UnqKGsZmahZFKhJSLQijC2?$>Yhh9s}RETVdAu9pfotyauGk^)`(MYQYcRtO5l zJ*bOSrLZWO>|diMnN{d4Ol%aKngla`ptl(_Vl{5K=0}W?UMuwnDZ0;JoweXL1y$*} zlcmXhG#Zxfz-qCcwg=%w)S?NkM^V!rhmEa^XTDTK!vqTP5*=*TGbE_ICYx_N{bd8h zR7j5ceW#xF@>DVY(5=q_#d5BHc@o&iAM=JB%Tb_JABY<4cN{e`|IPba?zz=>xif%; z|4rGh2USrj9u#fekvz2r7dhQ=FIlj4+XU^9I@B3v_%l2y`dRkEey$w~yf=O>baMDS zkP8N+1HjUKEp=?dDa}H^b5O%n)a7%H{K`%jdd_MBJ(G=I9~$k(UyWfGF&x!D!Cwh7 z8r`_3I=QEQwS9(zXWEWEExY+OD@)qFaU2c~U=gEZtHJ;iGM~{V>LW zG&GEMxpj1uhm3ox(~iqJYwbmHYX+ueuS$QLyZ)Rxen|B7+WjzEy&KO+tosCZafrX` zALDT%!WpX7{poWL@FO8U>LxS~ZmuZ2P>xwg6-^Kd7kQEtpr!P#eh5bHL|WGiRGTMC zKiJ`4Y~Ebw8#bwUKxfI!eUDCf@=F92IdaPs8+wQi$jnhMWr4n7qUQB#%_m}Edahkz zHP)P-_-jd$AbJ*27qcyI0&?N68CA){T3_)`WJeb9BgS;N*l#331{y(C0QOImn`UMT_T zb0>Wi%;M<9dc&Mo{UHAzwAX;H++>Lf??#?M6uh!ixU9d*CMz9kO|We0YSFPtKn-C_ zZ0s|~2lR6>l~9P-vEL{hpAsufe?{GjV}5=>d@z<=G|w#VRZCfx70x5X?k)5y>63#d zJFV6i$iuIZj81UmtG{2&zduRU7q2r^lVBi~ewOAC4{D9C56mMbZW58yDk=S13E6S+Z|q`3 zb%)D z<>*Ezr>{rIMydF=cBYfo_=B;0i(&h^uwcUr1H+Me;Y}A7Xplm_LUD1xM%ZLnVXk@CZLpnB0VQmm$qkU0ac82NJBPWB zT{NksC#Qe=nVk|nF{iQNN0!-%K87f@P}4;$d6nRkp-ON5Pj}zJ)KnX7Z2>7NAOZ>+ z5GkR(B1I{IAfP~^5~T%2ihv@bB1MXXAWbQe7En+i0%`>A0a6GF zH}5;&_ap8-GiTPBdCr{KXZF*dz1B_)nkh2-FR3Hv#P46WQ8Wajuw}Fo+7stF3@7bq zNh!>cQVxd)sig~So>Gsi6aSiUU_Z8;5^IUO<@)R=CD@e#3re#^_mUj-o_5<;HU*F? z;q~k1Ot(&X)zxFs2#Qf!I7c0i`*Zh_bY|2z$1CKJthBs7XRBqS$~!P;y?8!pPAhXg zWhCX2bn=LNC%kq!ABhD@dBSBT=Pe_!;I;3506o#*r-%#5oY*#--70gn^^NF-8hH*A zwMz9_Rx|mM&#~uEyG(W4FJ?GqpN-^>5H?j;X->6s-cI0OIj!>Gb)3LrjiPwOrQ|?v zEtFj@dg)UBZ9?4YV9V7(|9$Vl(%};_0@f0OTKOmUHEeK}x*f7Rc^e7=MGw7O@*ztz z#|JslDcx;T0nIa$FU(&UwhC;FygBHR`snH0jEdGd!JbZg`#8&YCwUyGdZyq3+-2#- zWry^_>NM?VD~~@Bm51vCPVkQ(-rXPMY<9kUDa7e;DbIlf4ebfRBeglo2kx+vfhK(X zKkHVE9Jh1gJO_9OEAQ>&|I%Rpegps|yqt|Vq(XP(!OGjmeIa+_hc6e|e9V{xj{W@* z@B4>j$;pQi1HQu24ra%E;wTWQCH?6I_F_V=Yvtu;~-e3qxCCd84!NG zbseuu+|SHIS`293*2p^khyw$+mW@U^pn1CL&Q%Wl<}?ou+}4=6$AN;TAk?9WLNkq0cALBWKvPw%1#?!YsIIsQ`W z_tNHeuff5Lj{!&2SPB{08|4_n+8uXfQ$B_e4i0{QU-K73J0A9;b5bF)80#L=bzh|R zmj@c&P1YG4-jAvH6#By7hfH838AYqtGB*eY5!c!;n$GqIb2Y_pmT6P$dA5G3NzTCw zAIE$ht8G+EbIdk2-mXhOANL3IEYb;-$bO4jV-tC6y9`5vw`r&MyXQ7f3*<DuN-G!t#Z;PdP_T4AJ6y0O&du<@ z#z5XShguB0GMDw=3`|Tx*L12ylEvGYrCcCm5_LK)+8m8=n0Y6{Js+cz98C;r8IVP0 zZQa|()&MrOlMuTNI9(KSaNs;c-scP@#-FEfASa}ktLRmA(aV-Q*0GKxB+d&uoYo@RSGzI0Cn1^umpeIn2>WOw z#mpaZ>?hO8FUcq6gbg1_`S>I2hy=LoD9EZ7q4j+u>>s_3iz0~ybU>_aK7zagoa{Ev zB>M%7#pw*;1B$wUCPd5DMN7suWXeXtQd|KVVE84Y+fEbwNwzyLW6*LlMGsM)?NZ}{@;i;y9kKAfYW#)eFz27)3_-WincoyfZB8Z~= zbMTz9Vb=iP`e3vY$u(1g;k9QdjEmyGv|p{SsCql0cqc939)_Qvw6M(??$|aoIf)SL z1$yR+T^7^->%|NtywrS;Jl4d-R43tL7Jh_m0JnlD1=w6L>nBdIl%oKTcg)3$u{3> zIU8wv^=@K!7&~u-a5FZ^KyQd3H=K3w}PZtRV6nvMxh_>{+A?Wj!o$>_`v)?eO` z@4Yrdbmv=yTIMXN_~l>gpz@peLW!l3;}VI_{X3i?MnBA3bzrkJ^Wm3az~j;T>kx+( zf`<4C{wdQO(l0gj4`|B~fZHTjQvWo=OIc5d@0wyHgvV}X@cjsIr`G6JE z%{+!cw$=v9>4ZOUq2x1Kgpx40U2HzQ!` zcL+4=RtA}iQ0_JrF-^M)yCa$n{0gnWPWudw&}Yzy`&knUd7j{twLhzyiG#IP-K85D zAvM89Vnq?mOjAa;#9x?!<;wdh0SOXbXwIQfn)N;G>L@usAQo7a$nw7t!F5;Bn``~0 zbIAR{gGwyF`dEDHcT71A8Ha*iNbtq;vKV#^$h<3i$YOFvgzpvc%AUR!0Wj=UKlPW; z1nkWvON0FSouE;F-`QHDy}B;IXL*W_>%2cK;dKq+9ZVs4WHdI@mHIW6Wb%g@-TP-K zcAPJ*OAj+Y<`*j$C>~TrX4)>B(GvA`ijVo)0DicWpBrS9HspF0ANc8*Bq-S^aU@4MO8h7oaOa-bx`)-RmLz7HuluE8xSLa1NfS8P8b>-x9 z%a!^nU^r?8)T-pl&m#lmKB{OT81oLev+vpEw^MbL>K%&Rw++*`Rh@%w;0;)7lXSb& z5!u!?>*9vvtM0z93cG`4np-=U(t2Y{I1y4#pH*hzG2h6yk#hZYkJC$8TvKI@D&~8) z;;M!cPoKd-~%M@Y!b(je@ zHN5n9Xz7MXbqSGS6S_h}2*Q$H4|^!qmGw6OEO`7EM8C%M4{0REI}}NQGr8lo{9MHB zl+LJ$VS-g6JaW!A?&@hu;~wt#LbA3iTvY=RJHlcV9WXNU!~@)P2iS~?}` zU;Ro1mzpe{U_SH6#DOzqr+#%RxV-w{~1XoyO;88wXI7zV7kK9)C{!_b4F`rxS2E7m4kO0B-7-bNy)P!B zcS_(F?1_n!!HJ*MzY9^0xFo=>5S!RjDD(|zWlQWQ{7)ynctA5||7;U^Qgmp+i=>c= zk6L><_`LuXZs~AeLG6k4NP@LRdP9Y^uwOW?wJqxPx5S==2`{ma1Nv^x-i8IvLoZBC zPy#yqhf-I1))OOd&IS*fM9b}dJ6${HvdK`ivu>orA^MA8N-c>FLBy?IyNrG?r@X1N z84l<_FohU2)N$f1GF}4I{eL@zz6K4g*FlOUslN!9vf$qaEuV;PcPrZ^O1aBCd{0l7 z733`~xORqeqx|+S`FCK7*U+2~C(RW&@}np;-H5M17`4=1udxN*Nat73`idePeLg4% zp|Gtx^x@MFxXdyY>_@OX7lYNedYk~@si;1Q2jv=*j9ukf?D8pk#RdPUb0l_7}0G#j#Ze2;Wt`d#=b z#*mAf`FqGe?Z!r{S0C;VgtsUMPK_UM$<5R&eK^|$dvZ@xZ|1!$%pED%vPiSHF>z5X zt$eDzTQJnDZ~kj7#INug?|fe_H6o+gq9!S+@Qhl&{n%U#T~B7hldfXFJ-o$Qc3N1L zwD@hu&AgOmW?GxJu5OZ9-&Us#R4X% zMBq*V3dP2}0S{qQ@mG-<1Q+VAFzxp2EfP`&iI;Ys$l)VI@e6eA#EFLV+xqPcpp1?lVr5N8IG^ZtG zaCi{*J;Ih^{PRz}BxhV4C$UK?)YnO<@a|JGHk=c1>T=-QUM2AUZcsbARz~5?prw=# zW8%Ku>RrYk7w*wukHmMO_Gv2@huJIDuOeUkIB~?^uMX>Bri8R|@`GhNKhktO7o;t^ z3;3xVe(IrTtwC#GOh&WO%O|&U-O8KL0vMNd5x!xvnW?Ox)W;e*B&UZ{(oxNp1vS#b zt@tPcV5%h(5rYbyG&W-Q&@}_!G{`5E^&2Q5Bx>B-_Y=3WffXpvnn=oSHVMG$mtQYd z);Q?=6{72DpLMP?R=UEAm5G z6gNz`tV-IVN*;pg?xz$@U`=#8SvD=5WF~uK8dk(hvJLI25o|PEP((~%sSGahE&>cR z4gPYGsu^f|YrOX9luAFcy(=J5f07t%4=S1!qE*5^?qD9o4~71d)+^Wc?KwZs(J>pI zasO?#NN-=*uuwvi(XQ-UocTILLi?dU@*rUB(O(76G6_)dLb@!wlW0?05*rUGHj;uz>Sxr}h-YEtJ35f}V)wQfqTMma*eOL! zky@>0AG_AreZOQa^Bt>otva{#`PWJYwHILwHelmXneoJ(%0SrT*Vc!s-U=n}sZhR_ zd%ua%e3^u;I$cf7Z7>xG`RF?hsIK8L`vw{f1fEhhEdGwj8wpkWo7oEy!<~3hH1PQ2 z>g~1-kL!2k76!EBM0FPJb#_pAo~$F5DU zvmk01ME!cTGS2-Mkss5=bQNxZnH+v6S+hP!yuOLvX#*9=`v@k1>PG3>I2u2=H5Ue2 z)@Qu~ezdHkvi*lN!i;a?HMj{Hw87L{>PuEkO1WIOs)?6oOp553C0>!`_KMh1hUu|` zFTYKiaqIj&oX3^u;A3Sd-{5ZU@D7LaYx}`5D#}zqmBPl)lMO)u%(O#Q{-B=vxYBP? zKdGPZm?2c@;jLX^3kBQgQgUv_7p157djaalWFOS2m&F%(bu(|B`Jq4Rmrgu_<<9+j=x>72W|TtZU7sDi>d?3dl}E z+PsD_@nb*+NI@O?0)AHXrf>ZiY?F-*j*R!pQ~q+u`o=LrW>OiU3K;s^*0dEL@hZq6 z_nB$~Mx69lkfJpXZ411-U4DWR)I8Ttn;&-X1>2Br*%!Dzjy+ErU~q&N9~O^Tzs&|j z8%+z9?jpS>eD80pb2&=<+iqr#?rfilB-Y7*0)hnJ{C!UZwXkyfMg6v}YGg$@ zF*BNkk!@9S?gys=ozN99@L^K^Vxo`~!GnrKMT222ZQ8plD8~*Z^S)5zwo!TUz14;5 zWQp*~QPYjzkyqlWo#`dDyrgDxRF=L~q5QBQvir(Aa^M|brp7yb$)y>O9a87#?yjX% zI6Ibq1Xacx2V3gmmouL`_DM(J%ZP7l9>yTWtm5iTS@%-Sm+bJYeD#}3YiO2qsj;5Z zL=kt8F62bxQNDX>1Du5Pmj0sR^m;HN9SV03iD-_Mv~62fL?h`EF2%KgyOisj)}zNh zn~`5g@p)7@qeUt4>G4-^F(@>P41I)+NelKHHUOzySa#-F%-@UWV#2|nr?pbvD?R*~ zf^lU-Ckx~L9wqq{MsE~J&Um;;&iHLn`IwfV2y)MAEt)frQ&mppzbsM)RnZPp7Hznf zhfI;#3Kl~tA0^C50|V-;Epn-fX&F1_9V}eZwbeGq?Jw{(rBOxU(z}V$X8ByJKD_+N zXRu1q%bUAOUgy*?z^nVuzI72Q;xE%y5Ub2&9700-N!6t6gU>9#bR7Rg&;917DnqRS zB}b{evya5OQ1&el$@b{up&9pJtBK^5qoo7E2#WyrH>sl8;RH~i_bVcV_21}lrcjUj zr#3`r0)2F?+t^}g#7trYAgIG9NiOG~_toSx50-N>ozH!>_@_Qt)W}ur7NJ|%T6Ev2 zEOkZL{vqv_63pxo4`-a#z#xQw`}22<(u#|jyizU;14kwU_UBBo;NNG#<*u)UB&8j_ zrLSggdOFq2ixIWdb!3aI2pMg7rnU}^)d_u$K%v9lOXb{L^AdHI0BwF`O}sxDlNvLpJclx>uZ8dR50o3+ z=BFyx0Yo9!+Fr;sNaUp&FK_qgg`W&&Pp0o#BiI_A+m6RMTA^_1Dh}0g$Th3lf)4qP zl5zj6AMg91;nnOy&*`(llW#F>O&XtX*0WFF-9Pc!D}3g_W1};j9QdC0C$g9W?(+X< eirdRQ5011fYLG!8mOl=0F31h@>v$ulu>S#x88LDI literal 0 HcmV?d00001 From 3df6141e0e8a01e85b95ad3b58b7b4c8ec048ccc Mon Sep 17 00:00:00 2001 From: shanshi Date: Thu, 11 Jul 2024 19:49:28 +0800 Subject: [PATCH 008/128] [feat] update db_handler and memory module --- .../prompts/__init__.py | 0 .../prompts/agent_selector_template_prompt.py | 0 .../prompts/checker_template_prompt.py | 0 .../prompts/code2doc_template_prompt.py | 0 .../prompts/code2test_template_prompt.py | 0 .../prompts/executor_template_prompt.py | 0 .../prompts/input_template_prompt.py | 0 .../prompts/intention_template_prompt.py | 0 .../prompts/metagpt_prompt.py | 0 .../prompts/planner_template_prompt.py | 0 .../prompts/qa_template_prompt.py | 0 .../prompts/react_code_prompt.py | 0 .../prompts/react_template_prompt.py | 0 .../prompts/react_tool_code_planner_prompt.py | 0 .../prompts/react_tool_code_prompt.py | 0 .../prompts/react_tool_prompt.py | 0 .../prompts/refine_template_prompt.py | 0 .../base_configs/prompts/simple_prompts.py | 206 +++++++++ .../prompts/summary_template_prompt.py | 0 muagent/chat/code_chat.py | 2 +- muagent/chat/knowledge_chat.py | 2 +- muagent/chat/search_chat.py | 2 +- .../codechat/code_analyzer/code_intepreter.py | 2 +- muagent/connector/actions/__init__.py | 6 - muagent/connector/actions/base_action.py | 16 - muagent/connector/agents/base_agent.py | 33 +- muagent/connector/agents/executor_agent.py | 4 +- muagent/connector/agents/react_agent.py | 2 +- muagent/connector/agents/selector_agent.py | 2 +- muagent/connector/configs/agent_config.py | 2 +- .../configs/agent_prompt/design_writer.yaml | 99 ----- .../configs/agent_prompt/prd_writer.yaml | 101 ----- .../configs/agent_prompt/review_code.yaml | 177 -------- .../configs/agent_prompt/task_write.yaml | 148 ------- .../configs/agent_prompt/write_code.yaml | 147 ------- muagent/connector/configs/generate_prompt.py | 57 +++ muagent/connector/memory/__init__.py | 5 + .../memory/hierarchical_memory_manager.py | 275 ++++++++++++ muagent/connector/memory_manager.py | 402 +++++++++--------- muagent/connector/phase/base_phase.py | 2 +- muagent/connector/schema/memory.py | 19 +- muagent/db_handler/__init__.py | 10 +- .../db_handler/graph_db_handler/__init__.py | 8 +- .../graph_db_handler/networkx_handler.py | 138 ++++++ .../db_handler/vector_db_handler/__init__.py | 10 +- .../vector_db_handler/chroma_handler.py | 8 +- .../vector_db_handler/local_faiss_handler.py | 142 +++++++ .../vector_db_handler/tbase_handler.py} | 14 +- muagent/llm_models/__init__.py | 4 +- muagent/llm_models/llm_config.py | 63 ++- muagent/orm/commands/code_base_cds.py | 2 +- muagent/orm/commands/document_base_cds.py | 2 +- muagent/orm/commands/document_file_cds.py | 2 +- .../{service => retrieval}/base_service.py | 0 muagent/retrieval/commands/__init__.py | 0 muagent/retrieval/commands/default_vs_cds.py | 37 -- .../faiss_db_service.py | 2 +- .../{service => retrieval}/service_factory.py | 2 +- muagent/retrieval/utils.py | 2 +- muagent/{orm => }/schemas/__init__.py | 0 muagent/schemas/db/__init__.py | 6 + muagent/schemas/db/db_config.py | 29 ++ .../schemas => schemas/kb}/base_schema.py | 0 muagent/schemas/memory/__init__.py | 6 + .../memory/auto_extract_graph_schema.py | 42 ++ muagent/service/cb_api.py | 12 +- muagent/service/kb_api.py | 2 +- muagent/service/migrate.py | 2 +- tests/connector/agent_test.py | 2 +- tests/connector/chain_test.py | 2 +- tests/connector/flow_test.py | 2 +- tests/connector/hierachical_memory_testy.py | 111 +++++ tests/connector/memory_manager_test.py | 137 +++--- tests/connector/phase_test.py | 2 +- tests/db_handler/networkx_handler_test.py | 75 ++++ 75 files changed, 1491 insertions(+), 1096 deletions(-) rename muagent/{connector/configs => base_configs}/prompts/__init__.py (100%) rename muagent/{connector/configs => base_configs}/prompts/agent_selector_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/checker_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/code2doc_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/code2test_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/executor_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/input_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/intention_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/metagpt_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/planner_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/qa_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/react_code_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/react_template_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/react_tool_code_planner_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/react_tool_code_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/react_tool_prompt.py (100%) rename muagent/{connector/configs => base_configs}/prompts/refine_template_prompt.py (100%) create mode 100644 muagent/base_configs/prompts/simple_prompts.py rename muagent/{connector/configs => base_configs}/prompts/summary_template_prompt.py (100%) delete mode 100644 muagent/connector/actions/__init__.py delete mode 100644 muagent/connector/actions/base_action.py delete mode 100644 muagent/connector/configs/agent_prompt/design_writer.yaml delete mode 100644 muagent/connector/configs/agent_prompt/prd_writer.yaml delete mode 100644 muagent/connector/configs/agent_prompt/review_code.yaml delete mode 100644 muagent/connector/configs/agent_prompt/task_write.yaml delete mode 100644 muagent/connector/configs/agent_prompt/write_code.yaml create mode 100644 muagent/connector/configs/generate_prompt.py create mode 100644 muagent/connector/memory/__init__.py create mode 100644 muagent/connector/memory/hierarchical_memory_manager.py create mode 100644 muagent/db_handler/graph_db_handler/networkx_handler.py create mode 100644 muagent/db_handler/vector_db_handler/local_faiss_handler.py rename muagent/{utils/tbase_util.py => db_handler/vector_db_handler/tbase_handler.py} (91%) rename muagent/{service => retrieval}/base_service.py (100%) delete mode 100644 muagent/retrieval/commands/__init__.py delete mode 100644 muagent/retrieval/commands/default_vs_cds.py rename muagent/{service => retrieval}/faiss_db_service.py (99%) rename muagent/{service => retrieval}/service_factory.py (98%) rename muagent/{orm => }/schemas/__init__.py (100%) create mode 100644 muagent/schemas/db/__init__.py create mode 100644 muagent/schemas/db/db_config.py rename muagent/{orm/schemas => schemas/kb}/base_schema.py (100%) create mode 100644 muagent/schemas/memory/__init__.py create mode 100644 muagent/schemas/memory/auto_extract_graph_schema.py create mode 100644 tests/connector/hierachical_memory_testy.py create mode 100644 tests/db_handler/networkx_handler_test.py diff --git a/muagent/connector/configs/prompts/__init__.py b/muagent/base_configs/prompts/__init__.py similarity index 100% rename from muagent/connector/configs/prompts/__init__.py rename to muagent/base_configs/prompts/__init__.py diff --git a/muagent/connector/configs/prompts/agent_selector_template_prompt.py b/muagent/base_configs/prompts/agent_selector_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/agent_selector_template_prompt.py rename to muagent/base_configs/prompts/agent_selector_template_prompt.py diff --git a/muagent/connector/configs/prompts/checker_template_prompt.py b/muagent/base_configs/prompts/checker_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/checker_template_prompt.py rename to muagent/base_configs/prompts/checker_template_prompt.py diff --git a/muagent/connector/configs/prompts/code2doc_template_prompt.py b/muagent/base_configs/prompts/code2doc_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/code2doc_template_prompt.py rename to muagent/base_configs/prompts/code2doc_template_prompt.py diff --git a/muagent/connector/configs/prompts/code2test_template_prompt.py b/muagent/base_configs/prompts/code2test_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/code2test_template_prompt.py rename to muagent/base_configs/prompts/code2test_template_prompt.py diff --git a/muagent/connector/configs/prompts/executor_template_prompt.py b/muagent/base_configs/prompts/executor_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/executor_template_prompt.py rename to muagent/base_configs/prompts/executor_template_prompt.py diff --git a/muagent/connector/configs/prompts/input_template_prompt.py b/muagent/base_configs/prompts/input_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/input_template_prompt.py rename to muagent/base_configs/prompts/input_template_prompt.py diff --git a/muagent/connector/configs/prompts/intention_template_prompt.py b/muagent/base_configs/prompts/intention_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/intention_template_prompt.py rename to muagent/base_configs/prompts/intention_template_prompt.py diff --git a/muagent/connector/configs/prompts/metagpt_prompt.py b/muagent/base_configs/prompts/metagpt_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/metagpt_prompt.py rename to muagent/base_configs/prompts/metagpt_prompt.py diff --git a/muagent/connector/configs/prompts/planner_template_prompt.py b/muagent/base_configs/prompts/planner_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/planner_template_prompt.py rename to muagent/base_configs/prompts/planner_template_prompt.py diff --git a/muagent/connector/configs/prompts/qa_template_prompt.py b/muagent/base_configs/prompts/qa_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/qa_template_prompt.py rename to muagent/base_configs/prompts/qa_template_prompt.py diff --git a/muagent/connector/configs/prompts/react_code_prompt.py b/muagent/base_configs/prompts/react_code_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/react_code_prompt.py rename to muagent/base_configs/prompts/react_code_prompt.py diff --git a/muagent/connector/configs/prompts/react_template_prompt.py b/muagent/base_configs/prompts/react_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/react_template_prompt.py rename to muagent/base_configs/prompts/react_template_prompt.py diff --git a/muagent/connector/configs/prompts/react_tool_code_planner_prompt.py b/muagent/base_configs/prompts/react_tool_code_planner_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/react_tool_code_planner_prompt.py rename to muagent/base_configs/prompts/react_tool_code_planner_prompt.py diff --git a/muagent/connector/configs/prompts/react_tool_code_prompt.py b/muagent/base_configs/prompts/react_tool_code_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/react_tool_code_prompt.py rename to muagent/base_configs/prompts/react_tool_code_prompt.py diff --git a/muagent/connector/configs/prompts/react_tool_prompt.py b/muagent/base_configs/prompts/react_tool_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/react_tool_prompt.py rename to muagent/base_configs/prompts/react_tool_prompt.py diff --git a/muagent/connector/configs/prompts/refine_template_prompt.py b/muagent/base_configs/prompts/refine_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/refine_template_prompt.py rename to muagent/base_configs/prompts/refine_template_prompt.py diff --git a/muagent/base_configs/prompts/simple_prompts.py b/muagent/base_configs/prompts/simple_prompts.py new file mode 100644 index 0000000..2239148 --- /dev/null +++ b/muagent/base_configs/prompts/simple_prompts.py @@ -0,0 +1,206 @@ +agent_prompt_en = ''' +Please ensure your selection is one of the listed roles. Available roles for selection: +{agents} +Please ensure select the Role from agent names, such as {agent_names}''' + +agent_prompt_zh = ''' +Please ensure your selection is one of the listed roles. Available roles for selection: +{agents} +Please ensure select the Role from agent names, such as {agent_names}''' + + +summary_prompt_zh = ''' +Your job is to summarize a history of previous messages in a conversation between an AI persona and a human. +The conversation you are given is a fixed context window and may not be complete. +Messages sent by the AI are marked with the 'assistant' role. +The AI 'assistant' can also make calls to functions, whose outputs can be seen in messages with the 'function' role. +Things the AI says in the message content are considered inner monologue and are not seen by the user. +The only AI messages seen by the user are from when the AI uses 'send_message'. +Messages the user sends are in the 'user' role. +The 'user' role is also used for important system events, such as login events and heartbeat events (heartbeats run the AI's program without user action, allowing the AI to act without prompting from the user sending them a message). +Summarize what happened in the conversation from the perspective of the AI (use the first person). +Keep your summary less than 100 words, do NOT exceed this word limit. +Only output the summary, do NOT include anything else in your output. + +--- conversation +{conversation} +--- +''' + + +summary_prompt_en = ''' +Your job is to summarize a history of previous messages in a conversation between an AI persona and a human. +The conversation you are given is a fixed context window and may not be complete. +Messages sent by the AI are marked with the 'assistant' role. +The AI 'assistant' can also make calls to functions, whose outputs can be seen in messages with the 'function' role. +Things the AI says in the message content are considered inner monologue and are not seen by the user. +The only AI messages seen by the user are from when the AI uses 'send_message'. +Messages the user sends are in the 'user' role. +The 'user' role is also used for important system events, such as login events and heartbeat events (heartbeats run the AI's program without user action, allowing the AI to act without prompting from the user sending them a message). +Summarize what happened in the conversation from the perspective of the AI (use the first person). +Keep your summary less than 100 words, do NOT exceed this word limit. +Only output the summary, do NOT include anything else in your output. + +--- conversation +{conversation} +--- +''' + + +memory_extract_prompt_en = '''## 角色 +你是一个结构化信息抽取的专家，你需要根据定义的节点、边类型，从输入的长对话文本中抽取节点和边关系。 + +## 节点和边的数据结构 +{schemas} + +## 要求 +1、根据 节点和边的数据结构 完成信息抽取 +2、edges中出现的left和right节点一定要在node中出现过 + +## 输出结构 +{ + 'nodes': [ + {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ..., + {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ], + 'edges': [ + {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ..., + {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ], +} + +## 输入 +{conversation} + +## 输出 +''' + + + +memory_extract_prompt_zh = '''## 角色 +你是一个结构化信息抽取的专家，你需要根据定义的节点、边类型，从输入的长对话文本中抽取节点和边关系。 + +## 节点和边的数据结构 +{schemas} + +## 要求 +1、根据 节点和边的数据结构 完成信息抽取 +2、edges中出现的left和right节点一定要在node中出现过 + +## 输出结构 +{ + 'nodes': [ + {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ..., + {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ], + 'edges': [ + {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ..., + {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + ], +} + +## 输入 +{conversation} + +## 输出 +''' + + +memory_auto_schema_prompt_en = """## 角色 +你是一个知识图谱专家，擅长从对话文本中抽象出实体和边的数据结构定义 + +## 目的 +从对话中总结出节点和边的抽象定义，包括节点类型及相关属性、边类型及相关属性 + +## 节点要求 +1、利用本体论帮助定义一致和标准化的节点类型。实体节点的类型可按照实体的自然属性（如人、地点等）、社会属性（如组织、事件等）或功能属性（如产品、服务等）等进行分类。 +2、也可以参考现有的分类体系，如图书馆分类法、行业标准等，可以为定义节点类型提供参考 +3、同时需要定义节点类型之间的关系，例如继承、关联等。以便于构建复杂的知识结构。 +4、同时采用层次化的方法来组织节点类型，例如“人”可以进一步细分为“政治家”、“艺术家”、“科学家”等。 +5、定义通用属性和特定属性，每个节点类型可能具有多维度的属性，例如“地点”可以有“经纬度”、“海拔”、“人口”等属性。 + + +## 边要求 +1、利用本体论可以帮助定义一致和标准化的边类型。边代表实体间的关系，例如“出生在”、“属于”等。 +2、参考现有的分类体系和关系定义，如RDF Schema、OWL等，可以为定义边类型提供参考。 +3、将关系进行分类，例如将所有表示位置的关系归为一类，所有表示组织的归属关系归为另一类。 +4、除了基本的关系，如“出生在”、“属于”，还应考虑更复杂或特定领域的关系。 +5、某些关系可能需要附加的属性来提供更多信息，例如“开始时间”、“结束时间”等。 + + +## 分析要求 +1、节点和边的定义应该遵循一定的语义规则，以确保知识图谱的一致性和可理解性。 +2、从长文本对话的多层次结构进行分析 + + +## 输出结构 +{ + 'nodes': [ + {'type': "{节点类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ..., + {'type': "{节点类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ], + 'edge': [ + {'type': "{边类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ..., + {'type': "{边类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ], +} + +## 输入 +{conversation} + +## 输出 +""" + + +memory_auto_schema_prompt_zh = """## 角色 +你是一个知识图谱专家，擅长从对话文本中抽象出实体和边的数据结构定义 + +## 目的 +从对话中总结出节点和边的抽象定义，包括节点类型及相关属性、边类型及相关属性 + +## 节点要求 +1、利用本体论帮助定义一致和标准化的节点类型。实体节点的类型可按照实体的自然属性（如人、地点等）、社会属性（如组织、事件等）或功能属性（如产品、服务等）等进行分类。 +2、也可以参考现有的分类体系，如图书馆分类法、行业标准等，可以为定义节点类型提供参考 +3、同时需要定义节点类型之间的关系，例如继承、关联等。以便于构建复杂的知识结构。 +4、同时采用层次化的方法来组织节点类型，例如“人”可以进一步细分为“政治家”、“艺术家”、“科学家”等。 +5、定义通用属性和特定属性，每个节点类型可能具有多维度的属性，例如“地点”可以有“经纬度”、“海拔”、“人口”等属性。 + + +## 边要求 +1、利用本体论可以帮助定义一致和标准化的边类型。边代表实体间的关系，例如“出生在”、“属于”等。 +2、参考现有的分类体系和关系定义，如RDF Schema、OWL等，可以为定义边类型提供参考。 +3、将关系进行分类，例如将所有表示位置的关系归为一类，所有表示组织的归属关系归为另一类。 +4、除了基本的关系，如“出生在”、“属于”，还应考虑更复杂或特定领域的关系。 +5、某些关系可能需要附加的属性来提供更多信息，例如“开始时间”、“结束时间”等。 + + +## 分析要求 +1、节点和边的定义应该遵循一定的语义规则，以确保知识图谱的一致性和可理解性。 +2、从长文本对话的多层次结构进行分析 + + +## 输出结构 +{ + 'nodes': [ + {'type': "{节点类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ..., + {'type': "{节点类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ], + 'edge': [ + {'type': "{边类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ..., + {'type': "{边类型}", 'attributes': [{'name': '{属性名称}', 'description': '{属性描述}'}, ..., {'name': '{属性名称}', 'description': '{属性描述}'}]}, + ], +} + +## 输入 +{conversation} + +## 输出 +""" \ No newline at end of file diff --git a/muagent/connector/configs/prompts/summary_template_prompt.py b/muagent/base_configs/prompts/summary_template_prompt.py similarity index 100% rename from muagent/connector/configs/prompts/summary_template_prompt.py rename to muagent/base_configs/prompts/summary_template_prompt.py diff --git a/muagent/chat/code_chat.py b/muagent/chat/code_chat.py index 2a7dba1..ae95ec0 100644 --- a/muagent/chat/code_chat.py +++ b/muagent/chat/code_chat.py @@ -18,7 +18,7 @@ # from configs.model_config import ( # llm_model_dict, LLM_MODEL, PROMPT_TEMPLATE, # VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD, CODE_PROMPT_TEMPLATE) -from muagent.connector.configs.prompts import CODE_PROMPT_TEMPLATE +from muagent.base_configs.prompts import CODE_PROMPT_TEMPLATE from muagent.chat.utils import History, wrap_done from muagent.utils import BaseResponse from .base_chat import Chat diff --git a/muagent/chat/knowledge_chat.py b/muagent/chat/knowledge_chat.py index 981dd04..33c265f 100644 --- a/muagent/chat/knowledge_chat.py +++ b/muagent/chat/knowledge_chat.py @@ -12,7 +12,7 @@ # llm_model_dict, LLM_MODEL, PROMPT_TEMPLATE, # VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD) from muagent.base_configs.env_config import KB_ROOT_PATH -from muagent.connector.configs.prompts import ORIGIN_TEMPLATE_PROMPT +from muagent.base_configs.prompts import ORIGIN_TEMPLATE_PROMPT from muagent.chat.utils import History, wrap_done from muagent.utils import BaseResponse from muagent.llm_models.llm_config import LLMConfig, EmbedConfig diff --git a/muagent/chat/search_chat.py b/muagent/chat/search_chat.py index 6f3e5ec..5876ee3 100644 --- a/muagent/chat/search_chat.py +++ b/muagent/chat/search_chat.py @@ -10,7 +10,7 @@ # from configs.model_config import ( # PROMPT_TEMPLATE, SEARCH_ENGINE_TOP_K, BING_SUBSCRIPTION_KEY, BING_SEARCH_URL, # VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD) -from muagent.connector.configs.prompts import ORIGIN_TEMPLATE_PROMPT +from muagent.base_configs.prompts import ORIGIN_TEMPLATE_PROMPT from muagent.chat.utils import History, wrap_done from muagent.utils import BaseResponse from muagent.llm_models.llm_config import LLMConfig, EmbedConfig diff --git a/muagent/codechat/code_analyzer/code_intepreter.py b/muagent/codechat/code_analyzer/code_intepreter.py index cc839a6..10097ce 100644 --- a/muagent/codechat/code_analyzer/code_intepreter.py +++ b/muagent/codechat/code_analyzer/code_intepreter.py @@ -11,7 +11,7 @@ ) # from configs.model_config import CODE_INTERPERT_TEMPLATE -from muagent.connector.configs.prompts import CODE_INTERPERT_TEMPLATE +from muagent.base_configs.prompts import CODE_INTERPERT_TEMPLATE from muagent.llm_models.openai_model import getChatModelFromConfig from muagent.llm_models.llm_config import LLMConfig diff --git a/muagent/connector/actions/__init__.py b/muagent/connector/actions/__init__.py deleted file mode 100644 index 13bbc3c..0000000 --- a/muagent/connector/actions/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .base_action import BaseAction - - -__all__ = [ - "BaseAction" -] \ No newline at end of file diff --git a/muagent/connector/actions/base_action.py b/muagent/connector/actions/base_action.py deleted file mode 100644 index d64f97b..0000000 --- a/muagent/connector/actions/base_action.py +++ /dev/null @@ -1,16 +0,0 @@ - -from langchain.schema import BaseRetriever, Document - -class BaseAction: - - - def __init__(self, ): - pass - - def step(self, ): - pass - - def astep(self, ): - pass - - \ No newline at end of file diff --git a/muagent/connector/agents/base_agent.py b/muagent/connector/agents/base_agent.py index aad2beb..148307c 100644 --- a/muagent/connector/agents/base_agent.py +++ b/muagent/connector/agents/base_agent.py @@ -15,7 +15,7 @@ from muagent.connector.prompt_manager.prompt_manager import PromptManager from muagent.connector.memory_manager import BaseMemoryManager, LocalMemoryManager, TbaseMemoryManager from muagent.base_configs.env_config import JUPYTER_WORK_PATH, KB_ROOT_PATH -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler class BaseAgent: @@ -192,37 +192,6 @@ def end_action_step(self, message: Message) -> Message: def token_usage(self, ): '''calculate the usage of token''' pass - - # def select_memory_by_key(self, memory: Memory) -> Memory: - # return Memory( - # messages=[self.select_message_by_key(message) for message in memory.messages - # if self.select_message_by_key(message) is not None] - # ) - - # def select_memory_by_agent_key(self, memory: Memory) -> Memory: - # return Memory( - # messages=[self.select_message_by_agent_key(message) for message in memory.messages - # if self.select_message_by_agent_key(message) is not None] - # ) - - # def select_message_by_agent_key(self, message: Message) -> Message: - # # assume we focus all agents - # if self.focus_agents == []: - # return message - # return None if message is None or message.role_name not in self.focus_agents else self.select_message_by_key(message) - - # def select_message_by_key(self, message: Message) -> Message: - # # assume we focus all key contents - # if message is None: - # return message - - # if self.focus_message_keys == []: - # return message - - # message_c = copy.deepcopy(message) - # message_c.parsed_output = {k: v for k,v in message_c.parsed_output.items() if k in self.focus_message_keys} - # message_c.parsed_output_list = [{k: v for k,v in parsed_output.items() if k in self.focus_message_keys} for parsed_output in message_c.parsed_output_list] - # return message_c def get_memory(self, content_key="role_content"): return self.memory.to_tuple_messages(content_key="step_content") diff --git a/muagent/connector/agents/executor_agent.py b/muagent/connector/agents/executor_agent.py index 387c77b..1e39756 100644 --- a/muagent/connector/agents/executor_agent.py +++ b/muagent/connector/agents/executor_agent.py @@ -10,9 +10,9 @@ ) from muagent.connector.memory_manager import BaseMemoryManager from muagent.llm_models import LLMConfig, EmbedConfig -from muagent.connector.configs.prompts import PLAN_EXECUTOR_PROMPT +from muagent.base_configs.prompts import PLAN_EXECUTOR_PROMPT from muagent.base_configs.env_config import JUPYTER_WORK_PATH, KB_ROOT_PATH -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler from .base_agent import BaseAgent diff --git a/muagent/connector/agents/react_agent.py b/muagent/connector/agents/react_agent.py index 559d713..88c9e90 100644 --- a/muagent/connector/agents/react_agent.py +++ b/muagent/connector/agents/react_agent.py @@ -11,7 +11,7 @@ ) from muagent.connector.memory_manager import BaseMemoryManager from muagent.llm_models import LLMConfig, EmbedConfig -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler from .base_agent import BaseAgent from muagent.base_configs.env_config import JUPYTER_WORK_PATH, KB_ROOT_PATH diff --git a/muagent/connector/agents/selector_agent.py b/muagent/connector/agents/selector_agent.py index 1a78571..a006d65 100644 --- a/muagent/connector/agents/selector_agent.py +++ b/muagent/connector/agents/selector_agent.py @@ -11,7 +11,7 @@ ) from muagent.connector.memory_manager import BaseMemoryManager from muagent.llm_models import LLMConfig, EmbedConfig -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler from muagent.base_configs.env_config import JUPYTER_WORK_PATH, KB_ROOT_PATH from .base_agent import BaseAgent diff --git a/muagent/connector/configs/agent_config.py b/muagent/connector/configs/agent_config.py index 307bb49..df45c4d 100644 --- a/muagent/connector/configs/agent_config.py +++ b/muagent/connector/configs/agent_config.py @@ -1,5 +1,5 @@ from enum import Enum -from .prompts import * +from muagent.base_configs.prompts import * # from .prompts import ( # REACT_PROMPT_INPUT, CHECK_PROMPT_INPUT, EXECUTOR_PROMPT_INPUT, CONTEXT_PROMPT_INPUT, QUERY_CONTEXT_PROMPT_INPUT,PLAN_PROMPT_INPUT, # RECOGNIZE_INTENTION_PROMPT, diff --git a/muagent/connector/configs/agent_prompt/design_writer.yaml b/muagent/connector/configs/agent_prompt/design_writer.yaml deleted file mode 100644 index e1135ef..0000000 --- a/muagent/connector/configs/agent_prompt/design_writer.yaml +++ /dev/null @@ -1,99 +0,0 @@ -You are a Architect, named Bob, your goal is Design a concise, usable, complete python system, and the constraint is Try to specify good open source tools as much as possible. - -# Context -## Original Requirements: -Create a snake game. - -## Product Goals: -Develop a highly addictive and engaging snake game. -Provide a user-friendly and intuitive user interface. -Implement various levels and challenges to keep the players entertained. -## User Stories: -As a user, I want to be able to control the snake's movement using arrow keys or touch gestures. -As a user, I want to see my score and progress displayed on the screen. -As a user, I want to be able to pause and resume the game at any time. -As a user, I want to be challenged with different obstacles and levels as I progress. -As a user, I want to have the option to compete with other players and compare my scores. -## Competitive Analysis: -Python Snake Game: A simple snake game implemented in Python with basic features and limited levels. -Snake.io: A multiplayer online snake game with competitive gameplay and high engagement. -Slither.io: Another multiplayer online snake game with a larger player base and addictive gameplay. -Snake Zone: A mobile snake game with various power-ups and challenges. -Snake Mania: A classic snake game with modern graphics and smooth controls. -Snake Rush: A fast-paced snake game with time-limited challenges. -Snake Master: A snake game with unique themes and customizable snakes. - -## Requirement Analysis: -The product should be a highly addictive and engaging snake game with a user-friendly interface. It should provide various levels and challenges to keep the players entertained. The game should have smooth controls and allow the users to compete with each other. - -## Requirement Pool: -``` -[ - ["Implement different levels with increasing difficulty", "P0"], - ["Allow users to control the snake using arrow keys or touch gestures", "P0"], - ["Display the score and progress on the screen", "P1"], - ["Provide an option to pause and resume the game", "P1"], - ["Integrate leaderboards to enable competition among players", "P2"] -] -``` -## UI Design draft: -The game will have a simple and clean interface. The main screen will display the snake, obstacles, and the score. The snake's movement can be controlled using arrow keys or touch gestures. There will be buttons to pause and resume the game. The level and difficulty will be indicated on the screen. The design will have a modern and visually appealing style with smooth animations. - -## Anything UNCLEAR: -There are no unclear points. - -## Format example ---- -## Implementation approach -We will ... - -## Python package name -```python -"snake_game" -``` - -## File list -```python -[ - "main.py", -] -``` - -## Data structures and interface definitions -```mermaid -classDiagram - class Game{ - +int score - } - ... - Game "1" -- "1" Food: has -``` - -## Program call flow -```mermaid -sequenceDiagram - participant M as Main - ... - G->>M: end game -``` - -## Anything UNCLEAR -The requirement is clear to me. ---- ------ -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools -Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. - -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores - -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here - -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. \ No newline at end of file diff --git a/muagent/connector/configs/agent_prompt/prd_writer.yaml b/muagent/connector/configs/agent_prompt/prd_writer.yaml deleted file mode 100644 index 0aec402..0000000 --- a/muagent/connector/configs/agent_prompt/prd_writer.yaml +++ /dev/null @@ -1,101 +0,0 @@ -You are a Product Manager, named Alice, your goal is Efficiently create a successful product, and the constraint is . - -# Context -## Original Requirements -Create a snake game. - -## Search Information -### Search Results - -### Search Summary - -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` - -## Format example ---- -## Original Requirements -The boss ... - -## Product Goals -```python -[ - "Create a ...", -] -``` - -## User Stories -```python -[ - "As a user, ...", -] -``` - -## Competitive Analysis -```python -[ - "Python Snake Game: ...", -] -``` - -## Competitive Quadrant Chart -```mermaid -quadrantChart - title Reach and engagement of campaigns - ... - "Our Target Product": [0.6, 0.7] -``` - -## Requirement Analysis -The product should be a ... - -## Requirement Pool -```python -[ - ["End game ...", "P0"] -] -``` - -## UI Design draft -Give a basic function description, and a draft - -## Anything UNCLEAR -There are no unclear points. ---- ------ -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. - -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. \ No newline at end of file diff --git a/muagent/connector/configs/agent_prompt/review_code.yaml b/muagent/connector/configs/agent_prompt/review_code.yaml deleted file mode 100644 index 32567d8..0000000 --- a/muagent/connector/configs/agent_prompt/review_code.yaml +++ /dev/null @@ -1,177 +0,0 @@ - -NOTICE -Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". - -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` - -## Rewrite Code: point.py Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. ------ -# Context -## Implementation approach -For the snake game, we can use the Pygame library, which is an open-source and easy-to-use library for game development in Python. Pygame provides a simple and efficient way to handle graphics, sound, and user input, making it suitable for developing a snake game. - -## Python package name -``` -"snake_game" -``` -## File list -```` -[ - "main.py", -] -``` -## Data structures and interface definitions -``` -classDiagram - class Game: - -int score - -bool paused - +__init__() - +start_game() - +handle_input(key: int) - +update_game() - +draw_game() - +game_over() - - class Snake: - -list[Point] body - -Point dir - -bool alive - +__init__(start_pos: Point) - +move() - +change_direction(dir: Point) - +grow() - +get_head() -> Point - +get_body() -> list[Point] - +is_alive() -> bool - - class Point: - -int x - -int y - +__init__(x: int, y: int) - +set_coordinate(x: int, y: int) - +get_coordinate() -> tuple[int, int] - - class Food: - -Point pos - -bool active - +__init__() - +generate_new_food() - +get_position() -> Point - +is_active() -> bool - - Game "1" -- "1" Snake: contains - Game "1" -- "1" Food: has -``` - -## Program call flow -``` -sequenceDiagram - participant M as Main - participant G as Game - participant S as Snake - participant F as Food - - M->>G: Start game - G->>G: Initialize game - loop - M->>G: Handle user input - G->>S: Handle input - G->>F: Check if snake eats food - G->>S: Update snake movement - G->>G: Check game over condition - G->>G: Update score - G->>G: Draw game - M->>G: Update display - end - G->>G: Game over -``` -## Required Python third-party packages -``` -""" -pygame==2.0.1 -""" -``` -## Required Other language third-party packages -``` -""" -No third-party packages required for other languages. -""" -``` - -## Logic Analysis -``` -[ - ["main.py", "Main"], - ["game.py", "Game"], - ["snake.py", "Snake"], - ["point.py", "Point"], - ["food.py", "Food"] -] -``` -## Task list -``` -[ - "point.py", - "food.py", - "snake.py", - "game.py", - "main.py" -] -``` -## Shared Knowledge -``` -""" -The 'point.py' module contains the implementation of the Point class, which represents a point in a 2D coordinate system. - -The 'food.py' module contains the implementation of the Food class, which represents the food in the game. - -The 'snake.py' module contains the implementation of the Snake class, which represents the snake in the game. - -The 'game.py' module contains the implementation of the Game class, which manages the game logic. - -The 'main.py' module is the entry point of the application and starts the game. -""" -``` -## Anything UNCLEAR -We need to clarify the main entry point of the application and ensure that all required third-party libraries are properly initialized. - -## Code: point.py -``` -class Point: - def __init__(self, x: int, y: int): - self.x = x - self.y = y - - def set_coordinate(self, x: int, y: int): - self.x = x - self.y = y - - def get_coordinate(self) -> tuple[int, int]: - return self.x, self.y -``` ------ - -## Format example ------ -## Code Review -1. The code ... -2. ... -3. ... -4. ... -5. ... - -## Rewrite Code: point.py -```python -## point.py -... -``` ------ diff --git a/muagent/connector/configs/agent_prompt/task_write.yaml b/muagent/connector/configs/agent_prompt/task_write.yaml deleted file mode 100644 index faf2c2c..0000000 --- a/muagent/connector/configs/agent_prompt/task_write.yaml +++ /dev/null @@ -1,148 +0,0 @@ -You are a Project Manager, named Eve, your goal isImprove team efficiency and deliver with quality and quantity, and the constraint is . - -# Context -## Implementation approach -For the snake game, we can use the Pygame library, which is an open-source and easy-to-use library for game development in Python. Pygame provides a simple and efficient way to handle graphics, sound, and user input, making it suitable for developing a snake game. - -## Python package name -``` -"snake_game" -``` -## File list -```` -[ - "main.py", - "game.py", - "snake.py", - "food.py" -] -``` -## Data structures and interface definitions -``` -classDiagram - class Game{ - -int score - -bool game_over - +start_game() : void - +end_game() : void - +update() : void - +draw() : void - +handle_events() : void - } - class Snake{ - -list[Tuple[int, int]] body - -Tuple[int, int] direction - +move() : void - +change_direction(direction: Tuple[int, int]) : void - +is_collision() : bool - +grow() : void - +draw() : void - } - class Food{ - -Tuple[int, int] position - +generate() : void - +draw() : void - } - class Main{ - -Game game - +run() : void - } - Game "1" -- "1" Snake: contains - Game "1" -- "1" Food: has - Main "1" -- "1" Game: has -``` -## Program call flow -``` -sequenceDiagram - participant M as Main - participant G as Game - participant S as Snake - participant F as Food - - M->G: run() - G->G: start_game() - G->G: handle_events() - G->G: update() - G->G: draw() - G->G: end_game() - - G->S: move() - S->S: change_direction() - S->S: is_collision() - S->S: grow() - S->S: draw() - - G->F: generate() - F->F: draw() -``` -## Anything UNCLEAR -The design and implementation of the snake game are clear based on the given requirements. - -## Format example ---- -## Required Python third-party packages -```python -""" -flask==1.1.2 -bcrypt==3.2.0 -""" -``` - -## Required Other language third-party packages -```python -""" -No third-party ... -""" -``` - -## Full API spec -```python -""" -openapi: 3.0.0 -... -description: A JSON object ... -""" -``` - -## Logic Analysis -```python -[ - ["game.py", "Contains ..."], -] -``` - -## Task list -```python -[ - "game.py", -] -``` - -## Shared Knowledge -```python -""" -'game.py' contains ... -""" -``` - -## Anything UNCLEAR -We need ... how to start. ---- ------ -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules -Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Required Python third-party packages: Provided in requirements.txt format - -## Required Other language third-party packages: Provided in requirements.txt format - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. \ No newline at end of file diff --git a/muagent/connector/configs/agent_prompt/write_code.yaml b/muagent/connector/configs/agent_prompt/write_code.yaml deleted file mode 100644 index 4193b8b..0000000 --- a/muagent/connector/configs/agent_prompt/write_code.yaml +++ /dev/null @@ -1,147 +0,0 @@ -NOTICE -Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". - -## Code: snake.py Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. - ------ -# Context -## Implementation approach -For the snake game, we can use the Pygame library, which is an open-source and easy-to-use library for game development in Python. Pygame provides a simple and efficient way to handle graphics, sound, and user input, making it suitable for developing a snake game. - -## Python package name -``` -"snake_game" -``` -## File list -```` -[ - "main.py", - "game.py", - "snake.py", - "food.py" -] -``` -## Data structures and interface definitions -``` -classDiagram - class Game{ - -int score - -bool game_over - +start_game() : void - +end_game() : void - +update() : void - +draw() : void - +handle_events() : void - } - class Snake{ - -list[Tuple[int, int]] body - -Tuple[int, int] direction - +move() : void - +change_direction(direction: Tuple[int, int]) : void - +is_collision() : bool - +grow() : void - +draw() : void - } - class Food{ - -Tuple[int, int] position - +generate() : void - +draw() : void - } - class Main{ - -Game game - +run() : void - } - Game "1" -- "1" Snake: contains - Game "1" -- "1" Food: has - Main "1" -- "1" Game: has -``` -## Program call flow -``` -sequenceDiagram - participant M as Main - participant G as Game - participant S as Snake - participant F as Food - - M->G: run() - G->G: start_game() - G->G: handle_events() - G->G: update() - G->G: draw() - G->G: end_game() - - G->S: move() - S->S: change_direction() - S->S: is_collision() - S->S: grow() - S->S: draw() - - G->F: generate() - F->F: draw() -``` -## Anything UNCLEAR -The design and implementation of the snake game are clear based on the given requirements. - -## Required Python third-party packages -``` -""" -pygame==2.0.1 -""" -``` -## Required Other language third-party packages -``` -""" -No third-party packages required for other languages. -""" -``` - -## Logic Analysis -``` -[ - ["main.py", "Main"], - ["game.py", "Game"], - ["snake.py", "Snake"], - ["food.py", "Food"] -] -``` -## Task list -``` -[ - "snake.py", - "food.py", - "game.py", - "main.py" -] -``` -## Shared Knowledge -``` -""" -'game.py' contains the main logic for the snake game, including starting the game, handling user input, updating the game state, and drawing the game state. - -'snake.py' contains the logic for the snake, including moving the snake, changing its direction, checking for collisions, growing the snake, and drawing the snake. - -'food.py' contains the logic for the food, including generating a new food position and drawing the food. - -'main.py' initializes the game and runs the game loop. -""" -``` -## Anything UNCLEAR -We need to clarify the main entry point of the application and ensure that all required third-party libraries are properly initialized. - ------ -## Format example ------ -## Code: snake.py -```python -## snake.py -... -``` ------ \ No newline at end of file diff --git a/muagent/connector/configs/generate_prompt.py b/muagent/connector/configs/generate_prompt.py new file mode 100644 index 0000000..37c392c --- /dev/null +++ b/muagent/connector/configs/generate_prompt.py @@ -0,0 +1,57 @@ +from muagent.base_configs.prompts.simple_prompts import * + + +def replacePrompt(prompt: str, keys: list[str] = []): + prompt = prompt.replace("{", "{{").replace("}", "}}") + for key in keys: + prompt = prompt.replace(f"{{{{key}}}}", f"{{key}}") + return prompt + +def cleanPrompt(prompt): + while "\n " in prompt: + prompt = prompt.replace("\n ", "\n") + return prompt + + +def createAgentSelectorPrompt(agents, agent_names, language="en", **kwargs) -> str: + prompt = agent_prompt_zh if language == "zh" else agent_prompt_en + prompt = replacePrompt(prompt, keys=["agents", "agent_names"]) + prompt = prompt.format(**{"agents": agents, "agent_names": agent_names}) + # if language == "zh": + # prompt = agent_prompt_zh.format(**{"agents": agents, "agent_names": agent_names}) + # else: + # prompt = agent_prompt_en.format(**{"agents": agents, "agent_names": agent_names}) + return cleanPrompt(prompt) + + +def createSummaryPrompt(conversation, language="en", **kwargs) -> str: + prompt = summary_prompt_zh if language == "zh" else summary_prompt_en + prompt = replacePrompt(prompt, keys=["conversation"]) + prompt = prompt.format(**{"conversation": conversation,}) + # if language == "zh": + # prompt = summary_prompt_zh.format(**{"conversation": conversation}) + # else: + # prompt = summary_prompt_en.format(**{"conversation": conversation}) + return cleanPrompt(prompt) + + +def createMKGSchemaPrompt(conversation, language="en", **kwargs) -> str: + prompt = memory_auto_schema_prompt_zh if language == "zh" else memory_auto_schema_prompt_en + prompt = replacePrompt(prompt, keys=["conversation"]) + prompt = prompt.format(**{"conversation": conversation,}) + # if language == "zh": + # prompt = memory_auto_schema_prompt_zh.format(**{"conversation": conversation}) + # else: + # prompt = memory_auto_schema_prompt_en.format(**{"conversation": conversation}) + return cleanPrompt(prompt) + + +def createMKGPrompt(conversation, schemas, language="en", **kwargs) -> str: + prompt = memory_extract_prompt_zh if language == "zh" else memory_extract_prompt_en + prompt = replacePrompt(prompt, keys=["conversation", "schemas"]) + prompt = prompt.format(**{"conversation": conversation, "schemas": schemas}) + # if language == "zh": + # prompt = memory_extract_prompt_zh.format(**{"conversation": conversation, "schemas": schemas}) + # else: + # prompt = memory_extract_prompt_en.format(**{"conversation": conversation, "schemas": schemas}) + return cleanPrompt(prompt) \ No newline at end of file diff --git a/muagent/connector/memory/__init__.py b/muagent/connector/memory/__init__.py new file mode 100644 index 0000000..719b23b --- /dev/null +++ b/muagent/connector/memory/__init__.py @@ -0,0 +1,5 @@ +from .hierarchical_memory_manager import HierarchicalMemoryManager + +__all__ = [ + "HierarchicalMemoryManager" +] \ No newline at end of file diff --git a/muagent/connector/memory/hierarchical_memory_manager.py b/muagent/connector/memory/hierarchical_memory_manager.py new file mode 100644 index 0000000..5be5710 --- /dev/null +++ b/muagent/connector/memory/hierarchical_memory_manager.py @@ -0,0 +1,275 @@ +from typing import List, Union, Dict +import os +import json + +from langchain_openai import ChatOpenAI +from langchain.llms.base import LLM +from langchain_community.docstore.document import Document + + +from muagent.connector.configs.generate_prompt import * +from muagent.connector.schema import Memory, Message +from muagent.schemas.db import DBConfig, GBConfig, VBConfig, TBConfig +from muagent.schemas.memory import * +from muagent.db_handler import * +from muagent.connector.memory_manager import BaseMemoryManager +from muagent.llm_models import * +from muagent.base_configs.env_config import KB_ROOT_PATH +from muagent.orm import table_init + +from muagent.utils.common_utils import * + + +class HierarchicalMemoryManager(BaseMemoryManager): + def __init__( + self, + embed_config: EmbedConfig, + llm_config: LLMConfig, + db_config: DBConfig = None, + vb_config: VBConfig = None, + gb_config: GBConfig = None, + tb_config: TBConfig = None, + do_init: bool = False, + kb_root_path: str = KB_ROOT_PATH, + ): + self.db_config = db_config + self.vb_config = vb_config + self.gb_config = gb_config + self.tb_config = tb_config + self.do_init = do_init + self.kb_root_path = kb_root_path + self.embed_config: EmbedConfig = embed_config + self.llm_config: LLMConfig = llm_config + + # default + self.chat_index: str = "default" + self.user_name = "default" + self.uuid_name = "_".join([self.chat_index, self.user_name]) + self.kb_name = f"{self.chat_index}/{self.user_name}" + self.uuid_file = os.path.join(self.kb_root_path, f"{self.chat_index}/{self.user_name}/conversation.jsonl") + self.recall_memory_dict: Dict[str, Memory] = {} # overall turn conversation + self.gb_memory_dict: Dict[str, Memory] = {} # lastet 10 turn conversation + self.memory_uuids = set() + self.save_message_keys = [ + 'chat_index', 'message_index', 'user_name', 'role_name', 'role_type', 'input_query', 'role_content', 'step_content', + 'parsed_output', 'parsed_output_list', 'customed_kargs', "db_docs", "code_docs", "search_docs", 'start_datetime', 'end_datetime', + "keyword", "vector", + ] + # init from config + self.model = getChatModelFromConfig(self.llm_config) + self.init_handler() + self.load(do_init) + + def init_handler(self, ): + """Initializes Database VectorBase GraphDB TbaseDB""" + self.init_vb() + # self.init_db() + # self.init_tb() + self.init_gb() + + def reinit_handler(self, do_init: bool=False): + self.init_vb() + # self.init_db() + # self.init_tb() + self.init_gb() + + def clear_local(self, re_init: bool = False, handler_type: str = None): + if self.vb: # 存到了本地需要清理 + self.vb.clear_vs_local() + self.load(re_init) + + def init_tb(self, do_init: bool=None): + tb_dict = {"TbaseHandler": TbaseHandler} + tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) + tbase_args = { + "host": self.tb_config.host, + "port": self.tb_config.port, + "username": self.tb_config.username, + "password": self.tb_config.password, + } + self.vb = tb_class(tbase_args, self.tb_config.index_name, tb_config=self.tb_config) + + def init_gb(self, do_init: bool=None): + gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler} + gb_class = gb_dict.get(self.gb_config.gb_type, NetworkxHandler) + self.gb: NetworkxHandler = gb_class(kb_root_path=KB_ROOT_PATH, gb_config=self.gb_config) + + def init_db(self, do_init: bool=None): + self.db = None + # db_dict = {"LocalFaissHandler": LocalFaissHandler} + # db_class = db_dict.get(self.db_config.db_type) + # self.db = db_class(self.db_config) + + def init_vb(self, do_init: bool=None): + table_init() + vb_dict = {"LocalFaissHandler": LocalFaissHandler} + vb_class = vb_dict.get(self.vb_config.vb_type, LocalFaissHandler) + self.vb: LocalFaissHandler = vb_class(self.embed_config, vb_config=self.vb_config) + + def append(self, message: Message): + """ + Appends a message to the recall memory, current memory, and summary memory. + + Args: + - message: An instance of Message class representing the message to be appended. + """ + # update the newest uuid_name + self.check_uuid_name(message) + datetimes = self.recall_memory_dict[self.uuid_name].get_datetimes() + contents = self.recall_memory_dict[self.uuid_name].get_contents() + # if message not in chat history, no need to update + if (message.end_datetime not in datetimes) or \ + ((message.input_query not in contents) and (message.role_content not in contents)): + self.append2gb(message) + self.append2vb(message) + # 5、offline：summary node + # 6、offline：node cluster + # 7、offline: summary node/relation cluster + + def append2gb(self, message: Message): + message_limit = 9 + # 0、check the sustain the requirement + self.gb_memory_dict[self.uuid_name].append_with_limit(message, limit=message_limit) + # 1、acquire the conversation + messages = self.gb_memory_dict[self.uuid_name].get_messages(max(int(message_limit*2/3), 1)) + ## The number of messages must meet a certain size + if len(messages) >= 6: + # TODO add user-name as sep + conversation = "\n".join([m.to_str_content(content_key="step_content") for m in messages]) + # 2、offline: get schema from content by llm, merge schema from origin shcemas + self.node_schmeas, self.edge_schemas = self._get_schema_by_llm(conversation) + # 3、offline: get graph node and graph relation from content by llm + nodes, edges = self._get_ner_by_llm(conversation, self.node_schmeas, self.edge_schemas) + # 4、insert graph node and graph into graph db + self.gb.add_nodes(nodes) + self.gb.add_edges(edges) + # + if True: # resave the local + self.gb.save(self.kb_name) + + self.gb_memory_dict[self.uuid_name].clear(3) + + def append2vb(self, message: Message) -> None: + self.recall_memory_dict[self.uuid_name].append(message) + # + docs, json_messages = self.message_process([message]) + # + if True: # resave the local + save_to_json_file(json_messages, self.uuid_file) + + if self.embed_config: + self.vb.add_docs(docs, kb_name=self.kb_name) + + def extend(self, memory: Memory): + """ + Extends the recall memory, current memory, and summary memory. + + Args: + - memory: An instance of Memory class representing the memory to be extended. + """ + for message in memory.messages: + self.append(message) + + def get_memory_pool(self, chat_index: str): + """ + return memory_pool + """ + pass + + def _get_schema_by_llm( + self, conversation: str, node_schemas: List[GNodeAbs], edge_schemas: List[GRelationAbs] + ) -> tuple[List[GNodeAbs], List[GRelationAbs]]: + """ + """ + # acquire the prompt + prompt = createMKGSchemaPrompt(conversation=conversation) + # + content = self.model.predict(prompt) + # convert schema + raw_schema = json.loads(content) + nodes = raw_schema["nodes"] + edges = raw_schema["edges"] + # merge schemas + self._merge_node_schems([GNodeAbs(**node) for node in nodes], node_schemas) + self._merge_node_schems([GRelationAbs(**edge) for edge in edges], edge_schemas) + return node_schemas, edge_schemas + + def _merge_node_schems(self, new_schemas: List[GNodeAbs], old_schemas: List[GNodeAbs]): + old_schemas.extend(new_schemas) + return old_schemas + + def _merge_edge_schems(self, new_schemas: List[GRelationAbs], old_schemas: List[GRelationAbs]): + old_schemas.extend(new_schemas) + return old_schemas + + def _get_ner_by_llm( + self, conversation, node_schemas: List[GNodeAbs], edge_schemas: List[GRelationAbs] + ) -> tuple[list[GNode], list[GRelation]]: + """ + """ + # + schemas = { + "nodes": [s.dict() for s in node_schemas], + "edges": [s.dict() for s in edge_schemas] + } + schemas_json = json.dumps(schemas, indent=2, ensure_ascii=False) + prompt = createMKGPrompt(conversation=conversation, schemas=schemas_json) + # + content = self.model.predict(prompt) + # convert content to node or relation + raw_schema = json.loads(content) + nodes = raw_schema["nodes"] + nodes_dict = {node["name"]: node for node in nodes} + edges = raw_schema["edges"] + + nodes = [GNode(**node) for node in nodes] + edges = [ + GRelation(**{**edge, **{'left': nodes_dict.get(edge['left'], {}), 'right': nodes_dict.get(edge['right'], {})}}) + for edge in edges + ] + return nodes, edges + + def message_process(self, messages: List[Message]): + '''convert messages to vb/local data-format''' + messages = [ + {k: v for k, v in m.dict().items() if k in self.save_message_keys} + for m in messages + ] + docs = [{"page_content": m["step_content"] or m["role_content"] or m["input_query"], "metadata": m} for m in messages] + docs = [Document(**doc) for doc in docs] + # convert messages to local data-format + memory_messages = self.recall_memory_dict[self.uuid_name].dict() + json_messages = { + k: [ + {kkk: vvv for kkk, vvv in vv.items() if kkk in self.save_message_keys} + for vv in v + ] + for k, v in memory_messages.items() + } + + return docs, json_messages + + def check_uuid_name(self, message: Message = None): + if message.chat_index != self.chat_index: + self.chat_index = message.chat_index + self.user_name = message.user_name + # self.init_vb() + + self.uuid_name = "_".join([self.chat_index, self.user_name]) + self.kb_name = f"{self.chat_index}/{self.user_name}" + self.uuid_file = os.path.join(self.kb_root_path, f"{self.chat_index}/{self.user_name}/conversation.jsonl") + + self.memory_uuids.add(self.uuid_name) + if self.uuid_name not in self.recall_memory_dict: + self.recall_memory_dict[self.uuid_name] = Memory(messages=[]) + if self.uuid_name not in self.gb_memory_dict: + self.gb_memory_dict[self.uuid_name] = Memory(messages=[]) + + def get_uuid_from_chatindex(self, chat_index: str) -> str: + for i in self.recall_memory_dict: + if chat_index in i: + return i + return "" + + def get_kbname_from_chatindex(self, chat_index: str) -> str: + return self.get_uuid_from_chatindex(chat_index).replace("_", "/") \ No newline at end of file diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index 6096464..3c4c0a1 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -10,12 +10,14 @@ from .schema import Memory, Message -from muagent.service.service_factory import KBServiceFactory +from muagent.connector.configs.generate_prompt import * +from muagent.schemas.db import DBConfig, GBConfig, VBConfig, TBConfig +from muagent.db_handler import * +from muagent.retrieval.service_factory import KBServiceFactory from muagent.llm_models import getChatModelFromConfig from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.retrieval.utils import load_embeddings_from_path from muagent.utils.common_utils import * -from muagent.connector.configs.prompts import CONV_SUMMARY_PROMPT_SPEC from muagent.orm import table_init from muagent.base_configs.env_config import KB_ROOT_PATH # from configs.model_config import KB_ROOT_PATH, EMBEDDING_MODEL, EMBEDDING_DEVICE, SCORE_THRESHOLD @@ -51,39 +53,39 @@ class BaseMemoryManager(ABC): - datetime_retrieval: Retrieves messages based on datetime. - recursive_summary: Performs recursive summarization of messages. """ - def __init__( self, - user_name: str = "default", - unique_name: str = "default", - memory_type: str = "recall", + embed_config: EmbedConfig, + llm_config: LLMConfig, + db_config: DBConfig, + vb_config: VBConfig, + gb_config: GBConfig, + tb_config: TBConfig, do_init: bool = False, ): """ Initializes the LocalMemoryManager with the given parameters. Args: - - user_name: A string representing the user name. Default is "default". - - unique_name: A string representing the unique name. Default is "default". - - memory_type: A string representing the memory type. Default is "recall". + - embed_config: EmbedConfig, the embedding model config + - llm_config: LLMConfig, the LLM model config + - db_config: DBConfig, the Database config + - vb_config: VBConfig, the vector base config + - gb_config: GBConfig, the graph base config - do_init: A boolean indicating whether to initialize. Default is False. """ - self.user_name = user_name - self.unique_name = unique_name - self.memory_type = memory_type self.do_init = do_init - # self.current_memory = Memory(messages=[]) - # self.recall_memory = Memory(messages=[]) - # self.summary_memory = Memory(messages=[]) - self.current_memory_dict: Dict[str, Memory] = {} self.recall_memory_dict: Dict[str, Memory] = {} - self.summary_memory_dict: Dict[str, Memory] = {} self.save_message_keys = [ 'chat_index', 'role_name', 'role_type', 'role_prompt', 'input_query', 'datetime', 'role_content', 'step_content', 'parsed_output', 'spec_parsed_output', 'parsed_output_list', 'task', 'db_docs', 'code_docs', 'search_docs', 'phase_name', 'chain_name', 'customed_kargs'] self.init_vb() + + def init_db(self, ): + """Initializes Database VectorBase GraphDB TbaseDB""" + def re_init(self, do_init: bool=False): self.init_vb() @@ -111,15 +113,6 @@ def extend(self, memory: Memory): """ pass - def save(self, save_dir: str = ""): - """ - Saves the memory to the specified directory. - - Args: - - save_dir: A string representing the directory to save the memory. Default is KB_ROOT_PATH. - """ - pass - def load(self, load_dir: str = "") -> Memory: """ Loads the memory from the specified directory and returns a Memory instance. @@ -132,12 +125,21 @@ def load(self, load_dir: str = "") -> Memory: """ pass - def get_memory_pool(self, chat_index: str): + def get_memory_pool(self, chat_index: str) -> Memory: """ return memory_pool """ pass + + def search_messages(self, text: str=None, n=5, **kwargs) -> List[Message]: + """ + return the search messages + Args: + - text: A string representing the text for retrieval. Default is None. + - n: An integer representing the number of messages. Default is 5. + """ + def router_retrieval(self, text: str=None, datetime: str = None, n=5, top_k=5, retrieval_type: str = "embedding", **kwargs) -> List[Message]: """ Routes the retrieval based on the retrieval type. @@ -212,6 +214,12 @@ def recursive_summary(self, messages: List[Message], split_n: int = 20) -> List[ """ pass + def reranker(self, ): + """ + rerank the retrieval message from memory + """ + pass + class LocalMemoryManager(BaseMemoryManager): @@ -219,158 +227,156 @@ def __init__( self, embed_config: EmbedConfig, llm_config: LLMConfig, - user_name: str = "default", - unique_name: str = "default", - memory_type: str = "recall", + db_config: DBConfig = None, + vb_config: VBConfig = None, + gb_config: GBConfig = None, + tb_config: TBConfig = None, do_init: bool = False, kb_root_path: str = KB_ROOT_PATH, ): - self.user_name = user_name - self.unique_name = unique_name - self.memory_type = memory_type - self.chat_index: str = "default" + # self.user_name = user_name + # self.unique_name = unique_name + # self.memory_type = memory_type + self.db_config = db_config + self.vb_config = vb_config + self.gb_config = gb_config + self.tb_config = tb_config self.do_init = do_init self.kb_root_path = kb_root_path self.embed_config: EmbedConfig = embed_config self.llm_config: LLMConfig = llm_config - # self.current_memory = Memory(messages=[]) - # self.recall_memory = Memory(messages=[]) - # self.summary_memory = Memory(messages=[]) - self.current_memory_dict: Dict[str, Memory] = {} + + # default + self.chat_index: str = "default" + self.user_name = "default" + self.uuid_name = "_".join([self.chat_index, self.user_name]) + self.kb_name = f"{self.chat_index}/{self.user_name}" + self.uuid_file = os.path.join(self.kb_root_path, f"{self.chat_index}/{self.user_name}/conversation.jsonl") self.recall_memory_dict: Dict[str, Memory] = {} - self.summary_memory_dict: Dict[str, Memory] = {} + self.memory_uuids = set() self.save_message_keys = [ - 'chat_index', 'role_name', 'role_type', 'role_prompt', 'input_query', - 'datetime', 'role_content', 'step_content', 'parsed_output', 'spec_parsed_output', 'parsed_output_list', - 'task', 'db_docs', 'code_docs', 'search_docs', 'phase_name', 'chain_name', 'customed_kargs'] + 'chat_index', 'message_index', 'user_name', 'role_name', 'role_type', 'input_query', 'role_content', 'step_content', + 'parsed_output', 'parsed_output_list', 'customed_kargs', "db_docs", "code_docs", "search_docs", 'start_datetime', 'end_datetime', + "keyword", "vector", + ] + # init from config + self.model = getChatModelFromConfig(self.llm_config) + self.init_handler() + self.load(do_init) + + def init_handler(self, ): + """Initializes Database VectorBase GraphDB TbaseDB""" self.init_vb() + # self.init_db() + # self.init_tb() + # self.init_gb() - def re_init(self, do_init: bool=False): - self.init_vb(do_init) + def reinit_handler(self, do_init: bool=False): + self.init_vb() + # self.init_db() + # self.init_tb() + # self.init_gb() + + def clear_local(self, re_init: bool = False, handler_type: str = None): + if self.vb: # 存到了本地需要清理 + self.vb.clear_vs_local() + self.load(re_init) + + def init_tb(self, do_init: bool=None): + tb_dict = {"TbaseHandler": TbaseHandler} + tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) + tbase_args = { + "host": self.tb_config.host, + "port": self.tb_config.port, + "username": self.tb_config.username, + "password": self.tb_config.password, + } + self.vb = tb_class(tbase_args, self.tb_config.index_name) + + def init_gb(self, do_init: bool=None): + pass + gb_dict = {"NebulaHandler": NebulaHandler} + gb_class = gb_dict.get(self.gb_config.gb_type, NebulaHandler) + self.gb = gb_class(self.db_config) + + def init_db(self, do_init: bool=None): + pass + db_dict = {"LocalFaissHandler": LocalFaissHandler} + db_class = db_dict.get(self.db_config.db_type) + self.db = db_class(self.db_config) def init_vb(self, do_init: bool=None): - # vb_name = f"{self.user_name}/{self.unique_name}/{self.memory_type}" - vb_name = f"{self.chat_index}/{self.unique_name}/{self.memory_type}" - # default to recreate a new vb table_init() - vb = KBServiceFactory.get_service_by_name(vb_name, self.embed_config, self.kb_root_path) - if vb: - status = vb.clear_vs() - - check_do_init = do_init if do_init else self.do_init - if check_do_init: - self.load(self.kb_root_path, check_do_init) - else: - self.load(self.kb_root_path) - self.save_to_vs() + vb_dict = {"LocalFaissHandler": LocalFaissHandler} + vb_class = vb_dict.get(self.vb_config.vb_type, LocalFaissHandler) + self.vb: LocalFaissHandler = vb_class(self.embed_config, vb_config=self.vb_config) def append(self, message: Message) -> None: - self.check_chat_index(message.chat_index) - # uuid_name = "_".join([self.user_name, self.unique_name, self.memory_type]) - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - datetimes = self.recall_memory_dict[uuid_name].get_datetimes() - contents = self.recall_memory_dict[uuid_name].get_contents() + # update the newest uuid_name + self.check_uuid_name(message) + datetimes = self.recall_memory_dict[self.uuid_name].get_datetimes() + contents = self.recall_memory_dict[self.uuid_name].get_contents() # if message not in chat history, no need to update if (message.end_datetime not in datetimes) or ((message.input_query not in contents) and (message.role_content not in contents)): - self.recall_memory_dict[uuid_name].append(message) - # - if message.role_type == "summary": - self.summary_memory_dict[uuid_name].append(message) - else: - self.current_memory_dict[uuid_name].append(message) - - self.save(self.kb_root_path) - self.save_new_to_vs([message]) - - # def extend(self, memory: Memory): - # self.recall_memory.extend(memory) - # self.current_memory.extend(self.recall_memory.filter_by_role_type(["summary"])) - # self.summary_memory.extend(self.recall_memory.select_by_role_type(["summary"])) - # self.save(self.kb_root_path) - # self.save_new_to_vs(memory.messages) - - def save(self, save_dir: str = "./"): - # file_path = os.path.join(save_dir, f"{self.user_name}/{self.unique_name}/{self.memory_type}/converation.jsonl") - # uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - file_path = os.path.join(save_dir, f"{self.chat_index}/{self.unique_name}/{self.memory_type}/converation.jsonl") - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - memory_messages = self.recall_memory_dict[uuid_name].dict() - memory_messages = {k: [ - {kkk: vvv for kkk, vvv in vv.items() if kkk in self.save_message_keys} - for vv in v ] - for k, v in memory_messages.items() - } + self.append2vb(message) + + def append2vb(self, message: Message) -> None: + self.recall_memory_dict[self.uuid_name].append(message) # - save_to_json_file(memory_messages, file_path) - - def load(self, load_dir: str = None, re_init=None) -> Memory: - load_dir = load_dir or self.kb_root_path - # file_path = os.path.join(load_dir, f"{self.user_name}/{self.unique_name}/{self.memory_type}/converation.jsonl") - # uuid_name = "_".join([self.user_name, self.unique_name, self.memory_type]) - file_path = os.path.join(load_dir, f"{self.chat_index}/{self.unique_name}/{self.memory_type}/converation.jsonl") - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - if os.path.exists(file_path) and not re_init: - recall_memory = Memory(**read_json_file(file_path)) - self.recall_memory_dict[uuid_name] = recall_memory - self.current_memory_dict[uuid_name] = Memory(messages=recall_memory.filter_by_role_type(["summary"])) - self.summary_memory_dict[uuid_name] = Memory(messages=recall_memory.select_by_role_type(["summary"])) - else: - self.recall_memory_dict[uuid_name] = Memory(messages=[]) - self.current_memory_dict[uuid_name] = Memory(messages=[]) - self.summary_memory_dict[uuid_name] = Memory(messages=[]) + docs, json_messages = self.message_process([message]) + # + if True: # resave the local + save_to_json_file(json_messages, self.uuid_file) - def save_new_to_vs(self, messages: List[Message]): if self.embed_config: - # vb_name = f"{self.user_name}/{self.unique_name}/{self.memory_type}" - vb_name = f"{self.chat_index}/{self.unique_name}/{self.memory_type}" - # default to faiss, todo: add new vstype - vb = KBServiceFactory.get_service(vb_name, "faiss", self.embed_config, self.kb_root_path) - embeddings = load_embeddings_from_path(self.embed_config.embed_model_path, self.embed_config.model_device, self.embed_config.langchain_embeddings) - messages = [ + self.vb.add_docs(docs, kb_name=self.kb_name) + + def extend(self, memory: Memory): + for message in memory.messages: + self.append(message) + + def message_process(self, messages: List[Message]): + '''convert messages to vb/local data-format''' + # convert messages to vb data-format + messages = [ {k: v for k, v in m.dict().items() if k in self.save_message_keys} - for m in messages] - docs = [{"page_content": m["step_content"] or m["role_content"] or m["input_query"], "metadata": m} for m in messages] - docs = [Document(**doc) for doc in docs] - vb.do_add_doc(docs, embeddings) + for m in messages + ] + docs = [{"page_content": m["step_content"] or m["role_content"] or m["input_query"], "metadata": m} for m in messages] + docs = [Document(**doc) for doc in docs] + # convert messages to local data-format + memory_messages = self.recall_memory_dict[self.uuid_name].dict() + json_messages = { + k: [ + {kkk: vvv for kkk, vvv in vv.items() if kkk in self.save_message_keys} + for vv in v + ] + for k, v in memory_messages.items() + } - def save_to_vs(self): - '''only after load''' - if self.embed_config: - # vb_name = f"{self.user_name}/{self.unique_name}/{self.memory_type}" - # uuid_name = "_".join([self.user_name, self.unique_name, self.memory_type]) - vb_name = f"{self.chat_index}/{self.unique_name}/{self.memory_type}" - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - # default to recreate a new vb - vb = KBServiceFactory.get_service_by_name(vb_name, self.embed_config, self.kb_root_path) - if vb: - status = vb.clear_vs() - # create_kb(vb_name, "faiss", embed_model) - - # default to faiss, todo: add new vstype - vb = KBServiceFactory.get_service(vb_name, "faiss", self.embed_config, self.kb_root_path) - embeddings = load_embeddings_from_path(self.embed_config.embed_model_path, self.embed_config.model_device, self.embed_config.langchain_embeddings) - messages = self.recall_memory_dict[uuid_name].dict() - messages = [ - {kkk: vvv for kkk, vvv in vv.items() if kkk in self.save_message_keys} - for k, v in messages.items() for vv in v] - docs = [{"page_content": m["step_content"] or m["role_content"] or m["input_query"], "metadata": m} for m in messages] - docs = [Document(**doc) for doc in docs] - vb.do_add_doc(docs, embeddings) - - # def load_from_vs(self, embed_model=EMBEDDING_MODEL) -> Memory: - # vb_name = f"{self.user_name}/{self.unique_name}/{self.memory_type}" - - # create_kb(vb_name, "faiss", embed_model) - # # default to faiss, todo: add new vstype - # vb = KBServiceFactory.get_service(vb_name, "faiss", embed_model) - # docs = vb.get_all_documents() - # print(docs) + return docs, json_messages + + def load(self, re_init=False) -> Memory: + + if not re_init: + for root, dirs, files in os.walk(self.kb_root_path): + for file in files: + if file != 'conversation.jsonl': continue + file_path = os.path.join(root, file) + # get uuid_name + relative_path = os.path.relpath(root, self.kb_root_path) + path_parts = relative_path.split(os.sep) + uuid_name = "_".join(path_parts) + # load to local cache + recall_memory = Memory(**read_json_file(file_path)) + self.recall_memory_dict[uuid_name] = recall_memory + else: + self.recall_memory_dict = {} def get_memory_pool(self, chat_index: str = "") -> Memory: - self.check_chat_index(chat_index) - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - return self.recall_memory_dict[uuid_name] + uuid_name = self.get_uuid_from_chatindex(chat_index) + return self.recall_memory_dict.get(uuid_name, Memory(messages=[])) def router_retrieval(self, chat_index: str = "default", text: str=None, datetime: str = None, @@ -396,23 +402,25 @@ def router_retrieval(self, def embedding_retrieval(self, text: str, top_k=1, score_threshold=1.0, chat_index: str = "default", **kwargs) -> List[Message]: if text is None: return [] - vb_name = f"{chat_index}/{self.unique_name}/{self.memory_type}" - # logger.debug(f"vb_name={vb_name}") - vb = KBServiceFactory.get_service(vb_name, "faiss", self.embed_config, self.kb_root_path) - docs = vb.search_docs(text, top_k=top_k, score_threshold=score_threshold) + + kb_name = self.get_vbname_from_chatindex(chat_index) + docs = self.vb.search(text, top_k=top_k, score_threshold=score_threshold, kb_name=kb_name) return [Message(**doc.metadata) for doc, score in docs] def text_retrieval(self, text: str, chat_index: str = "default", **kwargs) -> List[Message]: if text is None: return [] - uuid_name = "_".join([chat_index, self.unique_name, self.memory_type]) - # logger.debug(f"uuid_name={uuid_name}") - return self._text_retrieval_from_cache(self.recall_memory_dict[uuid_name].messages, text, score_threshold=0.3, topK=5, **kwargs) + + uuid_name = self.get_uuid_from_chatindex(chat_index) + messages = self.recall_memory_dict.get(uuid_name, Memory(messages=[])).messages + return self._text_retrieval_from_cache(messages, text, score_threshold=0.3, topK=5, **kwargs) def datetime_retrieval(self, chat_index: str, datetime: str, text: str = None, n: int = 5, key: str = "start_datetime", **kwargs) -> List[Message]: if datetime is None: return [] - uuid_name = "_".join([chat_index, self.unique_name, self.memory_type]) - # logger.debug(f"uuid_name={uuid_name}") - return self._datetime_retrieval_from_cache(self.recall_memory_dict[uuid_name].messages, datetime, text, n, **kwargs) + + uuid_name = self.get_uuid_from_chatindex(chat_index) + logger.debug(f"uuid_name={uuid_name}") + messages = self.recall_memory_dict.get(uuid_name, Memory(messages=[])).messages + return self._datetime_retrieval_from_cache(messages, datetime, text, n, **kwargs) def _text_retrieval_from_cache(self, messages: List[Message], text: str = None, score_threshold=0.3, topK=5, tag_topK=5, **kwargs) -> List[Message]: keywords = extract_tags(text, topK=tag_topK) @@ -444,22 +452,20 @@ def recursive_summary(self, messages: List[Message], split_n: int = 20, chat_ind return messages newest_messages = messages[-split_n:] - summary_messages = messages[:len(messages)-split_n] + summary_messages = messages[:max(0, len(messages)-split_n)] while (len(newest_messages) != 0) and (newest_messages[0].role_type != "user"): message = newest_messages.pop(0) summary_messages.append(message) # summary - # model = getChatModel(temperature=0.2) - model = getChatModelFromConfig(self.llm_config) summary_content = '\n\n'.join([ m.role_type + "\n" + "\n".join(([f"*{k}* {v}" for parsed_output in m.parsed_output_list for k, v in parsed_output.items() if k not in ['Action Status']])) for m in summary_messages if m.role_type not in ["summary"] ]) - summary_prompt = CONV_SUMMARY_PROMPT_SPEC.format(conversation=summary_content) - content = model.predict(summary_prompt) + summary_prompt = createSummaryPrompt(conversation=summary_content) + content = self.model.predict(summary_prompt) summary_message = Message( chat_index=chat_index, role_name="summaryer", @@ -472,23 +478,32 @@ def recursive_summary(self, messages: List[Message], split_n: int = 20, chat_ind summary_message.parsed_output_list.append({"summary": content}) newest_messages.insert(0, summary_message) return newest_messages - - def check_chat_index(self, chat_index: str): - # logger.debug(f"self.user_name is {self.user_name}, self.chat_dex is {self.chat_index}") - if chat_index != self.chat_index: - self.chat_index = chat_index - self.init_vb() - uuid_name = "_".join([self.chat_index, self.unique_name, self.memory_type]) - if uuid_name not in self.recall_memory_dict: - self.recall_memory_dict[uuid_name] = Memory(messages=[]) - self.current_memory_dict[uuid_name] = Memory(messages=[]) - self.summary_memory_dict[uuid_name] = Memory(messages=[]) + def check_uuid_name(self, message: Message = None): + if message.chat_index != self.chat_index: + self.chat_index = message.chat_index + self.user_name = message.user_name + # self.init_vb() + + self.uuid_name = "_".join([self.chat_index, self.user_name]) + self.kb_name = f"{self.chat_index}/{self.user_name}" + self.uuid_file = os.path.join(self.kb_root_path, f"{self.chat_index}/{self.user_name}/conversation.jsonl") + + self.memory_uuids.add(self.uuid_name) + if self.uuid_name not in self.recall_memory_dict: + self.recall_memory_dict[self.uuid_name] = Memory(messages=[]) + + def get_uuid_from_chatindex(self, chat_index: str) -> str: + for i in self.recall_memory_dict: + if chat_index in i: + return i + return "" - # logger.debug(f"self.user_name is {self.user_name}") + def get_vbname_from_chatindex(self, chat_index: str) -> str: + return self.get_uuid_from_chatindex(chat_index).replace("_", "/") -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler from muagent.llm_models.get_embedding import get_embedding from redis.commands.search.field import ( TextField, @@ -641,28 +656,6 @@ def get_memory_pool_by_all(self, search_key_contents: dict): query = f"({')('.join(querys)})" if len(querys) >=2 else "".join(querys) r = self.th.search(query) return self.tbasedoc2Memory(r) - - def router_retrieval(self, - chat_index: str = "default", text: str=None, datetime: str = None, - n=5, top_k=5, retrieval_type: str = "embedding", **kwargs - ) -> List[Message]: - - retrieval_func_dict = { - "embedding": self.embedding_retrieval, "text": self.text_retrieval, "datetime": self.datetime_retrieval - } - - # 确保提供了合法的检索类型 - if retrieval_type not in retrieval_func_dict: - raise ValueError(f"Invalid retrieval_type: '{retrieval_type}'. Available types: {list(retrieval_func_dict.keys())}") - - retrieval_func = retrieval_func_dict[retrieval_type] - # - params = locals() - params.pop("self") - params.pop("retrieval_type") - params.update(params.pop('kwargs', {})) - # - return retrieval_func(**params) def embedding_retrieval(self, text: str, top_k=1, score_threshold=1.0, chat_index: str = "default", **kwargs) -> List[Message]: if text is None: return [] @@ -745,7 +738,8 @@ def recursive_summary(self, messages: List[Message], chat_index: str, message_in for m in summary_messages if m.role_type not in ["summary"] ]) - summary_prompt = CONV_SUMMARY_PROMPT_SPEC.format(conversation=summary_content) + # summary_prompt = CONV_SUMMARY_PROMPT_SPEC.format(conversation=summary_content) + summary_prompt = createSummaryPrompt(conversation=summary_content) content = model.predict(summary_prompt) summary_message = Message( chat_index=chat_index, diff --git a/muagent/connector/phase/base_phase.py b/muagent/connector/phase/base_phase.py index e2ff835..d1ef766 100644 --- a/muagent/connector/phase/base_phase.py +++ b/muagent/connector/phase/base_phase.py @@ -20,7 +20,7 @@ from muagent.connector.message_process import MessageUtils from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.base_configs.env_config import JUPYTER_WORK_PATH, KB_ROOT_PATH -from muagent.utils.tbase_util import TbaseHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler role_configs = load_role_configs(AGETN_CONFIGS) diff --git a/muagent/connector/schema/memory.py b/muagent/connector/schema/memory.py index e2a2891..5bb85f4 100644 --- a/muagent/connector/schema/memory.py +++ b/muagent/connector/schema/memory.py @@ -20,17 +20,26 @@ def append(self, message: Message): def extend(self, memory: 'Memory'): self.messages.extend(memory.messages) + def append_with_limit(self, message: Message, limit: int = 10): + self.messages.append(message) + self.messages = self.messages[-limit:] + + def extend_with_limit(self, memory: 'Memory', limit: int = 10): + self.messages.extend(memory.messages) + self.messages = self.messages[-limit:] + def update(self, role_name: str, role_type: str, role_content: str): self.messages.append(Message(role_name, role_type, role_content, role_content)) def sort_by_key(self, key: str): self.messages = sorted(self.messages, key=lambda x: getattr(x, key, f"No this {key}")) - def clear(self, ): - self.messages = [] - - def delete(self, ): - pass + def clear(self, k: int = None): + '''save the messages by k limit''' + if k is None: + self.messages = [] + else: + self.messages = self.messages[-k:] def get_messages(self, k=0) -> List[Message]: """Return the most recent k memories, return all when k=0""" diff --git a/muagent/db_handler/__init__.py b/muagent/db_handler/__init__.py index 83c9d92..80a0546 100644 --- a/muagent/db_handler/__init__.py +++ b/muagent/db_handler/__init__.py @@ -4,4 +4,12 @@ @file: __init__.py.py @time: 2023/11/16 下午3:15 @desc: -''' \ No newline at end of file +''' + +from .graph_db_handler import NebulaHandler, NetworkxHandler +from .vector_db_handler import LocalFaissHandler, TbaseHandler, ChromaHandler + + +__all__ = [ + "NebulaHandler", "ChromaHandler", "TbaseHandler", "LocalFaissHandler", "NetworkxHandler" +] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/__init__.py b/muagent/db_handler/graph_db_handler/__init__.py index 6d3396a..1730697 100644 --- a/muagent/db_handler/graph_db_handler/__init__.py +++ b/muagent/db_handler/graph_db_handler/__init__.py @@ -4,4 +4,10 @@ @file: __init__.py.py @time: 2023/11/20 下午3:07 @desc: -''' \ No newline at end of file +''' +from .nebula_handler import NebulaHandler +from .networkx_handler import NetworkxHandler + +__all__ = [ + "NebulaHandler", "NetworkxHandler" +] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/networkx_handler.py b/muagent/db_handler/graph_db_handler/networkx_handler.py new file mode 100644 index 0000000..c48fc97 --- /dev/null +++ b/muagent/db_handler/graph_db_handler/networkx_handler.py @@ -0,0 +1,138 @@ +import os +import networkx as nx +from typing import List, Tuple, Dict + +from muagent.schemas.memory import * +from muagent.base_configs.env_config import KB_ROOT_PATH +from muagent.schemas.db import GBConfig + + + +class NetworkxHandler: + def __init__( + self, + kb_root_path: str = KB_ROOT_PATH, + gb_config: GBConfig = None, + ): + self.graph = nx.Graph() # 使用有向图，根据需要也可以使用无向图 nx.Graph() + self.kb_root_path = kb_root_path + self.gb_config = gb_config + self.kb_name = "default" + + def add_node(self, node: GNode): + insert_nodes = self.node_process([node]) + self.graph.add_nodes_from(insert_nodes) + + def add_nodes(self, nodes: List[GNode]): + insert_nodes = self.node_process(nodes) + self.graph.add_nodes_from(insert_nodes) + + def add_edge(self, grelation: GRelation): + insert_relations = self.relation_process([grelation]) + self.graph.add_edges_from(insert_relations) + + def add_edges(self, grelations: List[GRelation]): + insert_relations = self.relation_process(grelations) + self.graph.add_edges_from(insert_relations) + + def search_nodes_by_nodeid(self, nodeid: str) -> GNode: + if self.missing_node(nodeid): return None + + node_attrs = self.graph.nodes.get(nodeid, {}) + return GNode(id=nodeid, attributes=node_attrs) + + def search_edges_by_nodeid(self, nodeid: str) -> List[GRelation]: + if self.missing_node(nodeid): return [] + + return [ + GRelation( + id=f"{nodeid}-{neighbor}", + left=self.search_nodes_by_nodeid(nodeid), + right=self.search_nodes_by_nodeid(neighbor), + attributes=self.graph.get_edge_data(nodeid, neighbor) + ) + for neighbor, attr in self.graph.adj[nodeid].items() + ] + + def search_edges_by_nodeids(self, left: str, right: str) -> GRelation: + if self.missing_node(left) or self.missing_node(right): return None + if self.missing_edge(left, right): return None + + return GRelation( + id=f"{left}-{right}", + left=self.search_nodes_by_nodeid(left), + right=self.search_nodes_by_nodeid(right), + attributes=self.graph.get_edge_data(left, right) + ) + + def search_nodes_by_attr(self, **attributes) -> List[GNode]: + return [GNode(id=node, attributes=attr) for node, attr in self.graph.nodes(data=True) if all(attr.get(k) == v for k, v in attributes.items())] + + def search_edges_by_attr(self, **attributes) -> List[GRelation]: + return [GRelation( + id=f"{left}-{right}", + left=self.search_nodes_by_nodeid(left), + right=self.search_nodes_by_nodeid(right), + attributes=attr + ) for left, right, attr in self.graph.edges(data=True) if all(attr.get(k) == v for k, v in attributes.items())] + + def save(self, kb_name: str): + self.kb_name = kb_name or self.kb_name + self.save_to_local(self.kb_name) + + def load(self, kb_name: str): + if self.kb_name != kb_name: + self.kb_name = kb_name if self.kb_name == "default" else self.kb_name + self.load_from_local(kb_name) + + def save_to_local(self, kb_name: str): + dir_path = os.path.join(self.kb_root_path, kb_name) + # 将图保存到本地文件 + nx.write_graphml(self.graph, os.path.join(dir_path, 'graph.graphml')) + + def load_from_local(self, kb_name: str): + dir_path = os.path.join(self.kb_root_path, kb_name) + # 从本地文件加载图 + if os.path.exists(os.path.join(dir_path, 'graph.graphml')): + self.graph = nx.read_graphml(os.path.join(dir_path, 'graph.graphml')) + + def delete_node(self, nodeid: str): + self.graph.remove_node(nodeid) + + def delete_nodes(self, nodeids: List[str]): + self.graph.remove_nodes_from(nodeids) + + def delete_edges_by_nodeid(self, nodeid: str): + edges = list(self.graph.edges(nodeid)) + self.graph.remove_edges_from(edges) + + def delete_edge(self, left: str, right: str): + self.graph.remove_edges_from([(left, right)]) + + def delete_edges(self, edges: List[Tuple]): + self.graph.remove_edges_from(edges) + + def clear(self): + self.graph.clear() + + def node_process(self, nodes: List[GNode]) -> List[Tuple]: + node_list = [] + for node in nodes: + node_id = node.id + node_attrs = node.attributes + node_list.append((node_id, node_attrs)) + + return node_list + + def relation_process(self, relations: List[GRelation]) -> List[Tuple]: + relation_list = [] + for relation in relations: + edge_attrs = relation.attributes + relation_list.append((relation.left.id, relation.right.id, edge_attrs)) + return relation_list + + def missing_edge(self, left: str, right: str) -> bool: + return not self.graph.has_edge(left, right) + + def missing_node(self, nodeid: str) -> bool: + return nodeid not in self.graph.nodes \ No newline at end of file diff --git a/muagent/db_handler/vector_db_handler/__init__.py b/muagent/db_handler/vector_db_handler/__init__.py index 2b67ecf..7a7d61a 100644 --- a/muagent/db_handler/vector_db_handler/__init__.py +++ b/muagent/db_handler/vector_db_handler/__init__.py @@ -4,4 +4,12 @@ @file: __init__.py.py @time: 2023/11/20 下午3:08 @desc: -''' \ No newline at end of file +''' + +from .chroma_handler import ChromaHandler +from .tbase_handler import TbaseHandler +from .local_faiss_handler import LocalFaissHandler + +__all__ = [ + "ChromaHandler", "TbaseHandler", "LocalFaissHandler" +] \ No newline at end of file diff --git a/muagent/db_handler/vector_db_handler/chroma_handler.py b/muagent/db_handler/vector_db_handler/chroma_handler.py index f86141c..e5ec944 100644 --- a/muagent/db_handler/vector_db_handler/chroma_handler.py +++ b/muagent/db_handler/vector_db_handler/chroma_handler.py @@ -8,9 +8,15 @@ from loguru import logger import chromadb +from muagent.schemas.db import VBConfig + class ChromaHandler: - def __init__(self, path: str, collection_name: str = ''): + def __init__( + self, path: str, + collection_name: str = '', + vb_config: VBConfig = None + ): ''' init client @param path: path of data diff --git a/muagent/db_handler/vector_db_handler/local_faiss_handler.py b/muagent/db_handler/vector_db_handler/local_faiss_handler.py new file mode 100644 index 0000000..f18605c --- /dev/null +++ b/muagent/db_handler/vector_db_handler/local_faiss_handler.py @@ -0,0 +1,142 @@ +from loguru import logger +from typing import List +from functools import lru_cache +import os, shutil + +from langchain.embeddings.base import Embeddings +from langchain_community.docstore.document import Document + +from muagent.utils.server_utils import torch_gc +from muagent.retrieval.base_service import SupportedVSType +from muagent.retrieval.faiss_m import FAISS +from muagent.llm_models.llm_config import EmbedConfig +from muagent.schemas.db import VBConfig +from muagent.retrieval.utils import load_embeddings_from_path + +from muagent.base_configs.env_config import ( + KB_ROOT_PATH, FAISS_NORMALIZE_L2, SCORE_THRESHOLD +) + + + +class LocalFaissHandler: + + def __init__( + self, + embed_config: EmbedConfig, + vb_config: VBConfig = None + ): + + self.vb_config = vb_config + self.embed_config = embed_config + self.embeddings = load_embeddings_from_path( + self.embed_config.embed_model_path, + self.embed_config.model_device, + self.embed_config.langchain_embeddings + ) + + # INIT + self.search_index: FAISS = None + self.kb_name = "default" + self.kb_root_path = vb_config.kb_root_path or KB_ROOT_PATH + self.kb_path = "default" + self.vs_path = "default" + # DEFAULT + self.distance_strategy = "EUCLIDEAN_DISTANCE" + # init search_index + self.create_vs() + + @lru_cache(1) + def create_vs(self, kb_name: str = None): + kb_name = kb_name or self.kb_name + self.mkdir_vspath(kb_name, self.kb_root_path) + if "index.faiss" in os.listdir(self.vs_path): + self.load_vs_from_localdir(self.vs_path) + else: + self.create_empty_vs() + + def add_docs( + self, + docs: List[Document], + kb_name: str = None, + **kwargs, + ): + if kb_name: + self.create_vs(kb_name) + # 可能需要临时修改处理 + self.search_index.embedding_function = self.embeddings.embed_documents + # logger.info("loaded docs, docs' lens is {}".format(len(docs))) + self.search_index.add_documents(docs) + torch_gc() + self.search_index.save_local(self.vs_path) + + def clear_vs(self): + if os.path.exists(self.kb_path): + shutil.rmtree(self.kb_path) + os.makedirs(self.kb_path) + + def clear_vs_local(self, kb_name: str = None): + def _del(_path): + if os.path.exists(_path): + shutil.rmtree(_path) + os.makedirs(_path) + return True + + if kb_name: + kb_path = LocalFaissHandler.get_kb_path(kb_name, self.kb_root_path) + return _del(kb_path) + + for dir in os.listdir(self.kb_root_path): + dir_path = os.path.join(self.kb_root_path, dir) + if not os.path.isdir(dir_path): continue + _del(dir_path) + return True + + def search( + self, + query: str, + top_k: int, + score_threshold: float = SCORE_THRESHOLD, + kb_name: str = None, + ) -> List[Document]: + + if kb_name: + self.create_vs(kb_name) + + docs = self.search_index.similarity_search_with_score(query, k=top_k, score_threshold=score_threshold) + return docs + + def get_all_documents(self, kb_name: str = None): + if kb_name: + self.create_vs() + return self.search_index.get_all_documents() + + # method for initing vs + def create_empty_vs(self, ): + doc = Document(page_content="init", metadata={}) + self.search_index = FAISS.from_documents([doc], self.embeddings, normalize_L2=FAISS_NORMALIZE_L2, distance_strategy=self.distance_strategy) + ids = [k for k, v in self.search_index.docstore._dict.items()] + self.search_index.delete(ids) + + def load_vs_from_localdir(self, vs_path): + if "index.faiss" in os.listdir(vs_path): + self.search_index = FAISS.load_local(vs_path, self.embeddings, normalize_L2=FAISS_NORMALIZE_L2, distance_strategy=self.distance_strategy) + else: + self.create_empty_vs() + + def mkdir_vspath(self, kb_name, kb_root_path): + self.vs_path = LocalFaissHandler.get_vs_path(kb_name, kb_root_path) + self.kb_path = LocalFaissHandler.get_kb_path(kb_name, kb_root_path) + if not os.path.exists(self.vs_path): + os.makedirs(self.vs_path) + + def vs_type(self) -> str: + return SupportedVSType.FAISS + + @staticmethod + def get_vs_path(kb_name: str, kb_root_path: str): + return os.path.join(LocalFaissHandler.get_kb_path(kb_name, kb_root_path), "vector_store") + + @staticmethod + def get_kb_path(kb_name: str, kb_root_path: str): + return os.path.join(kb_root_path, kb_name) \ No newline at end of file diff --git a/muagent/utils/tbase_util.py b/muagent/db_handler/vector_db_handler/tbase_handler.py similarity index 91% rename from muagent/utils/tbase_util.py rename to muagent/db_handler/vector_db_handler/tbase_handler.py index 8bd452a..fa68375 100644 --- a/muagent/utils/tbase_util.py +++ b/muagent/db_handler/vector_db_handler/tbase_handler.py @@ -16,9 +16,18 @@ TagField ) +from muagent.schemas.db import TBConfig + + class TbaseHandler: - def __init__(self, tbase_args, index_name="wyp311395", definition_value="message", ): + def __init__( + self, + tbase_args, + index_name="test", + definition_value="message", + tb_config: TBConfig = None + ): self.client = redis.Redis( host=tbase_args['host'], port=tbase_args['port'], @@ -27,6 +36,7 @@ def __init__(self, tbase_args, index_name="wyp311395", definition_value="message ) self.index_name = index_name self.definition_value = definition_value + self.tb_config = tb_config def create_index(self, index_name=None, schema=None, definition: list =None): ''' @@ -116,7 +126,7 @@ def get(self, content): res = self.client.hgetall(id) return res - def fuzzy_delete(self, collection_name, delete_str, index_name="wyp311395"): + def fuzzy_delete(self, collection_name, delete_str, index_name="test"): ''' delete by metaid :param index_name: diff --git a/muagent/llm_models/__init__.py b/muagent/llm_models/__init__.py index 11bbe7a..4edab6c 100644 --- a/muagent/llm_models/__init__.py +++ b/muagent/llm_models/__init__.py @@ -1,8 +1,8 @@ -from .openai_model import getExtraModel, getChatModelFromConfig +from .openai_model import getExtraModel, getChatModelFromConfig, CustomLLMModel from .llm_config import LLMConfig, EmbedConfig __all__ = [ - "getExtraModel", "getChatModelFromConfig", + "getExtraModel", "getChatModelFromConfig", "CustomLLMModel", "LLMConfig", "EmbedConfig" ] \ No newline at end of file diff --git a/muagent/llm_models/llm_config.py b/muagent/llm_models/llm_config.py index 504639d..02bb074 100644 --- a/muagent/llm_models/llm_config.py +++ b/muagent/llm_models/llm_config.py @@ -40,33 +40,52 @@ def __str__(self): return ', '.join(f"{k}: {v}" for k,v in vars(self).items()) -@dataclass + +@dataclass(frozen=True) class EmbedConfig: - def __init__( - self, - api_key: str = "", - api_base_url: str = "", - embed_model: str = "", - embed_model_path: str = "", - embed_engine: str = "", - model_device: str = "cpu", - langchain_embeddings: Embeddings = None, - **kwargs - ): - self.embed_model: str = embed_model - self.embed_model_path: str = embed_model_path - self.embed_engine: str = embed_engine - self.model_device: str = model_device - self.api_key: str = api_key - self.api_base_url: str = api_base_url - # custom embeddings - self.langchain_embeddings = langchain_embeddings - # - self.check_config() + + embed_model: str = "" + embed_model_path: str = "" + embed_engine: str = "openai" + model_device: str = "cpu" + api_key: str = "" + api_base_url: str = "" + # custom embeddings + langchain_embeddings: Embeddings = None def check_config(self, ): pass def __str__(self): return ', '.join(f"{k}: {v}" for k,v in vars(self).items()) + +# @dataclass +# class EmbedConfig: +# def __init__( +# self, +# api_key: str = "", +# api_base_url: str = "", +# embed_model: str = "", +# embed_model_path: str = "", +# embed_engine: str = "", +# model_device: str = "cpu", +# langchain_embeddings: Embeddings = None, +# **kwargs +# ): +# self.embed_model: str = embed_model +# self.embed_model_path: str = embed_model_path +# self.embed_engine: str = embed_engine +# self.model_device: str = model_device +# self.api_key: str = api_key +# self.api_base_url: str = api_base_url +# # custom embeddings +# self.langchain_embeddings = langchain_embeddings +# # +# self.check_config() + +# def check_config(self, ): +# pass + +# def __str__(self): +# return ', '.join(f"{k}: {v}" for k,v in vars(self).items()) \ No newline at end of file diff --git a/muagent/orm/commands/code_base_cds.py b/muagent/orm/commands/code_base_cds.py index e74a0f1..f1d0e31 100644 --- a/muagent/orm/commands/code_base_cds.py +++ b/muagent/orm/commands/code_base_cds.py @@ -7,7 +7,7 @@ ''' from loguru import logger from muagent.orm.db import with_session -from muagent.orm.schemas.base_schema import CodeBaseSchema +from muagent.schemas.kb.base_schema import CodeBaseSchema @with_session diff --git a/muagent/orm/commands/document_base_cds.py b/muagent/orm/commands/document_base_cds.py index f19b71b..43f8700 100644 --- a/muagent/orm/commands/document_base_cds.py +++ b/muagent/orm/commands/document_base_cds.py @@ -1,5 +1,5 @@ from muagent.orm.db import with_session -from muagent.orm.schemas.base_schema import KnowledgeBaseSchema +from muagent.schemas.kb.base_schema import KnowledgeBaseSchema # @with_session diff --git a/muagent/orm/commands/document_file_cds.py b/muagent/orm/commands/document_file_cds.py index 269dc72..6816009 100644 --- a/muagent/orm/commands/document_file_cds.py +++ b/muagent/orm/commands/document_file_cds.py @@ -1,5 +1,5 @@ from muagent.orm.db import with_session -from muagent.orm.schemas.base_schema import KnowledgeFileSchema, KnowledgeBaseSchema +from muagent.schemas.kb.base_schema import KnowledgeFileSchema, KnowledgeBaseSchema from muagent.orm.utils import DocumentFile diff --git a/muagent/service/base_service.py b/muagent/retrieval/base_service.py similarity index 100% rename from muagent/service/base_service.py rename to muagent/retrieval/base_service.py diff --git a/muagent/retrieval/commands/__init__.py b/muagent/retrieval/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/muagent/retrieval/commands/default_vs_cds.py b/muagent/retrieval/commands/default_vs_cds.py deleted file mode 100644 index 94707cb..0000000 --- a/muagent/retrieval/commands/default_vs_cds.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import List - -from langchain.embeddings.base import Embeddings -from langchain.schema import Document - - - -class BaseVSCService: - def do_create_kb(self): - pass - - def do_drop_kb(self): - pass - - def do_add_doc(self, docs: List[Document], embeddings: Embeddings): - pass - - def do_clear_vs(self): - pass - - def vs_type(self) -> str: - return "default" - - def do_init(self): - pass - - def do_search(self): - pass - - def do_insert_multi_knowledge(self): - pass - - def do_insert_one_knowledge(self): - pass - - def do_delete_doc(self): - pass diff --git a/muagent/service/faiss_db_service.py b/muagent/retrieval/faiss_db_service.py similarity index 99% rename from muagent/service/faiss_db_service.py rename to muagent/retrieval/faiss_db_service.py index 77720f8..dac966c 100644 --- a/muagent/service/faiss_db_service.py +++ b/muagent/retrieval/faiss_db_service.py @@ -33,7 +33,7 @@ def _embeddings_hash(self): _VECTOR_STORE_TICKS = {} -# @lru_cache(CACHED_VS_NUM) +@lru_cache(CACHED_VS_NUM) def load_vector_store( knowledge_base_name: str, embed_config: EmbedConfig, diff --git a/muagent/service/service_factory.py b/muagent/retrieval/service_factory.py similarity index 98% rename from muagent/service/service_factory.py rename to muagent/retrieval/service_factory.py index 14a3253..cacdfc7 100644 --- a/muagent/service/service_factory.py +++ b/muagent/retrieval/service_factory.py @@ -2,7 +2,7 @@ import os from loguru import logger -# from configs.model_config import EMBEDDING_MODEL, KB_ROOT_PATH + from muagent.base_configs.env_config import KB_ROOT_PATH from .faiss_db_service import FaissKBService diff --git a/muagent/retrieval/utils.py b/muagent/retrieval/utils.py index 9a42dbf..50b783b 100644 --- a/muagent/retrieval/utils.py +++ b/muagent/retrieval/utils.py @@ -14,7 +14,7 @@ def load_embeddings(model: str, device: str, embedding_model_dict: dict): return embeddings -# @lru_cache(1) +@lru_cache(1) def load_embeddings_from_path(model_path: str, device: str, langchain_embeddings: Embeddings = None): if langchain_embeddings: return langchain_embeddings diff --git a/muagent/orm/schemas/__init__.py b/muagent/schemas/__init__.py similarity index 100% rename from muagent/orm/schemas/__init__.py rename to muagent/schemas/__init__.py diff --git a/muagent/schemas/db/__init__.py b/muagent/schemas/db/__init__.py new file mode 100644 index 0000000..abdf7ea --- /dev/null +++ b/muagent/schemas/db/__init__.py @@ -0,0 +1,6 @@ +from .db_config import DBConfig, GBConfig, VBConfig, TBConfig + + +__all__ = [ + "DBConfig", "GBConfig", "VBConfig", "TBConfig" +] \ No newline at end of file diff --git a/muagent/schemas/db/db_config.py b/muagent/schemas/db/db_config.py new file mode 100644 index 0000000..538afc2 --- /dev/null +++ b/muagent/schemas/db/db_config.py @@ -0,0 +1,29 @@ +from pydantic import BaseModel +from typing import List, Dict + + + +class DBConfig(BaseModel): + db_type: str + extra_kwargs: Dict = {} + + +class GBConfig(BaseModel): + gb_type: str + extra_kwargs: Dict = {} + + +class TBConfig(BaseModel): + tb_type: str + index_name: str + host: str + port: str + username: str + password: str + extra_kwargs: Dict = {} + + +class VBConfig(BaseModel): + vb_type: str + kb_root_path: str = None + extra_kwargs: Dict = {} \ No newline at end of file diff --git a/muagent/orm/schemas/base_schema.py b/muagent/schemas/kb/base_schema.py similarity index 100% rename from muagent/orm/schemas/base_schema.py rename to muagent/schemas/kb/base_schema.py diff --git a/muagent/schemas/memory/__init__.py b/muagent/schemas/memory/__init__.py new file mode 100644 index 0000000..7ea6809 --- /dev/null +++ b/muagent/schemas/memory/__init__.py @@ -0,0 +1,6 @@ +from .auto_extract_graph_schema import * + + +__all__ = [ + "GNodeAbs", "GRelationAbs", "Attribute", "GNode", "GRelation", "ThemeEnums" +] \ No newline at end of file diff --git a/muagent/schemas/memory/auto_extract_graph_schema.py b/muagent/schemas/memory/auto_extract_graph_schema.py new file mode 100644 index 0000000..57e6e0a --- /dev/null +++ b/muagent/schemas/memory/auto_extract_graph_schema.py @@ -0,0 +1,42 @@ +from pydantic import BaseModel +from typing import List, Dict +from enum import Enum + + + +class Attribute(BaseModel): + name: str + description: str + + +class GNodeAbs(BaseModel): + type: str + attributes: List[Attribute] + + +class GRelationAbs(BaseModel): + type: str + attributes: List[Attribute] + + +class GNode(BaseModel): + id: str = None + type: str + attributes: Dict[str, str] + + +class GRelation(BaseModel): + id: str = None + type: str + left: GNode + right: GNode + attributes: Dict[str, str] + + +class ThemeEnums(Enum): + ''' + the memory themes + ''' + Person: str = "person" + Event: str = "event" + diff --git a/muagent/service/cb_api.py b/muagent/service/cb_api.py index 974a32f..4d9f791 100644 --- a/muagent/service/cb_api.py +++ b/muagent/service/cb_api.py @@ -10,11 +10,7 @@ from typing import List, Dict import shutil -from fastapi.responses import StreamingResponse, FileResponse -from fastapi import File, Form, Body, Query, UploadFile -from langchain_community.docstore.document import Document - -from .service_factory import KBServiceFactory +from fastapi import Body from muagent.utils.server_utils import BaseResponse, ListResponse from muagent.utils.path_utils import * from muagent.orm.commands import * @@ -26,12 +22,6 @@ CHROMA_PERSISTENT_PATH ) - -# from configs.model_config import ( -# CB_ROOT_PATH -# ) - -# from muagent.codebase_handler.codebase_handler import CodeBaseHandler from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler diff --git a/muagent/service/kb_api.py b/muagent/service/kb_api.py index 68da570..34d49a2 100644 --- a/muagent/service/kb_api.py +++ b/muagent/service/kb_api.py @@ -10,7 +10,7 @@ from langchain_community.docstore.document import Document -from .service_factory import KBServiceFactory +from muagent.retrieval.service_factory import KBServiceFactory from muagent.utils.server_utils import BaseResponse, ListResponse from muagent.utils.path_utils import * from muagent.orm.commands import * diff --git a/muagent/service/migrate.py b/muagent/service/migrate.py index 0c19b9d..ed561a3 100644 --- a/muagent/service/migrate.py +++ b/muagent/service/migrate.py @@ -6,7 +6,7 @@ from muagent.orm.utils import DocumentFile from muagent.orm.commands import add_doc_to_db from muagent.utils.path_utils import * -from .service_factory import KBServiceFactory +from muagent.retrieval.service_factory import KBServiceFactory ''' diff --git a/tests/connector/agent_test.py b/tests/connector/agent_test.py index 8f2f96c..3245792 100644 --- a/tests/connector/agent_test.py +++ b/tests/connector/agent_test.py @@ -153,7 +153,7 @@ **Role:** Select the role from agent names. """ -from muagent.connector.configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT +from muagent.base_configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT tool_role = Role(role_type="assistant", role_name="tool_reacter", prompt=REACT_TOOL_PROMPT) tool_react_agent = ReactAgent( diff --git a/tests/connector/chain_test.py b/tests/connector/chain_test.py index 5624ccf..2971fb4 100644 --- a/tests/connector/chain_test.py +++ b/tests/connector/chain_test.py @@ -69,7 +69,7 @@ **Role:** Select the role from agent names. """ -from muagent.connector.configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT +from muagent.base_configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT tool_role = Role(role_type="assistant", role_name="tool_reacter", prompt=REACT_TOOL_PROMPT) tool_react_agent = ReactAgent( diff --git a/tests/connector/flow_test.py b/tests/connector/flow_test.py index d2b4b06..cbc2223 100644 --- a/tests/connector/flow_test.py +++ b/tests/connector/flow_test.py @@ -68,7 +68,7 @@ from muagent.connector.antflow import AgentFlow, ChainFlow, PhaseFlow -from muagent.connector.configs.prompts import QA_TEMPLATE_PROMPT +from muagent.base_configs.prompts import QA_TEMPLATE_PROMPT from muagent.connector.configs import BASE_PROMPT_CONFIGS from muagent.retrieval.base_retrieval import BaseCodeRetrieval import os diff --git a/tests/connector/hierachical_memory_testy.py b/tests/connector/hierachical_memory_testy.py new file mode 100644 index 0000000..9c90478 --- /dev/null +++ b/tests/connector/hierachical_memory_testy.py @@ -0,0 +1,111 @@ +import os, sys +from loguru import logger + +os.environ["do_create_dir"] = "1" + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +# test local code +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) +from muagent.connector.memory_manager import BaseMemoryManager, LocalMemoryManager +from muagent.connector.memory import HierarchicalMemoryManager +from muagent.connector.memory_manager import LocalMemoryManager, Message +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +from muagent.schemas.db import DBConfig, GBConfig, VBConfig, TBConfig + +llm_config = LLMConfig( + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +) + +embed_config = EmbedConfig( + embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +) + + +# prepare your message +message1 = Message( + chat_index="default", role_name="test1", role_type="user", role_content="hello", + parsed_output_list=[{"input": "hello"}], user_name="default" +) + +text = "hi! how can I help you?" +message2 = Message( + chat_index="shuimo", role_name="test2", role_type="assistant", role_content=text, parsed_output_list=[{"answer": text}], + user_name="shuimo" +) + +text = "they say hello and hi to each other" +message3 = Message( + chat_index="shanshi", role_name="test3", role_type="summary", role_content=text, + parsed_output_list=[{"summary": text}], + user_name="shanshi" + ) + +# append or extend test +vb_config = VBConfig(vb_type="LocalFaissHandler") +gb_config = GBConfig(vb_type="HierarchicalMemoryManager") +hierach_memory_manager = HierarchicalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, gb_config=gb_config, do_init=True) +# append can ignore user_name +hierach_memory_manager.append(message=message1) +hierach_memory_manager.append(message=message2) +hierach_memory_manager.append(message=message3) + +# # test init_local +# local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=True) +# print(local_memory_manager.get_memory_pool("default").messages) +# print(local_memory_manager.get_memory_pool("shuimo").messages) +# print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # test load from local +# local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # test after clear local vs and jsonl +# local_memory_manager.clear_local(re_init=True) +# print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # test init_local +# local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# print(local_memory_manager.get_memory_pool("shanshi").messages) + + +local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# embedding retrieval test +text = "say hi to each other, i want some help" +# retrieval_type=datetime => retrieval from datetime and jieba +print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, datetime="2024-06-26 14:15:00", n=4, top_k=5, retrieval_type= "datetime")) +# retrieval_type=eembedding => retrieval from embedding +print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "embedding")) +# retrieval_type=text => retrieval from jieba +print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "text")) + +# # recursive_summary test +print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shanshi").messages, split_n=1, chat_index="shanshi")) + +# print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shuimo").messages, split_n=1, chat_index="shanshi")) + +# print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("default").messages, split_n=1, chat_index="shanshi")) + diff --git a/tests/connector/memory_manager_test.py b/tests/connector/memory_manager_test.py index 5a759dd..7b69260 100644 --- a/tests/connector/memory_manager_test.py +++ b/tests/connector/memory_manager_test.py @@ -30,65 +30,80 @@ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) sys.path.append(src_dir) -from muagent.connector.memory_manager import LocalMemoryManager, Message -from muagent.llm_models.llm_config import EmbedConfig, LLMConfig - - -llm_config = LLMConfig( - model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, -) - -embed_config = EmbedConfig( - embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path -) - - -# prepare your message -message1 = Message( - chat_index="default", role_name="test1", role_type="user", role_content="hello", - parsed_output_list=[{"input": "hello"}], user_name="default" -) - -text = "hi! how can I help you?" -message2 = Message( - chat_index="shuimo", role_name="test2", role_type="assistant", role_content=text, parsed_output_list=[{"answer": text}], - user_name="shuimo" -) - -text = "they say hello and hi to each other" -message3 = Message( - chat_index="shanshi", role_name="test3", role_type="summary", role_content=text, - parsed_output_list=[{"summary": text}], - user_name="shanshi" - ) - -# append or extend test -local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, do_init=True) -# append can ignore user_name -local_memory_manager.append(message=message1) -local_memory_manager.append(message=message2) -local_memory_manager.append(message=message3) -# -# local_memory_manager = LocalMemoryManager(embed_config=None, llm_config=None, do_init=False) -local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, do_init=False) -local_memory_manager.load() -print(local_memory_manager.get_memory_pool("default").messages) -print(local_memory_manager.get_memory_pool("shuimo").messages) -print(local_memory_manager.get_memory_pool("shanshi").messages) - -# embedding retrieval test -text = "say hi to each other, i want some help" -# retrieval_type=datetime => retrieval from datetime and jieba -print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, datetime="2024-03-12 17:48:00", n=4, top_k=5, retrieval_type= "datetime")) -# retrieval_type=eembedding => retrieval from embedding -print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "embedding")) -# retrieval_type=text => retrieval from jieba -print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "text")) - -# recursive_summary test -print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shanshi").messages, split_n=1, chat_index="shanshi")) - -print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shuimo").messages, split_n=1, chat_index="shanshi")) - -print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("default").messages, split_n=1, chat_index="shanshi")) +from muagent.connector.memory_manager import BaseMemoryManager, LocalMemoryManager +# from muagent.connector.memory_manager import LocalMemoryManager, Message +# from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +# from muagent.schemas.db import DBConfig, GBConfig, VBConfig, TBConfig + +# llm_config = LLMConfig( +# model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +# ) + +# embed_config = EmbedConfig( +# embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +# ) + + +# # prepare your message +# message1 = Message( +# chat_index="default", role_name="test1", role_type="user", role_content="hello", +# parsed_output_list=[{"input": "hello"}], user_name="default" +# ) + +# text = "hi! how can I help you?" +# message2 = Message( +# chat_index="shuimo", role_name="test2", role_type="assistant", role_content=text, parsed_output_list=[{"answer": text}], +# user_name="shuimo" +# ) + +# text = "they say hello and hi to each other" +# message3 = Message( +# chat_index="shanshi", role_name="test3", role_type="summary", role_content=text, +# parsed_output_list=[{"summary": text}], +# user_name="shanshi" +# ) + +# # append or extend test +# vb_config = VBConfig(vb_type="LocalFaissHandler") +# local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=True) +# # append can ignore user_name +# local_memory_manager.append(message=message1) +# local_memory_manager.append(message=message2) +# local_memory_manager.append(message=message3) + +# # # test init_local +# # local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=True) +# # print(local_memory_manager.get_memory_pool("default").messages) +# # print(local_memory_manager.get_memory_pool("shuimo").messages) +# # print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # # test load from local +# # local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# # print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # # test after clear local vs and jsonl +# # local_memory_manager.clear_local(re_init=True) +# # print(local_memory_manager.get_memory_pool("shanshi").messages) + +# # # test init_local +# # local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# # print(local_memory_manager.get_memory_pool("shanshi").messages) + + +# local_memory_manager = LocalMemoryManager(embed_config=embed_config, llm_config=llm_config, vb_config=vb_config, do_init=False) +# # embedding retrieval test +# text = "say hi to each other, i want some help" +# # retrieval_type=datetime => retrieval from datetime and jieba +# print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, datetime="2024-06-26 14:15:00", n=4, top_k=5, retrieval_type= "datetime")) +# # retrieval_type=eembedding => retrieval from embedding +# print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "embedding")) +# # retrieval_type=text => retrieval from jieba +# print(local_memory_manager.router_retrieval(chat_index="shanshi", text=text, top_k=5, retrieval_type= "text")) + +# # # recursive_summary test +# print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shanshi").messages, split_n=1, chat_index="shanshi")) + +# # print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("shuimo").messages, split_n=1, chat_index="shanshi")) + +# # print(local_memory_manager.recursive_summary(local_memory_manager.get_memory_pool("default").messages, split_n=1, chat_index="shanshi")) diff --git a/tests/connector/phase_test.py b/tests/connector/phase_test.py index 62d8d64..122f992 100644 --- a/tests/connector/phase_test.py +++ b/tests/connector/phase_test.py @@ -71,7 +71,7 @@ **Role:** Select the role from agent names. """ -from muagent.connector.configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT +from muagent.base_configs.prompts import REACT_CODE_PROMPT, REACT_TOOL_PROMPT tool_role = Role(role_type="assistant", role_name="tool_reacter", prompt=REACT_TOOL_PROMPT) tool_react_agent = ReactAgent( diff --git a/tests/db_handler/networkx_handler_test.py b/tests/db_handler/networkx_handler_test.py new file mode 100644 index 0000000..e0d4f10 --- /dev/null +++ b/tests/db_handler/networkx_handler_test.py @@ -0,0 +1,75 @@ +import time + +from tqdm import tqdm +from loguru import logger + +import sys, os + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +print(src_dir) +sys.path.append(src_dir) +from muagent.db_handler import NetworkxHandler +from muagent.schemas.memory import GNode, GRelation + + +node1 = GNode(**{"id": "node1", "attributes": {"name": "test" }}) +node2 = GNode(**{"id": "node2", "attributes": {"name": "test" }}) +node3 = GNode(**{"id": "node3", "attributes": {"name": "test3" }}) +node4 = GNode(**{"id": "node4", "attributes": {"name": "test3" }}) + +edge1 = GRelation(**{"id": "edge1", "left": node1, "right": node2, "attributes": {"name": "test" }}) +edge2 = GRelation(**{"id": "edge2", "left": node2, "right": node3, "attributes": {"name": "test2" }}) + + +nh = NetworkxHandler() +nh.add_node(node1) +nh.add_nodes([node2]) +nh.add_nodes([node3, node4]) + +nh.add_edge(edge1) +nh.add_edges([edge2]) + +# +print(nh.search_nodes_by_nodeid("node1")) +print(nh.search_edges_by_nodeid("node2")) +print(nh.search_edges_by_nodeids("node1", "node2")) + +# +print(nh.search_nodes_by_attr(**{"name": "test3"})) +print(nh.search_edges_by_attr(**{"name": "test2"})) + +# +nh.save_to_local() + +# +nh.clear() +print(nh.search_nodes_by_attr(**{"name": "test3"})) +print(nh.search_edges_by_attr(**{"name": "test2"})) + +# +nh.load_from_local() +print(nh.search_nodes_by_attr(**{"name": "test3"})) +print(nh.search_edges_by_attr(**{"name": "test2"})) + +# +nh.delete_node("node1") +print(nh.search_nodes_by_nodeid("node1")) + +# +nh.delete_nodes(["node1", "node2"]) +print(nh.search_nodes_by_nodeid("node1")) +print(nh.search_nodes_by_nodeid("node2")) + +# +nh.delete_edge("node1", "node2") +print(nh.search_edges_by_nodeids("node1", "node2")) + +# +nh.delete_edges_by_nodeid("node1") +print(nh.search_edges_by_nodeids("node1", "node2")) + +# +nh.delete_edges([("node1", "node2")]) +print(nh.search_edges_by_nodeids("node1", "node2")) \ No newline at end of file From bb96a7f4254d431c5ae55de59f2013c262b8fc38 Mon Sep 17 00:00:00 2001 From: shanshi Date: Fri, 12 Jul 2024 15:32:07 +0800 Subject: [PATCH 009/128] add aliyun sls_handler --- muagent/db_handler/__init__.py | 5 +- muagent/db_handler/sls_db_handler/__init__.py | 6 ++ .../sls_db_handler/aliyun_sls_hanlder.py | 99 +++++++++++++++++++ muagent/schemas/db/__init__.py | 4 +- muagent/schemas/db/db_config.py | 5 + muagent/schemas/ekg/__init__.py | 9 ++ muagent/schemas/ekg/ekg_graph.py | 98 ++++++++++++++++++ requirements.txt | 1 + tests/db_handler/slshandler_test.py | 83 ++++++++++++++++ 9 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 muagent/db_handler/sls_db_handler/__init__.py create mode 100644 muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py create mode 100644 muagent/schemas/ekg/__init__.py create mode 100644 muagent/schemas/ekg/ekg_graph.py create mode 100644 tests/db_handler/slshandler_test.py diff --git a/muagent/db_handler/__init__.py b/muagent/db_handler/__init__.py index 80a0546..0249809 100644 --- a/muagent/db_handler/__init__.py +++ b/muagent/db_handler/__init__.py @@ -8,8 +8,11 @@ from .graph_db_handler import NebulaHandler, NetworkxHandler from .vector_db_handler import LocalFaissHandler, TbaseHandler, ChromaHandler +from .sls_db_handler import AliYunSLSHandler __all__ = [ - "NebulaHandler", "ChromaHandler", "TbaseHandler", "LocalFaissHandler", "NetworkxHandler" + "NebulaHandler", "NetworkxHandler", + "ChromaHandler", "TbaseHandler", "LocalFaissHandler", + "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/sls_db_handler/__init__.py b/muagent/db_handler/sls_db_handler/__init__.py new file mode 100644 index 0000000..237191b --- /dev/null +++ b/muagent/db_handler/sls_db_handler/__init__.py @@ -0,0 +1,6 @@ +from .aliyun_sls_hanlder import AliYunSLSHandler + + +__all__ = [ + "AliYunSLSHandler" +] \ No newline at end of file diff --git a/muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py b/muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py new file mode 100644 index 0000000..51a9077 --- /dev/null +++ b/muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py @@ -0,0 +1,99 @@ +from typing import List, Dict, Tuple +from datetime import datetime, timedelta +import time + +from aliyun.log import * + +from muagent.schemas.db import SLSConfig + + + +class AliYunSLSHandler: + + def __init__( + self, + sls_config: SLSConfig = SLSConfig(sls_Type="aliyunSls") + ): + self.project = sls_config.extra_kwargs.get("project") + self.logstore = sls_config.extra_kwargs.get("logstore") + self.endpoint = sls_config.extra_kwargs.get("endpoint") + self.accessKeyId = sls_config.extra_kwargs.get("accessKeyId") + self.accessKey = sls_config.extra_kwargs.get("accessKey") + + self.client = LogClient(self.endpoint, self.accessKeyId, self.accessKey) + + def add_data(self, data: List[Tuple[str, str]]) -> str: + # 将数据写入到 sls 中 + # > data_list: list of list of tuple [ [ ('k1', 'v1'), ('k2', 'v2')] ] + log_item_list = [ + LogItem(timestamp=int(time.time()), contents=data) + ] + response = self.put_logs(self.logstore, log_item_list, ) + return response.log_print() + + def add_datas(self, data_list: List[List[Tuple[str, str]]]) -> str: + # 将数据写入到 sls 中 + # > data_list: list of list of tuple [ [ ('k1', 'v1'), ('k2', 'v2')] ] + log_item_list = [ + LogItem(timestamp=int(time.time()), contents=data) + for data in data_list + ] + response = self.put_logs(self.logstore, log_item_list, ) + return response.log_print() + + def create_project(self, project, project_description=''): + """创建项目""" + return self.client.create_project(project, project_description) + + def create_logstore(self, logstore=None, ttl=30, shard_count=2): + """创建日志库""" + logstore = logstore or self.logstore + return self.client.create_logstore(self.project, logstore, ttl, shard_count) + + def delete_logstore(self, logstore): + """删除日志库""" + return self.client.delete_logstore(self.project, logstore) + + def create_index(self, logstore, index_detail): + """创建日志索引""" + return self.client.create_index(self.project, logstore, index_detail) + + def list_logstores(self, project: str = None) -> List[str]: + """获取项目中所有日志库名称""" + request = ListLogstoresRequest(project or self.project) + r = self.client.list_logstores(request) + return r.logstores + + def list_shards(self, project: str = None, logstore: str = None) -> List[int]: + response = self.client.list_shards(project or self.project, logstore or self.logstore) + shards = response.get_shards_info() + return [i["shardID"] for i in shards] + + def put_logs(self, logstore, logitems: List[LogItem], topic=None, source=None): + """写入日志""" + req = PutLogsRequest(self.project, logstore, topic, source, logitems, compress=False) + return self.client.put_logs(req) + + def get_logs(self, logstore, query, from_time, to_time, topic=None): + """查询日志""" + req = GetLogsRequest(self.project, logstore, from_time, to_time, topic, query) + return self.client.get_logs(req) + + def pull_logs(self, logstore=None, count=10) -> List: + """拉取日志""" + logstore = logstore or self.logstore + + end_time = datetime.utcnow() + start_time = end_time - timedelta(minutes=3) + + shards = self.list_shards(logstore=logstore) + shard_id = shards[0] + + # 获得开始cursor，传入 Shard ID 和开始时间 + start_cursor = self.client.get_cursor(self.project, logstore, shard_id, start_time.timestamp()).get_cursor() + # 获得结束cursor，这使用"end"表示当前最新的cursor + end_cursor = self.client.get_cursor(self.project, logstore, shard_id, "end").get_cursor() + + r = self.client.pull_logs(self.project, logstore, shard_id, start_cursor, count, end_cursor) + return r.get_loggroup_list() + \ No newline at end of file diff --git a/muagent/schemas/db/__init__.py b/muagent/schemas/db/__init__.py index abdf7ea..f6267cd 100644 --- a/muagent/schemas/db/__init__.py +++ b/muagent/schemas/db/__init__.py @@ -1,6 +1,6 @@ -from .db_config import DBConfig, GBConfig, VBConfig, TBConfig +from .db_config import DBConfig, GBConfig, VBConfig, TBConfig, SLSConfig __all__ = [ - "DBConfig", "GBConfig", "VBConfig", "TBConfig" + "DBConfig", "GBConfig", "VBConfig", "TBConfig", "SLSConfig" ] \ No newline at end of file diff --git a/muagent/schemas/db/db_config.py b/muagent/schemas/db/db_config.py index 538afc2..86841f7 100644 --- a/muagent/schemas/db/db_config.py +++ b/muagent/schemas/db/db_config.py @@ -26,4 +26,9 @@ class TBConfig(BaseModel): class VBConfig(BaseModel): vb_type: str kb_root_path: str = None + extra_kwargs: Dict = {} + + +class SLSConfig(BaseModel): + sls_Type: str extra_kwargs: Dict = {} \ No newline at end of file diff --git a/muagent/schemas/ekg/__init__.py b/muagent/schemas/ekg/__init__.py new file mode 100644 index 0000000..21d134d --- /dev/null +++ b/muagent/schemas/ekg/__init__.py @@ -0,0 +1,9 @@ +from .ekg_graph import * + + +__all__ = [ + "EKGEdge", "EKGNode", + "EKGTaskNode", "EKGIntentNode", "EKGAnalysisNode", "EKGScheduleNode", "EKGPhenomenonNode", + "EKGEdgeTbase", "EKGEdgeTbase", "EKGGraphSls", + "SHAPE2TYPE" +] \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py new file mode 100644 index 0000000..c737618 --- /dev/null +++ b/muagent/schemas/ekg/ekg_graph.py @@ -0,0 +1,98 @@ +from pydantic import BaseModel +from typing import List, Dict +from enum import Enum + + +SHAPE2TYPE = { + 'rect': 'task', + 'parallelogram': 'analysis', + 'diamond': 'phenomenon', + 'circle': 'schedule', + 'process': 'task', + 'data': 'analysis', + 'decision': 'phenomenon', + 'start-end': 'schedule' +} + + +class NodeTypesEnum(Enum): + TASK = 'task' + ANALYSIS = 'analysis' + PHENOMENON = 'phenomenon' + SCHEDULE = 'schedule' + INTENT = 'intent' + TOOL = 'tool' + TEAM = 'team' + OWNER = 'owner' + + +class EKGNode(BaseModel): + # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} + id: str + # depend on user's difine + name: str + # depend on user's difine + description: str + + +class EKGEdge(EKGNode): + # ekg_edge:{graph_id}:{start_id}:{end_id} + id: str + # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} + star_id: str + # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} + end_id: str + + +class EKGIntentNode(EKGNode): + pass + + +class EKGScheduleNode(EKGNode): + pass + + +class EKGTaskNode(EKGNode): + tool: str + needCheck: bool + accessCriteira: str + + +class EKGAnalysisNode(EKGNode): + accessCriteira: str + + +class EKGPhenomenonNode(EKGNode): + pass + + +class EKGGraphSls(BaseModel): + # node_{NodeTypesEnum} + type: str + name: str + id: str + description: str + start_id: str = '' + end_id: str = '' + # ADD/DELETE + operation_type: str = '' + # {tool_id},{tool_id},{tool_id} + tool: str = '' + access_criteria: str = '' + + +class EKGNodeTbase(BaseModel): + node_id: str + node_type: str + # node_str = 'graph_id={graph_id}', use for searching by graph_id + node_str: str + node_vector: List + + +class EKGEdgeTbase(BaseModel): + edge_id: str + edge_type: str + edge_source: str + edge_target: str + # edge_str = 'graph_id={graph_id}', use for searching by graph_id + edge_str: str \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f8be989..8c23d47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ SQLAlchemy==2.0.19 docker redis==5.0.1 pydantic<=1.10.14 +aliyun-log-python-sdk==0.9.0 # pydantic # duckduckgo-search diff --git a/tests/db_handler/slshandler_test.py b/tests/db_handler/slshandler_test.py new file mode 100644 index 0000000..a70b62f --- /dev/null +++ b/tests/db_handler/slshandler_test.py @@ -0,0 +1,83 @@ +import sys, os +from loguru import logger + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + endpoint = os.environ["endpoint"] + accessKeyId = os.environ["accessKeyId"] + accessKey = os.environ["accessKey"] +except Exception as e: + # set your config + endpoint = "" + accessKeyId = "" + accessKey = "" + logger.error(f"{e}") + + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +print(src_dir) +sys.path.append(src_dir) +from muagent.db_handler.sls_db_handler import AliYunSLSHandler +from muagent.schemas.db import SLSConfig + + +# 使用示例 +project = "ls_devops_eg_node_edge" +logstore = "wyp311395_test" + + + +# 初始化 SLSHandler 实例 +sls_config = SLSConfig( + sls_Type="AliYunSLSHandler", + extra_kwargs={ + 'project': project, + 'logstore': logstore, + 'accessKeyId': accessKeyId, + 'accessKey': accessKey, + 'endpoint': endpoint + } +) +sls = AliYunSLSHandler(sls_config) + + +# list_logstores +print(sls.list_logstores()) + +# pull logs +print(sls.list_shards()) + +# add datas +t = [{ + 'type': 'node_intent', + 'name': 'wenjin', + 'id': 'wenjin_id', + 'description': 'wenjin_description', + 'operation_type': 'ADD', + 'path': '', + 'start_id': '', + 'end_id': '' +}] + +write_data_list = [ + [(key, v) for key, v in tt.items()] + for tt in t +] + +# sls.add_data(write_data_list[0]) +sls.add_datas(write_data_list) + +# get logs +print(sls.get_logs()) + +# pull logs +print(sls.pull_logs()) + + + From f006d3cc9518451e1715025d3d77af24260a9541 Mon Sep 17 00:00:00 2001 From: shanshi Date: Mon, 15 Jul 2024 15:08:31 +0800 Subject: [PATCH 010/128] update ekg service --- .../base_configs/prompts/simple_prompts.py | 12 +- .../memory/hierarchical_memory_manager.py | 2 +- .../db_handler/graph_db_handler/__init__.py | 4 +- .../aliyun_sls_hanlder.py | 18 +- .../graph_db_handler/networkx_handler.py | 38 +-- muagent/db_handler/sls_db_handler/__init__.py | 6 - .../db_handler/sql_db_hanlder/__initl__.py | 5 + .../sql_db_hanlder/sqlalchemy_handler.py | 236 ++++++++++++++++++ muagent/retrieval/base_service.py | 4 +- muagent/retrieval/faiss_db_service.py | 2 +- muagent/retrieval/service_factory.py | 2 +- muagent/schemas/db/db_config.py | 2 +- muagent/schemas/ekg/__init__.py | 14 +- muagent/schemas/ekg/ekg_create.py | 11 + muagent/schemas/ekg/ekg_graph.py | 32 ++- .../utils.py => schemas/kb/file_schema.py} | 0 .../memory/auto_extract_graph_schema.py | 12 +- muagent/schemas/readme.md | 18 ++ muagent/service/__init__.py | 0 muagent/service/cb_api.py | 2 +- .../ekg_construct/ekg_construct_base.py | 150 +++++++++++ muagent/service/kb_api.py | 4 +- muagent/service/migrate.py | 4 +- .../ui_file_service}/__init__.py | 0 .../ui_file_service}/code_base_cds.py | 0 .../ui_file_service}/document_base_cds.py | 0 .../ui_file_service}/document_file_cds.py | 2 +- requirements.txt | 2 +- setup.py | 3 +- 29 files changed, 520 insertions(+), 65 deletions(-) rename muagent/db_handler/{sls_db_handler => graph_db_handler}/aliyun_sls_hanlder.py (83%) delete mode 100644 muagent/db_handler/sls_db_handler/__init__.py create mode 100644 muagent/db_handler/sql_db_hanlder/__initl__.py create mode 100644 muagent/db_handler/sql_db_hanlder/sqlalchemy_handler.py create mode 100644 muagent/schemas/ekg/ekg_create.py rename muagent/{orm/utils.py => schemas/kb/file_schema.py} (100%) create mode 100644 muagent/schemas/readme.md delete mode 100644 muagent/service/__init__.py create mode 100644 muagent/service/ekg_construct/ekg_construct_base.py rename muagent/{orm/commands => service/ui_file_service}/__init__.py (100%) rename muagent/{orm/commands => service/ui_file_service}/code_base_cds.py (100%) rename muagent/{orm/commands => service/ui_file_service}/document_base_cds.py (100%) rename muagent/{orm/commands => service/ui_file_service}/document_file_cds.py (98%) diff --git a/muagent/base_configs/prompts/simple_prompts.py b/muagent/base_configs/prompts/simple_prompts.py index 2239148..d4e5785 100644 --- a/muagent/base_configs/prompts/simple_prompts.py +++ b/muagent/base_configs/prompts/simple_prompts.py @@ -55,7 +55,7 @@ ## 要求 1、根据 节点和边的数据结构 完成信息抽取 -2、edges中出现的left和right节点一定要在node中出现过 +2、edges中出现的start_id和end_id节点一定要在node中出现过 ## 输出结构 { @@ -65,9 +65,9 @@ {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ], 'edges': [ - {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + {'type': '{边类型}', 'start_id': '{实体名称}', 'end_id': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ..., - {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + {'type': '{边类型}', 'start_id': '{实体名称}', 'end_id': '{实体名称}', 'name': '{边名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ], } @@ -87,7 +87,7 @@ ## 要求 1、根据 节点和边的数据结构 完成信息抽取 -2、edges中出现的left和right节点一定要在node中出现过 +2、edges中出现的start_id和end_id节点一定要在node中出现过 ## 输出结构 { @@ -97,9 +97,9 @@ {'type': '{节点类型}', 'name': '{节点名称}', 'attributes': {'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ], 'edges': [ - {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + {'type': '{边类型}', 'start_id': '{实体名称}', 'end_id': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ..., - {'type': '{边类型}', 'left': '{实体名称}', 'right': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, + {'type': '{边类型}', 'start_id': '{实体名称}', 'end_id': '{实体名称}', 'attributes': {'name': '{边名称}', 'attribute1': '{属性值1}', ..., 'attributeN': '{属性值N}'}, ], } diff --git a/muagent/connector/memory/hierarchical_memory_manager.py b/muagent/connector/memory/hierarchical_memory_manager.py index 5be5710..03e0630 100644 --- a/muagent/connector/memory/hierarchical_memory_manager.py +++ b/muagent/connector/memory/hierarchical_memory_manager.py @@ -224,7 +224,7 @@ def _get_ner_by_llm( nodes = [GNode(**node) for node in nodes] edges = [ - GRelation(**{**edge, **{'left': nodes_dict.get(edge['left'], {}), 'right': nodes_dict.get(edge['right'], {})}}) + GRelation(**{**edge, **{'start_id': nodes_dict.get(edge['start_id'], {}), 'end_id': nodes_dict.get(edge['end_id'], {})}}) for edge in edges ] return nodes, edges diff --git a/muagent/db_handler/graph_db_handler/__init__.py b/muagent/db_handler/graph_db_handler/__init__.py index 1730697..f36951b 100644 --- a/muagent/db_handler/graph_db_handler/__init__.py +++ b/muagent/db_handler/graph_db_handler/__init__.py @@ -7,7 +7,9 @@ ''' from .nebula_handler import NebulaHandler from .networkx_handler import NetworkxHandler +from .aliyun_sls_hanlder import AliYunSLSHandler __all__ = [ - "NebulaHandler", "NetworkxHandler" + "NebulaHandler", "NetworkxHandler", + "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py similarity index 83% rename from muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py rename to muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py index 51a9077..fe5397c 100644 --- a/muagent/db_handler/sls_db_handler/aliyun_sls_hanlder.py +++ b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py @@ -12,7 +12,7 @@ class AliYunSLSHandler: def __init__( self, - sls_config: SLSConfig = SLSConfig(sls_Type="aliyunSls") + sls_config: SLSConfig = SLSConfig(sls_type="aliyunSls") ): self.project = sls_config.extra_kwargs.get("project") self.logstore = sls_config.extra_kwargs.get("logstore") @@ -22,6 +22,22 @@ def __init__( self.client = LogClient(self.endpoint, self.accessKeyId, self.accessKey) + # def add_node(self, node: GNode): + # insert_nodes = self.node_process([node]) + # self.graph.add_nodes_from(insert_nodes) + + # def add_nodes(self, nodes: List[GNode]): + # insert_nodes = self.node_process(nodes) + # self.graph.add_nodes_from(insert_nodes) + + # def add_edge(self, grelation: GRelation): + # insert_relations = self.relation_process([grelation]) + # self.graph.add_edges_from(insert_relations) + + # def add_edges(self, grelations: List[GRelation]): + # insert_relations = self.relation_process(grelations) + # self.graph.add_edges_from(insert_relations) + def add_data(self, data: List[Tuple[str, str]]) -> str: # 将数据写入到 sls 中 # > data_list: list of list of tuple [ [ ('k1', 'v1'), ('k2', 'v2')] ] diff --git a/muagent/db_handler/graph_db_handler/networkx_handler.py b/muagent/db_handler/graph_db_handler/networkx_handler.py index c48fc97..fb7a1b2 100644 --- a/muagent/db_handler/graph_db_handler/networkx_handler.py +++ b/muagent/db_handler/graph_db_handler/networkx_handler.py @@ -47,34 +47,42 @@ def search_edges_by_nodeid(self, nodeid: str) -> List[GRelation]: return [ GRelation( id=f"{nodeid}-{neighbor}", - left=self.search_nodes_by_nodeid(nodeid), - right=self.search_nodes_by_nodeid(neighbor), + start_id=nodeid, + end_id=neighbor, attributes=self.graph.get_edge_data(nodeid, neighbor) ) for neighbor, attr in self.graph.adj[nodeid].items() ] - def search_edges_by_nodeids(self, left: str, right: str) -> GRelation: - if self.missing_node(left) or self.missing_node(right): return None - if self.missing_edge(left, right): return None + def search_edges_by_nodeids(self, start_id: str, end_id: str) -> GRelation: + if self.missing_node(start_id) or self.missing_node(end_id): return None + if self.missing_edge(start_id, end_id): return None return GRelation( - id=f"{left}-{right}", - left=self.search_nodes_by_nodeid(left), - right=self.search_nodes_by_nodeid(right), - attributes=self.graph.get_edge_data(left, right) + id=f"{start_id}-{end_id}", + start_id=start_id, + end_id=end_id, + attributes=self.graph.get_edge_data(start_id, end_id) ) def search_nodes_by_attr(self, **attributes) -> List[GNode]: - return [GNode(id=node, attributes=attr) for node, attr in self.graph.nodes(data=True) if all(attr.get(k) == v for k, v in attributes.items())] + return [ + GNode(id=node, attributes=attr) + for node, attr in self.graph.nodes(data=True) + if all(attr.get(k) == v for k, v in attributes.items()) + ] def search_edges_by_attr(self, **attributes) -> List[GRelation]: - return [GRelation( + return [ + GRelation( id=f"{left}-{right}", - left=self.search_nodes_by_nodeid(left), - right=self.search_nodes_by_nodeid(right), + start_id=left, + end_id=right, attributes=attr - ) for left, right, attr in self.graph.edges(data=True) if all(attr.get(k) == v for k, v in attributes.items())] + ) + for left, right, attr in self.graph.edges(data=True) + if all(attr.get(k) == v for k, v in attributes.items()) + ] def save(self, kb_name: str): self.kb_name = kb_name or self.kb_name @@ -128,7 +136,7 @@ def relation_process(self, relations: List[GRelation]) -> List[Tuple]: relation_list = [] for relation in relations: edge_attrs = relation.attributes - relation_list.append((relation.left.id, relation.right.id, edge_attrs)) + relation_list.append((relation.start_id, relation.end_id, edge_attrs)) return relation_list def missing_edge(self, left: str, right: str) -> bool: diff --git a/muagent/db_handler/sls_db_handler/__init__.py b/muagent/db_handler/sls_db_handler/__init__.py deleted file mode 100644 index 237191b..0000000 --- a/muagent/db_handler/sls_db_handler/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .aliyun_sls_hanlder import AliYunSLSHandler - - -__all__ = [ - "AliYunSLSHandler" -] \ No newline at end of file diff --git a/muagent/db_handler/sql_db_hanlder/__initl__.py b/muagent/db_handler/sql_db_hanlder/__initl__.py new file mode 100644 index 0000000..bcd999f --- /dev/null +++ b/muagent/db_handler/sql_db_hanlder/__initl__.py @@ -0,0 +1,5 @@ +from .sqlalchemy_handler import SqlalchemyHandler + +__all__ = [ + "SqlalchemyHandler" +] \ No newline at end of file diff --git a/muagent/db_handler/sql_db_hanlder/sqlalchemy_handler.py b/muagent/db_handler/sql_db_hanlder/sqlalchemy_handler.py new file mode 100644 index 0000000..8781052 --- /dev/null +++ b/muagent/db_handler/sql_db_hanlder/sqlalchemy_handler.py @@ -0,0 +1,236 @@ +from contextlib import contextmanager +from sqlalchemy.engine import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + + +from muagent.schemas.kb.base_schema import KnowledgeBaseSchema +from muagent.base_configs.env_config import SQLALCHEMY_DATABASE_URI + +# connect the sql databse +_engine = create_engine(SQLALCHEMY_DATABASE_URI) +Base = declarative_base() + +session_factory = sessionmaker(bind=_engine) + + + +# def init_session(): +# session = session_factory() + +# try: +# yield session +# finally: +# try: +# session.commit() +# except Exception as e: +# session.rollback() +# raise e +# finally: +# session.close() + + + +# def with_session(func): +# def wrapper(*args, **kwargs): +# session = session_factory() +# try: +# return func(session, *args, **kwargs) +# finally: +# try: +# session.commit() +# except Exception as e: +# session.rollback() +# raise e +# finally: +# session.close() +# return wrapper + + +# @contextmanager +# def session_scope(): +# """上下文管理器用于自动获取 Session, 避免错误""" +# session = session_factory(autoflush=True) +# try: +# session.commit() +# except Exception as e: +# session.rollback() +# raise e +# finally: +# session.close() + + +def db_session_commit_rollback_close(func): + def wrapper(self, *args, **kwargs): + try: + result = func(self, *args, **kwargs) # 执行函数逻辑 + self.session.commit() # 提交事务 + return result + except Exception as e: + self.session.rollback() # 事务回滚 + print(f"An error occurred: {e}") # 这里可以替换为日志记录或者返回错误信息 + raise + finally: + self.session.close() # 关闭会话 + self.session = session_factory() # 重新创建会话以便复用handler + return wrapper + + +from sqlalchemy import create_engine, Column, Integer, String +# 定义模型 +class User(Base): + __tablename__ = 'users' + + id = Column(Integer, primary_key=True) + name = Column(String) + fullname = Column(String) + + def __repr__(self): + return f"" + +t = User() + + + +class SqlalchemyHandler(): + + def __init__(self, ): + # 创建 Session 实例 + self.session = session_factory() + + @db_session_commit_rollback_close + def add(self, schema, kwargs): + # 创建新用户对象 + data = schema(**kwargs) + # 将对象添加到会话并提交保存到数据库 + self.session.add(data) + return True + + @db_session_commit_rollback_close + def query(self, schema, kwargs: dict, query_type="first"): + if query_type == "first": + # 查询单个对象 + data = self.session.query(schema).filter_by(**kwargs).first() + datas = [data] + else: + # 查询所有对象 + datas = self.session.query(schema).all() + + for data in datas: + print(data) + return datas + + @db_session_commit_rollback_close + def update(self, schema, kwargs: dict): + # 查询后更新对象 + data = self.session.query(schema).filter_by(**kwargs).first() + if data: + for key, value in kwargs.items(): + # 只有在字段内时，才更新该属性 + if hasattr(data, key): + setattr(data, key, value) + return True + + @db_session_commit_rollback_close + def delete(self, schema, kwargs: dict, filter_keys: list = [],): + filter_kwargs = {k:v for k,v in kwargs.items() if k in filter_keys} + # 查询后删除对象 + data = self.session.query(schema).filter_by(**filter_kwargs).first() + if data: + self.session.delete(data) + return True + return False + + @db_session_commit_rollback_close + def add_kb_to_db(self, schema, kwargs: dict, filter_keys: list = [],): + filter_kwargs = {k:v for k,v in kwargs.items() if k in filter_keys} + # 创建知识库实例 + data = self.session.query(schema).filter_by(**filter_kwargs).first() + if not data: + data = schema(**kwargs) + self.session.add(data) + else: # update kb with new vs_type and embed_model + for key, value in kwargs.items(): + # 只有在字段内时，才更新该属性 + if hasattr(data, key): + setattr(data, key, value) + return True + + @with_session + def add_kb_to_db(session, kb_name, vs_type, embed_model): + # 创建知识库实例 + kb = session.query(KnowledgeBaseSchema).filter_by(kb_name=kb_name).first() + if not kb: + kb = KnowledgeBaseSchema(kb_name=kb_name, vs_type=vs_type, embed_model=embed_model) + session.add(kb) + else: # update kb with new vs_type and embed_model + kb.vs_type = vs_type + kb.embed_model = embed_model + return True + + + @with_session + def list_kbs_from_db(session, min_file_count: int = -1): + kbs = session.query(KnowledgeBaseSchema.kb_name).filter(KnowledgeBaseSchema.file_count > min_file_count).all() + kbs = [kb[0] for kb in kbs] + return kbs + + + @with_session + def kb_exists(session, kb_name): + kb = session.query(KnowledgeBaseSchema).filter_by(kb_name=kb_name).first() + status = True if kb else False + return status + + + @with_session + def load_kb_from_db(session, kb_name): + kb = session.query(KnowledgeBaseSchema).filter_by(kb_name=kb_name).first() + if kb: + kb_name, vs_type, embed_model = kb.kb_name, kb.vs_type, kb.embed_model + else: + kb_name, vs_type, embed_model = None, None, None + return kb_name, vs_type, embed_model + + + @with_session + def delete_kb_from_db(session, kb_name): + kb = session.query(KnowledgeBaseSchema).filter_by(kb_name=kb_name).first() + if kb: + session.delete(kb) + return True + + + @with_session + def get_kb_detail(session, kb_name: str) -> dict: + kb: KnowledgeBaseSchema = session.query(KnowledgeBaseSchema).filter_by(kb_name=kb_name).first() + if kb: + return { + "kb_name": kb.kb_name, + "vs_type": kb.vs_type, + "embed_model": kb.embed_model, + "file_count": kb.file_count, + "create_time": kb.create_time, + } + else: + return {} + + @classmethod + def create_tables(cls): + Base.metadata.create_all(bind=_engine) + + @classmethod + def reset_tables(cls, ): + Base.metadata.drop_all(bind=_engine) + SqlalchemyHandler.create_tables() + + @classmethod + def table_init(cls, ): + if (not SqlalchemyHandler.check_tables_exist("knowledge_base")) or (not SqlalchemyHandler.check_tables_exist ("knowledge_file")) or \ + (not SqlalchemyHandler.check_tables_exist ("code_base")): + SqlalchemyHandler.create_tables() + + @classmethod + def check_tables_exist(cls, table_name) -> bool: + table_exist = _engine.dialect.has_table(_engine.connect(), table_name, schema=None) + return table_exist diff --git a/muagent/retrieval/base_service.py b/muagent/retrieval/base_service.py index b180918..2fd75e5 100644 --- a/muagent/retrieval/base_service.py +++ b/muagent/retrieval/base_service.py @@ -13,9 +13,9 @@ from muagent.base_configs.env_config import ( VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD, kbs_config ) -from muagent.orm.commands import * +from muagent.service.ui_file_service import * from muagent.utils.path_utils import * -from muagent.orm.utils import DocumentFile +from muagent.schemas.kb.file_schema import DocumentFile from muagent.retrieval.utils import load_embeddings, load_embeddings_from_path from muagent.retrieval.text_splitter import LCTextSplitter from muagent.llm_models.llm_config import EmbedConfig diff --git a/muagent/retrieval/faiss_db_service.py b/muagent/retrieval/faiss_db_service.py index dac966c..ef191c5 100644 --- a/muagent/retrieval/faiss_db_service.py +++ b/muagent/retrieval/faiss_db_service.py @@ -16,7 +16,7 @@ from .base_service import KBService, SupportedVSType from muagent.utils.path_utils import * -from muagent.orm.utils import DocumentFile +from muagent.schemas.kb.file_schema import DocumentFile from muagent.utils.server_utils import torch_gc from muagent.retrieval.utils import load_embeddings, load_embeddings_from_path from muagent.retrieval.faiss_m import FAISS diff --git a/muagent/retrieval/service_factory.py b/muagent/retrieval/service_factory.py index cacdfc7..8007239 100644 --- a/muagent/retrieval/service_factory.py +++ b/muagent/retrieval/service_factory.py @@ -7,7 +7,7 @@ from .faiss_db_service import FaissKBService from .base_service import KBService, SupportedVSType -from muagent.orm.commands import * +from muagent.service.ui_file_service import * from muagent.utils.path_utils import * from muagent.llm_models.llm_config import EmbedConfig diff --git a/muagent/schemas/db/db_config.py b/muagent/schemas/db/db_config.py index 86841f7..4c9821a 100644 --- a/muagent/schemas/db/db_config.py +++ b/muagent/schemas/db/db_config.py @@ -30,5 +30,5 @@ class VBConfig(BaseModel): class SLSConfig(BaseModel): - sls_Type: str + sls_type: str extra_kwargs: Dict = {} \ No newline at end of file diff --git a/muagent/schemas/ekg/__init__.py b/muagent/schemas/ekg/__init__.py index 21d134d..5016a16 100644 --- a/muagent/schemas/ekg/__init__.py +++ b/muagent/schemas/ekg/__init__.py @@ -1,9 +1,13 @@ from .ekg_graph import * - +from .ekg_create import * __all__ = [ - "EKGEdge", "EKGNode", - "EKGTaskNode", "EKGIntentNode", "EKGAnalysisNode", "EKGScheduleNode", "EKGPhenomenonNode", - "EKGEdgeTbase", "EKGEdgeTbase", "EKGGraphSls", - "SHAPE2TYPE" + "EKGEdgeSchema", "EKGNodeSchema", + "EKGTaskNodeSchema", "EKGIntentNodeSchema", "EKGAnalysisNodeSchema", "EKGScheduleNodeSchema", "EKGPhenomenonNodeSchema", + "EKGEdgeTbaseSchema", "EKGEdgeTbaseSchema", "EKGTbaseData", + "EKGGraphSlsSchema", "EKGSlsData", + "SHAPE2TYPE", + + + "EKGIntentResp", ] \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_create.py b/muagent/schemas/ekg/ekg_create.py new file mode 100644 index 0000000..70491b8 --- /dev/null +++ b/muagent/schemas/ekg/ekg_create.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel +from typing import List, Dict +from enum import Enum + + + +class EKGIntentResp(BaseModel): + intent_leaf_nodes: List[str] + intent_nodes: List[str] + + \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index c737618..9c8c6b0 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -26,7 +26,7 @@ class NodeTypesEnum(Enum): OWNER = 'owner' -class EKGNode(BaseModel): +class EKGNodeSchema(BaseModel): # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} id: str # depend on user's difine @@ -35,7 +35,7 @@ class EKGNode(BaseModel): description: str -class EKGEdge(EKGNode): +class EKGEdgeSchema(EKGNodeSchema): # ekg_edge:{graph_id}:{start_id}:{end_id} id: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} @@ -44,29 +44,29 @@ class EKGEdge(EKGNode): end_id: str -class EKGIntentNode(EKGNode): +class EKGIntentNodeSchema(EKGNodeSchema): pass -class EKGScheduleNode(EKGNode): +class EKGScheduleNodeSchema(EKGNodeSchema): pass -class EKGTaskNode(EKGNode): +class EKGTaskNodSchema(EKGNodeSchema): tool: str needCheck: bool accessCriteira: str -class EKGAnalysisNode(EKGNode): +class EKGAnalysisNodeSchema(EKGNodeSchema): accessCriteira: str -class EKGPhenomenonNode(EKGNode): +class EKGPhenomenonNodeSchema(EKGNodeSchema): pass -class EKGGraphSls(BaseModel): +class EKGGraphSlsSchema(BaseModel): # node_{NodeTypesEnum} type: str name: str @@ -81,7 +81,7 @@ class EKGGraphSls(BaseModel): access_criteria: str = '' -class EKGNodeTbase(BaseModel): +class EKGNodeTbaseSchema(BaseModel): node_id: str node_type: str # node_str = 'graph_id={graph_id}', use for searching by graph_id @@ -89,10 +89,20 @@ class EKGNodeTbase(BaseModel): node_vector: List -class EKGEdgeTbase(BaseModel): +class EKGEdgeTbaseSchema(BaseModel): edge_id: str edge_type: str edge_source: str edge_target: str # edge_str = 'graph_id={graph_id}', use for searching by graph_id - edge_str: str \ No newline at end of file + edge_str: str + + +class EKGTbaseData(BaseModel): + nodes: list[EKGNodeTbaseSchema] + edges: list[EKGEdgeTbaseSchema] + + +class EKGSlsData(BaseModel): + nodes: list[EKGGraphSlsSchema] + edges: list[EKGGraphSlsSchema] \ No newline at end of file diff --git a/muagent/orm/utils.py b/muagent/schemas/kb/file_schema.py similarity index 100% rename from muagent/orm/utils.py rename to muagent/schemas/kb/file_schema.py diff --git a/muagent/schemas/memory/auto_extract_graph_schema.py b/muagent/schemas/memory/auto_extract_graph_schema.py index 57e6e0a..5fd9eb5 100644 --- a/muagent/schemas/memory/auto_extract_graph_schema.py +++ b/muagent/schemas/memory/auto_extract_graph_schema.py @@ -10,26 +10,26 @@ class Attribute(BaseModel): class GNodeAbs(BaseModel): + # node type for extract type: str attributes: List[Attribute] class GRelationAbs(BaseModel): + # relation type for extract type: str attributes: List[Attribute] class GNode(BaseModel): - id: str = None - type: str + id: str attributes: Dict[str, str] class GRelation(BaseModel): - id: str = None - type: str - left: GNode - right: GNode + id: str + start_id: str + end_id: str attributes: Dict[str, str] diff --git a/muagent/schemas/readme.md b/muagent/schemas/readme.md new file mode 100644 index 0000000..8b27998 --- /dev/null +++ b/muagent/schemas/readme.md @@ -0,0 +1,18 @@ + +## xxSchemas +xxSchemas within DB, such as mysql\sls\chroma\faiss\tbase\nebula + +## Config +xxConfig as the class initial config + +## xxResp +xxResp as the http output + +## xxRequest +xxRequest as the http input + +## xx +xx as the function/method output + +## xxParam +xxParam as the function/method input diff --git a/muagent/service/__init__.py b/muagent/service/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/muagent/service/cb_api.py b/muagent/service/cb_api.py index 4d9f791..6bee44d 100644 --- a/muagent/service/cb_api.py +++ b/muagent/service/cb_api.py @@ -13,7 +13,7 @@ from fastapi import Body from muagent.utils.server_utils import BaseResponse, ListResponse from muagent.utils.path_utils import * -from muagent.orm.commands import * +from muagent.service.ui_file_service import * from muagent.db_handler.graph_db_handler.nebula_handler import NebulaHandler from muagent.db_handler.vector_db_handler.chroma_handler import ChromaHandler from muagent.base_configs.env_config import ( diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py new file mode 100644 index 0000000..00b7047 --- /dev/null +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -0,0 +1,150 @@ +from loguru import logger + + +from muagent.schemas.ekg import * +from muagent.schemas.db import * +from muagent.db_handler import * +from muagent.orm import table_init + +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig + +from muagent.base_configs.env_config import KB_ROOT_PATH + + + +class EKGConstructService: + + def __init__( + self, + embed_config: EmbedConfig, + llm_config: LLMConfig, + db_config: DBConfig = None, + vb_config: VBConfig = None, + gb_config: GBConfig = None, + tb_config: TBConfig = None, + sls_config: SLSConfig = None, + do_init: bool = False, + kb_root_path: str = KB_ROOT_PATH, + ): + + self.db_config = db_config + self.vb_config = vb_config + self.gb_config = gb_config + self.tb_config = tb_config + self.sls_config = sls_config + self.do_init = do_init + self.kb_root_path = kb_root_path + self.embed_config: EmbedConfig = embed_config + self.llm_config: LLMConfig = llm_config + + def init_handler(self, ): + """Initializes Database VectorBase GraphDB TbaseDB""" + self.init_vb() + # self.init_db() + # self.init_tb() + # self.init_gb() + + def reinit_handler(self, do_init: bool=False): + self.init_vb() + # self.init_db() + # self.init_tb() + # self.init_gb() + + def init_tb(self, do_init: bool=None): + tb_dict = {"TbaseHandler": TbaseHandler} + tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) + tbase_args = { + "host": self.tb_config.host, + "port": self.tb_config.port, + "username": self.tb_config.username, + "password": self.tb_config.password, + } + self.vb = tb_class(tbase_args, self.tb_config.index_name) + + def init_gb(self, do_init: bool=None): + pass + gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler} + gb_class = gb_dict.get(self.gb_config.gb_type, NetworkxHandler) + self.gb = gb_class(self.db_config) + + def init_db(self, do_init: bool=None): + pass + db_dict = {"LocalFaissHandler": LocalFaissHandler} + db_class = db_dict.get(self.db_config.db_type) + self.db = db_class(self.db_config) + + def init_vb(self, do_init: bool=None): + table_init() + vb_dict = {"LocalFaissHandler": LocalFaissHandler} + vb_class = vb_dict.get(self.vb_config.vb_type, LocalFaissHandler) + self.vb: LocalFaissHandler = vb_class(self.embed_config, vb_config=self.vb_config) + + def init_sls(self, do_init: bool=None): + sls_dict = {"AliYunSLSHandler": AliYunSLSHandler} + sls_class = sls_dict.get(self.sls_config.sls_type, AliYunSLSHandler) + self.vb: AliYunSLSHandler = sls_class(self.embed_config, vb_config=self.vb_config) + + def create_ekg(self, ): + ekg_router = {} + + def text2graph(self, ): + # graph_id, alarms, steps + + # alarms, steps ==|llm|==> node_dict, edge_list, abnormal_dict + + # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + + # dsl2graph => write2kg + + pass + + def dsl2graph(self, ): + # dsl, write2kg, intent_node, graph_id + + # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + + # dsl2graph => write2kg + pass + + def yuque2graph(self, **kwargs): + # yuque_url, write2kg, intent_node + + # get_graph(yuque_url) + # graph_id from md(yuque_content) + + # yuque_dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + + # dsl2graph => write2kg + pass + + def write2kg(self, graph_id: str, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData): + # dsl2graph => write2kg + ## delete tbase/graph by graph_id + ### diff the tabse within newest by graph_id + ### diff the graph within newest by graph_id + ## update tbase/graph by graph_id + pass + + def get_intent(self, content: dict, ) -> EKGIntentResp: + '''according content search intent''' + pass + + def get_intents(self, contents: list[dict], ) -> EKGIntentResp: + '''according contents search intents''' + pass + + def get_node_edge_dict(self, cotents: list[dict], ) -> EKGSlsData: + '''according contents generate ekg's raw datas''' + # code2graph + pass + + # def transform2sls(self, ekg_sls_data: EKGSlsData) -> list[EKGGraphSlsSchema]: + # pass + + def transform2tbase(self, ekg_sls_data: EKGSlsData) -> EKGTbaseData: + pass + + def transform2dsl(self, ekg_sls_data: EKGSlsData): + '''define your personal dsl format and code''' + pass + diff --git a/muagent/service/kb_api.py b/muagent/service/kb_api.py index 34d49a2..42e9c0e 100644 --- a/muagent/service/kb_api.py +++ b/muagent/service/kb_api.py @@ -13,8 +13,8 @@ from muagent.retrieval.service_factory import KBServiceFactory from muagent.utils.server_utils import BaseResponse, ListResponse from muagent.utils.path_utils import * -from muagent.orm.commands import * -from muagent.orm.utils import DocumentFile +from muagent.service.ui_file_service import * +from muagent.schemas.kb.file_schema import DocumentFile from muagent.base_configs.env_config import KB_ROOT_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.utils.server_utils import run_async diff --git a/muagent/service/migrate.py b/muagent/service/migrate.py index ed561a3..b5f450e 100644 --- a/muagent/service/migrate.py +++ b/muagent/service/migrate.py @@ -3,8 +3,8 @@ # from configs.model_config import EMBEDDING_MODEL, DEFAULT_VS_TYPE, KB_ROOT_PATH -from muagent.orm.utils import DocumentFile -from muagent.orm.commands import add_doc_to_db +from muagent.schemas.kb.file_schema import DocumentFile +from muagent.service.ui_file_service import add_doc_to_db from muagent.utils.path_utils import * from muagent.retrieval.service_factory import KBServiceFactory diff --git a/muagent/orm/commands/__init__.py b/muagent/service/ui_file_service/__init__.py similarity index 100% rename from muagent/orm/commands/__init__.py rename to muagent/service/ui_file_service/__init__.py diff --git a/muagent/orm/commands/code_base_cds.py b/muagent/service/ui_file_service/code_base_cds.py similarity index 100% rename from muagent/orm/commands/code_base_cds.py rename to muagent/service/ui_file_service/code_base_cds.py diff --git a/muagent/orm/commands/document_base_cds.py b/muagent/service/ui_file_service/document_base_cds.py similarity index 100% rename from muagent/orm/commands/document_base_cds.py rename to muagent/service/ui_file_service/document_base_cds.py diff --git a/muagent/orm/commands/document_file_cds.py b/muagent/service/ui_file_service/document_file_cds.py similarity index 98% rename from muagent/orm/commands/document_file_cds.py rename to muagent/service/ui_file_service/document_file_cds.py index 6816009..9cdd27e 100644 --- a/muagent/orm/commands/document_file_cds.py +++ b/muagent/service/ui_file_service/document_file_cds.py @@ -1,6 +1,6 @@ from muagent.orm.db import with_session from muagent.schemas.kb.base_schema import KnowledgeFileSchema, KnowledgeBaseSchema -from muagent.orm.utils import DocumentFile +from muagent.schemas.kb.file_schema import DocumentFile diff --git a/requirements.txt b/requirements.txt index 8c23d47..87cc536 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ SQLAlchemy==2.0.19 docker redis==5.0.1 pydantic<=1.10.14 -aliyun-log-python-sdk==0.9.0 +# aliyun-log-python-sdk==0.9.0 # pydantic # duckduckgo-search diff --git a/setup.py b/setup.py index e6c8321..1ef0b2c 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "nebula3-python==3.1.0", "SQLAlchemy==2.0.19", "redis==5.0.1", - "pydantic<=1.10.14" + "pydantic<=1.10.14", + "aliyun-log-python-sdk==0.9.0" ], ) \ No newline at end of file From 10c9baf447ab04588e4c5789c2a812116a5bb01f Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 24 Jul 2024 11:29:48 +0800 Subject: [PATCH 011/128] update geabase_handler --- muagent/db_handler/__init__.py | 5 +- .../db_handler/graph_db_handler/__init__.py | 4 +- .../graph_db_handler/aliyun_sls_hanlder.py | 69 +++- .../graph_db_handler/base_gb_handler.py | 77 ++++ .../graph_db_handler/geabase_handler.py | 357 ++++++++++++++++++ .../graph_db_handler/networkx_handler.py | 7 +- muagent/db_handler/utils.py | 13 + muagent/schemas/ekg/ekg_graph.py | 1 + muagent/schemas/memory/__init__.py | 2 +- .../memory/auto_extract_graph_schema.py | 19 +- muagent/service/__init__.py | 0 muagent/utils/common_utils.py | 11 + requirements.txt | 2 +- tests/db_handler/geabase_hanlder_test.py | 134 +++++++ tests/db_handler/networkx_handler_test.py | 19 +- tests/db_handler/slshandler_test.py | 2 +- 16 files changed, 686 insertions(+), 36 deletions(-) create mode 100644 muagent/db_handler/graph_db_handler/base_gb_handler.py create mode 100644 muagent/db_handler/graph_db_handler/geabase_handler.py create mode 100644 muagent/db_handler/utils.py create mode 100644 muagent/service/__init__.py create mode 100644 tests/db_handler/geabase_hanlder_test.py diff --git a/muagent/db_handler/__init__.py b/muagent/db_handler/__init__.py index 0249809..73c6222 100644 --- a/muagent/db_handler/__init__.py +++ b/muagent/db_handler/__init__.py @@ -6,13 +6,12 @@ @desc: ''' -from .graph_db_handler import NebulaHandler, NetworkxHandler +from .graph_db_handler import NebulaHandler, NetworkxHandler, AliYunSLSHandler, GeaBaseHandler from .vector_db_handler import LocalFaissHandler, TbaseHandler, ChromaHandler -from .sls_db_handler import AliYunSLSHandler __all__ = [ - "NebulaHandler", "NetworkxHandler", + "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", "ChromaHandler", "TbaseHandler", "LocalFaissHandler", "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/__init__.py b/muagent/db_handler/graph_db_handler/__init__.py index f36951b..54fb0bb 100644 --- a/muagent/db_handler/graph_db_handler/__init__.py +++ b/muagent/db_handler/graph_db_handler/__init__.py @@ -8,8 +8,10 @@ from .nebula_handler import NebulaHandler from .networkx_handler import NetworkxHandler from .aliyun_sls_hanlder import AliYunSLSHandler +from .geabase_handler import GeaBaseHandler + __all__ = [ - "NebulaHandler", "NetworkxHandler", + "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py index fe5397c..680025d 100644 --- a/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py +++ b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py @@ -3,13 +3,13 @@ import time from aliyun.log import * - +from muagent.schemas.memory import * from muagent.schemas.db import SLSConfig class AliYunSLSHandler: - + '''can ADD/DELETE within graph''' def __init__( self, sls_config: SLSConfig = SLSConfig(sls_type="aliyunSls") @@ -22,22 +22,6 @@ def __init__( self.client = LogClient(self.endpoint, self.accessKeyId, self.accessKey) - # def add_node(self, node: GNode): - # insert_nodes = self.node_process([node]) - # self.graph.add_nodes_from(insert_nodes) - - # def add_nodes(self, nodes: List[GNode]): - # insert_nodes = self.node_process(nodes) - # self.graph.add_nodes_from(insert_nodes) - - # def add_edge(self, grelation: GRelation): - # insert_relations = self.relation_process([grelation]) - # self.graph.add_edges_from(insert_relations) - - # def add_edges(self, grelations: List[GRelation]): - # insert_relations = self.relation_process(grelations) - # self.graph.add_edges_from(insert_relations) - def add_data(self, data: List[Tuple[str, str]]) -> str: # 将数据写入到 sls 中 # > data_list: list of list of tuple [ [ ('k1', 'v1'), ('k2', 'v2')] ] @@ -112,4 +96,51 @@ def pull_logs(self, logstore=None, count=10) -> List: r = self.client.pull_logs(self.project, logstore, shard_id, start_cursor, count, end_cursor) return r.get_loggroup_list() - \ No newline at end of file + + def add_node(self, node: GNode): + insert_nodes = self.node_process([node], "ADD") + self.add_datas(insert_nodes) + + def add_nodes(self, nodes: List[GNode]): + insert_nodes = self.node_process(nodes, "ADD") + self.add_datas(insert_nodes) + + def add_edge(self, edge: GRelation): + insert_relations = self.relation_process([edge], "ADD") + self.add_datas(insert_relations) + + def add_edges(self, edges: List[GRelation]): + insert_relations = self.relation_process(edges, "ADD") + self.add_datas(insert_relations) + + def delete_node(self, node: GNode): + insert_nodes = self.node_process([node], "DELETE") + self.add_datas(insert_nodes) + + def delete_nodes(self, nodes: List[GNode]): + insert_nodes = self.node_process(nodes, "DELETE") + self.add_datas(insert_nodes) + + def delete_edge(self, edge: GRelation): + insert_relations = self.relation_process([edge], "DELETE") + self.add_datas(insert_relations) + + def delete_edges(self, edges: List[Tuple]): + insert_relations = self.relation_process(edges, "DELETE") + self.add_datas(insert_relations) + + def node_process(self, nodes: List[GNode], crud_type) -> List[List[Tuple]]: + node_list = [ + [('id', node.id)] + \ + [(k, v) if k!="operation_type" else (k, crud_type) for k,v in node.attributes.items()] + for node in nodes + ] + return node_list + + def relation_process(self, relations: List[GRelation], crud_type) -> List[List[Tuple]]: + relation_list = [ + [('start_id', relation.start_id), ('end_id', relation.start_id)] +\ + [(k, v) if k!="operation_type" else (k, crud_type) for k,v in relation.attributes.items()] + for node in nodes + ] + return relation_list \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/base_gb_handler.py b/muagent/db_handler/graph_db_handler/base_gb_handler.py new file mode 100644 index 0000000..e04760c --- /dev/null +++ b/muagent/db_handler/graph_db_handler/base_gb_handler.py @@ -0,0 +1,77 @@ +from typing import List, Dict +import uuid +from loguru import logger +import json + +from muagent.schemas.memory import * + + + + +class GBHandler: + + def __init__(self) -> None: + pass + + def add_node(self, node: GNode): + return self.add_nodes([node]) + + def add_nodes(self, nodes: List[GNode]): + pass + + def add_edge(self, edge: GEdge): + return self.add_edges([edge]) + + def add_edges(self, edges: List[GEdge]): + pass + + def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None): + pass + + def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None): + pass + + def delete_node(self, attributes: dict, node_type: str = None, ID: int = None): + pass + + def delete_nodes(self, attributes: dict, node_type: str = None, ID: int = None): + pass + + def delete_edge(self, src_id, dst_id, edge_type: str = None): + pass + + def delete_edges(self, src_id, dst_id, edge_type: str = None): + pass + + def search_node_by_nodeid(self, nodeid: str, node_type: str = None) -> GNode: + pass + + def search_edges_by_nodeid(self, nodeid: str, node_type: str = None) -> List[GEdge]: + pass + + def search_edge_by_nodeids(self, start_id: str, end_id: str, edge_type: str = None) -> GEdge: + pass + + def search_nodes_by_attr(self, attributes: dict) -> List[GNode]: + pass + + def search_edges_by_attr(self, attributes: dict, edge_type: str = None) -> List[GEdge]: + pass + + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + pass + + def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + pass + + def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> Dict: + pass + + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + pass + + def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + pass + + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + pass diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py new file mode 100644 index 0000000..cdba745 --- /dev/null +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -0,0 +1,357 @@ +from typing import List, Dict +import uuid +from loguru import logger +import json + +from gdbc2.geabase_client import GeaBaseClient, Node, Edge, MutateBatchOperation, GeaBaseUtil +from gdbc2.geabase_env import GeaBaseEnv + +from muagent.db_handler.utils import deduplicate_dict +from muagent.schemas.db import GBConfig +from muagent.schemas.memory import * +from muagent.utils.common_utils import double_hashing + + +class GeaBaseHandler: + def __init__( + self, + gb_config: GBConfig = None + ): + self.metaserver_address = gb_config.extra_kwargs.get("metaserver_address") + self.project = gb_config.extra_kwargs.get("project") + self.city = gb_config.extra_kwargs.get("city") + self.lib_path = gb_config.extra_kwargs.get("lib_path") + + GeaBaseEnv.init(self.lib_path) + self.geabase_client = GeaBaseClient( + self.metaserver_address, self.project,self.city + ) + + # option 指定 + self.option = GeaBaseEnv.QueryRequestOption.newBuilder().gqlType(GeaBaseEnv.QueryProtocol.GQLType.GQL_ISO).build() + + def add_node(self, node: GNode) -> dict: + return self.add_nodes([node]) + + def add_nodes(self, nodes: List[GNode]) -> dict: + node_str_list = [] + for node in nodes: + node_type = node.attributes.get("type", ) + node_attributes = {"@id": double_hashing(node.id), "id": node.id} + node_attributes.update(node.attributes) + _ = node_attributes.pop("type") + logger.debug(f"{node_attributes}") + node_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in node_attributes.items()]) + node_str_list.append(f"(:{node_type} {{{node_str}}})") + + gql = f"INSERT {','.join(node_str_list)}" + return self.execute(gql) + + def add_edge(self, grelation: GRelation) -> dict: + return self.add_edges([grelation]) + + def add_edges(self, edges: List[GRelation]) -> dict: + '''不支持批量edge插入''' + edge_str_list = [] + for edge in edges: + edge_type = edge.attributes.get("type", ) + src_id, dst_id = double_hashing(edge.start_id,), double_hashing(edge.end_id,) + edge_attributes = {"@src_id": src_id, "@dst_id": dst_id, "@timestamp": 1} + edge_attributes.update(edge.attributes) + _ = edge_attributes.pop("type") + edge_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in edge_attributes.items()]) + edge_str_list.append(f"()-[:{edge_type} {{{edge_str}}}]->()") + + gql = f"INSERT {','.join(edge_str_list)}" + return self.execute(gql) + + def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None): + # demo: "MATCH (n:opsgptkg_employee {@ID: xxxx}) SET n.originname = 'xxx', n.description = 'xxx'" + set_str = ", ".join([f"n.{k}='{v}'" if isinstance(v, str) else f"n.{k}={v}" for k, v in set_attributes.items()]) + + if (ID is None) or (not isinstance(ID, int)): + ID = self.get_current_nodeID(attributes, node_type) + # ID = double_hashing(ID) + gql = f"MATCH (n:{node_type}) WHERE n.@ID={ID} SET {set_str}" + return self.execute(gql) + + def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None): + # geabase 不支持直接根据边关系进行检索 + src_id, dst_id = self.get_current_edgeID(src_id, dst_id, edge_type) + # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) + set_str = ", ".join([f"e.{k}='{v}'" if isinstance(v, str) else f"e.{k}={v}" for k, v in set_attributes.items()]) + # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; + gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, @timestamp: 1}}]->() SET {set_str}" + return self.execute(gql) + + def delete_node(self, attributes: dict, node_type: str = None, ID: int = None): + if (ID is None) or (not isinstance(ID, int)): + ID = self.get_current_nodeID(attributes, node_type) + # ID = double_hashing(ID) + gql = f"MATCH (n:{node_type}) WHERE n.@ID={ID} DELETE n" + return self.execute(gql) + + def delete_nodes(self, attributes: dict, node_type: str = None, ID: int = None): + pass + + def delete_edge(self, src_id, dst_id, edge_type: str = None): + # geabase 不支持直接根据边关系进行检索 + src_id, dst_id = self.get_current_edgeID(src_id, dst_id, edge_type) + # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) + # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; + gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, @timestamp: 1}}]->() DELETE e" + return self.execute(gql) + + def delete_edges(self, src_id, dst_id, edge_type: str = None): + pass + + def execute(self, gql: str, option=None, return_keys: list = []) -> Dict: + option = option or self.option + logger.info(f"{gql}") + result = self.geabase_client.executeGQL(gql, option) + result = json.loads(str(result.getJsonGQLResponse())) + return result + + def get_current_nodeID(self, attributes: dict, node_type: str) -> int: + result = self.get_current_node(attributes, node_type) + return result.get("ID") + + def get_current_edgeID(self, src_id, dst_id, edeg_type:str = None): + if not isinstance(src_id, int) or not isinstance(dst_id, int): + result = self.get_current_edge(src_id, dst_id, edeg_type) + return result.get("srcId"), result.get("dstId") + else: + return src_id, dst_id + + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + # + return_str = ", ".join([f"n0.{k}" for k in return_keys]) if return_keys else "n0" + where_str = ' and '.join([f"n0.{k}='{v}'" for k,v in attributes.items()]) + gql = f"MATCH (n0:{node_type}) WHERE {where_str} RETURN {return_str}" + # + result = self.execute(gql, return_keys=return_keys) + result = self.decode_result(result, gql) + result = result.get("n0", []) or result.get("n0.attr", []) + + return result[0] + + def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + # + return_str = ", ".join([f"n0.{k}" for k in return_keys]) if return_keys else "n0" + where_str = ' and '.join([f"n0.{k}='{v}'" for k,v in attributes.items()]) + gql = f"MATCH (n0:{node_type}) WHERE {where_str} RETURN {return_str}" + # + result = self.execute(gql, return_keys=return_keys) + result = self.decode_result(result, gql) + return result.get("n0", []) or result.get("n0.attr", []) + + def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> Dict: + # + src_type, dst_type = edge_type.split("_route_") + gql = f"MATCH (n0: {src_type} {{id: '{src_id}'}})-[e]->(n1: {dst_type} {{id: '{dst_id}'}}) RETURN e" + # + result = self.execute(gql, return_keys=return_keys) + result = self.decode_result(result, gql) + result = result.get("e", []) or result.get("e.attr", []) + + return result[0] + + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + # + return_str = ", ".join([f"n1.{k}" for k in return_keys]) if return_keys else "n1" + where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) + gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" + # + result = self.execute(gql, return_keys=return_keys) + result = self.decode_result(result, gql) + return result.get("n1", []) or result.get("n1.attr", []) + + def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + # + return_str = ", ".join([f"e.{k}" for k in return_keys]) if return_keys else "e" + where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) + gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" + # + result = self.execute(gql, return_keys=return_keys) + result = self.decode_result(result, gql) + + return result.get("e", []) or result.get("e.attr", []) + + def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}): + result = self.get_neighbor_nodes(attributes, node_type,) + filter_result = [i for i in result if all([item in i.items() for item in check_attributes.items()])] + return len(filter_result) > 0 + + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + ''' + hop >= 2， 表面需要至少两跳 + ''' + hop_max = 10 + hop_list = [] + # + where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) + gql = f"MATCH p = (n0:{node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + last_node_ids, last_node_types = [], [] + while hop > 1: + if last_node_ids == []: + # + result = self.execute(gql) + result = self.decode_result(result, gql) + else: + for _node_id, _node_type in zip(last_node_ids, last_node_types): + where_str = f"n0.id='{_node_id}'" + gql = f"MATCH p = (n0:{_node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + # + _result = self.execute(gql) + _result = self.decode_result(_result, gql) + + result = self.merge_hotinfos(result, _result) + # + last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes) + hop -= hop_max + + return result + + def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + # + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.get("n1", []) or result.get("n1.attr", []) + + def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + # + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.get("e", []) or result.get("e.attr", []) + + def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + # + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.get("p", []) + + def deduplicate_paths(self, result, block_attributes: dict = {}): + # 获取数据 + n0, n1, e, p = result["n0"], result["n1"], result["e"], result["p"] + block_node_ids = [ + i["id"] + for i in n0+n1 + # 这里block为空时也会生效，属于合理情况 + # if block_attributes=={} or all(item in i.items() for item in block_attributes.items()) + if block_attributes and all(item in i.items() for item in block_attributes.items()) + ] + # 路径去重 + path_strs = ["&&".join(_p) for _p in p] + new_p = [] + new_path_strs_set = set() + for path_str, _p in zip(path_strs, p): + if not any(path_str in other for other in path_strs if path_str != other): + if path_str not in new_path_strs_set and all([_pid not in block_node_ids for _pid in _p]): + new_p.append(_p) + new_path_strs_set.add(path_str) + + # 根据保留路径进行合并 + nodeid2type = {i["id"]: i["type"] for i in n0+n1} + unique_node_ids = [j for i in new_p for j in i] + last_node_ids = [i[-1] for i in new_p] + last_node_types = [nodeid2type[i] for i in last_node_ids] + new_n0 = deduplicate_dict([i for i in n0 if i["id"] in unique_node_ids]) + new_n1 = deduplicate_dict([i for i in n1 if i["id"] in unique_node_ids]) + new_e = deduplicate_dict([i for i in e if i["source_id"] in unique_node_ids and i["target_id"] in unique_node_ids]) + + return last_node_ids, last_node_types, {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} + + def merge_hotinfos(self, result1, result2) -> Dict: + new_n0 = result1["n0"] + result2["n0"] + new_n1 = result1["n1"] + result2["n1"] + new_e = result1["e"] + result2["e"] + new_p = result1["p"] + result2["p"] + [ + p_old_1 + p_old_2[1:] + for p_old_1 in result1["p"] + for p_old_2 in result2["p"] + if p_old_2[0] == p_old_1[-1] + ] + new_result = {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} + return new_result + + def decode_result(self, geabase_result, gql: str) -> Dict: + return_keys = gql.split("RETURN")[-1].split(',') + save_keys = [k.strip() if "." not in k else k.strip().split(".")[0]+".attr" for k in return_keys] + + decode_geabase_result_func_by_key = { + "p": self.decode_path, + "n0": self.decode_vertex, + "n1": self.decode_vertex, + "e": self.decode_edge, + "n0.attr": self.decode_attribute, + "n1.attr": self.decode_attribute, + "e.attr": self.decode_attribute, + } + + output = {k: [] for k in save_keys} + if "resultSet" in geabase_result: + # decode geabase result + for row in geabase_result['resultSet']['rows']: + attr_dict = {} + for col_data, rk, sk in zip(row["columns"], return_keys, save_keys): + _decode_func = decode_geabase_result_func_by_key.get(sk, self.decode_attribute) + # print(sk, json.dumps(col_data, ensure_ascii=False, indent=2)) + decode_reuslt = _decode_func(col_data, rk) + if ".attr" in sk: + attr_dict.setdefault(sk, {}).update(decode_reuslt) + + # print(sk, decode_reuslt) + + if sk=="e": + output[sk].extend(decode_reuslt) + elif ".attr" in sk: + pass + else: + output[sk].append(decode_reuslt) + + for sk, v in attr_dict.items(): + v = {kk.split(".")[-1]: vv for kk, vv in v.items()} + output[sk].append(v) + + return output + + def decode_path(self, col_data, k) -> List: + path = [] + steps = col_data.get("pathVal", {}).get("steps", []) + for step in steps: + props = step["props"] + if path == []: + path.append(props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1)) + + path.append(props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1)) + + return path + + def decode_vertex(self, col_data, k) -> Dict: + vertextVal = col_data.get("vertexVal", {}) + node_val_json = { + **{"ID": vertextVal.get("id", ""), "type": vertextVal.get("type", "")}, + **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in vertextVal.get("props", {}).items()} + } + return node_val_json + + def decode_edge(self, col_data, k) -> Dict: + def _decode_edge(data): + edgeVal= data.get("edgeVal", {}) + edge_val_json = { + **{"srcId": edgeVal.get("srcId", ""), "dstId": edgeVal.get("dstId", ""), "type": edgeVal.get("type", "")}, + **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in edgeVal.get("props", {}).items()} + } + # 存在业务逻辑 + edge_val_json["source_id"] = edge_val_json.pop("original_src_id1__") + edge_val_json["target_id"] = edge_val_json.pop("original_dst_id2__") + return edge_val_json + + edge_val_jsons = [] + if "listVal" in col_data: + for val in col_data["listVal"].get("vals", []): + edge_val_jsons.append(_decode_edge(val)) + elif "edgeVal" in col_data: + edge_val_jsons = [_decode_edge(col_data)] + + return edge_val_jsons + + def decode_attribute(self, col_data, k) -> Dict: + return {k: col_data.get("strVal", "") or col_data.get("intVal", "0")} \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/networkx_handler.py b/muagent/db_handler/graph_db_handler/networkx_handler.py index fb7a1b2..9325b94 100644 --- a/muagent/db_handler/graph_db_handler/networkx_handler.py +++ b/muagent/db_handler/graph_db_handler/networkx_handler.py @@ -46,7 +46,7 @@ def search_edges_by_nodeid(self, nodeid: str) -> List[GRelation]: return [ GRelation( - id=f"{nodeid}-{neighbor}", + # id=f"{nodeid}-{neighbor}", start_id=nodeid, end_id=neighbor, attributes=self.graph.get_edge_data(nodeid, neighbor) @@ -59,7 +59,7 @@ def search_edges_by_nodeids(self, start_id: str, end_id: str) -> GRelation: if self.missing_edge(start_id, end_id): return None return GRelation( - id=f"{start_id}-{end_id}", + # id=f"{start_id}-{end_id}", start_id=start_id, end_id=end_id, attributes=self.graph.get_edge_data(start_id, end_id) @@ -75,7 +75,7 @@ def search_nodes_by_attr(self, **attributes) -> List[GNode]: def search_edges_by_attr(self, **attributes) -> List[GRelation]: return [ GRelation( - id=f"{left}-{right}", + # id=f"{left}-{right}", start_id=left, end_id=right, attributes=attr @@ -95,6 +95,7 @@ def load(self, kb_name: str): def save_to_local(self, kb_name: str): dir_path = os.path.join(self.kb_root_path, kb_name) + os.makedirs(dir_path, exist_ok=True) # 将图保存到本地文件 nx.write_graphml(self.graph, os.path.join(dir_path, 'graph.graphml')) diff --git a/muagent/db_handler/utils.py b/muagent/db_handler/utils.py new file mode 100644 index 0000000..1b4a52f --- /dev/null +++ b/muagent/db_handler/utils.py @@ -0,0 +1,13 @@ + +def deduplicate_dict(dict_list: list = []): + seen = set() + unique_dicts = [] + + for d in dict_list: + # 使用 frozenset 进行较快的哈希，并避免重复转换 + d_tuple = tuple(sorted(d.items())) + if d_tuple not in seen: + seen.add(d_tuple) + unique_dicts.append(d) + + return unique_dicts \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 9c8c6b0..6e6ed68 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -22,6 +22,7 @@ class NodeTypesEnum(Enum): SCHEDULE = 'schedule' INTENT = 'intent' TOOL = 'tool' + TOOL_INSTANCE = 'tool_instance' TEAM = 'team' OWNER = 'owner' diff --git a/muagent/schemas/memory/__init__.py b/muagent/schemas/memory/__init__.py index 7ea6809..015f3ea 100644 --- a/muagent/schemas/memory/__init__.py +++ b/muagent/schemas/memory/__init__.py @@ -2,5 +2,5 @@ __all__ = [ - "GNodeAbs", "GRelationAbs", "Attribute", "GNode", "GRelation", "ThemeEnums" + "GNodeAbs", "GEdgeAbs", "GRelationAbs", "Attribute", "GNode", "GEdge", "GRelation", "ThemeEnums" ] \ No newline at end of file diff --git a/muagent/schemas/memory/auto_extract_graph_schema.py b/muagent/schemas/memory/auto_extract_graph_schema.py index 5fd9eb5..6a9bfc2 100644 --- a/muagent/schemas/memory/auto_extract_graph_schema.py +++ b/muagent/schemas/memory/auto_extract_graph_schema.py @@ -16,6 +16,13 @@ class GNodeAbs(BaseModel): class GRelationAbs(BaseModel): + # todo: 废弃 + # relation type for extract + type: str + attributes: List[Attribute] + + +class GEdgeAbs(BaseModel): # relation type for extract type: str attributes: List[Attribute] @@ -23,14 +30,20 @@ class GRelationAbs(BaseModel): class GNode(BaseModel): id: str - attributes: Dict[str, str] + attributes: Dict + + +class GEdge(BaseModel): + start_id: str + end_id: str + attributes: Dict class GRelation(BaseModel): - id: str + # todo: 废弃 start_id: str end_id: str - attributes: Dict[str, str] + attributes: Dict class ThemeEnums(Enum): diff --git a/muagent/service/__init__.py b/muagent/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/muagent/utils/common_utils.py b/muagent/utils/common_utils.py index 4583886..074de60 100644 --- a/muagent/utils/common_utils.py +++ b/muagent/utils/common_utils.py @@ -112,3 +112,14 @@ def get_uploadfile(file: Union[str, Path, bytes], filename=None) -> UploadFile: return UploadFile(file=temp_file, filename=filename) +def string_to_long_sha256(s: str) -> int: + # 使用 SHA-256 哈希函数 + hash_object = hashlib.sha256(s.encode()) + # 转换为16进制然后转换为整数 + return int(hash_object.hexdigest(), 16) + + +def double_hashing(s: str, modulus: int = 10e12) -> int: + hash1 = string_to_long_sha256(s) + hash2 = string_to_long_sha256(s[::-1]) # 用字符串的反序进行第二次hash + return int((hash1 + hash2) % modulus) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 87cc536..1adf00a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,5 +24,5 @@ pydantic<=1.10.14 # aliyun-log-python-sdk==0.9.0 # pydantic # duckduckgo-search - +urllib3==1.26.6 sseclient \ No newline at end of file diff --git a/tests/db_handler/geabase_hanlder_test.py b/tests/db_handler/geabase_hanlder_test.py new file mode 100644 index 0000000..348c79e --- /dev/null +++ b/tests/db_handler/geabase_hanlder_test.py @@ -0,0 +1,134 @@ +import time +import sys, os +from loguru import logger + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + endpoint = os.environ["endpoint"] + accessKeyId = os.environ["accessKeyId"] + accessKey = os.environ["accessKey"] +except Exception as e: + # set your config + endpoint = "" + accessKeyId = "" + accessKey = "" + logger.error(f"{e}") + +# import muagent +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) +from muagent.db_handler import GeaBaseHandler +from muagent.schemas.memory import GNode, GRelation +from muagent.schemas.db import GBConfig + +import copy + + + +# init SLSHandler instance +gb_config = GBConfig( + gb_type="GeaBaseHandler", + extra_kwargs={ + 'metaserver_address': os.environ['metaserver_address'], + 'project': os.environ['project'], + 'city': os.environ['city'], + 'lib_path': os.environ['lib_path'], + } +) +geabase_handler = GeaBaseHandler(gb_config) + +# it's a case, you should use your node and edge attributes +node1 = GNode(**{ + "id": "antshanshi311395_1", + "attributes": { + "type": "opsgptkg_intent", + "path": "shanshi_test", + "name": "shanshi_test", + "description":'shanshi_test', + "gdb_timestamp": 1719276995619 + } +}) + +edge1 = GRelation(**{ + "start_id": "antshanshi311395_1", + "end_id": "antshanshi311395_2", + "attributes": { + "type": "opsgptkg_intent_route_opsgptkg_intent", + "original_dst_id2__": "antshanshi311395_1", + "original_src_id1__": "antshanshi311395_2", + "gdb_timestamp": 1719276995619 + } +}) + +edge2 = GRelation(**{ + "start_id": "antshanshi311395_2", + "end_id": "antshanshi311395_3", + "attributes": { + "type": "opsgptkg_intent_route_opsgptkg_intent", + "original_dst_id2__": "antshanshi311395_2", + "original_src_id1__": "antshanshi311395_3", + "gdb_timestamp": 1719276995619 + } +}) + +node2 = copy.deepcopy(node1) +node2.id = "antshanshi311395_2" + +node3 = copy.deepcopy(node1) +node3.id = "antshanshi311395_3" + +# 添加节点 +t = geabase_handler.add_node(node1) +print(t) + +t = geabase_handler.add_nodes([node2, node3]) +print(t) + +t = geabase_handler.add_nodes([node2, node3]) +print(t) + +t = geabase_handler.add_edges([edge1, edge2]) +print(t) + +# 测试节点的查改删 +t = geabase_handler.get_current_node(attributes={"id": "antshanshi311395_1",}, node_type="opsgptkg_intent", ) +print(t) + +t = geabase_handler.update_node(attributes={"id": "antshanshi311395_1",}, node_type="opsgptkg_intent", set_attributes={"name": "shanshi_test_rename"}, ) +print(t) + +t = geabase_handler.get_current_node(attributes={"id": "antshanshi311395_1",}, node_type="opsgptkg_intent", ) +print(t) + +t = geabase_handler.delete_node(attributes={"id": "antshanshi311395_1",}, node_type="opsgptkg_intent", ) +print(t) + +t = geabase_handler.get_current_node(attributes={"id": "antshanshi311395_1",}, node_type="opsgptkg_intent", ) +print(t) + +# 测试边的查改删 +t = geabase_handler.get_current_edge( + edge_type="opsgptkg_intent_route_opsgptkg_intent", src_id="antshanshi311395_1", dst_id="antshanshi311395_2") +print(t) + +t = geabase_handler.update_edge( + edge_type="opsgptkg_intent_route_opsgptkg_intent", set_attributes={"gdb_timestamp": 1719276995623}, src_id="antshanshi311395_1", dst_id="antshanshi311395_2") +print(t) + +t = geabase_handler.get_current_edge( + edge_type="opsgptkg_intent_route_opsgptkg_intent", src_id="antshanshi311395_1", dst_id="antshanshi311395_2") +print(t) + +t = geabase_handler.delete_edge( + edge_type="opsgptkg_intent_route_opsgptkg_intent", src_id="antshanshi311395_1", dst_id="antshanshi311395_2") +print(t) + +t = geabase_handler.get_current_edge( + edge_type="opsgptkg_intent_route_opsgptkg_intent", src_id="antshanshi311395_1", dst_id="antshanshi311395_2") +print(t) diff --git a/tests/db_handler/networkx_handler_test.py b/tests/db_handler/networkx_handler_test.py index e0d4f10..52a3fe1 100644 --- a/tests/db_handler/networkx_handler_test.py +++ b/tests/db_handler/networkx_handler_test.py @@ -10,6 +10,7 @@ ) print(src_dir) sys.path.append(src_dir) +from muagent import db_handler from muagent.db_handler import NetworkxHandler from muagent.schemas.memory import GNode, GRelation @@ -19,8 +20,8 @@ node3 = GNode(**{"id": "node3", "attributes": {"name": "test3" }}) node4 = GNode(**{"id": "node4", "attributes": {"name": "test3" }}) -edge1 = GRelation(**{"id": "edge1", "left": node1, "right": node2, "attributes": {"name": "test" }}) -edge2 = GRelation(**{"id": "edge2", "left": node2, "right": node3, "attributes": {"name": "test2" }}) +edge1 = GRelation(**{"start_id": "node1", "end_id": "node2", "attributes": {"id": "edge1", "name": "test" }}) +edge2 = GRelation(**{"start_id": "node2", "end_id": "node3", "attributes": {"id": "edge2", "name": "test2" }}) nh = NetworkxHandler() @@ -32,44 +33,54 @@ nh.add_edges([edge2]) # +print("search by nodeid") print(nh.search_nodes_by_nodeid("node1")) print(nh.search_edges_by_nodeid("node2")) print(nh.search_edges_by_nodeids("node1", "node2")) # +print("search by attr") print(nh.search_nodes_by_attr(**{"name": "test3"})) print(nh.search_edges_by_attr(**{"name": "test2"})) # -nh.save_to_local() +print("save to local") +nh.save_to_local("networkx_test") # +print("clear cache for search by attr") nh.clear() print(nh.search_nodes_by_attr(**{"name": "test3"})) print(nh.search_edges_by_attr(**{"name": "test2"})) # -nh.load_from_local() +print("load from local for search by attr") +nh.load_from_local("networkx_test") print(nh.search_nodes_by_attr(**{"name": "test3"})) print(nh.search_edges_by_attr(**{"name": "test2"})) # +print("search by nodeid after delete") nh.delete_node("node1") print(nh.search_nodes_by_nodeid("node1")) # +print("search by nodeids after delete") nh.delete_nodes(["node1", "node2"]) print(nh.search_nodes_by_nodeid("node1")) print(nh.search_nodes_by_nodeid("node2")) # +print("search edge by nodeids after delete") nh.delete_edge("node1", "node2") print(nh.search_edges_by_nodeids("node1", "node2")) # +print("search edge by nodeid after delete") nh.delete_edges_by_nodeid("node1") print(nh.search_edges_by_nodeids("node1", "node2")) # +print("search edge by nodeids after delete") nh.delete_edges([("node1", "node2")]) print(nh.search_edges_by_nodeids("node1", "node2")) \ No newline at end of file diff --git a/tests/db_handler/slshandler_test.py b/tests/db_handler/slshandler_test.py index a70b62f..ad6d342 100644 --- a/tests/db_handler/slshandler_test.py +++ b/tests/db_handler/slshandler_test.py @@ -23,7 +23,7 @@ ) print(src_dir) sys.path.append(src_dir) -from muagent.db_handler.sls_db_handler import AliYunSLSHandler +from muagent.db_handler import AliYunSLSHandler from muagent.schemas.db import SLSConfig From 53a50d93a9aaaa0660482c11e889864b4fde864e Mon Sep 17 00:00:00 2001 From: lightislost Date: Sun, 4 Aug 2024 16:52:27 +0800 Subject: [PATCH 012/128] update text2graph and alarm2graph --- .../base_configs/prompts/simple_prompts.py | 101 ++- muagent/connector/configs/generate_prompt.py | 11 +- .../memory/hierarchical_memory_manager.py | 2 +- muagent/db_handler/__init__.py | 4 +- .../db_handler/graph_db_handler/__init__.py | 3 +- .../graph_db_handler/aliyun_sls_hanlder.py | 4 +- .../graph_db_handler/base_gb_handler.py | 14 +- .../graph_db_handler/geabase_handler.py | 190 +++-- .../graph_db_handler/networkx_handler.py | 2 +- .../vector_db_handler/tbase_handler.py | 25 +- muagent/schemas/common/__init__.py | 8 + .../auto_extract_graph_schema.py | 133 ++-- muagent/schemas/ekg/__init__.py | 6 +- muagent/schemas/ekg/ekg_graph.py | 158 +++- muagent/schemas/memory/__init__.py | 6 - muagent/schemas/readme.md | 4 +- muagent/service/ekg_construct/__init__.py | 5 + .../ekg_construct/ekg_construct_base.py | 719 ++++++++++++++++-- .../service/ekg_construct/ekg_db_service.py | 243 ++++++ muagent/utils/common_utils.py | 2 + tests/db_handler/geabase_hanlder_test.py | 25 +- tests/db_handler/networkx_handler_test.py | 2 +- 22 files changed, 1398 insertions(+), 269 deletions(-) create mode 100644 muagent/schemas/common/__init__.py rename muagent/schemas/{memory => common}/auto_extract_graph_schema.py (70%) delete mode 100644 muagent/schemas/memory/__init__.py create mode 100644 muagent/service/ekg_construct/__init__.py create mode 100644 muagent/service/ekg_construct/ekg_db_service.py diff --git a/muagent/base_configs/prompts/simple_prompts.py b/muagent/base_configs/prompts/simple_prompts.py index d4e5785..e2b7588 100644 --- a/muagent/base_configs/prompts/simple_prompts.py +++ b/muagent/base_configs/prompts/simple_prompts.py @@ -203,4 +203,103 @@ {conversation} ## 输出 -""" \ No newline at end of file +""" + + + +text2EKG_prompt_en = '''你是一个结构化信息抽取的专家，你需要根据输入的文档，抽取其中的关键节点及节点间的连接顺序。请用json结构返回。 + +json结构定义如下: +{ + "nodes": { + "节点序号": { + "type": "节点类型", + "content": "节点内容" + } + }, + "edges": [ + { + "start": "起始节点序号", + "end": "终止节点序号" + } + ] +} +其中 nodes 用来存放抽取的节点，每个 node 的 key 通过从0开始对递增序列表示，value 是一个字典，包含 type 和 content 两个属性， type 对应下面定义的三种节点类型，content 为抽取的节点内容。 +edges 用来存放节点间的连接顺序，它是一个列表，每个元素是一个字典，包含 start 和 end 两个属性， start 为起始 node 的 节点序号, end 为结束 node 的 节点序号。 + +节点类型定义如下: +Schedule: + 表示整篇输入文档所做的事情，是对整篇输入文档的总结； + 第一个节点永远是Schedule节点。 +Task: + 表示需要执行的任务。 +Phenomenon: + 表示依据Task节点的执行结果，得到的事实结论。 + Phenomenon节点只能连接在Task节点之后。 +Analysis: + 表示依据Phenomenon节点的事实进行推断的过程； + Analysis节点只能连接在Phenomenon节点之后。 + +以下是一个例子： +input: 路径：排查网络问题 +1. 通过观察sofagw网关监控发现，BOLT失败数突增 +2. 且失败曲线与退保成功率曲线相关性较高，判定是网络问题。 + +output: { + "nodes": { + "0": { + "type": "Schedule", + "content": "排查网络问题" + }, + "1": { + "type": "Task", + "content": "查询sofagw网关监控BOLT失败数" + }, + "2": { + "type": "Task", + "content": "查询sofagw网关监控退保成功率" + }, + "3": { + "type": "Task", + "content": "判断两条时序相关性" + }, + "4": { + "type": "Phenomenon", + "content": "相关性较高" + }, + "5": { + "type": "Analysis", + "content": "网络问题" + } + }, + "edges": [ + { + "start": "0", + "end": "1" + }, + { + "start": "1", + "end": "2" + }, + { + "start": "2", + "end": "3" + }, + { + "start": "3", + "end": "4" + }, + { + "start": "4", + "end": "5" + } + ] +} + +请根据上述说明和例子来对以下的输入文档抽取结构化信息: + +input: {text} + +output:''' + +text2EKG_prompt_zh = text2EKG_prompt_en \ No newline at end of file diff --git a/muagent/connector/configs/generate_prompt.py b/muagent/connector/configs/generate_prompt.py index 37c392c..955eee2 100644 --- a/muagent/connector/configs/generate_prompt.py +++ b/muagent/connector/configs/generate_prompt.py @@ -4,7 +4,7 @@ def replacePrompt(prompt: str, keys: list[str] = []): prompt = prompt.replace("{", "{{").replace("}", "}}") for key in keys: - prompt = prompt.replace(f"{{{{key}}}}", f"{{key}}") + prompt = prompt.replace(f"{{{{{key}}}}}", f"{{{key}}}") return prompt def cleanPrompt(prompt): @@ -54,4 +54,13 @@ def createMKGPrompt(conversation, schemas, language="en", **kwargs) -> str: # prompt = memory_extract_prompt_zh.format(**{"conversation": conversation, "schemas": schemas}) # else: # prompt = memory_extract_prompt_en.format(**{"conversation": conversation, "schemas": schemas}) + return cleanPrompt(prompt) + + +def createText2EKGPrompt(text, language="en", **kwargs) -> str: + prompt = text2EKG_prompt_zh if language == "zh" else text2EKG_prompt_en + prompt = replacePrompt(prompt, keys=["text"]) + from loguru import logger + logger.debug(f"{prompt}") + prompt = prompt.format(**{"text": text,}) return cleanPrompt(prompt) \ No newline at end of file diff --git a/muagent/connector/memory/hierarchical_memory_manager.py b/muagent/connector/memory/hierarchical_memory_manager.py index 03e0630..180dc35 100644 --- a/muagent/connector/memory/hierarchical_memory_manager.py +++ b/muagent/connector/memory/hierarchical_memory_manager.py @@ -10,7 +10,7 @@ from muagent.connector.configs.generate_prompt import * from muagent.connector.schema import Memory, Message from muagent.schemas.db import DBConfig, GBConfig, VBConfig, TBConfig -from muagent.schemas.memory import * +from muagent.schemas.common import * from muagent.db_handler import * from muagent.connector.memory_manager import BaseMemoryManager from muagent.llm_models import * diff --git a/muagent/db_handler/__init__.py b/muagent/db_handler/__init__.py index 73c6222..aebb417 100644 --- a/muagent/db_handler/__init__.py +++ b/muagent/db_handler/__init__.py @@ -6,12 +6,12 @@ @desc: ''' -from .graph_db_handler import NebulaHandler, NetworkxHandler, AliYunSLSHandler, GeaBaseHandler +from .graph_db_handler import NebulaHandler, NetworkxHandler, AliYunSLSHandler, GeaBaseHandler, GBHandler from .vector_db_handler import LocalFaissHandler, TbaseHandler, ChromaHandler __all__ = [ - "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", + "GBHandler", "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", "ChromaHandler", "TbaseHandler", "LocalFaissHandler", "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/__init__.py b/muagent/db_handler/graph_db_handler/__init__.py index 54fb0bb..3a1b238 100644 --- a/muagent/db_handler/graph_db_handler/__init__.py +++ b/muagent/db_handler/graph_db_handler/__init__.py @@ -5,6 +5,7 @@ @time: 2023/11/20 下午3:07 @desc: ''' +from .base_gb_handler import GBHandler from .nebula_handler import NebulaHandler from .networkx_handler import NetworkxHandler from .aliyun_sls_hanlder import AliYunSLSHandler @@ -12,6 +13,6 @@ __all__ = [ - "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", + "GBHandler", "NebulaHandler", "NetworkxHandler", "GeaBaseHandler", "AliYunSLSHandler" ] \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py index 680025d..d0843d9 100644 --- a/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py +++ b/muagent/db_handler/graph_db_handler/aliyun_sls_hanlder.py @@ -3,7 +3,7 @@ import time from aliyun.log import * -from muagent.schemas.memory import * +from muagent.schemas.common import * from muagent.schemas.db import SLSConfig @@ -141,6 +141,6 @@ def relation_process(self, relations: List[GRelation], crud_type) -> List[List[T relation_list = [ [('start_id', relation.start_id), ('end_id', relation.start_id)] +\ [(k, v) if k!="operation_type" else (k, crud_type) for k,v in relation.attributes.items()] - for node in nodes + for relation in relations ] return relation_list \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/base_gb_handler.py b/muagent/db_handler/graph_db_handler/base_gb_handler.py index e04760c..5e55d29 100644 --- a/muagent/db_handler/graph_db_handler/base_gb_handler.py +++ b/muagent/db_handler/graph_db_handler/base_gb_handler.py @@ -3,7 +3,7 @@ from loguru import logger import json -from muagent.schemas.memory import * +from muagent.schemas.common import * @@ -58,20 +58,20 @@ def search_nodes_by_attr(self, attributes: dict) -> List[GNode]: def search_edges_by_attr(self, attributes: dict, edge_type: str = None) -> List[GEdge]: pass - def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: pass - def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: pass - def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> Dict: + def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> GEdge: pass - def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: pass - def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: pass - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}) -> Graph: pass diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index cdba745..1582c1b 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -6,13 +6,14 @@ from gdbc2.geabase_client import GeaBaseClient, Node, Edge, MutateBatchOperation, GeaBaseUtil from gdbc2.geabase_env import GeaBaseEnv +from .base_gb_handler import GBHandler from muagent.db_handler.utils import deduplicate_dict from muagent.schemas.db import GBConfig -from muagent.schemas.memory import * +from muagent.schemas.common import * from muagent.utils.common_utils import double_hashing -class GeaBaseHandler: +class GeaBaseHandler(GBHandler): def __init__( self, gb_config: GBConfig = None @@ -30,42 +31,49 @@ def __init__( # option 指定 self.option = GeaBaseEnv.QueryRequestOption.newBuilder().gqlType(GeaBaseEnv.QueryProtocol.GQLType.GQL_ISO).build() + def execute(self, gql: str, option=None, return_keys: list = []) -> Dict: + option = option or self.option + logger.info(f"{gql}") + result = self.geabase_client.executeGQL(gql, option) + result = json.loads(str(result.getJsonGQLResponse())) + return result + def add_node(self, node: GNode) -> dict: return self.add_nodes([node]) def add_nodes(self, nodes: List[GNode]) -> dict: node_str_list = [] for node in nodes: - node_type = node.attributes.get("type", ) + node_type = node.type node_attributes = {"@id": double_hashing(node.id), "id": node.id} node_attributes.update(node.attributes) - _ = node_attributes.pop("type") - logger.debug(f"{node_attributes}") + # _ = node_attributes.pop("type") + # logger.debug(f"{node_attributes}") node_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in node_attributes.items()]) node_str_list.append(f"(:{node_type} {{{node_str}}})") gql = f"INSERT {','.join(node_str_list)}" return self.execute(gql) - def add_edge(self, grelation: GRelation) -> dict: - return self.add_edges([grelation]) + def add_edge(self, edge: GEdge) -> dict: + return self.add_edges([edge]) - def add_edges(self, edges: List[GRelation]) -> dict: + def add_edges(self, edges: List[GEdge]) -> dict: '''不支持批量edge插入''' edge_str_list = [] for edge in edges: - edge_type = edge.attributes.get("type", ) + edge_type = edge.type src_id, dst_id = double_hashing(edge.start_id,), double_hashing(edge.end_id,) - edge_attributes = {"@src_id": src_id, "@dst_id": dst_id, "@timestamp": 1} + edge_attributes = {"@src_id": src_id, "@dst_id": dst_id} edge_attributes.update(edge.attributes) - _ = edge_attributes.pop("type") + # _ = edge_attributes.pop("type") edge_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in edge_attributes.items()]) edge_str_list.append(f"()-[:{edge_type} {{{edge_str}}}]->()") gql = f"INSERT {','.join(edge_str_list)}" return self.execute(gql) - def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None): + def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None) -> dict: # demo: "MATCH (n:opsgptkg_employee {@ID: xxxx}) SET n.originname = 'xxx', n.description = 'xxx'" set_str = ", ".join([f"n.{k}='{v}'" if isinstance(v, str) else f"n.{k}={v}" for k, v in set_attributes.items()]) @@ -75,114 +83,129 @@ def update_node(self, attributes: dict, set_attributes: dict, node_type: str = N gql = f"MATCH (n:{node_type}) WHERE n.@ID={ID} SET {set_str}" return self.execute(gql) - def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None): + def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None) -> dict: # geabase 不支持直接根据边关系进行检索 - src_id, dst_id = self.get_current_edgeID(src_id, dst_id, edge_type) + src_id, dst_id, timestamp = self.get_current_edgeID(src_id, dst_id, edge_type) + src_type, dst_type = self.get_nodetypes_by_edgetype(edge_type) # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) set_str = ", ".join([f"e.{k}='{v}'" if isinstance(v, str) else f"e.{k}={v}" for k, v in set_attributes.items()]) # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; - gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, @timestamp: 1}}]->() SET {set_str}" + # gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, timestamp:{timestamp}}}]->() SET {set_str}" + gql = f"MATCH (n0:{src_type} {{@id: {src_id}}})-[e]->(n1:{dst_type} {{@id:{dst_id}}}) SET {set_str}" return self.execute(gql) - def delete_node(self, attributes: dict, node_type: str = None, ID: int = None): + def delete_node(self, attributes: dict, node_type: str = None, ID: int = None) -> dict: if (ID is None) or (not isinstance(ID, int)): ID = self.get_current_nodeID(attributes, node_type) # ID = double_hashing(ID) gql = f"MATCH (n:{node_type}) WHERE n.@ID={ID} DELETE n" return self.execute(gql) - def delete_nodes(self, attributes: dict, node_type: str = None, ID: int = None): - pass + def delete_nodes(self, attributes: dict, node_type: str = None, IDs: List[int] = None) -> dict: + if (IDs is None) or len(IDs)==0: + IDs = self.get_nodeIDs(attributes, node_type) + # ID = double_hashing(ID) + gql = f"MATCH (n:{node_type}) WHERE n.@ID in {IDs} DELETE n" + return self.execute(gql) - def delete_edge(self, src_id, dst_id, edge_type: str = None): + def delete_edge(self, src_id, dst_id, edge_type: str = None) -> dict: # geabase 不支持直接根据边关系进行检索 - src_id, dst_id = self.get_current_edgeID(src_id, dst_id, edge_type) + src_id, dst_id, timestamp = self.get_current_edgeID(src_id, dst_id, edge_type) + src_type, dst_type = self.get_nodetypes_by_edgetype(edge_type) # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; - gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, @timestamp: 1}}]->() DELETE e" + gql = f"MATCH (n0:{src_type} {{@id: {src_id}}})-[e]->(n1:{dst_type} {{@id:{dst_id}}}) DELETE e" return self.execute(gql) - def delete_edges(self, src_id, dst_id, edge_type: str = None): - pass - - def execute(self, gql: str, option=None, return_keys: list = []) -> Dict: - option = option or self.option - logger.info(f"{gql}") - result = self.geabase_client.executeGQL(gql, option) - result = json.loads(str(result.getJsonGQLResponse())) - return result + def delete_edges(self, id_pairs: List, edge_type: str = None): + # geabase 不支持直接根据边关系进行检索 + src_id, dst_id, timestamp = self.get_current_edgeID(src_id, dst_id, edge_type) + # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) + gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}}}]->() DELETE e" + gql = f"MATCH (n:opsgptkg_intent )-[r]->(t1) DELETE r" + return self.execute(gql) + def get_nodeIDs(self, attributes: dict, node_type: str) -> List[int]: + result = self.get_current_nodes(attributes, node_type) + return [i.attributes.get("ID") for i in result] + def get_current_nodeID(self, attributes: dict, node_type: str) -> int: result = self.get_current_node(attributes, node_type) + return result.attributes.get("ID") return result.get("ID") def get_current_edgeID(self, src_id, dst_id, edeg_type:str = None): if not isinstance(src_id, int) or not isinstance(dst_id, int): result = self.get_current_edge(src_id, dst_id, edeg_type) - return result.get("srcId"), result.get("dstId") + logger.debug(f"{result}") + return result.attributes.get("srcId"), result.attributes.get("dstId"), result.attributes.get("timestamp") + return result.get("srcId"), result.get("dstId"), else: - return src_id, dst_id + return src_id, dst_id, 1 - def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: - # - return_str = ", ".join([f"n0.{k}" for k in return_keys]) if return_keys else "n0" - where_str = ' and '.join([f"n0.{k}='{v}'" for k,v in attributes.items()]) - gql = f"MATCH (n0:{node_type}) WHERE {where_str} RETURN {return_str}" - # - result = self.execute(gql, return_keys=return_keys) - result = self.decode_result(result, gql) - result = result.get("n0", []) or result.get("n0.attr", []) - - return result[0] + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: + return self.get_current_nodes(attributes, node_type, return_keys)[0] - def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> Dict: + def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: # - return_str = ", ".join([f"n0.{k}" for k in return_keys]) if return_keys else "n0" + extra_keys = list(set(return_keys + ["@ID", "id", "@node_type"])) + return_str = ", ".join([f"n0.{k}" for k in extra_keys]) if return_keys else "n0" where_str = ' and '.join([f"n0.{k}='{v}'" for k,v in attributes.items()]) gql = f"MATCH (n0:{node_type}) WHERE {where_str} RETURN {return_str}" # result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) - return result.get("n0", []) or result.get("n0.attr", []) + + nodes = result.get("n0", []) or result.get("n0.attr", []) + return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] - def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> Dict: - # - src_type, dst_type = edge_type.split("_route_") + def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> GEdge: + # todo 业务逻辑 + src_type, dst_type = self.get_nodetypes_by_edgetype(edge_type) + # todo 看是否能调整不需要节点类型 gql = f"MATCH (n0: {src_type} {{id: '{src_id}'}})-[e]->(n1: {dst_type} {{id: '{dst_id}'}}) RETURN e" # result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) - result = result.get("e", []) or result.get("e.attr", []) + + edges = result.get("e", []) or result.get("e.attr", []) + return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges][0] return result[0] - def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: # - return_str = ", ".join([f"n1.{k}" for k in return_keys]) if return_keys else "n1" + extra_keys = list(set(return_keys + ["@ID", "id", "@node_type"])) + return_str = ", ".join([f"n1.{k}" for k in extra_keys]) if return_keys else "n1" where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" # result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) + nodes = result.get("n1", []) or result.get("n1.attr", []) + return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] return result.get("n1", []) or result.get("n1.attr", []) - def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[Dict]: + def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: # - return_str = ", ".join([f"e.{k}" for k in return_keys]) if return_keys else "e" + extra_keys = list(set(return_keys + ["@SRCID", "@DSTID", "@edge_type"])) + return_str = ", ".join([f"e.{k}" for k in extra_keys]) if return_keys else "e" where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" # result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) - + + edges = result.get("e", []) or result.get("e.attr", []) + return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges] return result.get("e", []) or result.get("e.attr", []) - def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}): + def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}) -> bool: result = self.get_neighbor_nodes(attributes, node_type,) - filter_result = [i for i in result if all([item in i.items() for item in check_attributes.items()])] + filter_result = [i for i in result if all([item in i.attributes.items() for item in check_attributes.items()])] return len(filter_result) > 0 - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}) -> Graph: ''' hop >= 2， 表面需要至少两跳 ''' @@ -192,6 +215,8 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) gql = f"MATCH p = (n0:{node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" last_node_ids, last_node_types = [], [] + + result = {} while hop > 1: if last_node_ids == []: # @@ -207,27 +232,32 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b result = self.merge_hotinfos(result, _result) # - last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes) + last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes) hop -= hop_max - + + nodes = [GNode(id=node["id"], type=node["type"], attributes=node) for node in result.get("n1", [])] + edges = [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in result.get("e", [])] + return Graph(nodes=nodes, edges=edges, paths=result.get("p", [])) return result - def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.nodes return result.get("n1", []) or result.get("n1.attr", []) - def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GEdge]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.edges return result.get("e", []) or result.get("e.attr", []) - def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []): + def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[str]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) - return result.get("p", []) + return result.paths - def deduplicate_paths(self, result, block_attributes: dict = {}): + def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}): # 获取数据 n0, n1, e, p = result["n0"], result["n1"], result["e"], result["p"] block_node_ids = [ @@ -236,6 +266,10 @@ def deduplicate_paths(self, result, block_attributes: dict = {}): # 这里block为空时也会生效，属于合理情况 # if block_attributes=={} or all(item in i.items() for item in block_attributes.items()) if block_attributes and all(item in i.items() for item in block_attributes.items()) + ] + [ + i["id"] + for i in n0+n1 + if select_attributes and not all(item not in i.items() for item in select_attributes.items()) ] # 路径去重 path_strs = ["&&".join(_p) for _p in p] @@ -254,8 +288,8 @@ def deduplicate_paths(self, result, block_attributes: dict = {}): last_node_types = [nodeid2type[i] for i in last_node_ids] new_n0 = deduplicate_dict([i for i in n0 if i["id"] in unique_node_ids]) new_n1 = deduplicate_dict([i for i in n1 if i["id"] in unique_node_ids]) - new_e = deduplicate_dict([i for i in e if i["source_id"] in unique_node_ids and i["target_id"] in unique_node_ids]) - + new_e = deduplicate_dict([i for i in e if i["start_id"] in unique_node_ids and i["end_id"] in unique_node_ids]) + return last_node_ids, last_node_types, {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} def merge_hotinfos(self, result1, result2) -> Dict: @@ -308,8 +342,14 @@ def decode_result(self, geabase_result, gql: str) -> Dict: for sk, v in attr_dict.items(): v = {kk.split(".")[-1]: vv for kk, vv in v.items()} - output[sk].append(v) + if "@node_type" in v: + v["type"] = v.pop("@node_type") + + if "@edge_type" in v: + v["type"] = v.pop("@edge_type") + output[sk].append(v) + return output def decode_path(self, col_data, k) -> List: @@ -340,8 +380,8 @@ def _decode_edge(data): **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in edgeVal.get("props", {}).items()} } # 存在业务逻辑 - edge_val_json["source_id"] = edge_val_json.pop("original_src_id1__") - edge_val_json["target_id"] = edge_val_json.pop("original_dst_id2__") + edge_val_json["start_id"] = edge_val_json.pop("original_src_id1__") + edge_val_json["end_id"] = edge_val_json.pop("original_dst_id2__") return edge_val_json edge_val_jsons = [] @@ -354,4 +394,12 @@ def _decode_edge(data): return edge_val_jsons def decode_attribute(self, col_data, k) -> Dict: - return {k: col_data.get("strVal", "") or col_data.get("intVal", "0")} \ No newline at end of file + return {k: col_data.get("strVal", "") or col_data.get("intVal", "0")} + + def get_nodetypes_by_edgetype(self, edge_type: str): + src_type, dst_type = edge_type.split("_opsgptkg")[0], "opsgptkg_" + edge_type.split("_opsgptkg_")[1] + for edge_bridge in ["_route_", "_extend_"]: + if edge_bridge in edge_type: + src_type, dst_type = edge_type.split(edge_bridge) + break + return src_type, dst_type \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/networkx_handler.py b/muagent/db_handler/graph_db_handler/networkx_handler.py index 9325b94..2838774 100644 --- a/muagent/db_handler/graph_db_handler/networkx_handler.py +++ b/muagent/db_handler/graph_db_handler/networkx_handler.py @@ -2,7 +2,7 @@ import networkx as nx from typing import List, Tuple, Dict -from muagent.schemas.memory import * +from muagent.schemas.common import * from muagent.base_configs.env_config import KB_ROOT_PATH from muagent.schemas.db import GBConfig diff --git a/muagent/db_handler/vector_db_handler/tbase_handler.py b/muagent/db_handler/vector_db_handler/tbase_handler.py index fa68375..1cb4f8a 100644 --- a/muagent/db_handler/vector_db_handler/tbase_handler.py +++ b/muagent/db_handler/vector_db_handler/tbase_handler.py @@ -1,5 +1,5 @@ from loguru import logger - +from typing import Union import numpy as np import redis @@ -23,20 +23,20 @@ class TbaseHandler: def __init__( self, - tbase_args, + tb_config: TBConfig, index_name="test", definition_value="message", - tb_config: TBConfig = None ): self.client = redis.Redis( - host=tbase_args['host'], - port=tbase_args['port'], - username=tbase_args['username'], - password=tbase_args['password'] + host=tb_config.host, + port=tb_config.port, + username=tb_config.username, + password=tb_config.password, ) self.index_name = index_name self.definition_value = definition_value self.tb_config = tb_config + self.expire_time = tb_config.extra_kwargs.get("expire_time", 86400) def create_index(self, index_name=None, schema=None, definition: list =None): ''' @@ -59,7 +59,13 @@ def create_index(self, index_name=None, schema=None, definition: list =None): # logger.debug(self.client.ft(index_name).info()) return True - def insert_data_hash(self, data_list, key="message_index", expire_time=86400): + def insert_data_hash( + self, + data_list: Union[list[dict], dict], + key: str = "message_index", + expire_time: int = 86400, + need_etime: bool = True + ): ''' insert data into hash index :param index_name: @@ -74,7 +80,8 @@ def insert_data_hash(self, data_list, key="message_index", expire_time=86400): for data in data_list: key_value = f"{self.definition_value}:" + data.get(key, "") r = self.client.hset(key_value, mapping=data) - rr = self.client.expire(key_value, expire_time) + if need_etime: + rr = self.client.expire(key_value, expire_time or self.expire_time) return len(data_list) def search(self, query, index_name: str = None, query_params: dict = {}): diff --git a/muagent/schemas/common/__init__.py b/muagent/schemas/common/__init__.py new file mode 100644 index 0000000..78cb1d6 --- /dev/null +++ b/muagent/schemas/common/__init__.py @@ -0,0 +1,8 @@ +from .auto_extract_graph_schema import * + + +__all__ = [ + "GNodeAbs", "GEdgeAbs", "GRelationAbs", "Attribute", + "GNode", "GEdge", "Graph", "GEdgeRequst", "GNodeRequest", "GRelation", + "ThemeEnums" +] \ No newline at end of file diff --git a/muagent/schemas/memory/auto_extract_graph_schema.py b/muagent/schemas/common/auto_extract_graph_schema.py similarity index 70% rename from muagent/schemas/memory/auto_extract_graph_schema.py rename to muagent/schemas/common/auto_extract_graph_schema.py index 6a9bfc2..4514dd9 100644 --- a/muagent/schemas/memory/auto_extract_graph_schema.py +++ b/muagent/schemas/common/auto_extract_graph_schema.py @@ -1,55 +1,78 @@ -from pydantic import BaseModel -from typing import List, Dict -from enum import Enum - - - -class Attribute(BaseModel): - name: str - description: str - - -class GNodeAbs(BaseModel): - # node type for extract - type: str - attributes: List[Attribute] - - -class GRelationAbs(BaseModel): - # todo: 废弃 - # relation type for extract - type: str - attributes: List[Attribute] - - -class GEdgeAbs(BaseModel): - # relation type for extract - type: str - attributes: List[Attribute] - - -class GNode(BaseModel): - id: str - attributes: Dict - - -class GEdge(BaseModel): - start_id: str - end_id: str - attributes: Dict - - -class GRelation(BaseModel): - # todo: 废弃 - start_id: str - end_id: str - attributes: Dict - - -class ThemeEnums(Enum): - ''' - the memory themes - ''' - Person: str = "person" - Event: str = "event" - +from pydantic import BaseModel +from typing import List, Dict +from enum import Enum + + + +class Attribute(BaseModel): + name: str + description: str + + +class GNodeAbs(BaseModel): + # node type for extract + type: str + attributes: List[Attribute] + + +class GRelationAbs(BaseModel): + # todo: 废弃 + # relation type for extract + type: str + attributes: List[Attribute] + + +class GEdgeAbs(BaseModel): + # relation type for extract + type: str + attributes: List[Attribute] + + +class GNode(BaseModel): + id: str + type: str + attributes: Dict + + +class GEdge(BaseModel): + start_id: str + end_id: str + type: str + attributes: Dict + + +class Graph(BaseModel): + nodes: List[GNode] + edges: List[GEdge] + paths: List[List[str]] = [] + + +class GNodeRequest(BaseModel): + id: str + type: str + attributes: Dict + operationType: str + + +class GEdgeRequst(BaseModel): + start_id: str + end_id: str + type: str + attributes: Dict + operationType: str + + +class GRelation(BaseModel): + # todo: 废弃 + start_id: str + end_id: str + attributes: Dict + + +class ThemeEnums(Enum): + ''' + the memory themes + ''' + Person: str = "person" + Event: str = "event" + diff --git a/muagent/schemas/ekg/__init__.py b/muagent/schemas/ekg/__init__.py index 5016a16..3d014ab 100644 --- a/muagent/schemas/ekg/__init__.py +++ b/muagent/schemas/ekg/__init__.py @@ -4,10 +4,10 @@ __all__ = [ "EKGEdgeSchema", "EKGNodeSchema", "EKGTaskNodeSchema", "EKGIntentNodeSchema", "EKGAnalysisNodeSchema", "EKGScheduleNodeSchema", "EKGPhenomenonNodeSchema", - "EKGEdgeTbaseSchema", "EKGEdgeTbaseSchema", "EKGTbaseData", + "EKGNodeTbaseSchema", "EKGEdgeTbaseSchema", "EKGTbaseData", "EKGGraphSlsSchema", "EKGSlsData", - "SHAPE2TYPE", - + "SHAPE2TYPE", "TYPE2SCHEMA", + "YuqueDslNodeData", "YuqueDslEdgeData", "YuqueDslDatas", "EKGIntentResp", ] \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 6e6ed68..cb9fa16 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -14,65 +14,122 @@ 'start-end': 'schedule' } - -class NodeTypesEnum(Enum): - TASK = 'task' - ANALYSIS = 'analysis' - PHENOMENON = 'phenomenon' - SCHEDULE = 'schedule' - INTENT = 'intent' - TOOL = 'tool' - TOOL_INSTANCE = 'tool_instance' - TEAM = 'team' - OWNER = 'owner' - - -class EKGNodeSchema(BaseModel): - # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} +##################################################################### +############################ Base Schema ############################# +##################################################################### +class NodeSchema(BaseModel): + ID: int = None # depend on id-str id: str # depend on user's difine name: str # depend on user's difine description: str + gdb_timestamp: int -class EKGEdgeSchema(EKGNodeSchema): - # ekg_edge:{graph_id}:{start_id}:{end_id} - id: str + +class EdgeSchema(BaseModel): # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - star_id: str + src_id: int = None + original_src_id1__: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - end_id: str + dst_id: int = None + original_dst_id2__: str + # + timestamp: int + gdb_timestamp: int + + +##################################################################### +############################ EKG Schema ############################# +##################################################################### + +class NodeTypesEnum(Enum): + TASK = 'opsgptkg_task' + ANALYSIS = 'opsgptkg_analysis' + PHENOMENON = 'opsgptkg_phenomenon' + SCHEDULE = 'opsgptkg_schedule' + INTENT = 'opsgptkg_intent' + TOOL = 'opsgptkg_tool' + TOOL_INSTANCE = 'opsgptkg_tool_instance' + TEAM = 'opsgptkg_team' + OWNER = 'opsgptkg_owner' + edge = 'edge' + + +# EKG Node and Edge Schemas +class EKGNodeSchema(NodeSchema): + teamid: str + version:str # yyyy-mm-dd HH:MM:SS + extra: str = '' + + +class EKGEdgeSchema(EdgeSchema): + teamid: str + version:str # yyyy-mm-dd HH:MM:SS + extra: str = '' class EKGIntentNodeSchema(EKGNodeSchema): - pass + path: str = '' class EKGScheduleNodeSchema(EKGNodeSchema): - pass + # do action or not + switch: bool -class EKGTaskNodSchema(EKGNodeSchema): +class EKGTaskNodeSchema(EKGNodeSchema): tool: str needCheck: bool - accessCriteira: str + # when to access + accessCriteria: str + # + owner: str class EKGAnalysisNodeSchema(EKGNodeSchema): - accessCriteira: str + # when to access + accessCriteria: str + # do summary or not + summarySwtich: bool + # summary template + dslTemplate: str class EKGPhenomenonNodeSchema(EKGNodeSchema): pass +class EKGPhenomenonNodeSchema(EKGNodeSchema): + pass + + +# Ekg Tool Schemas +class ToolSchema(NodeSchema): + teamid: str + version:str # yyyy-mm-dd HH:MM:SS + extra: str = '' + + +class EKGPToolTypeSchema(ToolSchema): + toolprotocol: str + status: bool + + +class EKGPToolSchema(EKGPToolTypeSchema): + input: str # jsonstr + output: str # jsonstr + + + +# SLS / Tbase class EKGGraphSlsSchema(BaseModel): # node_{NodeTypesEnum} - type: str - name: str - id: str - description: str + type: str = '' + id: str = '' + name: str = '' + description: str = '' start_id: str = '' end_id: str = '' # ADD/DELETE @@ -80,22 +137,26 @@ class EKGGraphSlsSchema(BaseModel): # {tool_id},{tool_id},{tool_id} tool: str = '' access_criteria: str = '' + teamid: str = '' class EKGNodeTbaseSchema(BaseModel): node_id: str node_type: str - # node_str = 'graph_id={graph_id}', use for searching by graph_id + # node_str = 'graph_id={graph_id}'/teamid, use for searching by graph_id/teamid node_str: str node_vector: List class EKGEdgeTbaseSchema(BaseModel): + # {start_id}:{end_id} edge_id: str edge_type: str + # start_id edge_source: str + # end_id edge_target: str - # edge_str = 'graph_id={graph_id}', use for searching by graph_id + # edge_str = 'graph_id={graph_id}'/teamid, use for searching by graph_id/teamid edge_str: str @@ -106,4 +167,37 @@ class EKGTbaseData(BaseModel): class EKGSlsData(BaseModel): nodes: list[EKGGraphSlsSchema] - edges: list[EKGGraphSlsSchema] \ No newline at end of file + edges: list[EKGGraphSlsSchema] + + +TYPE2SCHEMA = { + NodeTypesEnum.ANALYSIS.value: EKGAnalysisNodeSchema, + NodeTypesEnum.TASK.value: EKGTaskNodeSchema, + NodeTypesEnum.INTENT.value: EKGIntentNodeSchema, + NodeTypesEnum.PHENOMENON.value: EKGPhenomenonNodeSchema, + NodeTypesEnum.SCHEDULE.value: EKGScheduleNodeSchema, + NodeTypesEnum.TOOL.value: EKGPToolTypeSchema, + NodeTypesEnum.TOOL_INSTANCE.value: EKGPToolSchema, + NodeTypesEnum.edge.value: EKGEdgeSchema +} + + +##################### +##### yuque dsl ##### +##################### + +class YuqueDslNodeData(BaseModel): + id: str + label: str + type: str + +class YuqueDslEdgeData(BaseModel): + id: str + source: str + target: str + label: str + + +class YuqueDslDatas(BaseModel): + nodes: List[YuqueDslNodeData] + edges: List[YuqueDslEdgeData] \ No newline at end of file diff --git a/muagent/schemas/memory/__init__.py b/muagent/schemas/memory/__init__.py deleted file mode 100644 index 015f3ea..0000000 --- a/muagent/schemas/memory/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .auto_extract_graph_schema import * - - -__all__ = [ - "GNodeAbs", "GEdgeAbs", "GRelationAbs", "Attribute", "GNode", "GEdge", "GRelation", "ThemeEnums" -] \ No newline at end of file diff --git a/muagent/schemas/readme.md b/muagent/schemas/readme.md index 8b27998..7cd6427 100644 --- a/muagent/schemas/readme.md +++ b/muagent/schemas/readme.md @@ -11,8 +11,8 @@ xxResp as the http output ## xxRequest xxRequest as the http input -## xx -xx as the function/method output +## xxData +xxData as the function/method output ## xxParam xxParam as the function/method input diff --git a/muagent/service/ekg_construct/__init__.py b/muagent/service/ekg_construct/__init__.py new file mode 100644 index 0000000..58a61dd --- /dev/null +++ b/muagent/service/ekg_construct/__init__.py @@ -0,0 +1,5 @@ +from .ekg_construct_base import EKGConstructService + +__all__ = [ + "EKGConstructService" +] \ No newline at end of file diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 00b7047..1e76863 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -1,15 +1,37 @@ from loguru import logger +import re +import json +from typing import List, Dict +import numpy as np +import random +import uuid from muagent.schemas.ekg import * from muagent.schemas.db import * +from muagent.schemas.common import * from muagent.db_handler import * from muagent.orm import table_init +from muagent.connector.configs.generate_prompt import * + from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +from muagent.llm_models import * + from muagent.base_configs.env_config import KB_ROOT_PATH +from muagent.llm_models.get_embedding import get_embedding +from muagent.utils.common_utils import getCurrentDatetime, getCurrentTimestap + + +def getClassFields(model): + # 收集所有字段，包括继承自父类的字段 + all_fields = set(model.__annotations__.keys()) + for base in model.__bases__: + if hasattr(base, '__annotations__'): + all_fields.update(getClassFields(base)) + return all_fields class EKGConstructService: @@ -37,76 +59,341 @@ def __init__( self.embed_config: EmbedConfig = embed_config self.llm_config: LLMConfig = llm_config + self.model = getChatModelFromConfig(self.llm_config) + + + self.init_handler() + def init_handler(self, ): """Initializes Database VectorBase GraphDB TbaseDB""" self.init_vb() # self.init_db() - # self.init_tb() - # self.init_gb() + self.init_tb() + self.init_gb() def reinit_handler(self, do_init: bool=False): self.init_vb() # self.init_db() - # self.init_tb() - # self.init_gb() + self.init_tb() + self.init_gb() def init_tb(self, do_init: bool=None): - tb_dict = {"TbaseHandler": TbaseHandler} - tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) - tbase_args = { - "host": self.tb_config.host, - "port": self.tb_config.port, - "username": self.tb_config.username, - "password": self.tb_config.password, - } - self.vb = tb_class(tbase_args, self.tb_config.index_name) + if self.tb_config: + tb_dict = {"TbaseHandler": TbaseHandler} + tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) + self.tb = tb_class(tb_config=self.tb_config, index_name=self.tb_config.index_name) + # # create index + # if not self.tb.is_index_exists(): + # res = self.tb.create_index(schema=self.tb_config.extra_kwargs.get("schema", )) + # logger.info(f"tb init: {res}") + else: + self.tb = None def init_gb(self, do_init: bool=None): - pass - gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler} - gb_class = gb_dict.get(self.gb_config.gb_type, NetworkxHandler) - self.gb = gb_class(self.db_config) + if self.gb_config: + gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler, "GeaBaseHandler": GeaBaseHandler,} + gb_class = gb_dict.get(self.gb_config.gb_type, NetworkxHandler) + self.gb: GBHandler = gb_class(self.gb_config) + else: + self.gb = None def init_db(self, do_init: bool=None): - pass - db_dict = {"LocalFaissHandler": LocalFaissHandler} - db_class = db_dict.get(self.db_config.db_type) - self.db = db_class(self.db_config) + if self.db_config: + table_init() + db_dict = {"LocalFaissHandler": LocalFaissHandler} + db_class = db_dict.get(self.db_config.db_type) + self.db = db_class(self.db_config) + else: + self.db = None def init_vb(self, do_init: bool=None): - table_init() - vb_dict = {"LocalFaissHandler": LocalFaissHandler} - vb_class = vb_dict.get(self.vb_config.vb_type, LocalFaissHandler) - self.vb: LocalFaissHandler = vb_class(self.embed_config, vb_config=self.vb_config) + if self.vb_config: + vb_dict = {"LocalFaissHandler": LocalFaissHandler} + vb_class = vb_dict.get(self.vb_config.vb_type, LocalFaissHandler) + self.vb: LocalFaissHandler = vb_class(self.embed_config, vb_config=self.vb_config) + else: + self.vb = None def init_sls(self, do_init: bool=None): - sls_dict = {"AliYunSLSHandler": AliYunSLSHandler} - sls_class = sls_dict.get(self.sls_config.sls_type, AliYunSLSHandler) - self.vb: AliYunSLSHandler = sls_class(self.embed_config, vb_config=self.vb_config) + if self.sls_config: + sls_dict = {"AliYunSLSHandler": AliYunSLSHandler} + sls_class = sls_dict.get(self.sls_config.sls_type, AliYunSLSHandler) + self.sls: AliYunSLSHandler = sls_class(self.embed_config, vb_config=self.vb_config) + else: + self.sls = None + + def add_nodes(self, nodes: List[GNode], teamid: str): + nodetype2fields_dict = {} + for node in nodes: + node_type = node.type + node.attributes["teamid"] = teamid + node.attributes["gdb_timestamp"] = getCurrentTimestap() + node.attributes["version"] = getCurrentDatetime() + node.attributes.setdefault("extra", '{}') + + # check the data's key-value + schema = TYPE2SCHEMA.get(node_type,) + if node_type in nodetype2fields_dict: + fields = nodetype2fields_dict[node_type] + else: + fields = list(getClassFields(schema)) + nodetype2fields_dict[node_type] = fields + + flag = any([ + field not in node.attributes + for field in fields + if field not in ["start_id", "end_id", "id", "ID"] + ]) + if flag: + raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") + + tbase_nodes = [{ + "node_id": f'''ekg_node:{teamid}:{node.id}''', + "node_type": node.type, + "node_str": f"graph_id={teamid}", + "node_vector": np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() + } + for node in nodes + ] + result = {"tbase_nodes": tbase_nodes, "nodes": nodes} + return result + # try: + # gb_result = self.gb.add_nodes(nodes) + # tb_result = self.tb.insert_data_hash(tbase_nodes, key_name='node_id', need_etime=False) + # except Exception as e: + # pass + + # return gb_result or tb_result + + def add_edges(self, edges: List[GEdge], teamid: str): + edgetype2fields_dict = {} + for edge in edges: + edge_type = edge.type + edge.attributes["teamid"] = teamid + edge.attributes["@timestamp"] = getCurrentTimestap() + edge.attributes["gdb_timestamp"] = getCurrentTimestap() + edge.attributes["version"] = getCurrentDatetime() + edge.attributes["extra"] = '{}' + + # todo 根据边类型进行数据校验 + schema = TYPE2SCHEMA.get("edge",) + if edge_type in edgetype2fields_dict: + fields = edgetype2fields_dict[edge_type] + else: + fields = list(getClassFields(schema)) + edgetype2fields_dict[edge_type] = fields + + flag = any([field not in edge.attributes for field in fields if field not in ["dst_id", "src_id", "timestamp", "id"]]) + if flag: + raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") + + tbase_edges = [{ + 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", + 'edge_type': edge.type, + 'edge_source': edge.start_id, + 'edge_target': edge.end_id, + 'edge_str': f'graph_id={teamid}' + } + for edge in edges + ] + result = {"tbase_edges": tbase_edges, "edges": edges} + return result + try: + gb_result = self.gb.add_edges(edges) + tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) + except Exception as e: + pass + + return gb_result or tb_result + + def delete_nodes(self, nodes: List[GNode], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') + tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + delete_nodeids = [node.id for node in nodes] + tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] + delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] + + if len(tbase_missing_nodeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + + # delete the nodeids in tbase + tb_result = [] + for nodeid in delete_tbase_nodeids: + self.tb.delete(nodeid) + resp = self.tb.delete(nodeid) + tb_result.append(resp) + # logger.info(f'id={nodeid}, delete resp={resp}') + + # delete the nodeids in geabase + gb_result = self.gb.delete_nodes(delete_tbase_nodeids) + + return gb_result or tb_result + + def delete_edges(self, edges: List[GEdge], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='ekg_edge') + tbase_edgeids = [data['id'] for data in r.docs] # 存疑 + delete_edgeids = [f"edge:{edge.start_id}:{edge.end_id}" for edge in edges] + tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] + delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] + + if len(tbase_missing_edgeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") + + # delete the edgeids in tbase + tb_result = [] + for edgeid in delete_tbase_edgeids: + self.tb.delete(edgeid) + resp = self.tb.delete(edgeid) + tb_result.append(resp) + # logger.info(f'id={edgeid}, delete resp={resp}') + + # delete the nodeids in geabase + gb_result = self.gb.delete_edges(delete_tbase_edgeids) + + def update_nodes(self, nodes: List[GNode], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') + tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + update_nodeids = [node.id for node in nodes] + tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] + update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] + + if len(tbase_missing_nodeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + + # delete the nodeids in tbase + tb_result = [] + for node in nodes: + if node.id not in update_tbase_nodeids: continue + data = node.attributes + data.update({"node_id": node.id}) + resp = self.tb.insert_data_hash(data, key="node_id", need_etime=False) + tb_result.append(resp) + # logger.info(f'id={nodeid}, delete resp={resp}') + + # update the nodeids in geabase + gb_result = [] + for node in nodes: + if node.id not in update_tbase_nodeids: continue + resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) + gb_result.append(resp) + return gb_result or tb_result + + def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: + return self.gb.get_current_node({'id': nodeid}, node_type=node_type) + + def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str, hop: int = 10) -> Graph: + if hop >= 15: + raise Exception(f"hop can't be larger than 15, now hop is {hop}") + # filter the node which dont match teamid + result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop, select_attributes={"teamid": teamid}) + return result - def create_ekg(self, ): - ekg_router = {} + def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: - def text2graph(self, ): - # graph_id, alarms, steps + if text is None: return [] - # alarms, steps ==|llm|==> node_dict, edge_list, abnormal_dict + # if self.embed_config: + # raise Exception(f"can't use vector search, because there is no {self.embed_config}") - # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + # 直接检索文本 + # r = self.tb.search(text) + # nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) + # nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) + + if self.embed_config: + vector_dict = get_embedding( + self.embed_config.embed_engine, [text], + self.embed_config.embed_model_path, self.embed_config.model_device, + self.embed_config + ) + query_embedding = np.array(vector_dict[text]).astype(dtype=np.float32).tobytes() + base_query = f'(@teamid:{teamid})=>[KNN {top_k} @vector $vector AS distance]' + query_params = {"vector": query_embedding} + else: + query_embedding = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() + base_query = f'(@teamid:{teamid})' + query_params = {} + + r = self.tb.vector_search(base_query, query_params=query_params) + + return + + def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, teamid: str): + rootid = f"{teamid}" + result = self.gb.get_hop_infos({"@ID": nodeid}, node_type=node_type, hop=15) + + # 根据nodeid和teamid来检索path + + def create_ekg( + self, + text: str, + teamid: str, + service_name: str, + intent_text: str = None, + intent_nodes: List[str] = [], + all_intent_list: List=[], + do_save: bool = False + ): - # dsl2graph => write2kg + if intent_nodes: + ancestor_list = intent_nodes + elif intent_text: + ancestor_list, all_intent_list = self.get_intents(intent_text) + else: + raise Exception(f"must have intent infomation") - pass + if service_name == "dsl2graph": + reuslt = self.dsl2graph() + else: + # text2graph + result = self.text2graph(text, ancestor_list, all_intent_list, teamid) - def dsl2graph(self, ): - # dsl, write2kg, intent_node, graph_id + # do write + if do_save: + self.write2kg(result) - # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + return result - # dsl2graph => write2kg - pass + def alarm2graph( + self, + alarms: List[dict], + alarm_analyse_content: dict, + teamid: str, + do_save: bool = False + ): + ancestor_list, all_intent_list = self.get_intents_by_alarms(alarms) + + graph_datas_by_pathid = {} + for path_id, diagnose_path in enumerate(alarm_analyse_content["diagnose_path"]): + + content = f"路径:{diagnose_path['name']}\n" + cur_count = 1 + for idx, step in enumerate(diagnose_path['diagnose_step']): + step_text = step['content'] if type(step['content']) == str else \ + step['content']['textInfo']['text'] + step_text = step_text.strip('[').strip(']') + step_text_no_time = EKGConstructService.remove_time(step_text) + # continue update + content += f'''{cur_count}. {step_text_no_time}\n''' + cur_count += 1 + + result = self.create_ekg( + content, teamid, service_name="text2graph", + intent_nodes=ancestor_list, all_intent_list=all_intent_list, + do_save= do_save + ) + graph_datas_by_pathid[path_id] = result + + return self.returndsl(graph_datas_by_pathid, intents=ancestor_list) def yuque2graph(self, **kwargs): + # get yuque loader + + # + self.create_ekg() # yuque_url, write2kg, intent_node # get_graph(yuque_url) @@ -117,7 +404,29 @@ def yuque2graph(self, **kwargs): # dsl2graph => write2kg pass - def write2kg(self, graph_id: str, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData): + def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], teamid: str) -> dict: + # generate graph by llm + result = self.get_graph_by_text(text, ) + # convert llm contet to database schema + sls_graph = self.transform2sls(result, intents, teamid=teamid) + tbase_graph = self.transform2tbase(sls_graph, teamid=teamid) + dsl_graph = self.transform2dsl(sls_graph, intents, all_intent_list, teamid=teamid) + return {"tbase_graph": tbase_graph, "sls_graph": sls_graph, "dsl_graph": dsl_graph} + + def dsl2graph(self, ) -> dict: + # dsl, write2kg, intent_node, graph_id + + # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict + + # dsl2graph => write2kg + pass + + def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData): + + # self.gb.add_nodes(result) + # self.gb.add_edges(result) + # self.tb.insert_data_hash(result) + # dsl2graph => write2kg ## delete tbase/graph by graph_id ### diff the tabse within newest by graph_id @@ -125,26 +434,314 @@ def write2kg(self, graph_id: str, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGT ## update tbase/graph by graph_id pass - def get_intent(self, content: dict, ) -> EKGIntentResp: - '''according content search intent''' - pass - - def get_intents(self, contents: list[dict], ) -> EKGIntentResp: + def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: + # 返回值需要返回 dsl 结构的数据用于展示，这里稍微做下数据处理，但主要就需要 dsl 对应的值 + res = {'dsl': '', 'details': {}, 'intent_node_list': intents} + + merge_dsl_nodes, merge_dsl_edges = [], [] + id_sets = set() + for path_id, graph_datas in graph_datas_by_path.items(): + res['details'][path_id] = { + 'dsl': graph_datas["dsl_graph"], + 'sls': graph_datas["sls_graph"] + } + merge_dsl_nodes.extent([node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets]) + id_sets.update([i.id for i in graph_datas["dsl_graph"].nodes]) + merge_dsl_edges.extent([edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets]) + id_sets.update([i.id for i in graph_datas["dsl_graph"].edges]) + res["dsl"] = {"nodes": merge_dsl_nodes, "edges": merge_dsl_edges} + return res + + def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: '''according contents search intents''' - pass + ancestor_list = set() + all_intent_list = [] + for alarm in alarm_list: + ancestor, all_intent = self.get_intent_by_alarm(alarm) + if ancestor is None: + continue + ancestor_list.add(ancestor) + all_intent_list.append(all_intent) + + return list(ancestor_list), all_intent_list + + def get_intents_by_alarms(self, alarm_list: list[dict], ) -> EKGIntentResp: + '''according contents search intents''' + ancestor_list = set() + all_intent_list = [] + for alarm in alarm_list: + ancestor, all_intent = self.get_intent_by_alarm(alarm) + if ancestor is None: + continue + ancestor_list.add(ancestor) + all_intent_list.append(all_intent) + + return list(ancestor_list), all_intent_list - def get_node_edge_dict(self, cotents: list[dict], ) -> EKGSlsData: - '''according contents generate ekg's raw datas''' - # code2graph - pass - - # def transform2sls(self, ekg_sls_data: EKGSlsData) -> list[EKGGraphSlsSchema]: - # pass - - def transform2tbase(self, ekg_sls_data: EKGSlsData) -> EKGTbaseData: - pass - - def transform2dsl(self, ekg_sls_data: EKGSlsData): + def get_graph_by_text(self, text: str) -> EKGSlsData: + '''according text generate ekg's raw datas''' + prompt = createText2EKGPrompt(text) + content = self.model.predict(prompt) + # get json part from answer + pat_str = r'\{.*\}' + match = re.search(pat_str, content, re.DOTALL) + json_str = match.group(0) + try: + node_edge_dict = json.loads(json_str) + except: + node_edge_dict = eval(json_str) + + return node_edge_dict + + def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str='') -> EKGSlsData: + # type类型处理也要注意下 + sls_nodes, sls_edges = [], [] + for node_idx, node_info in node_edge_dict['nodes'].items(): + node_type = node_info['type'].lower() + node_id = str(uuid.uuid4()) + node_info['node_id_new'] = node_id + + ekg_slsdata = EKGGraphSlsSchema( + id=node_id, + type='node_' + node_type, + name=node_info['content'], + description=node_info['content'], + tool='', + need_check='false', + operation_type='ADD', + teamid=teamid + ) + sls_nodes.append(ekg_slsdata) + + # 追加边关系 + if node_idx == '0': + for pid in pnode_ids: + sls_edges.append( + EKGGraphSlsSchema( + start_id=pid, + type=f'edge_route_intent_{node_type}', # 需要注意与老逻辑的兼容 + end_id=node_id, + operation_type='ADD', + teamid=teamid + ) + ) + # edges + for node_pair in node_edge_dict['edges']: + start_node = node_edge_dict['nodes'][node_pair['start']] + end_node = node_edge_dict['nodes'][node_pair['end']] + # + start_id = start_node['node_id_new'] + end_id = end_node['node_id_new'] + src_type, dst_type = start_node['type'].lower(), end_node['type'].lower() + # 需要注意与老逻辑的兼容 + edge_type = f'edge_route_{src_type}_{dst_type}' + sls_edges.append( + EKGGraphSlsSchema( + start_id=start_id, + type=edge_type, + end_id=end_id, + operation_type='ADD', + teamid=teamid + ) + ) + return EKGSlsData(nodes=sls_nodes, edges=sls_edges) + + def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData: + tbase_nodes, tbase_edges = [], [] + + for node in ekg_sls_data.nodes: + tbase_nodes.append( + EKGNodeTbaseSchema( + node_id=node.id, + node_type=node.type, + node_str=f'graph_id={teamid}', + # 后续可用embedding完成替换 + node_vector=[random.random() for _ in range(768)], + ) + ) + for edge in ekg_sls_data.edges: + tbase_edges.append( + EKGEdgeTbaseSchema( + edge_id=edge.id, + edge_type=edge.type, + edge_source=edge.start_id, + edge_target=edge.end_id, + edge_str=f'graph_id={teamid}', + ) + ) + return EKGTbaseData(nodes=tbase_nodes, edges=tbase_edges) + + def transform2dsl(self, ekg_sls_data: EKGSlsData, pnode_ids: List[str], all_intents: List[str], teamid: str) -> YuqueDslDatas: '''define your personal dsl format and code''' - pass + def get_md5(s): + import hashlib + md = hashlib.md5() + md5_content = s + md.update(md5_content.encode('utf-8')) + res = md.hexdigest() + return res + + type_dict = { + 'schedule': 'start-end', + 'task': 'process', + 'analysis': 'data', + 'phenomenon': 'decision' + } + nodes, edges = [], [] + schedule_id = '' + for node in ekg_sls_data.nodes: + # 需要注意下 dsl的id md编码 + nodes.append( + YuqueDslNodeData(id=node.id, type=type_dict.get(node.type.split("node_")[-1]), label=node.description) + ) + # 记录 schedule id 用于添加意图节点的边 + if node.type.split("node_")[-1] == 'schedule': + schedule_id = node.id + + # 添加意图节点 + # 需要记录哪些是被添加过的 + added_intent = set() + intent_names_dict = {} + for pid in pnode_ids: + dsl_pid = get_md5(pid) + dsl_pid = f'ekg_node:{teamid}:intent:{dsl_pid}' + if dsl_pid not in intent_names_dict: + intent_names_dict[dsl_pid] = self.gb.get_current_node( + {'id': pid}, 'opsgptkg_intent').attributes["name"] + + nodes.append( + YuqueDslNodeData( + id=node.id, type='display', + label=intent_names_dict.get(dsl_pid, pid),) + ) + added_intent.add(dsl_pid) + + + for intent_list in all_intents: + for intent in intent_list: + # 存在业务逻辑需要注意 + if 'SRE_Agent' in intent: continue + + intent_id = get_md5(intent) + intent_id = f'ekg_node:{teamid}:intent:{intent_id}' + if intent_id not in intent_names_dict: + intent_names_dict[intent_id] = self.gb.get_current_node( + {'id': intent}, 'opsgptkg_intent').attributes["name"] + + if intent_id not in added_intent: + nodes.append( + YuqueDslNodeData( + id=intent_id, type='display', + label=intent_names_dict.get(intent_id, intent),) + ) + added_intent.add(intent_id) + + for edge in ekg_sls_data.edges: + edges.append( + YuqueDslEdgeData( + id=f'{edge.start_id}___{edge.end_id}', + source=edge.start_id, + target=edge.end_id, + label='' + ) + ) + + # 添加意图边 + added_edges = set() + for pid in pnode_ids: + # 处理意图节点展示样式 + dsl_pid = get_md5(pid) + dsl_pid = f'ekg_node:{teamid}:intent:{dsl_pid}' + + # 处理意图节点展示样式 + edge_id = f'{dsl_pid}___{schedule_id}' + if edge_id not in added_edges: + edges.append( + YuqueDslEdgeData( + id=edge_id, + source=dsl_pid, + target=schedule_id, + label='' + ) + ) + added_edges.add(edge_id) + + for intent_list in all_intents: + for idx in range(len(intent_list[0:-1])): + if 'SRE_Agent' in intent_list[idx]: + continue + + if 'SRE_Agent' in intent_list[idx+1]: + continue + + start_id = get_md5(intent_list[idx]) + start_id = f'ekg_node:{teamid}:intent:{start_id}' + + end_id = get_md5(intent_list[idx+1]) + end_id = f'ekg_node:{teamid}:intent:{end_id}' + edge_id = f'{start_id}___{end_id}' + + if edge_id not in added_edges: + edges.append( + YuqueDslEdgeData( + id=edge_id, + source=start_id, + target=end_id, + label='' + ) + ) + added_edges.add(edge_id) + + return YuqueDslDatas(nodes=nodes, edges=edges) + + @staticmethod + def preprocess_json_contingency(content_dict, remove_time_flag=True): + # 专门处理告警数据合并成文本 + # 将对应的 content json 预处理为需要的样子，由于可能含有多个 path，用 dict 存储结果 + diagnose_path_list = content_dict.get('diagnose_path', []) + res = {} + if diagnose_path_list: + for idx, diagnose_path in enumerate(diagnose_path_list): + path_id = idx + path_name = diagnose_path['name'] + content = f'路径:{path_name}\n' + cur_count = 1 + + for idx, step in enumerate(diagnose_path['diagnose_step']): + step_name = step['name'] + + if type(step['content']) == str: + step_text = step['content'] + else: + step_text = step['content']['textInfo']['text'] + + step_text = step_text.strip('[') + step_text = step_text.strip(']') + + # step_text = step['step_summary'] + + step_text_no_time = EKGConstructService.remove_time(step_text) + to_append = f'''{cur_count}. {step_text_no_time}\n''' + cur_count += 1 + + content += to_append + + res[path_id] = { + 'path_id': path_id, + 'path_name': path_name, + 'content': content + } + return res + + @staticmethod + def remove_time(text): + re_pat = '''\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}(:\d{2})*''' + text_res = re.split(re_pat, text) + res = '' + for i in text_res: + if i: + i_strip = i.strip('，。\n') + i_strip = f'{i_strip}' + res += i_strip + return res \ No newline at end of file diff --git a/muagent/service/ekg_construct/ekg_db_service.py b/muagent/service/ekg_construct/ekg_db_service.py new file mode 100644 index 0000000..6793c8b --- /dev/null +++ b/muagent/service/ekg_construct/ekg_db_service.py @@ -0,0 +1,243 @@ +from typing import List, Dict +import numpy as np +import random + +from .ekg_construct_base import * +from muagent.schemas.common import * + +from muagent.llm_models.get_embedding import get_embedding +from muagent.utils.common_utils import getCurrentDatetime, getCurrentTimestap + + +def getClassFields(model): + # 收集所有字段，包括继承自父类的字段 + all_fields = set(model.__annotations__.keys()) + for base in model.__bases__: + if hasattr(base, '__annotations__'): + all_fields.update(getClassFields(base)) + return all_fields + + +class EKGDBService(EKGConstructService): + + def __init__( + self, + embed_config: EmbedConfig, + llm_config: LLMConfig, + db_config: DBConfig = None, + vb_config: VBConfig = None, + gb_config: GBConfig = None, + tb_config: TBConfig = None, + sls_config: SLSConfig = None, + do_init: bool = False, + kb_root_path: str = KB_ROOT_PATH + ): + super().__init__(embed_config, llm_config, db_config, vb_config, gb_config, tb_config, sls_config, do_init, kb_root_path) + + def add_nodes(self, nodes: List[GNode], teamid: str): + nodetype2fields_dict = {} + for node in nodes: + node_type = node.type + node.attributes["teamid"] = teamid + node.attributes["gdb_timestamp"] = getCurrentTimestap() + node.attributes["version"] = getCurrentDatetime() + node.attributes.setdefault("extra", '{}') + + # todo 根据节点类型进行数据校验 + schema = node_type + if node_type in nodetype2fields_dict: + fields = nodetype2fields_dict[node_type] + else: + fields = list(getClassFields(schema)) + nodetype2fields_dict[node_type] = fields + + flag = any([ + field not in node.attributes + for field in fields + if field not in ["start_id", "end_id", "id"] + ]) + if flag: + raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") + + tbase_nodes = [{ + "node_id": f'''ekg_node:{teamid}:{node.id}''', + "node_type": node.type, + "node_str": f"graph_id={teamid}", + "node_vector": np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() + } + for node in nodes + ] + + try: + gb_result = self.gb.add_nodes(nodes) + tb_result = self.tb.insert_data_hash(tbase_nodes, key_name='node_id', need_etime=False) + except Exception as e: + pass + + return gb_result or tb_result + + def add_edges(self, edges: List[GEdge], teamid: str): + edgetype2fields_dict = {} + for edge in edges: + edge_type = edge.type + edge.attributes["teamid"] = teamid + edge.attributes["@timestamp"] = getCurrentTimestap() + edge.attributes["gdb_timestamp"] = getCurrentTimestap() + edge.attributes["version"] = getCurrentDatetime() + edge.attributes["extra"] = '{}' + + # todo 根据边类型进行数据校验 + schema = edge_type + if edge_type in edgetype2fields_dict: + fields = edgetype2fields_dict[edge_type] + else: + fields = list(getClassFields(schema)) + edgetype2fields_dict[edge_type] = fields + + flag = any([field not in edge.attributes for field in fields if field not in ["start_id", "end_id", "id"]]) + if flag: + raise Exception(f"edge is wrong, type is {edge_type}, data is {edge.attributes}") + + tbase_edges = [{ + 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", + 'edge_type': edge.type, + 'edge_source': edge.start_id, + 'edge_target': edge.end_id, + 'edge_str': f'graph_id={teamid}' + } + for edge in edges + ] + + try: + gb_result = self.gb.add_edges(edges) + tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) + except Exception as e: + pass + + return gb_result or tb_result + + def delete_nodes(self, nodes: List[GNode], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') + tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + delete_nodeids = [node.id for node in nodes] + tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] + delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] + + if len(tbase_missing_nodeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + + # delete the nodeids in tbase + tb_result = [] + for nodeid in delete_tbase_nodeids: + self.tb.delete(nodeid) + resp = self.tb.delete(nodeid) + tb_result.append(resp) + # logger.info(f'id={nodeid}, delete resp={resp}') + + # delete the nodeids in geabase + gb_result = self.gb.delete_nodes(delete_tbase_nodeids) + + return gb_result or tb_result + + def delete_edges(self, edges: List[GEdge], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='ekg_edge') + tbase_edgeids = [data['id'] for data in r.docs] # 存疑 + delete_edgeids = [f"edge:{edge.start_id}:{edge.end_id}" for edge in edges] + tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] + delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] + + if len(tbase_missing_edgeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") + + # delete the edgeids in tbase + tb_result = [] + for edgeid in delete_tbase_edgeids: + self.tb.delete(edgeid) + resp = self.tb.delete(edgeid) + tb_result.append(resp) + # logger.info(f'id={edgeid}, delete resp={resp}') + + # delete the nodeids in geabase + gb_result = self.gb.delete_edges(delete_tbase_edgeids) + + def update_nodes(self, nodes: List[GNode], teamid: str): + # delete tbase nodes + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') + tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + update_nodeids = [node.id for node in nodes] + tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] + update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] + + if len(tbase_missing_nodeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + + # delete the nodeids in tbase + tb_result = [] + for node in nodes: + if node.id not in update_tbase_nodeids: continue + data = node.attributes + data.update({"node_id": node.id}) + resp = self.tb.insert_data_hash(data, key="node_id", need_etime=False) + tb_result.append(resp) + # logger.info(f'id={nodeid}, delete resp={resp}') + + # update the nodeids in geabase + gb_result = [] + for node in nodes: + if node.id not in update_tbase_nodeids: continue + resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) + gb_result.append(resp) + return gb_result or tb_result + + def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: + result = self.gb.get_current_node({'id': nodeid}, node_type=node_type) + nodeid = result.pop("id") + node_type = result.pop("node_type") + return GNode(id=nodeid, type=node_type, attributes=result) + + def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str, hop: int = 10) -> Graph: + if hop >= 15: + raise Exception(f"hop can't be larger than 15, now hop is {hop}") + # filter the node which dont match teamid + result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop, select_attributes={"teamid": teamid}) + return result + + def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: + + if text is None: return [] + + # if self.embed_config: + # raise Exception(f"can't use vector search, because there is no {self.embed_config}") + + # 直接检索文本 + # r = self.tb.search(text) + # nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) + # nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) + + if self.embed_config: + vector_dict = get_embedding( + self.embed_config.embed_engine, [text], + self.embed_config.embed_model_path, self.embed_config.model_device, + self.embed_config + ) + query_embedding = np.array(vector_dict[text]).astype(dtype=np.float32).tobytes() + base_query = f'(@teamid:{teamid})=>[KNN {top_k} @vector $vector AS distance]' + query_params = {"vector": query_embedding} + else: + query_embedding = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() + base_query = f'(@teamid:{teamid})' + query_params = {} + + r = self.tb.vector_search(base_query, query_params=query_params) + + return + + def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, teamid: str): + rootid = f"{teamid}" + result = self.gb.get_hop_infos({"@ID": nodeid}, node_type=node_type, hop=15) + + # 根据nodeid和teamid来检索path + + diff --git a/muagent/utils/common_utils.py b/muagent/utils/common_utils.py index 074de60..73cf6f4 100644 --- a/muagent/utils/common_utils.py +++ b/muagent/utils/common_utils.py @@ -16,6 +16,8 @@ def getCurrentDatetime(): return datetime.now().strftime("%Y-%m-%d %H:%M:%S") +def getCurrentTimestap(): + return int(datetime.now().timestamp()) def addMinutesToTime(input_time: str, n: int = 5, dateformat=DATE_FORMAT): dt = datetime.strptime(input_time, dateformat) diff --git a/tests/db_handler/geabase_hanlder_test.py b/tests/db_handler/geabase_hanlder_test.py index 348c79e..b7768a4 100644 --- a/tests/db_handler/geabase_hanlder_test.py +++ b/tests/db_handler/geabase_hanlder_test.py @@ -24,7 +24,7 @@ ) sys.path.append(src_dir) from muagent.db_handler import GeaBaseHandler -from muagent.schemas.memory import GNode, GRelation +from muagent.schemas.common import GNode, GEdge from muagent.schemas.db import GBConfig import copy @@ -46,8 +46,8 @@ # it's a case, you should use your node and edge attributes node1 = GNode(**{ "id": "antshanshi311395_1", + "type": "opsgptkg_intent", "attributes": { - "type": "opsgptkg_intent", "path": "shanshi_test", "name": "shanshi_test", "description":'shanshi_test', @@ -55,24 +55,26 @@ } }) -edge1 = GRelation(**{ +edge1 = GEdge(**{ "start_id": "antshanshi311395_1", "end_id": "antshanshi311395_2", + "type": "opsgptkg_intent_route_opsgptkg_intent", "attributes": { - "type": "opsgptkg_intent_route_opsgptkg_intent", - "original_dst_id2__": "antshanshi311395_1", - "original_src_id1__": "antshanshi311395_2", + "@timestamp": 1719276995619, + "original_src_id1__": "antshanshi311395_1", + "original_dst_id2__": "antshanshi311395_2", "gdb_timestamp": 1719276995619 } }) -edge2 = GRelation(**{ +edge2 = GEdge(**{ "start_id": "antshanshi311395_2", "end_id": "antshanshi311395_3", + "type": "opsgptkg_intent_route_opsgptkg_intent", "attributes": { - "type": "opsgptkg_intent_route_opsgptkg_intent", - "original_dst_id2__": "antshanshi311395_2", - "original_src_id1__": "antshanshi311395_3", + "@timestamp": 1719276995619, + "original_src_id1__": "antshanshi311395_2", + "original_dst_id2__": "antshanshi311395_3", "gdb_timestamp": 1719276995619 } }) @@ -90,9 +92,6 @@ t = geabase_handler.add_nodes([node2, node3]) print(t) -t = geabase_handler.add_nodes([node2, node3]) -print(t) - t = geabase_handler.add_edges([edge1, edge2]) print(t) diff --git a/tests/db_handler/networkx_handler_test.py b/tests/db_handler/networkx_handler_test.py index 52a3fe1..25eace2 100644 --- a/tests/db_handler/networkx_handler_test.py +++ b/tests/db_handler/networkx_handler_test.py @@ -12,7 +12,7 @@ sys.path.append(src_dir) from muagent import db_handler from muagent.db_handler import NetworkxHandler -from muagent.schemas.memory import GNode, GRelation +from muagent.schemas.common import GNode, GRelation node1 = GNode(**{"id": "node1", "attributes": {"name": "test" }}) From 8639800ffc0bf2a3ad36bdc3557d2fa4d85ad74e Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 7 Aug 2024 16:58:33 +0800 Subject: [PATCH 013/128] add ekg_construct test --- muagent/connector/configs/generate_prompt.py | 2 - .../graph_db_handler/base_gb_handler.py | 5 +- .../graph_db_handler/geabase_handler.py | 52 ++-- .../{__initl__.py => __init__.py} | 8 +- .../ekg_construct/ekg_construct_base.py | 273 +++++++++++++----- tests/db_handler/tbase_handler_test.py | 27 ++ tests/service/ekg_construct_test.py | 186 ++++++++++++ 7 files changed, 449 insertions(+), 104 deletions(-) rename muagent/db_handler/sql_db_hanlder/{__initl__.py => __init__.py} (95%) create mode 100644 tests/db_handler/tbase_handler_test.py create mode 100644 tests/service/ekg_construct_test.py diff --git a/muagent/connector/configs/generate_prompt.py b/muagent/connector/configs/generate_prompt.py index 955eee2..e4f565f 100644 --- a/muagent/connector/configs/generate_prompt.py +++ b/muagent/connector/configs/generate_prompt.py @@ -60,7 +60,5 @@ def createMKGPrompt(conversation, schemas, language="en", **kwargs) -> str: def createText2EKGPrompt(text, language="en", **kwargs) -> str: prompt = text2EKG_prompt_zh if language == "zh" else text2EKG_prompt_en prompt = replacePrompt(prompt, keys=["text"]) - from loguru import logger - logger.debug(f"{prompt}") prompt = prompt.format(**{"text": text,}) return cleanPrompt(prompt) \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/base_gb_handler.py b/muagent/db_handler/graph_db_handler/base_gb_handler.py index 5e55d29..afac9d1 100644 --- a/muagent/db_handler/graph_db_handler/base_gb_handler.py +++ b/muagent/db_handler/graph_db_handler/base_gb_handler.py @@ -58,6 +58,9 @@ def search_nodes_by_attr(self, attributes: dict) -> List[GNode]: def search_edges_by_attr(self, attributes: dict, edge_type: str = None) -> List[GEdge]: pass + def get_nodes_by_ids(self, ids: List[int]) -> List[GNode]: + pass + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: pass @@ -73,5 +76,5 @@ def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_key def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: pass - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}) -> Graph: + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse: bool =False) -> Graph: pass diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 1582c1b..c66c4e9 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -132,19 +132,26 @@ def get_nodeIDs(self, attributes: dict, node_type: str) -> List[int]: def get_current_nodeID(self, attributes: dict, node_type: str) -> int: result = self.get_current_node(attributes, node_type) return result.attributes.get("ID") - return result.get("ID") def get_current_edgeID(self, src_id, dst_id, edeg_type:str = None): if not isinstance(src_id, int) or not isinstance(dst_id, int): result = self.get_current_edge(src_id, dst_id, edeg_type) logger.debug(f"{result}") return result.attributes.get("srcId"), result.attributes.get("dstId"), result.attributes.get("timestamp") - return result.get("srcId"), result.get("dstId"), else: return src_id, dst_id, 1 def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: return self.get_current_nodes(attributes, node_type, return_keys)[0] + + def get_nodes_by_ids(self, ids: List[int] = []) -> List[GNode]: + where_str = f'@id in {ids}' + gql = f"MATCH (n0 WHERE {where_str}) RETURN n0" + # + result = self.execute(gql, return_keys=[]) + result = self.decode_result(result, gql) + nodes = result.get("n0", []) or result.get("n0.attr", []) + return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: # @@ -170,8 +177,6 @@ def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: li edges = result.get("e", []) or result.get("e.attr", []) return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges][0] - - return result[0] def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: # @@ -184,7 +189,6 @@ def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_key result = self.decode_result(result, gql) nodes = result.get("n1", []) or result.get("n1.attr", []) return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] - return result.get("n1", []) or result.get("n1.attr", []) def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: # @@ -198,22 +202,23 @@ def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_key edges = result.get("e", []) or result.get("e.attr", []) return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges] - return result.get("e", []) or result.get("e.attr", []) def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}) -> bool: result = self.get_neighbor_nodes(attributes, node_type,) filter_result = [i for i in result if all([item in i.attributes.items() for item in check_attributes.items()])] return len(filter_result) > 0 - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}) -> Graph: + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse=False) -> Graph: ''' hop >= 2， 表面需要至少两跳 ''' hop_max = 10 - hop_list = [] # where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) - gql = f"MATCH p = (n0:{node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + if reverse: + gql = f"MATCH p = (n0:{node_type} WHERE {where_str})<-[e]-{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + else: + gql = f"MATCH p = (n0:{node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" last_node_ids, last_node_types = [], [] result = {} @@ -238,19 +243,16 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b nodes = [GNode(id=node["id"], type=node["type"], attributes=node) for node in result.get("n1", [])] edges = [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in result.get("e", [])] return Graph(nodes=nodes, edges=edges, paths=result.get("p", [])) - return result - + def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.nodes - return result.get("n1", []) or result.get("n1.attr", []) def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GEdge]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.edges - return result.get("e", []) or result.get("e.attr", []) def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[str]: # @@ -353,14 +355,28 @@ def decode_result(self, geabase_result, gql: str) -> Dict: return output def decode_path(self, col_data, k) -> List: - path = [] steps = col_data.get("pathVal", {}).get("steps", []) + connections = {} for step in steps: props = step["props"] - if path == []: - path.append(props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1)) - - path.append(props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1)) + # if path == []: + # path.append(props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1)) + # path.append(props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1)) + + start = props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1) + end = props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1) + connections[start] = end + + # 找到头部（1） + for k in connections: + if k not in connections.values(): + head = k + path = [head] + + # 根据连通关系构建路径 + while head in connections: + head = connections[head] + path.append(head) return path diff --git a/muagent/db_handler/sql_db_hanlder/__initl__.py b/muagent/db_handler/sql_db_hanlder/__init__.py similarity index 95% rename from muagent/db_handler/sql_db_hanlder/__initl__.py rename to muagent/db_handler/sql_db_hanlder/__init__.py index bcd999f..9d01107 100644 --- a/muagent/db_handler/sql_db_hanlder/__initl__.py +++ b/muagent/db_handler/sql_db_hanlder/__init__.py @@ -1,5 +1,5 @@ -from .sqlalchemy_handler import SqlalchemyHandler - -__all__ = [ - "SqlalchemyHandler" +from .sqlalchemy_handler import SqlalchemyHandler + +__all__ = [ + "SqlalchemyHandler" ] \ No newline at end of file diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 1e76863..6c1bcb1 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -5,6 +5,13 @@ import numpy as np import random import uuid +from redis.commands.search.field import ( + TextField, + NumericField, + VectorField, + TagField +) +from jieba.analyse import extract_tags from muagent.schemas.ekg import * @@ -78,14 +85,51 @@ def reinit_handler(self, do_init: bool=False): self.init_gb() def init_tb(self, do_init: bool=None): + + DIM = 768 # it depends on your embedding model vector + NODE_SCHEMA = [ + TextField("node_id", ), + TextField("node_type", ), + TextField("node_str", ), + VectorField("name_vector", + 'FLAT', + { + "TYPE": "FLOAT32", + "DIM": DIM, + "DISTANCE_METRIC": "COSINE" + }), + VectorField("desc_vector", + 'FLAT', + { + "TYPE": "FLOAT32", + "DIM": DIM, + "DISTANCE_METRIC": "COSINE" + }), + TagField(name='name_keyword', separator='|'), + TagField(name='desc_keyword', separator='|') + ] + + EDGE_SCHEMA = [ + TextField("edge_id", ), + TextField("edge_type", ), + TextField("edge_source", ), + TextField("edge_target", ), + TextField("edge_str", ), + ] + + if self.tb_config: tb_dict = {"TbaseHandler": TbaseHandler} tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) - self.tb = tb_class(tb_config=self.tb_config, index_name=self.tb_config.index_name) + self.tb: TbaseHandler = tb_class(tb_config=self.tb_config, index_name=self.tb_config.index_name, definition_value="opsgptkg") # # create index - # if not self.tb.is_index_exists(): - # res = self.tb.create_index(schema=self.tb_config.extra_kwargs.get("schema", )) - # logger.info(f"tb init: {res}") + if not self.tb.is_index_exists("opsgptkg_node"): + res = self.tb.create_index(index_name="opsgptkg_node", schema=NODE_SCHEMA) + logger.info(f"tb init: {res}") + + if not self.tb.is_index_exists("opsgptkg_edge"): + res = self.tb.create_index(index_name="opsgptkg_edge", schema=EDGE_SCHEMA) + logger.info(f"tb init: {res}") else: self.tb = None @@ -131,7 +175,7 @@ def add_nodes(self, nodes: List[GNode], teamid: str): node.attributes["version"] = getCurrentDatetime() node.attributes.setdefault("extra", '{}') - # check the data's key-value + # check the data's key-value by node_type schema = TYPE2SCHEMA.get(node_type,) if node_type in nodetype2fields_dict: fields = nodetype2fields_dict[node_type] @@ -147,23 +191,32 @@ def add_nodes(self, nodes: List[GNode], teamid: str): if flag: raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") - tbase_nodes = [{ - "node_id": f'''ekg_node:{teamid}:{node.id}''', - "node_type": node.type, - "node_str": f"graph_id={teamid}", - "node_vector": np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() - } - for node in nodes - ] - result = {"tbase_nodes": tbase_nodes, "nodes": nodes} - return result - # try: - # gb_result = self.gb.add_nodes(nodes) - # tb_result = self.tb.insert_data_hash(tbase_nodes, key_name='node_id', need_etime=False) - # except Exception as e: - # pass + tbase_nodes = [] + for node in nodes: + name = node.attributes.get("name", "") + description = node.attributes.get("description", "") + name_vector = self._get_embedding(name) + desc_vector = self._get_embedding(description) + tbase_nodes.append({ + # "node_id": f'''ekg_node:{teamid}:{node.id}''', + "node_id": f'''{node.id}''', + "node_type": node.type, + "node_str": f"graph_id={teamid}", + "name_vector": np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), + "desc_vector": np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), + "name_keyword": " | ".join(extract_tags(name, topK=None)), + "desc_keyword": " | ".join(extract_tags(description, topK=None)), + }) + + tb_result = {"error": True} + try: + # gb_result = self.gb.add_nodes(nodes) + tb_result = self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False) + except Exception as e: + logger.error(e) + return tb_result - # return gb_result or tb_result + return gb_result or tb_result def add_edges(self, edges: List[GEdge], teamid: str): edgetype2fields_dict = {} @@ -175,7 +228,7 @@ def add_edges(self, edges: List[GEdge], teamid: str): edge.attributes["version"] = getCurrentDatetime() edge.attributes["extra"] = '{}' - # todo 根据边类型进行数据校验 + # check the data's key-value by edge_type schema = TYPE2SCHEMA.get("edge",) if edge_type in edgetype2fields_dict: fields = edgetype2fields_dict[edge_type] @@ -188,7 +241,8 @@ def add_edges(self, edges: List[GEdge], teamid: str): raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") tbase_edges = [{ - 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", + # 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", + 'edge_id': f"{edge.start_id}__{edge.end_id}", 'edge_type': edge.type, 'edge_source': edge.start_id, 'edge_target': edge.end_id, @@ -196,23 +250,27 @@ def add_edges(self, edges: List[GEdge], teamid: str): } for edge in edges ] - result = {"tbase_edges": tbase_edges, "edges": edges} - return result + + tb_result = {"error": True} try: - gb_result = self.gb.add_edges(edges) + # gb_result = self.gb.add_edges(edges) tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) except Exception as e: - pass + logger.error(e) + + return tb_result return gb_result or tb_result def delete_nodes(self, nodes: List[GNode], teamid: str): # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') - tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='opsgptkg_node') + + tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 + tbase_nodeids_dict = {data["node_id"]:data['id'] for data in r.docs} # 附带了definition信息 delete_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] - delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] + delete_tbase_nodeids = [tbase_nodeids_dict[nodeid] for nodeid in delete_nodeids if nodeid in tbase_nodeids] if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") @@ -225,18 +283,19 @@ def delete_nodes(self, nodes: List[GNode], teamid: str): tb_result.append(resp) # logger.info(f'id={nodeid}, delete resp={resp}') - # delete the nodeids in geabase - gb_result = self.gb.delete_nodes(delete_tbase_nodeids) - + # # delete the nodeids in geabase + # gb_result = self.gb.delete_nodes(delete_tbase_nodeids) + return tb_result return gb_result or tb_result def delete_edges(self, edges: List[GEdge], teamid: str): # delete tbase nodes - r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='ekg_edge') - tbase_edgeids = [data['id'] for data in r.docs] # 存疑 - delete_edgeids = [f"edge:{edge.start_id}:{edge.end_id}" for edge in edges] + r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='opsgptkg_edge') + tbase_edgeids = [data['edge_id'] for data in r.docs] + tbase_edgeids_dict = {data["edge_id"]:data['id'] for data in r.docs} # id附带了definition信息 + delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] - delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] + delete_tbase_edgeids = [tbase_edgeids_dict[edgeid] for edgeid in delete_edgeids if edgeid in tbase_edgeids] if len(tbase_missing_edgeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") @@ -249,13 +308,14 @@ def delete_edges(self, edges: List[GEdge], teamid: str): tb_result.append(resp) # logger.info(f'id={edgeid}, delete resp={resp}') - # delete the nodeids in geabase - gb_result = self.gb.delete_edges(delete_tbase_edgeids) + # # delete the nodeids in geabase + # gb_result = self.gb.delete_edges(delete_tbase_edgeids) + return tb_result def update_nodes(self, nodes: List[GNode], teamid: str): # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') - tbase_nodeids = [data['id'] for data in r.docs] # 存疑 + r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='opsgptkg_node') + tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 update_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] @@ -263,30 +323,45 @@ def update_nodes(self, nodes: List[GNode], teamid: str): if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - # delete the nodeids in tbase tb_result = [] + random_vector = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() for node in nodes: - if node.id not in update_tbase_nodeids: continue - data = node.attributes - data.update({"node_id": node.id}) - resp = self.tb.insert_data_hash(data, key="node_id", need_etime=False) + if node.id not in tbase_nodeids: continue + + tbase_data = {} + for key in ["name", "description"]: + if key not in node.attributes: continue + + if key == "name": + text = node.attributes.get("name", "") + tbase_data["name_keyword"] = " | ".join(extract_tags(text, topK=None)) + tbase_data["name_vector"] = np.array(self._get_embedding(text)[text] + ).astype(dtype=np.float32).tobytes() + + if key == "description": + text = node.attributes.get("description", "") + tbase_data["desc_keyword"] = " | ".join(extract_tags(text, topK=None)) + tbase_data["desc_vector"] = np.array(self._get_embedding(text)[text] + ).astype(dtype=np.float32).tobytes() + tbase_data["node_id"] = node.id + + resp = self.tb.insert_data_hash(tbase_data, key="node_id", need_etime=False) tb_result.append(resp) - # logger.info(f'id={nodeid}, delete resp={resp}') # update the nodeids in geabase gb_result = [] - for node in nodes: - if node.id not in update_tbase_nodeids: continue - resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) - gb_result.append(resp) + # for node in nodes: + # if node.id not in update_tbase_nodeids: continue + # resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) + # gb_result.append(resp) return gb_result or tb_result def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: return self.gb.get_current_node({'id': nodeid}, node_type=node_type) def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str, hop: int = 10) -> Graph: - if hop >= 15: - raise Exception(f"hop can't be larger than 15, now hop is {hop}") + if hop > 14: + raise Exception(f"hop can't be larger than 14, now hop is {hop}") # filter the node which dont match teamid result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop, select_attributes={"teamid": teamid}) return result @@ -299,33 +374,59 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N # raise Exception(f"can't use vector search, because there is no {self.embed_config}") # 直接检索文本 - # r = self.tb.search(text) - # nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) - # nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) - + keywords = extract_tags(text) + keyword = "|".join(keywords) + + nodeids = [] + # if self.embed_config: - vector_dict = get_embedding( - self.embed_config.embed_engine, [text], - self.embed_config.embed_model_path, self.embed_config.model_device, - self.embed_config - ) + vector_dict = self._get_embedding(text) query_embedding = np.array(vector_dict[text]).astype(dtype=np.float32).tobytes() - base_query = f'(@teamid:{teamid})=>[KNN {top_k} @vector $vector AS distance]' - query_params = {"vector": query_embedding} - else: - query_embedding = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() - base_query = f'(@teamid:{teamid})' - query_params = {} - - r = self.tb.vector_search(base_query, query_params=query_params) - return + nodeid_with_dist = [] + for key in ["name_vector", "desc_vector"]: + base_query = f'(@node_str: graph_id={teamid})=>[KNN {top_k} @{key} $vector AS distance]' + query_params = {"vector": query_embedding} + r = self.tb.vector_search(base_query, query_params=query_params) - def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, teamid: str): - rootid = f"{teamid}" - result = self.gb.get_hop_infos({"@ID": nodeid}, node_type=node_type, hop=15) - - # 根据nodeid和teamid来检索path + for i in r.docs: + nodeid_with_dist.append((i["node_id"], float(i["distance"]))) + + nodeid_with_dist = sorted(nodeid_with_dist, key=lambda x:x[1], reverse=False) + for nodeid, dis in nodeid_with_dist: + if nodeid not in nodeids: + nodeids.append(nodeid) + + for key in ["name_keyword", "desc_keyword"]: + r = self.tb.search(f"(@{key}:{{{keyword}}})") + for i in r.docs: + if i["node_id"] not in nodeids: + nodeids.append(i["node_id"]) + + nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) + nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) + nodes = self.gb.get_nodes_by_ids(nodeids) + return nodes_by_name + nodes_by_desc + nodes + + def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: + # rootid = f"{teamid}" # todo check the rootid + result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) + + # paths must be ordered from start to end + paths = result.paths + new_paths = [] + for path in paths: + try: + start_idx = path.index(rootid) + end_idx = path.index(nodeid) + new_paths.append(path[start_idx:end_idx+1]) + except: + pass + + nodeid_set = set([nodeid for path in paths for nodeid in path]) + new_nodes = [node for node in result.nodes if node.id in nodeid_set] + new_edges = [edge for edge in result.edges if edge.start_id in nodeid_set and edge.end_id in nodeid_set] + return Graph(nodes=new_nodes, edges=new_edges, paths=new_paths) def create_ekg( self, @@ -445,9 +546,9 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: 'dsl': graph_datas["dsl_graph"], 'sls': graph_datas["sls_graph"] } - merge_dsl_nodes.extent([node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets]) + merge_dsl_nodes.extend([node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets]) id_sets.update([i.id for i in graph_datas["dsl_graph"].nodes]) - merge_dsl_edges.extent([edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets]) + merge_dsl_edges.extend([edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets]) id_sets.update([i.id for i in graph_datas["dsl_graph"].edges]) res["dsl"] = {"nodes": merge_dsl_nodes, "edges": merge_dsl_edges} return res @@ -464,7 +565,7 @@ def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: all_intent_list.append(all_intent) return list(ancestor_list), all_intent_list - + def get_intents_by_alarms(self, alarm_list: list[dict], ) -> EKGIntentResp: '''according contents search intents''' ancestor_list = set() @@ -482,6 +583,8 @@ def get_graph_by_text(self, text: str) -> EKGSlsData: '''according text generate ekg's raw datas''' prompt = createText2EKGPrompt(text) content = self.model.predict(prompt) + # logger.debug(f"{prompt}") + # logger.debug(f"{content}") # get json part from answer pat_str = r'\{.*\}' match = re.search(pat_str, content, re.DOTALL) @@ -695,6 +798,18 @@ def get_md5(s): return YuqueDslDatas(nodes=nodes, edges=edges) + def _get_embedding(self, text): + text_vector = {} + if self.embed_config: + text_vector = get_embedding( + self.embed_config.embed_engine, [text], + self.embed_config.embed_model_path, self.embed_config.model_device, + self.embed_config + ) + else: + text_vector = {text: [random.random() for _ in range(768)]} + return text_vector + @staticmethod def preprocess_json_contingency(content_dict, remove_time_flag=True): # 专门处理告警数据合并成文本 diff --git a/tests/db_handler/tbase_handler_test.py b/tests/db_handler/tbase_handler_test.py new file mode 100644 index 0000000..3253d75 --- /dev/null +++ b/tests/db_handler/tbase_handler_test.py @@ -0,0 +1,27 @@ +import time + +from tqdm import tqdm +from loguru import logger + +import sys, os + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +print(src_dir) + +sys.path.append(src_dir) + + +print("os.getcwd(): ", os.getcwd()) + +# import muagent +# from muagent import db_handler + +# from muagent.dbhandler.vectordbhandler import tbase_handler +from muagent.db_handler.graph_db_handler.base_gb_handler import GBHandler +# from muagent.db_handler.graph_db_handler.networkx_handler import NetworkxHandler +# from muagent.db_handler.graph_db_handler.geabase_handler import GeaBaseHandler +# from muagent.db_handler.graph_db_handler.aliyun_sls_hanlder import AliYunSLSHandler +# from muagent.db_handler.graph_db_handler.nebula_handler import NebulaHandler + diff --git a/tests/service/ekg_construct_test.py b/tests/service/ekg_construct_test.py new file mode 100644 index 0000000..7fd3a80 --- /dev/null +++ b/tests/service/ekg_construct_test.py @@ -0,0 +1,186 @@ +import time +import sys, os +from loguru import logger + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) + +import os, sys +from loguru import logger + +sys.path.append("/ossfs/workspace/muagent") +sys.path.append("/ossfs/workspace/notebooks/custom_funcs") +from muagent.db_handler import GeaBaseHandler +from muagent.schemas.common import GNode, GEdge +from muagent.schemas.db import GBConfig, TBConfig +from muagent.service.ekg_construct import EKGConstructService +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig + + + + + +# 初始化 GeaBaseHandler 实例 +gb_config = GBConfig( + gb_type="GeaBaseHandler", + extra_kwargs={ + 'metaserver_address': os.environ['metaserver_address'], + 'project': os.environ['project'], + 'city': os.environ['city'], + 'lib_path': os.environ['lib_path'], + } +) + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="muagent_test", + host=os.environ['host'], + port=os.environ['port'], + username=os.environ['username'], + password=os.environ['password'], + extra_kwargs={ + 'host': os.environ['host'], + 'port': os.environ['port'], + 'username': os.environ['username'] , + 'password': os.environ['password'], + 'definition_value': os.environ['definition_value'] + } +) + +# llm config +llm_config = LLMConfig( + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +) + + +# emebdding config +# embed_config = EmbedConfig( +# embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +# ) + +# embed_config = EmbedConfig( +# embed_model="default", +# langchain_embeddings=embeddings +# ) +embed_config = None + +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) + + +import copy +# it's a case, you should use your node and edge attributes +node1 = GNode(**{ + "id": "antshanshi311395_1", + "type": "opsgptkg_intent", + "attributes": { + "path": "shanshi_test", + "name": "shanshi_test", + "description":'shanshi_test', + "gdb_timestamp": 1719276995619 + } +}) + +edge1 = GEdge(**{ + "start_id": "antshanshi311395_1", + "end_id": "antshanshi311395_2", + "type": "opsgptkg_intent_route_opsgptkg_intent", + "attributes": { + "@timestamp": 1719276995619, + "original_src_id1__": "antshanshi311395_1", + "original_dst_id2__": "antshanshi311395_2", + "gdb_timestamp": 1719276995619 + } +}) + +edge2 = GEdge(**{ + "start_id": "antshanshi311395_2", + "end_id": "antshanshi311395_3", + "type": "opsgptkg_intent_route_opsgptkg_intent", + "attributes": { + "@timestamp": 1719276995619, + "original_src_id1__": "antshanshi311395_2", + "original_dst_id2__": "antshanshi311395_3", + "gdb_timestamp": 1719276995619 + } +}) + +node2 = copy.deepcopy(node1) +node2.id = "antshanshi311395_2" + +node3 = copy.deepcopy(node1) +node3.id = "antshanshi311395_3" + +node1_copy = copy.deepcopy(node1) +node1_copy.attributes = { + "description":'nav 表示 "文档"和"社区" 导航栏，order 表示 "文档" 和 "社区"的顺序', +} +node2_copy = copy.deepcopy(node2) +node2_copy.attributes = { + "description":'用于两种模式，一种是根据语雀url去更新图谱（已发布），一种是应急经验沉淀直接传一个流程图json来更新图谱,文档,文档,文档', +} + +# 需要修改数据库属性后才可以进行测试 +t = ekg_construct_service.add_nodes([node1, node2, node3], teamid="shanshi_test") + +# 需要修改数据库属性后才可以进行测试 +t = ekg_construct_service.add_edges([edge1, edge2], teamid="shanshi_test") + +# 需要修改数据库属性后才可以进行测试 +t = ekg_construct_service.update_nodes([node1_copy, node2_copy], teamid="shanshi_test") +t + +# search_nodes_by_text +text = 'nav 表示 "文档"和"社区" 导航栏，order 表示 "文档" 和 "社区"的顺序' +text = '用于两种模式，一种是根据语雀url去更新图谱（已发布），一种是应急经验沉淀直接传一个流程图json来更新图谱,文档,文档,文档' +teamid = "shanshi_test" +t = ekg_construct_service.search_nodes_by_text(text, teamid=teamid) +t + +# +t = ekg_construct_service.search_rootpath_by_nodeid(nodeid="antshanshi311395_3", node_type="opsgptkg_intent", rootid="antshanshi311395_2") +t + + +# # 根据节点ID查询节点明细,需要修改数据库属性后才可以进行测试 +# t = ekg_construct_service.get_node_by_id('antshanshi311395_2', 'opsgptkg_intent') +# t + +# # 根据节点ID查询后续节点/边/路径的明细,需要修改数据库属性后才可以进行测试 +# t = ekg_construct_service.get_graph_by_nodeid('antshanshi311395_2', 'opsgptkg_intent', teamid="shanshi_test", hop = 10) +# t + +# # 删除节点和边 +# t = ekg_construct_service.delete_edges([edge1, edge2], teamid="shanshi_test") +# t = ekg_construct_service.delete_nodes([node1, node2, node3], teamid="shanshi_test") + From b7e69054e282222dea3a4e649b0ca2e0100a0701 Mon Sep 17 00:00:00 2001 From: lightislost Date: Mon, 12 Aug 2024 14:04:45 +0800 Subject: [PATCH 014/128] update new attr and ekg_construct_test_2 --- .../graph_db_handler/geabase_handler.py | 8 +- .../vector_db_handler/tbase_handler.py | 4 +- muagent/schemas/ekg/ekg_graph.py | 112 ++++- .../ekg_construct/ekg_construct_base.py | 441 ++++++++++++------ tests/service/ekg_construct_test_2.py | 234 ++++++++++ 5 files changed, 628 insertions(+), 171 deletions(-) create mode 100644 tests/service/ekg_construct_test_2.py diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index c66c4e9..2d67112 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -34,6 +34,7 @@ def __init__( def execute(self, gql: str, option=None, return_keys: list = []) -> Dict: option = option or self.option logger.info(f"{gql}") + # return {"error": True} result = self.geabase_client.executeGQL(gql, option) result = json.loads(str(result.getJsonGQLResponse())) return result @@ -178,12 +179,15 @@ def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: li edges = result.get("e", []) or result.get("e.attr", []) return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges][0] - def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = [], reverse=False) -> List[GNode]: # extra_keys = list(set(return_keys + ["@ID", "id", "@node_type"])) return_str = ", ".join([f"n1.{k}" for k in extra_keys]) if return_keys else "n1" where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) - gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" + if reverse: + gql = f"MATCH (n0:{node_type} WHERE {where_str})<-[e]-(n1) RETURN {return_str}" + else: + gql = f"MATCH (n0:{node_type})-[e]->(n1) WHERE {where_str} RETURN {return_str}" # result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) diff --git a/muagent/db_handler/vector_db_handler/tbase_handler.py b/muagent/db_handler/vector_db_handler/tbase_handler.py index 1cb4f8a..24b05d8 100644 --- a/muagent/db_handler/vector_db_handler/tbase_handler.py +++ b/muagent/db_handler/vector_db_handler/tbase_handler.py @@ -84,7 +84,7 @@ def insert_data_hash( rr = self.client.expire(key_value, expire_time or self.expire_time) return len(data_list) - def search(self, query, index_name: str = None, query_params: dict = {}): + def search(self, query, index_name: str = None, query_params: dict = {}, limit=10): ''' search :param index_name: @@ -96,7 +96,7 @@ def search(self, query, index_name: str = None, query_params: dict = {}): index = self.client.ft(index_name) if type(query) == str: - query = Query(query) + query = Query(query).paging(0, limit) res = index.search(query, query_params=query_params) return res diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index cb9fa16..9a90a60 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -1,6 +1,7 @@ from pydantic import BaseModel from typing import List, Dict from enum import Enum +import json SHAPE2TYPE = { @@ -54,39 +55,77 @@ class NodeTypesEnum(Enum): TOOL_INSTANCE = 'opsgptkg_tool_instance' TEAM = 'opsgptkg_team' OWNER = 'opsgptkg_owner' - edge = 'edge' - + EDGE = 'edge' # EKG Node and Edge Schemas class EKGNodeSchema(NodeSchema): - teamid: str - version:str # yyyy-mm-dd HH:MM:SS + teamids: str + # version:str # yyyy-mm-dd HH:MM:SS extra: str = '' + class EKGEdgeSchema(EdgeSchema): - teamid: str - version:str # yyyy-mm-dd HH:MM:SS + # teamids: str + # version:str # yyyy-mm-dd HH:MM:SS extra: str = '' + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return extra_attr + class EKGIntentNodeSchema(EKGNodeSchema): path: str = '' + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return { + **{ + "name": self.name, + "description": self.description, + "teamids": self.teamids, + "path": self.path + }, + **extra_attr + } class EKGScheduleNodeSchema(EKGNodeSchema): # do action or not - switch: bool - + enable: bool + + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return { + **{ + "name": self.name, + "description": self.description, + "teamids": self.teamids, + "enable": self.enable + }, + **extra_attr + } class EKGTaskNodeSchema(EKGNodeSchema): - tool: str - needCheck: bool + # tool: str + # needCheck: bool # when to access accessCriteria: str # - owner: str - + # owner: str + + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return { + **{ + "name": self.name, + "description": self.description, + "teamids": self.teamids, + "accessCriteria": self.accessCriteria + }, + **extra_attr + } + class EKGAnalysisNodeSchema(EKGNodeSchema): # when to access @@ -96,19 +135,38 @@ class EKGAnalysisNodeSchema(EKGNodeSchema): # summary template dslTemplate: str + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return { + **{ + "name": self.name, + "description": self.description, + "teamids": self.teamids, + "accessCriteria": self.accessCriteria, + "summarySwtich": self.summarySwtich, + "dslTemplate": self.dslTemplate + }, + **extra_attr + } + class EKGPhenomenonNodeSchema(EKGNodeSchema): - pass - - -class EKGPhenomenonNodeSchema(EKGNodeSchema): - pass + def attrbutes(self, ): + extra_attr = json.loads(self.extra) + return { + **{ + "name": self.name, + "description": self.description, + "teamids": self.teamids, + }, + **extra_attr + } + # Ekg Tool Schemas class ToolSchema(NodeSchema): - teamid: str - version:str # yyyy-mm-dd HH:MM:SS + # version:str # yyyy-mm-dd HH:MM:SS extra: str = '' @@ -137,15 +195,21 @@ class EKGGraphSlsSchema(BaseModel): # {tool_id},{tool_id},{tool_id} tool: str = '' access_criteria: str = '' - teamid: str = '' + teamids: str = '' + extra: str = '' + enable: bool = False + dslTemplate: str = '' class EKGNodeTbaseSchema(BaseModel): node_id: str node_type: str - # node_str = 'graph_id={graph_id}'/teamid, use for searching by graph_id/teamid + # node_str = 'graph_id={graph_id}'/teamids, use for searching by graph_id/teamids node_str: str - node_vector: List + name_keyword: str + desc_keyword: str + name_vector: List + desc_vector: List class EKGEdgeTbaseSchema(BaseModel): @@ -156,7 +220,7 @@ class EKGEdgeTbaseSchema(BaseModel): edge_source: str # end_id edge_target: str - # edge_str = 'graph_id={graph_id}'/teamid, use for searching by graph_id/teamid + # edge_str = 'graph_id={graph_id}'/teamids, use for searching by graph_id/teamids edge_str: str @@ -178,7 +242,7 @@ class EKGSlsData(BaseModel): NodeTypesEnum.SCHEDULE.value: EKGScheduleNodeSchema, NodeTypesEnum.TOOL.value: EKGPToolTypeSchema, NodeTypesEnum.TOOL_INSTANCE.value: EKGPToolSchema, - NodeTypesEnum.edge.value: EKGEdgeSchema + NodeTypesEnum.EDGE.value: EKGEdgeSchema } diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 6c1bcb1..71cd977 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -30,6 +30,7 @@ from muagent.llm_models.get_embedding import get_embedding from muagent.utils.common_utils import getCurrentDatetime, getCurrentTimestap +from muagent.utils.common_utils import double_hashing def getClassFields(model): @@ -65,10 +66,12 @@ def __init__( self.kb_root_path = kb_root_path self.embed_config: EmbedConfig = embed_config self.llm_config: LLMConfig = llm_config + self.node_indexname = "shanshi_node" # "opsgptkg_node" + self.edge_indexname = "shanshi_edge" # "opsgptkg_edge" + # get llm model self.model = getChatModelFromConfig(self.llm_config) - - + # init db handler self.init_handler() def init_handler(self, ): @@ -121,14 +124,17 @@ def init_tb(self, do_init: bool=None): if self.tb_config: tb_dict = {"TbaseHandler": TbaseHandler} tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) - self.tb: TbaseHandler = tb_class(tb_config=self.tb_config, index_name=self.tb_config.index_name, definition_value="opsgptkg") + self.tb: TbaseHandler = tb_class( + tb_config=self.tb_config, index_name=self.tb_config.index_name, + definition_value=self.tb_config.extra_kwargs.get("definition_value", "muagent_ekg") + ) # # create index - if not self.tb.is_index_exists("opsgptkg_node"): - res = self.tb.create_index(index_name="opsgptkg_node", schema=NODE_SCHEMA) + if not self.tb.is_index_exists(self.node_indexname): + res = self.tb.create_index(index_name=self.node_indexname, schema=NODE_SCHEMA) logger.info(f"tb init: {res}") - if not self.tb.is_index_exists("opsgptkg_edge"): - res = self.tb.create_index(index_name="opsgptkg_edge", schema=EDGE_SCHEMA) + if not self.tb.is_index_exists(self.edge_indexname): + res = self.tb.create_index(index_name=self.edge_indexname, schema=EDGE_SCHEMA) logger.info(f"tb init: {res}") else: self.tb = None @@ -136,7 +142,7 @@ def init_tb(self, do_init: bool=None): def init_gb(self, do_init: bool=None): if self.gb_config: gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler, "GeaBaseHandler": GeaBaseHandler,} - gb_class = gb_dict.get(self.gb_config.gb_type, NetworkxHandler) + gb_class = gb_dict.get(self.gb_config.gb_type, NebulaHandler) self.gb: GBHandler = gb_class(self.gb_config) else: self.gb = None @@ -165,81 +171,82 @@ def init_sls(self, do_init: bool=None): self.sls: AliYunSLSHandler = sls_class(self.embed_config, vb_config=self.vb_config) else: self.sls = None + + def update_graph( + self, + origin_nodes: List[GNode], origin_edges: List[GEdge], + nodes: List[GNode], edges: List[GEdge], teamid: str + ): + # + origin_nodeids = set([node["id"] for node in origin_nodes]) + origin_edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in origin_edges]) + nodeids = set([node["id"] for node in nodes]) + edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in edges]) + unique_nodeids = origin_nodeids&nodeids + unique_edgeids = origin_edgeids&edgeids + nodeid2nodes_dict = {} + for node in origin_nodes + nodes: + nodeid2nodes_dict.setdefault(node["id"], []).append(node) + + edgeid2edges_dict = {} + for edge in origin_edges + edges: + edgeid2edges_dict.setdefault(f"{edge['start_id']}__{edge['end_id']}", []).append(edge) + + # get add nodes & edges + add_nodes = [node for node in nodes if node["id"] not in origin_nodeids] + add_edges = [edge for edge in edges if f"{edge['start_id']}__{edge['end_id']}" not in origin_edgeids] + + # get delete nodes & edges + delete_nodes = [node for node in origin_nodes if node["id"] not in nodeids] + delete_edges = [edge for edge in origin_edges if f"{edge['start_id']}__{edge['end_id']}" not in edgeids] + + # get update nodes & edges + update_nodes = [ + nodeid2nodes_dict[nodeid][1] + for nodeid in unique_nodeids + if nodeid2nodes_dict[nodeid][0]!=nodeid2nodes_dict[nodeid][1] + ] + update_edges = [ + edgeid2edges_dict[edgeid][1] + for edgeid in unique_edgeids + if edgeid2edges_dict[edgeid][0]!=edgeid2edges_dict[edgeid][1] + ] - def add_nodes(self, nodes: List[GNode], teamid: str): - nodetype2fields_dict = {} - for node in nodes: - node_type = node.type - node.attributes["teamid"] = teamid - node.attributes["gdb_timestamp"] = getCurrentTimestap() - node.attributes["version"] = getCurrentDatetime() - node.attributes.setdefault("extra", '{}') - # check the data's key-value by node_type - schema = TYPE2SCHEMA.get(node_type,) - if node_type in nodetype2fields_dict: - fields = nodetype2fields_dict[node_type] - else: - fields = list(getClassFields(schema)) - nodetype2fields_dict[node_type] = fields + self.add_nodes([GNode(**n) for n in add_nodes], teamid) + self.add_edges([GEdge(**e) for e in add_edges], teamid) + + self.delete_edges([GEdge(**e) for e in delete_edges], teamid) + self.delete_nodes([GNode(**n) for n in delete_nodes], teamid) + + self.update_nodes([GNode(**n) for n in update_nodes], teamid) + self.update_edges([GEdge(**e) for e in update_edges], teamid) + + + def add_nodes(self, nodes: List[GNode], teamid: str): + nodes = self._update_new_attr_for_nodes(nodes, teamid, do_check=True) - flag = any([ - field not in node.attributes - for field in fields - if field not in ["start_id", "end_id", "id", "ID"] - ]) - if flag: - raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") - tbase_nodes = [] for node in nodes: - name = node.attributes.get("name", "") - description = node.attributes.get("description", "") - name_vector = self._get_embedding(name) - desc_vector = self._get_embedding(description) tbase_nodes.append({ - # "node_id": f'''ekg_node:{teamid}:{node.id}''', - "node_id": f'''{node.id}''', - "node_type": node.type, - "node_str": f"graph_id={teamid}", - "name_vector": np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), - "desc_vector": np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), - "name_keyword": " | ".join(extract_tags(name, topK=None)), - "desc_keyword": " | ".join(extract_tags(description, topK=None)), - }) - - tb_result = {"error": True} + **{ + "node_id": f"{node.id}", + "node_type": node.type, + "node_str": f"graph_id={teamid}", + }, + **self._update_tbase_attr_for_nodes(node.attributes) + }) + + tb_result, gb_result = [], [] try: - # gb_result = self.gb.add_nodes(nodes) + gb_result = [self.gb.add_node(node) for node in nodes] tb_result = self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False) except Exception as e: logger.error(e) - return tb_result - return gb_result or tb_result def add_edges(self, edges: List[GEdge], teamid: str): - edgetype2fields_dict = {} - for edge in edges: - edge_type = edge.type - edge.attributes["teamid"] = teamid - edge.attributes["@timestamp"] = getCurrentTimestap() - edge.attributes["gdb_timestamp"] = getCurrentTimestap() - edge.attributes["version"] = getCurrentDatetime() - edge.attributes["extra"] = '{}' - - # check the data's key-value by edge_type - schema = TYPE2SCHEMA.get("edge",) - if edge_type in edgetype2fields_dict: - fields = edgetype2fields_dict[edge_type] - else: - fields = list(getClassFields(schema)) - edgetype2fields_dict[edge_type] = fields - - flag = any([field not in edge.attributes for field in fields if field not in ["dst_id", "src_id", "timestamp", "id"]]) - if flag: - raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") - + edges = self._update_new_attr_for_edges(edges, teamid, do_check=True) tbase_edges = [{ # 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", 'edge_id': f"{edge.start_id}__{edge.end_id}", @@ -251,132 +258,157 @@ def add_edges(self, edges: List[GEdge], teamid: str): for edge in edges ] - tb_result = {"error": True} + tb_result, gb_result = [], [] try: - # gb_result = self.gb.add_edges(edges) + gb_result = [self.gb.add_edge(edge) for edge in edges] tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) except Exception as e: logger.error(e) - - return tb_result - return gb_result or tb_result - def delete_nodes(self, nodes: List[GNode], teamid: str): + def delete_nodes(self, nodes: List[GNode], teamid: str=''): # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='opsgptkg_node') + # r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name=self.node_indexname) + r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 - tbase_nodeids_dict = {data["node_id"]:data['id'] for data in r.docs} # 附带了definition信息 + # tbase_nodeids_dict = {data["node_id"]:data['id'] for data in r.docs} # 附带了definition信息 delete_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] - delete_tbase_nodeids = [tbase_nodeids_dict[nodeid] for nodeid in delete_nodeids if nodeid in tbase_nodeids] + # delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] + # delete_tbase_nodeids = [tbase_nodeids_dict[nodeid] for nodeid in delete_nodeids if nodeid in tbase_nodeids] if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + node_neighbor_lens = [len(self.gb.get_neighbor_nodes({"id": node.id}, node.type)) for node in nodes] # delete the nodeids in tbase tb_result = [] - for nodeid in delete_tbase_nodeids: - self.tb.delete(nodeid) - resp = self.tb.delete(nodeid) + for node, node_len in zip(nodes, node_neighbor_lens): + if node_len >= 2: continue + resp = self.tb.delete(node.id) tb_result.append(resp) - # logger.info(f'id={nodeid}, delete resp={resp}') # # delete the nodeids in geabase - # gb_result = self.gb.delete_nodes(delete_tbase_nodeids) - return tb_result - return gb_result or tb_result + gb_result = [] + for node, node_len in zip(nodes, node_neighbor_lens): + if node_len >= 2: continue + gb_result.append(self.gb.delete_node({"id": node.id}, node.type, ID=double_hashing(node.id))) + return gb_result + tb_result def delete_edges(self, edges: List[GEdge], teamid: str): # delete tbase nodes - r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='opsgptkg_edge') + # r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name=self.edge_indexname) + r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.edge_indexname, limit=len(edges)) + tbase_edgeids = [data['edge_id'] for data in r.docs] - tbase_edgeids_dict = {data["edge_id"]:data['id'] for data in r.docs} # id附带了definition信息 + # tbase_edgeids_dict = {data["edge_id"]:data['id'] for data in r.docs} # id附带了definition信息 delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] - delete_tbase_edgeids = [tbase_edgeids_dict[edgeid] for edgeid in delete_edgeids if edgeid in tbase_edgeids] + # delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] + # delete_tbase_edgeids = [tbase_edgeids_dict[edgeid] for edgeid in delete_edgeids if edgeid in tbase_edgeids] if len(tbase_missing_edgeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") - # delete the edgeids in tbase tb_result = [] - for edgeid in delete_tbase_edgeids: - self.tb.delete(edgeid) + for edgeid in delete_edgeids: resp = self.tb.delete(edgeid) tb_result.append(resp) - # logger.info(f'id={edgeid}, delete resp={resp}') # # delete the nodeids in geabase - # gb_result = self.gb.delete_edges(delete_tbase_edgeids) - return tb_result + gb_result = [] + for edge in edges: + gb_result.append(self.gb.delete_edge(double_hashing(edge.start_id), double_hashing(edge.end_id), edge.type)) + return gb_result + tb_result def update_nodes(self, nodes: List[GNode], teamid: str): # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='opsgptkg_node') + # r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name=self.node_indexname) + r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) + teamids_by_nodeid = {data['node_id']: data["node_str"] for data in r.docs} + tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 update_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] - update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] + # update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + for node in nodes: + r = self.tb.search(f"@node_id: {node.id}", index_name=self.node_indexname) + teamids_by_nodeid.update({data['node_id']: data["node_str"] for data in r.docs}) tb_result = [] - random_vector = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() for node in nodes: - if node.id not in tbase_nodeids: continue - + # tbase_data = {} - for key in ["name", "description"]: - if key not in node.attributes: continue - - if key == "name": - text = node.attributes.get("name", "") - tbase_data["name_keyword"] = " | ".join(extract_tags(text, topK=None)) - tbase_data["name_vector"] = np.array(self._get_embedding(text)[text] - ).astype(dtype=np.float32).tobytes() - - if key == "description": - text = node.attributes.get("description", "") - tbase_data["desc_keyword"] = " | ".join(extract_tags(text, topK=None)) - tbase_data["desc_vector"] = np.array(self._get_embedding(text)[text] - ).astype(dtype=np.float32).tobytes() - tbase_data["node_id"] = node.id - + tbase_data["node_id"] = node.id + if teamid not in teamids_by_nodeid[node.id]: + tbase_data["teamids"] = teamids_by_nodeid[node.id] + f", {teamid}" + tbase_data.update(self._update_tbase_attr_for_nodes(node.attributes)) + # resp = self.tb.insert_data_hash(tbase_data, key="node_id", need_etime=False) tb_result.append(resp) # update the nodeids in geabase + nodes = self._update_new_attr_for_nodes(nodes, teamid, teamids_by_nodeid, do_check=False) gb_result = [] - # for node in nodes: - # if node.id not in update_tbase_nodeids: continue - # resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) - # gb_result.append(resp) + for node in nodes: + # if node.id not in update_tbase_nodeids: continue + resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=double_hashing(node.id)) + gb_result.append(resp) return gb_result or tb_result + def update_edges(self, edges: List[GEdge], teamid: str): + r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.node_indexname, limit=len(edges)) + teamids_by_edgeid = {data['edge_id']: data["edge_str"] for data in r.docs} + + tbase_edgeids = [data['edge_id'] for data in r.docs] + delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] + tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] + + if len(tbase_missing_edgeids) > 0: + logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") + for edge in edges: + r = self.tb.search(f"@edge_id: {edge.start_id}__{edge.end_id}", index_name=self.node_indexname) + teamids_by_edgeid.update({data['edge_id']: data["node_str"] for data in r.docs}) + + # update the nodeids in geabase + edges = self._update_new_attr_for_edges(edges, teamid, teamids_by_edgeid, do_check=False) + gb_result = [] + for edge in edges: + # if node.id not in update_tbase_nodeids: continue + resp = self.gb.update_edge( + double_hashing(edge.start_id), double_hashing(edge.end_id), + edge.attributes, edge_type=edge.type + ) + gb_result.append(resp) + return gb_result + def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: - return self.gb.get_current_node({'id': nodeid}, node_type=node_type) + node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) + extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + node.attributes.update(extra_attrs) + return node - def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str, hop: int = 10) -> Graph: + def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str=None, hop: int = 10) -> Graph: if hop > 14: raise Exception(f"hop can't be larger than 14, now hop is {hop}") # filter the node which dont match teamid - result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop, select_attributes={"teamid": teamid}) + result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop) + for node in result.nodes: + extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + node.attributes.update(extra_attrs) + for edge in result.edges: + extra_attrs = json.loads(edge.attributes.pop("extra", "{}")) + edge.attributes.update(extra_attrs) return result def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: if text is None: return [] - # if self.embed_config: - # raise Exception(f"can't use vector search, because there is no {self.embed_config}") - - # 直接检索文本 - keywords = extract_tags(text) - keyword = "|".join(keywords) - nodeids = [] # if self.embed_config: @@ -385,7 +417,8 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N nodeid_with_dist = [] for key in ["name_vector", "desc_vector"]: - base_query = f'(@node_str: graph_id={teamid})=>[KNN {top_k} @{key} $vector AS distance]' + # base_query = f'(@node_str: graph_id={teamid})=>[KNN {top_k} @{key} $vector AS distance]' + base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' query_params = {"vector": query_embedding} r = self.tb.vector_search(base_query, query_params=query_params) @@ -397,8 +430,11 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N if nodeid not in nodeids: nodeids.append(nodeid) + # search keyword by jieba spliting text + keywords = extract_tags(text) + keyword = "|".join(keywords) for key in ["name_keyword", "desc_keyword"]: - r = self.tb.search(f"(@{key}:{{{keyword}}})") + r = self.tb.search(f"(@{key}:{{{keyword}}})", limit=30) for i in r.docs: if i["node_id"] not in nodeids: nodeids.append(i["node_id"]) @@ -411,6 +447,12 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: # rootid = f"{teamid}" # todo check the rootid result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) + for node in result.nodes: + extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + node.attributes.update(extra_attrs) + for edge in result.edges: + extra_attrs = json.loads(edge.attributes.pop("extra", "{}")) + edge.attributes.update(extra_attrs) # paths must be ordered from start to end paths = result.paths @@ -523,11 +565,22 @@ def dsl2graph(self, ) -> dict: pass def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData): + # everytimes, it will add new nodes and edges - # self.gb.add_nodes(result) - # self.gb.add_edges(result) - # self.tb.insert_data_hash(result) + nodes = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] + nodes = [GNode(id=node.id, type=node.type, attributes=node.attrbutes()) for node in nodes] + gb_result = self.gb.add_nodes(nodes) + + edges = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] + edges = [GEdge(start_id=edge.start_id, end_id=edge.end_id, type=edge.type, attributes=edge.attrbutes()) for edge in edges] + gb_result = self.gb.add_edges(edges) + + nodes = [node.dict() for node in ekg_tbase_data.nodes] + tb_result = self.tb.insert_data_hash(nodes) + edges = [edge.dict() for edge in ekg_tbase_data.edges] + tb_result = self.tb.insert_data_hash(edges) + # dsl2graph => write2kg ## delete tbase/graph by graph_id ### diff the tabse within newest by graph_id @@ -565,7 +618,7 @@ def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: all_intent_list.append(all_intent) return list(ancestor_list), all_intent_list - + def get_intents_by_alarms(self, alarm_list: list[dict], ) -> EKGIntentResp: '''according contents search intents''' ancestor_list = set() @@ -612,7 +665,7 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= tool='', need_check='false', operation_type='ADD', - teamid=teamid + teamids=teamid ) sls_nodes.append(ekg_slsdata) @@ -625,7 +678,7 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= type=f'edge_route_intent_{node_type}', # 需要注意与老逻辑的兼容 end_id=node_id, operation_type='ADD', - teamid=teamid + teamids=teamid ) ) # edges @@ -644,7 +697,7 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= type=edge_type, end_id=end_id, operation_type='ADD', - teamid=teamid + teamids=teamid ) ) return EKGSlsData(nodes=sls_nodes, edges=sls_edges) @@ -653,13 +706,19 @@ def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData tbase_nodes, tbase_edges = [], [] for node in ekg_sls_data.nodes: + name = node.name + description = node.description + name_vector = self._get_embedding(name) + desc_vector = self._get_embedding(description) tbase_nodes.append( EKGNodeTbaseSchema( node_id=node.id, node_type=node.type, node_str=f'graph_id={teamid}', - # 后续可用embedding完成替换 - node_vector=[random.random() for _ in range(768)], + name_keyword=" | ".join(extract_tags(name, topK=None)), + desc_keyword=" | ".join(extract_tags(description, topK=None)), + name_vector=np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), + desc_vector= np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), ) ) for edge in ekg_sls_data.edges: @@ -709,6 +768,7 @@ def get_md5(s): for pid in pnode_ids: dsl_pid = get_md5(pid) dsl_pid = f'ekg_node:{teamid}:intent:{dsl_pid}' + dsl_pid = f'ekg_node:intent:{dsl_pid}' if dsl_pid not in intent_names_dict: intent_names_dict[dsl_pid] = self.gb.get_current_node( {'id': pid}, 'opsgptkg_intent').attributes["name"] @@ -728,6 +788,7 @@ def get_md5(s): intent_id = get_md5(intent) intent_id = f'ekg_node:{teamid}:intent:{intent_id}' + intent_id = f'ekg_node:intent:{intent_id}' if intent_id not in intent_names_dict: intent_names_dict[intent_id] = self.gb.get_current_node( {'id': intent}, 'opsgptkg_intent').attributes["name"] @@ -800,7 +861,7 @@ def get_md5(s): def _get_embedding(self, text): text_vector = {} - if self.embed_config: + if self.embed_config and text: text_vector = get_embedding( self.embed_config.embed_engine, [text], self.embed_config.embed_model_path, self.embed_config.model_device, @@ -859,4 +920,98 @@ def remove_time(text): i_strip = i.strip('，。\n') i_strip = f'{i_strip}' res += i_strip - return res \ No newline at end of file + return res + + def _update_tbase_attr_for_nodes(self, attrs): + tbase_attrs = {} + for k in ["name", "description"]: + if k in attrs: + text = attrs.get(k, "") + text_vector = self._get_embedding(text) + tbase_attrs[f"{k}_vector"] = np.array(text_vector[text]).astype(dtype=np.float32).tobytes() + tbase_attrs[f"{k}_keyword"] = " | ".join(extract_tags(text, topK=None)) + return tbase_attrs + + def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by_nodeid={}, do_check=False): + '''update new attributes for nodes''' + nodetype2fields_dict = {} + for node in nodes: + node_type = node.type + + if node.id in teamids_by_nodeid: + node.attributes["teamids"] = teamids_by_nodeid.get(node.id, "").split("=")[1] + f", {teamid}" + else: + node.attributes["teamids"] = f"{teamid}" + + node.attributes["gdb_timestamp"] = getCurrentTimestap() + # node.attributes["version"] = getCurrentDatetime() + + # check the data's key-value by node_type + schema = TYPE2SCHEMA.get(node_type,) + if node_type in nodetype2fields_dict: + fields = nodetype2fields_dict[node_type] + else: + fields = list(getClassFields(schema)) + nodetype2fields_dict[node_type] = fields + + flag = any([ + field not in node.attributes + for field in fields + if field not in ["start_id", "end_id", "ID", "id", "extra"] + ]) + if flag and do_check: + raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") + + # update extra infomations to extra + extra_fields = [k for k in node.attributes.keys() if k not in fields] + node.attributes.setdefault( + "extra", + json.dumps({ + k: node.attributes.pop(k, "") + for k in extra_fields + }, ensure_ascii=False) + ) + return nodes + + def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by_edgeid={}, do_check=False): + '''update new attributes for nodes''' + edgetype2fields_dict = {} + for edge in edges: + edge_type = edge.type + edge_id = f"{edge.start_id}__{edge.end_id}" + if edge_id in teamids_by_edgeid: + edge.attributes["teamids"] = teamids_by_edgeid.get(edge_id, "").split("=")[1] + f", {teamid}" + else: + edge.attributes["teamids"] = f"{teamid}" + + edge.attributes["@timestamp"] = getCurrentTimestap() + edge.attributes["gdb_timestamp"] = getCurrentTimestap() + edge.attributes['original_dst_id2__'] = edge.end_id + edge.attributes['original_src_id1__'] = edge.start_id + # edge.attributes["version"] = getCurrentDatetime() + # edge.attributes["extra"] = '{}' + + # check the data's key-value by edge_type + schema = TYPE2SCHEMA.get("edge",) + if edge_type in edgetype2fields_dict: + fields = edgetype2fields_dict[edge_type] + else: + fields = list(getClassFields(schema)) + edgetype2fields_dict[edge_type] = fields + + flag = any([ + field not in edge.attributes for field in fields + if field not in ["dst_id", "src_id", "timestamp", "ID", "id", "extra"]]) + if flag and do_check: + raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") + + # update extra infomations to extra + extra_fields = [k for k in edge.attributes.keys() if k not in fields+["@timestamp"]] + edge.attributes.setdefault( + "extra", + json.dumps({ + k: edge.attributes.pop(k, "") + for k in extra_fields + }, ensure_ascii=False) + ) + return edges \ No newline at end of file diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py new file mode 100644 index 0000000..0e6da7f --- /dev/null +++ b/tests/service/ekg_construct_test_2.py @@ -0,0 +1,234 @@ +import time +import sys, os +from loguru import logger + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) + +import os, sys +from loguru import logger + +sys.path.append("/ossfs/workspace/muagent") +sys.path.append("/ossfs/workspace/notebooks/custom_funcs") +from muagent.db_handler import GeaBaseHandler +from muagent.schemas.common import GNode, GEdge +from muagent.schemas.db import GBConfig, TBConfig +from muagent.service.ekg_construct import EKGConstructService +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig + + + + + +# 初始化 GeaBaseHandler 实例 +gb_config = GBConfig( + gb_type="GeaBaseHandler", + extra_kwargs={ + 'metaserver_address': os.environ['metaserver_address'], + 'project': os.environ['project'], + 'city': os.environ['city'], + 'lib_path': os.environ['lib_path'], + } +) + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="muagent_test", + host=os.environ['host'], + port=os.environ['port'], + username=os.environ['username'], + password=os.environ['password'], + extra_kwargs={ + 'host': os.environ['host'], + 'port': os.environ['port'], + 'username': os.environ['username'] , + 'password': os.environ['password'], + 'definition_value': os.environ['definition_value'] + } +) + +# llm config +llm_config = LLMConfig( + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +) + + +# emebdding config +# embed_config = EmbedConfig( +# embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +# ) + +# embed_config = EmbedConfig( +# embed_model="default", +# langchain_embeddings=embeddings +# ) +embed_config = None + +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) + + + +def generate_node(id, type): + extra_attr = {"tmp": "hello"} + if type == "opsgptkg_schedule": + extra_attr["enable"] = False + + if type == "opsgptkg_task": + extra_attr["accessCriteria"] = "hello" + + if type == "opsgptkg_analysis": + extra_attr["accessCriteria"] = "hello" + extra_attr["summarySwtich"] = False + extra_attr['dslTemplate'] = "hello" + + return GNode(**{ + "id": id, + "type": type, + "attributes": {**{ + "path": id, + "name": id, + "description": id, + }, **extra_attr} + }) + + +def generate_edge(node1, node2): + type_connect = "extend" if node1.type == "opsgptkg_intent" and node2.type == "opsgptkg_intent" else "route" + return GEdge(**{ + "start_id": node1.id, + "end_id": node2.id, + "type": f"{node1.type}_{type_connect}_{node2.type}", + "attributes": { + "lat": "hello", + "attr": "hello" + } + }) + + +nodetypes = [ + 'opsgptkg_intent', 'opsgptkg_schedule', 'opsgptkg_task', + 'opsgptkg_phenomenon', 'opsgptkg_analysis' +] + +nodes_dict = {} +for nodetype in nodetypes: + for i in range(8): + # print(f"shanshi_{nodetype}_{i}") + nodes_dict[f"shanshi_{nodetype}_{i}"] = generate_node(f"shanshi_{nodetype}_{i}", nodetype) + +edge_ids = [ + ["shanshi_opsgptkg_intent_0", "shanshi_opsgptkg_intent_1"], + ["shanshi_opsgptkg_intent_1", "shanshi_opsgptkg_intent_2"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_0"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_1"], + ["shanshi_opsgptkg_schedule_1", "shanshi_opsgptkg_analysis_3"], + ["shanshi_opsgptkg_schedule_0", "shanshi_opsgptkg_task_0"], + ["shanshi_opsgptkg_task_0", "shanshi_opsgptkg_task_1"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_analysis_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_1"], + ["shanshi_opsgptkg_phenomenon_0", "shanshi_opsgptkg_task_2"], + ["shanshi_opsgptkg_phenomenon_1", "shanshi_opsgptkg_task_3"], + ["shanshi_opsgptkg_task_2", "shanshi_opsgptkg_analysis_1"], + ["shanshi_opsgptkg_task_3", "shanshi_opsgptkg_analysis_2"], +] + +nodeid_set = set() +origin_edges = [] +origin_nodes = [] +for src_id, dst_id in edge_ids: + origin_edges.append(generate_edge(nodes_dict[src_id], nodes_dict[dst_id])) + if src_id not in nodeid_set: + nodeid_set.add(src_id) + origin_nodes.append(nodes_dict[src_id]) + if dst_id not in nodeid_set: + nodeid_set.add(dst_id) + origin_nodes.append(nodes_dict[dst_id]) + + + + +new_edge_ids = [ + ["shanshi_opsgptkg_intent_0", "shanshi_opsgptkg_intent_1"], + ["shanshi_opsgptkg_intent_1", "shanshi_opsgptkg_intent_2"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_0"], + # 新增 + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_2"], + ["shanshi_opsgptkg_schedule_2", "shanshi_opsgptkg_analysis_4"], + # + ["shanshi_opsgptkg_schedule_0", "shanshi_opsgptkg_task_0"], + ["shanshi_opsgptkg_task_0", "shanshi_opsgptkg_task_1"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_analysis_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_1"], + ["shanshi_opsgptkg_phenomenon_0", "shanshi_opsgptkg_task_2"], + ["shanshi_opsgptkg_phenomenon_1", "shanshi_opsgptkg_task_3"], + ["shanshi_opsgptkg_task_2", "shanshi_opsgptkg_analysis_1"], + ["shanshi_opsgptkg_task_3", "shanshi_opsgptkg_analysis_2"], +] + +nodeid_set = set() +edges = [] +nodes = [] +for src_id, dst_id in new_edge_ids: + edges.append(generate_edge(nodes_dict[src_id], nodes_dict[dst_id])) + if src_id not in nodeid_set: + nodeid_set.add(src_id) + nodes.append(nodes_dict[src_id]) + if dst_id not in nodeid_set: + nodeid_set.add(dst_id) + nodes.append(nodes_dict[dst_id]) + +for node in nodes: + if node.type == "opsgptkg_task": + node.attributes["name"] += "_update" + node.attributes["tmp"] += "_update" + node.attributes["description"] += "_update" + +for edge in edges: + if edge.type == "opsgptkg_task_route_opsgptkg_task": + edge.attributes["lat"] += "_update" + + +# +teamid = "shanshi_test" +origin_nodes = [GNode(**n) for n in origin_nodes] +origin_edges = [GEdge(**e) for e in origin_edges] +ekg_construct_service.add_nodes(origin_nodes, teamid) +ekg_construct_service.add_edges(origin_edges, teamid) + + +# +teamid = "shanshi_test_2" +ekg_construct_service.update_graph(origin_nodes, origin_edges, nodes, edges, teamid) \ No newline at end of file From a86ecc71a3945c6d716db42f9739028c3ddc039c Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 13 Aug 2024 13:55:52 +0800 Subject: [PATCH 015/128] update ekg construct graph test --- .../graph_db_handler/base_gb_handler.py | 48 +++--- .../graph_db_handler/geabase_handler.py | 62 +++++--- muagent/schemas/ekg/ekg_graph.py | 26 ++-- .../ekg_construct/ekg_construct_base.py | 140 +++++++++++++----- tests/service/ekg_construct_test_2.py | 23 ++- 5 files changed, 201 insertions(+), 98 deletions(-) diff --git a/muagent/db_handler/graph_db_handler/base_gb_handler.py b/muagent/db_handler/graph_db_handler/base_gb_handler.py index afac9d1..e21b0eb 100644 --- a/muagent/db_handler/graph_db_handler/base_gb_handler.py +++ b/muagent/db_handler/graph_db_handler/base_gb_handler.py @@ -13,56 +13,44 @@ class GBHandler: def __init__(self) -> None: pass - def add_node(self, node: GNode): + def add_node(self, node: GNode) -> dict: return self.add_nodes([node]) - def add_nodes(self, nodes: List[GNode]): + def add_nodes(self, nodes: List[GNode]) -> dict: pass - def add_edge(self, edge: GEdge): + def add_edge(self, edge: GEdge) -> dict: return self.add_edges([edge]) - def add_edges(self, edges: List[GEdge]): + def add_edges(self, edges: List[GEdge]) -> dict: pass - def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None): + def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None) -> dict: pass - def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None): + def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None) -> dict: pass - def delete_node(self, attributes: dict, node_type: str = None, ID: int = None): + def delete_node(self, attributes: dict, node_type: str = None, ID: int = None) -> dict: pass - def delete_nodes(self, attributes: dict, node_type: str = None, ID: int = None): + def delete_nodes(self, attributes: dict, node_type: str = None, IDs: List[int] = []) -> dict: pass - def delete_edge(self, src_id, dst_id, edge_type: str = None): + def delete_edge(self, src_id, dst_id, edge_type: str = None) -> dict: pass - - def delete_edges(self, src_id, dst_id, edge_type: str = None): - pass - - def search_node_by_nodeid(self, nodeid: str, node_type: str = None) -> GNode: - pass - - def search_edges_by_nodeid(self, nodeid: str, node_type: str = None) -> List[GEdge]: - pass - - def search_edge_by_nodeids(self, start_id: str, end_id: str, edge_type: str = None) -> GEdge: - pass - - def search_nodes_by_attr(self, attributes: dict) -> List[GNode]: - pass - - def search_edges_by_attr(self, attributes: dict, edge_type: str = None) -> List[GEdge]: + + def delete_edges(self, id_pairs: List, edge_type: str = None): pass - def get_nodes_by_ids(self, ids: List[int]) -> List[GNode]: + def get_nodeIDs(self, attributes: dict, node_type: str) -> List[int]: pass def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: pass + + def get_nodes_by_ids(self, ids: List[int] = []) -> List[GNode]: + pass def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: pass @@ -70,11 +58,11 @@ def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> GEdge: pass - def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = [], reverse=False) -> List[GNode]: pass def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: pass - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse: bool =False) -> Graph: - pass + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse=False) -> Graph: + pass diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 2d67112..e80db45 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -46,11 +46,11 @@ def add_nodes(self, nodes: List[GNode]) -> dict: node_str_list = [] for node in nodes: node_type = node.type - node_attributes = {"@id": double_hashing(node.id), "id": node.id} + node_attributes = {"id": node.id} + node_attributes["@id"] = node.attributes.pop("ID", "") or double_hashing(node.id) node_attributes.update(node.attributes) - # _ = node_attributes.pop("type") - # logger.debug(f"{node_attributes}") - node_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in node_attributes.items()]) + + node_str = ", ".join([f"{k}: '{v}'" if isinstance(v, (str, bool)) else f"{k}: {v}" for k, v in node_attributes.items()]) node_str_list.append(f"(:{node_type} {{{node_str}}})") gql = f"INSERT {','.join(node_str_list)}" @@ -64,11 +64,13 @@ def add_edges(self, edges: List[GEdge]) -> dict: edge_str_list = [] for edge in edges: edge_type = edge.type - src_id, dst_id = double_hashing(edge.start_id,), double_hashing(edge.end_id,) - edge_attributes = {"@src_id": src_id, "@dst_id": dst_id} + edge_attributes = { + "@src_id": edge.attributes.pop("SRCID", 0) or double_hashing(edge.start_id,), + "@dst_id": edge.attributes.pop("DSTID", 0) or double_hashing(edge.end_id,) + } edge_attributes.update(edge.attributes) - # _ = edge_attributes.pop("type") - edge_str = ", ".join([f"{k}: '{v}'" if isinstance(v, str) else f"{k}: {v}" for k, v in edge_attributes.items()]) + + edge_str = ", ".join([f"{k}: '{v}'" if isinstance(v, (str, bool)) else f"{k}: {v}" for k, v in edge_attributes.items()]) edge_str_list.append(f"()-[:{edge_type} {{{edge_str}}}]->()") gql = f"INSERT {','.join(edge_str_list)}" @@ -76,7 +78,7 @@ def add_edges(self, edges: List[GEdge]) -> dict: def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None) -> dict: # demo: "MATCH (n:opsgptkg_employee {@ID: xxxx}) SET n.originname = 'xxx', n.description = 'xxx'" - set_str = ", ".join([f"n.{k}='{v}'" if isinstance(v, str) else f"n.{k}={v}" for k, v in set_attributes.items()]) + set_str = ", ".join([f"n.{k}='{v}'" if isinstance(v, (str, bool)) else f"n.{k}={v}" for k, v in set_attributes.items()]) if (ID is None) or (not isinstance(ID, int)): ID = self.get_current_nodeID(attributes, node_type) @@ -89,7 +91,7 @@ def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = Non src_id, dst_id, timestamp = self.get_current_edgeID(src_id, dst_id, edge_type) src_type, dst_type = self.get_nodetypes_by_edgetype(edge_type) # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) - set_str = ", ".join([f"e.{k}='{v}'" if isinstance(v, str) else f"e.{k}={v}" for k, v in set_attributes.items()]) + set_str = ", ".join([f"e.{k}='{v}'" if isinstance(v, (str, bool)) else f"e.{k}={v}" for k, v in set_attributes.items()]) # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; # gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, timestamp:{timestamp}}}]->() SET {set_str}" gql = f"MATCH (n0:{src_type} {{@id: {src_id}}})-[e]->(n1:{dst_type} {{@id:{dst_id}}}) SET {set_str}" @@ -102,7 +104,7 @@ def delete_node(self, attributes: dict, node_type: str = None, ID: int = None) - gql = f"MATCH (n:{node_type}) WHERE n.@ID={ID} DELETE n" return self.execute(gql) - def delete_nodes(self, attributes: dict, node_type: str = None, IDs: List[int] = None) -> dict: + def delete_nodes(self, attributes: dict, node_type: str = None, IDs: List[int] = []) -> dict: if (IDs is None) or len(IDs)==0: IDs = self.get_nodeIDs(attributes, node_type) # ID = double_hashing(ID) @@ -138,7 +140,7 @@ def get_current_edgeID(self, src_id, dst_id, edeg_type:str = None): if not isinstance(src_id, int) or not isinstance(dst_id, int): result = self.get_current_edge(src_id, dst_id, edeg_type) logger.debug(f"{result}") - return result.attributes.get("srcId"), result.attributes.get("dstId"), result.attributes.get("timestamp") + return result.attributes.get("SRCID"), result.attributes.get("DSTID"), result.attributes.get("timestamp") else: return src_id, dst_id, 1 @@ -164,7 +166,8 @@ def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) - nodes = result.get("n0", []) or result.get("n0.attr", []) + nodes = result.get("n0", []) or result.get("n0.attr", []) + return self.convert2GNodes(nodes) return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> GEdge: @@ -177,6 +180,7 @@ def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: li result = self.decode_result(result, gql) edges = result.get("e", []) or result.get("e.attr", []) + return self.convert2GEdges(edges)[0] return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges][0] def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = [], reverse=False) -> List[GNode]: @@ -192,6 +196,7 @@ def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_key result = self.execute(gql, return_keys=return_keys) result = self.decode_result(result, gql) nodes = result.get("n1", []) or result.get("n1.attr", []) + return self.convert2GNodes(nodes) return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: @@ -205,6 +210,7 @@ def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_key result = self.decode_result(result, gql) edges = result.get("e", []) or result.get("e.attr", []) + return self.convert2GEdges(edges) return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges] def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}) -> bool: @@ -244,8 +250,10 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes) hop -= hop_max - nodes = [GNode(id=node["id"], type=node["type"], attributes=node) for node in result.get("n1", [])] - edges = [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in result.get("e", [])] + nodes = self.convert2GNodes(result.get("n1", [])) + edges = self.convert2GEdges(result.get("e", [])) + # nodes = [GNode(id=node["id"], type=node["type"], attributes=node) for node in result.get("n1", [])] + # edges = [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in result.get("e", [])] return Graph(nodes=nodes, edges=edges, paths=result.get("p", [])) def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: @@ -387,16 +395,17 @@ def decode_path(self, col_data, k) -> List: def decode_vertex(self, col_data, k) -> Dict: vertextVal = col_data.get("vertexVal", {}) node_val_json = { - **{"ID": vertextVal.get("id", ""), "type": vertextVal.get("type", "")}, + **{"ID": int(vertextVal.get("id", "")), "type": vertextVal.get("type", "")}, **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in vertextVal.get("props", {}).items()} } + node_val_json.pop("biz_node_id", "") return node_val_json def decode_edge(self, col_data, k) -> Dict: def _decode_edge(data): edgeVal= data.get("edgeVal", {}) edge_val_json = { - **{"srcId": edgeVal.get("srcId", ""), "dstId": edgeVal.get("dstId", ""), "type": edgeVal.get("type", "")}, + **{"SRCID": int(edgeVal.get("srcId", "")), "DSTID": int(edgeVal.get("dstId", "")), "type": edgeVal.get("type", "")}, **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in edgeVal.get("props", {}).items()} } # 存在业务逻辑 @@ -422,4 +431,21 @@ def get_nodetypes_by_edgetype(self, edge_type: str): if edge_bridge in edge_type: src_type, dst_type = edge_type.split(edge_bridge) break - return src_type, dst_type \ No newline at end of file + return src_type, dst_type + + def convert2GNodes(self, raw_nodes: List[Dict]) -> List[GNode]: + nodes = [] + for node in raw_nodes: + node_id = node.pop("id") + node_type = node.pop("type") + nodes.append(GNode(id=node_id, type=node_type, attributes=node)) + return nodes + + def convert2GEdges(self, raw_edges: List[Dict]) -> List[GEdge]: + edges = [] + for edge in raw_edges: + start_id = edge.pop("start_id") + end_id = edge.pop("end_id") + edge_type = edge.pop("type") + edges.append(GEdge(start_id=start_id, end_id=end_id, type=edge_type, attributes=edge)) + return edges \ No newline at end of file diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 9a90a60..6c9b3a5 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -31,10 +31,10 @@ class NodeSchema(BaseModel): class EdgeSchema(BaseModel): # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - src_id: int = None + SRCID: int = None original_src_id1__: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - dst_id: int = None + DSTID: int = None original_dst_id2__: str # timestamp: int @@ -107,10 +107,10 @@ def attrbutes(self, ): } class EKGTaskNodeSchema(EKGNodeSchema): - # tool: str - # needCheck: bool + tool: str + needcheck: bool # when to access - accessCriteria: str + accesscriteria: str # # owner: str @@ -121,7 +121,9 @@ def attrbutes(self, ): "name": self.name, "description": self.description, "teamids": self.teamids, - "accessCriteria": self.accessCriteria + "accesscriteria": self.accesscriteria, + "needcheck": self.needcheck, + "tool": self.tool }, **extra_attr } @@ -129,11 +131,11 @@ def attrbutes(self, ): class EKGAnalysisNodeSchema(EKGNodeSchema): # when to access - accessCriteria: str + accesscriteria: str # do summary or not - summarySwtich: bool + summaryswtich: bool # summary template - dslTemplate: str + dsltemplate: str def attrbutes(self, ): extra_attr = json.loads(self.extra) @@ -142,9 +144,9 @@ def attrbutes(self, ): "name": self.name, "description": self.description, "teamids": self.teamids, - "accessCriteria": self.accessCriteria, - "summarySwtich": self.summarySwtich, - "dslTemplate": self.dslTemplate + "accesscriteria": self.accesscriteria, + "summaryswtich": self.summaryswtich, + "dsltemplate": self.dsltemplate }, **extra_attr } diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 71cd977..556b05d 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -91,6 +91,7 @@ def init_tb(self, do_init: bool=None): DIM = 768 # it depends on your embedding model vector NODE_SCHEMA = [ + NumericField("ID", ), TextField("node_id", ), TextField("node_type", ), TextField("node_str", ), @@ -175,26 +176,26 @@ def init_sls(self, do_init: bool=None): def update_graph( self, origin_nodes: List[GNode], origin_edges: List[GEdge], - nodes: List[GNode], edges: List[GEdge], teamid: str + new_nodes: List[GNode], new_edges: List[GEdge], teamid: str ): # origin_nodeids = set([node["id"] for node in origin_nodes]) origin_edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in origin_edges]) - nodeids = set([node["id"] for node in nodes]) - edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in edges]) + nodeids = set([node["id"] for node in new_nodes]) + edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in new_edges]) unique_nodeids = origin_nodeids&nodeids unique_edgeids = origin_edgeids&edgeids nodeid2nodes_dict = {} - for node in origin_nodes + nodes: + for node in origin_nodes + new_nodes: nodeid2nodes_dict.setdefault(node["id"], []).append(node) edgeid2edges_dict = {} - for edge in origin_edges + edges: + for edge in origin_edges + new_edges: edgeid2edges_dict.setdefault(f"{edge['start_id']}__{edge['end_id']}", []).append(edge) # get add nodes & edges - add_nodes = [node for node in nodes if node["id"] not in origin_nodeids] - add_edges = [edge for edge in edges if f"{edge['start_id']}__{edge['end_id']}" not in origin_edgeids] + add_nodes = [node for node in new_nodes if node["id"] not in origin_nodeids] + add_edges = [edge for edge in new_edges if f"{edge['start_id']}__{edge['end_id']}" not in origin_edgeids] # get delete nodes & edges delete_nodes = [node for node in origin_nodes if node["id"] not in nodeids] @@ -211,16 +212,18 @@ def update_graph( for edgeid in unique_edgeids if edgeid2edges_dict[edgeid][0]!=edgeid2edges_dict[edgeid][1] ] + + # + add_node_result = self.add_nodes([GNode(**n) for n in add_nodes], teamid) + add_edge_result = self.add_edges([GEdge(**e) for e in add_edges], teamid) + delete_edge_result = self.delete_nodes([GNode(**n) for n in delete_nodes], teamid) + delete_node_result = self.delete_edges([GEdge(**e) for e in delete_edges], teamid) - self.add_nodes([GNode(**n) for n in add_nodes], teamid) - self.add_edges([GEdge(**e) for e in add_edges], teamid) - - self.delete_edges([GEdge(**e) for e in delete_edges], teamid) - self.delete_nodes([GNode(**n) for n in delete_nodes], teamid) + update_node_result = self.update_nodes([GNode(**n) for n in update_nodes], teamid) + update_edge_result = self.update_edges([GEdge(**e) for e in update_edges], teamid) - self.update_nodes([GNode(**n) for n in update_nodes], teamid) - self.update_edges([GEdge(**e) for e in update_edges], teamid) + return [add_node_result, add_edge_result, delete_edge_result, delete_node_result, update_node_result, update_edge_result] def add_nodes(self, nodes: List[GNode], teamid: str): @@ -230,6 +233,7 @@ def add_nodes(self, nodes: List[GNode], teamid: str): for node in nodes: tbase_nodes.append({ **{ + "ID": node.attributes.get("ID", 0) or double_hashing(node.id), "node_id": f"{node.id}", "node_type": node.type, "node_str": f"graph_id={teamid}", @@ -240,13 +244,13 @@ def add_nodes(self, nodes: List[GNode], teamid: str): tb_result, gb_result = [], [] try: gb_result = [self.gb.add_node(node) for node in nodes] - tb_result = self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False) + tb_result.append(self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False)) except Exception as e: logger.error(e) - return gb_result or tb_result + return gb_result + tb_result def add_edges(self, edges: List[GEdge], teamid: str): - edges = self._update_new_attr_for_edges(edges, teamid, do_check=True) + edges = self._update_new_attr_for_edges(edges, teamid) tbase_edges = [{ # 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", 'edge_id': f"{edge.start_id}__{edge.end_id}", @@ -261,10 +265,10 @@ def add_edges(self, edges: List[GEdge], teamid: str): tb_result, gb_result = [], [] try: gb_result = [self.gb.add_edge(edge) for edge in edges] - tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) + tb_result.append(self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False)) except Exception as e: logger.error(e) - return gb_result or tb_result + return gb_result + tb_result def delete_nodes(self, nodes: List[GNode], teamid: str=''): # delete tbase nodes @@ -281,19 +285,27 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - node_neighbor_lens = [len(self.gb.get_neighbor_nodes({"id": node.id}, node.type)) for node in nodes] + node_neighbor_lens = [ + len([ + n.id # reverse neighbor nodes which are not in delete nodes + for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=False) + if n.id not in delete_nodeids]) + for node in nodes + ] # delete the nodeids in tbase tb_result = [] for node, node_len in zip(nodes, node_neighbor_lens): - if node_len >= 2: continue + if node_len >= 1: continue resp = self.tb.delete(node.id) tb_result.append(resp) # # delete the nodeids in geabase gb_result = [] for node, node_len in zip(nodes, node_neighbor_lens): - if node_len >= 2: continue - gb_result.append(self.gb.delete_node({"id": node.id}, node.type, ID=double_hashing(node.id))) + if node_len >= 1: continue + gb_result.append(self.gb.delete_node( + {"id": node.id}, node.type, ID=node.attributes.get("ID") or double_hashing(node.id) + )) return gb_result + tb_result def delete_edges(self, edges: List[GEdge], teamid: str): @@ -319,7 +331,11 @@ def delete_edges(self, edges: List[GEdge], teamid: str): # # delete the nodeids in geabase gb_result = [] for edge in edges: - gb_result.append(self.gb.delete_edge(double_hashing(edge.start_id), double_hashing(edge.end_id), edge.type)) + gb_result.append(self.gb.delete_edge( + edge.attributes.get("SRCID") or double_hashing(edge.start_id), + edge.attributes.get("DSTID") or double_hashing(edge.end_id), + edge.type + )) return gb_result + tb_result def update_nodes(self, nodes: List[GNode], teamid: str): @@ -356,7 +372,10 @@ def update_nodes(self, nodes: List[GNode], teamid: str): gb_result = [] for node in nodes: # if node.id not in update_tbase_nodeids: continue - resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=double_hashing(node.id)) + resp = self.gb.update_node( + {}, node.attributes, node_type=node.type, + ID=node.attributes.get("ID") or double_hashing(node.id) + ) gb_result.append(resp) return gb_result or tb_result @@ -375,12 +394,13 @@ def update_edges(self, edges: List[GEdge], teamid: str): teamids_by_edgeid.update({data['edge_id']: data["node_str"] for data in r.docs}) # update the nodeids in geabase - edges = self._update_new_attr_for_edges(edges, teamid, teamids_by_edgeid, do_check=False) + edges = self._update_new_attr_for_edges(edges, teamid, teamids_by_edgeid, do_check=False, do_update=True) gb_result = [] for edge in edges: # if node.id not in update_tbase_nodeids: continue resp = self.gb.update_edge( - double_hashing(edge.start_id), double_hashing(edge.end_id), + edge.attributes.get("SRCID") or double_hashing(edge.start_id), + edge.attributes.get("DSTID") or double_hashing(edge.end_id), edge.attributes, edge_type=edge.type ) gb_result.append(resp) @@ -423,7 +443,7 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N r = self.tb.vector_search(base_query, query_params=query_params) for i in r.docs: - nodeid_with_dist.append((i["node_id"], float(i["distance"]))) + nodeid_with_dist.append((i["ID"], float(i["distance"]))) nodeid_with_dist = sorted(nodeid_with_dist, key=lambda x:x[1], reverse=False) for nodeid, dis in nodeid_with_dist: @@ -434,14 +454,17 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N keywords = extract_tags(text) keyword = "|".join(keywords) for key in ["name_keyword", "desc_keyword"]: - r = self.tb.search(f"(@{key}:{{{keyword}}})", limit=30) + r = self.tb.search(f"(@{key}:{{{keyword}}})", index_name=self.node_indexname, limit=30) for i in r.docs: - if i["node_id"] not in nodeids: - nodeids.append(i["node_id"]) + if i["ID"] not in nodeids: + nodeids.append(i["ID"]) nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) nodes = self.gb.get_nodes_by_ids(nodeids) + for node in nodes: + extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + node.attributes.update(extra_attrs) return nodes_by_name + nodes_by_desc + nodes def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: @@ -973,7 +996,7 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by ) return nodes - def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by_edgeid={}, do_check=False): + def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by_edgeid={}, do_check=True, do_update=False): '''update new attributes for nodes''' edgetype2fields_dict = {} for edge in edges: @@ -984,7 +1007,7 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by else: edge.attributes["teamids"] = f"{teamid}" - edge.attributes["@timestamp"] = getCurrentTimestap() + edge.attributes["@timestamp"] = edge.attributes.pop("timestamp", 0) or 1 # getCurrentTimestap() edge.attributes["gdb_timestamp"] = getCurrentTimestap() edge.attributes['original_dst_id2__'] = edge.end_id edge.attributes['original_src_id1__'] = edge.start_id @@ -1001,7 +1024,7 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by flag = any([ field not in edge.attributes for field in fields - if field not in ["dst_id", "src_id", "timestamp", "ID", "id", "extra"]]) + if field not in ["dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"]]) if flag and do_check: raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") @@ -1014,4 +1037,51 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by for k in extra_fields }, ensure_ascii=False) ) - return edges \ No newline at end of file + if do_update: + edge.attributes.pop("@timestamp") + edge.attributes.pop("extra") + return edges + + def get_intent_by_alarm(self, alarm: dict, ) -> EKGIntentResp: + '''according content search intent''' + import requests + error_type = alarm.get('errorType', '') + title = alarm.get('title', '') + content = alarm.get('content', '') + biz_code = alarm.get('bizCode', '') + + if not error_type or not title or not content or not biz_code: + return None, None + + alarm = { + 'type': 'ANTEMC_DINGTALK', + 'user_input': { + 'bizCode': biz_code, + 'title': title, + 'content': content, + 'execute_type': 'gql', + 'errorType': error_type + } + } + + body = { + 'features': { + 'query': alarm + } + } + intent_url = 'https://paiplusinferencepre.alipay.com/inference/ff998e48456308a9_EKG_route/0.1' + headers = { + 'Content-Type': 'application/json;charset=utf-8', + 'MPS-app-name': 'test', + 'MPS-http-version': '1.0' + } + ans = requests.post(intent_url, json=body, headers=headers) + + ans_json = ans.json() + output = ans_json.get('resultMap').get('output') + logger.debug(f"{body}") + logger.debug(f"{output}") + output_json = json.loads(output) + res = output_json[-1] + all_intent = output_json + return res, all_intent diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py index 0e6da7f..8c9b962 100644 --- a/tests/service/ekg_construct_test_2.py +++ b/tests/service/ekg_construct_test_2.py @@ -33,8 +33,6 @@ from loguru import logger sys.path.append("/ossfs/workspace/muagent") -sys.path.append("/ossfs/workspace/notebooks/custom_funcs") -from muagent.db_handler import GeaBaseHandler from muagent.schemas.common import GNode, GEdge from muagent.schemas.db import GBConfig, TBConfig from muagent.service.ekg_construct import EKGConstructService @@ -231,4 +229,23 @@ def generate_edge(node1, node2): # teamid = "shanshi_test_2" -ekg_construct_service.update_graph(origin_nodes, origin_edges, nodes, edges, teamid) \ No newline at end of file +ekg_construct_service.update_graph(origin_nodes, origin_edges, nodes, edges, teamid) + + + +# do search +node = ekg_construct_service.get_node_by_id(nodeid="shanshi_opsgptkg_task_3", node_type="opsgptkg_task") +print(node) +graph = ekg_construct_service.get_graph_by_nodeid(nodeid="shanshi_opsgptkg_intent_0", node_type="opsgptkg_intent") +print(len(graph.nodes), len(graph.edges)) + + +# search nodes by text +text = 'shanshi_test' +teamid = "shanshi_test" +nodes = ekg_construct_service.search_nodes_by_text(text, teamid=teamid) +print(len(nodes)) + +# search path by node and rootid +graph = ekg_construct_service.search_rootpath_by_nodeid(nodeid="shanshi_opsgptkg_analysis_3", node_type="opsgptkg_analysis", rootid="shanshi_opsgptkg_intent_0") +print(len(graph.nodes), len(graph.edges)) \ No newline at end of file From db6d0387406fc95bfcdf3c961e50708ebc16e0f2 Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 13 Aug 2024 15:19:17 +0800 Subject: [PATCH 016/128] delete tool/path/needcheck attributes --- muagent/schemas/ekg/ekg_graph.py | 8 ++-- .../ekg_construct/ekg_construct_base.py | 45 +------------------ tests/service/ekg_construct_test_2.py | 8 ++-- 3 files changed, 9 insertions(+), 52 deletions(-) diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 6c9b3a5..99265b6 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -76,7 +76,7 @@ def attrbutes(self, ): class EKGIntentNodeSchema(EKGNodeSchema): - path: str = '' + # path: str = '' def attrbutes(self, ): extra_attr = json.loads(self.extra) @@ -107,8 +107,8 @@ def attrbutes(self, ): } class EKGTaskNodeSchema(EKGNodeSchema): - tool: str - needcheck: bool + # tool: str + # needcheck: bool # when to access accesscriteria: str # @@ -123,7 +123,7 @@ def attrbutes(self, ): "teamids": self.teamids, "accesscriteria": self.accesscriteria, "needcheck": self.needcheck, - "tool": self.tool + # "tool": self.tool }, **extra_attr } diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 556b05d..ebfe186 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -1041,47 +1041,4 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by edge.attributes.pop("@timestamp") edge.attributes.pop("extra") return edges - - def get_intent_by_alarm(self, alarm: dict, ) -> EKGIntentResp: - '''according content search intent''' - import requests - error_type = alarm.get('errorType', '') - title = alarm.get('title', '') - content = alarm.get('content', '') - biz_code = alarm.get('bizCode', '') - - if not error_type or not title or not content or not biz_code: - return None, None - - alarm = { - 'type': 'ANTEMC_DINGTALK', - 'user_input': { - 'bizCode': biz_code, - 'title': title, - 'content': content, - 'execute_type': 'gql', - 'errorType': error_type - } - } - - body = { - 'features': { - 'query': alarm - } - } - intent_url = 'https://paiplusinferencepre.alipay.com/inference/ff998e48456308a9_EKG_route/0.1' - headers = { - 'Content-Type': 'application/json;charset=utf-8', - 'MPS-app-name': 'test', - 'MPS-http-version': '1.0' - } - ans = requests.post(intent_url, json=body, headers=headers) - - ans_json = ans.json() - output = ans_json.get('resultMap').get('output') - logger.debug(f"{body}") - logger.debug(f"{output}") - output_json = json.loads(output) - res = output_json[-1] - all_intent = output_json - return res, all_intent + diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py index 8c9b962..d337e95 100644 --- a/tests/service/ekg_construct_test_2.py +++ b/tests/service/ekg_construct_test_2.py @@ -103,12 +103,12 @@ def generate_node(id, type): extra_attr["enable"] = False if type == "opsgptkg_task": - extra_attr["accessCriteria"] = "hello" + extra_attr["accesscriteria"] = "hello" if type == "opsgptkg_analysis": - extra_attr["accessCriteria"] = "hello" - extra_attr["summarySwtich"] = False - extra_attr['dslTemplate'] = "hello" + extra_attr["accesscriteria"] = "hello" + extra_attr["summaryswtich"] = False + extra_attr['dsltemplate'] = "hello" return GNode(**{ "id": id, From 1a0330412c92d2528d94bf42d82274978250865a Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 13 Aug 2024 16:06:14 +0800 Subject: [PATCH 017/128] [bugfix][when extra is ''] --- .../db_handler/graph_db_handler/geabase_handler.py | 4 ++-- muagent/service/ekg_construct/ekg_construct_base.py | 12 ++++++------ tests/service/ekg_construct_test_2.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index e80db45..7eacf02 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -396,7 +396,7 @@ def decode_vertex(self, col_data, k) -> Dict: vertextVal = col_data.get("vertexVal", {}) node_val_json = { **{"ID": int(vertextVal.get("id", "")), "type": vertextVal.get("type", "")}, - **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in vertextVal.get("props", {}).items()} + **{k: v.get("strVal", "") if "strVal" in v else v.get("intVal", "0") for k, v in vertextVal.get("props", {}).items()} } node_val_json.pop("biz_node_id", "") return node_val_json @@ -406,7 +406,7 @@ def _decode_edge(data): edgeVal= data.get("edgeVal", {}) edge_val_json = { **{"SRCID": int(edgeVal.get("srcId", "")), "DSTID": int(edgeVal.get("dstId", "")), "type": edgeVal.get("type", "")}, - **{k: v.get("strVal", "") or v.get("intVal", "0") for k, v in edgeVal.get("props", {}).items()} + **{k: v.get("strVal", "") if "strVal" in v else v.get("intVal", "0") for k, v in edgeVal.get("props", {}).items()} } # 存在业务逻辑 edge_val_json["start_id"] = edge_val_json.pop("original_src_id1__") diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index ebfe186..ef90f9b 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -408,7 +408,7 @@ def update_edges(self, edges: List[GEdge], teamid: str): def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) - extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) return node @@ -418,10 +418,10 @@ def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str=None, hop # filter the node which dont match teamid result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop) for node in result.nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) for edge in result.edges: - extra_attrs = json.loads(edge.attributes.pop("extra", "{}")) + extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") edge.attributes.update(extra_attrs) return result @@ -463,7 +463,7 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) nodes = self.gb.get_nodes_by_ids(nodeids) for node in nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) return nodes_by_name + nodes_by_desc + nodes @@ -471,10 +471,10 @@ def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> # rootid = f"{teamid}" # todo check the rootid result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) for node in result.nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}")) + extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) for edge in result.edges: - extra_attrs = json.loads(edge.attributes.pop("extra", "{}")) + extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") edge.attributes.update(extra_attrs) # paths must be ordered from start to end diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py index d337e95..d7e4aee 100644 --- a/tests/service/ekg_construct_test_2.py +++ b/tests/service/ekg_construct_test_2.py @@ -247,5 +247,5 @@ def generate_edge(node1, node2): print(len(nodes)) # search path by node and rootid -graph = ekg_construct_service.search_rootpath_by_nodeid(nodeid="shanshi_opsgptkg_analysis_3", node_type="opsgptkg_analysis", rootid="shanshi_opsgptkg_intent_0") +graph = ekg_construct_service.search_rootpath_by_nodeid(nodeid="shanshi_opsgptkg_analysis_2", node_type="opsgptkg_analysis", rootid="shanshi_opsgptkg_intent_0") print(len(graph.nodes), len(graph.edges)) \ No newline at end of file From c9bbcb67e8dc2d363c3b6e37b7881423a929771d Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 14 Aug 2024 11:37:32 +0800 Subject: [PATCH 018/128] bugfix deduplicate_paths, and update leaf node attributes --- .../graph_db_handler/geabase_handler.py | 17 ++++++++---- .../ekg_construct/ekg_construct_base.py | 27 ++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 7eacf02..42d2788 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -285,15 +285,22 @@ def deduplicate_paths(self, result, block_attributes: dict = {}, select_attribut for i in n0+n1 if select_attributes and not all(item not in i.items() for item in select_attributes.items()) ] - # 路径去重 + p = [_p for _p in p if all([_pid not in block_node_ids for _pid in _p])] + # deduplicate the paths path_strs = ["&&".join(_p) for _p in p] new_p = [] - new_path_strs_set = set() for path_str, _p in zip(path_strs, p): if not any(path_str in other for other in path_strs if path_str != other): - if path_str not in new_path_strs_set and all([_pid not in block_node_ids for _pid in _p]): - new_p.append(_p) - new_path_strs_set.add(path_str) + new_p.append(_p) + # # 路径去重 + # path_strs = ["&&".join(_p) for _p in p] + # new_p = [] + # new_path_strs_set = set() + # for path_str, _p in zip(path_strs, p): + # if not any(path_str in other for other in path_strs if path_str != other): + # if path_str not in new_path_strs_set and all([_pid not in block_node_ids for _pid in _p]): + # new_p.append(_p) + # new_path_strs_set.add(path_str) # 根据保留路径进行合并 nodeid2type = {i["id"]: i["type"] for i in n0+n1} diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index ef90f9b..da30637 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -412,14 +412,35 @@ def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: node.attributes.update(extra_attrs) return node - def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str=None, hop: int = 10) -> Graph: - if hop > 14: + def get_graph_by_nodeid( + self, + nodeid: str, + node_type: str, + hop: int = 10, + block_attributes: dict = {} + ) -> Graph: + if hop<2: + raise Exception(f"hop must be smaller than 2, now hop is {hop}") + if hop >= 14: raise Exception(f"hop can't be larger than 14, now hop is {hop}") # filter the node which dont match teamid - result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop) + result = self.gb.get_hop_infos( + {'id': nodeid}, node_type=node_type, + hop=hop, block_attributes=block_attributes + ) + + if block_attributes: + leaf_nodeids = [node.id for node in result.nodes if node.type=="opsgptkg_schedule"] + else: + leaf_nodeids = [path[-1] for path in result.paths if len(path)==hop+1] + for node in result.nodes: extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) + if node.id in leaf_nodeids: + neighbor_nodes = self.gb.get_neighbor_nodes({"id": node.id}, node_type=node.type) + node.attributes["cnode_nums"] = len(neighbor_nodes) + for edge in result.edges: extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") edge.attributes.update(extra_attrs) From aaf8cf328babc1be461a9a92baa0fb6d06514a9e Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 14 Aug 2024 13:51:31 +0800 Subject: [PATCH 019/128] add task_node attribute=executetype --- muagent/schemas/ekg/ekg_graph.py | 3 ++- tests/service/ekg_construct_test_2.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 99265b6..1750f06 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -111,6 +111,7 @@ class EKGTaskNodeSchema(EKGNodeSchema): # needcheck: bool # when to access accesscriteria: str + executetype: str # # owner: str @@ -122,7 +123,7 @@ def attrbutes(self, ): "description": self.description, "teamids": self.teamids, "accesscriteria": self.accesscriteria, - "needcheck": self.needcheck, + "executetype": self.executetype, # "tool": self.tool }, **extra_attr diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py index d7e4aee..a14be26 100644 --- a/tests/service/ekg_construct_test_2.py +++ b/tests/service/ekg_construct_test_2.py @@ -104,6 +104,7 @@ def generate_node(id, type): if type == "opsgptkg_task": extra_attr["accesscriteria"] = "hello" + extra_attr["executetype"] = "hello" if type == "opsgptkg_analysis": extra_attr["accesscriteria"] = "hello" From 9de6d9ae2252856db1bafee8f0117edcb3a75861 Mon Sep 17 00:00:00 2001 From: lightislost Date: Thu, 15 Aug 2024 15:00:44 +0800 Subject: [PATCH 020/128] update nodeschema and text2graph --- muagent/schemas/ekg/ekg_graph.py | 99 ++++--------- .../ekg_construct/ekg_construct_base.py | 136 +++++++++++++----- 2 files changed, 126 insertions(+), 109 deletions(-) diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 1750f06..6f57802 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -1,6 +1,7 @@ from pydantic import BaseModel from typing import List, Dict from enum import Enum +import copy import json @@ -19,6 +20,7 @@ ############################ Base Schema ############################# ##################################################################### class NodeSchema(BaseModel): + type: str ID: int = None # depend on id-str id: str # depend on user's difine @@ -27,9 +29,16 @@ class NodeSchema(BaseModel): description: str gdb_timestamp: int + def attributes(self, ): + attrs = copy.deepcopy(vars(self)) + for k in ["ID", "type", "id"]: + attrs.pop(k) + attrs.update(json.loads(attrs.pop("extra", '{}') or '{}')) + return attrs class EdgeSchema(BaseModel): + type: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} SRCID: int = None original_src_id1__: str @@ -37,10 +46,15 @@ class EdgeSchema(BaseModel): DSTID: int = None original_dst_id2__: str # - timestamp: int + timestamp: int = None gdb_timestamp: int - + def attributes(self, ): + attrs = copy.deepcopy(vars(self)) + for k in ["SRCID", "DSTID", "type", "timestamp", "original_src_id1__", "original_dst_id2__"]: + attrs.pop(k) + attrs.update(json.loads(attrs.pop("extra", '{}') or '{}')) + return attrs ##################################################################### ############################ EKG Schema ############################# ##################################################################### @@ -64,47 +78,20 @@ class EKGNodeSchema(NodeSchema): extra: str = '' - class EKGEdgeSchema(EdgeSchema): # teamids: str # version:str # yyyy-mm-dd HH:MM:SS extra: str = '' - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return extra_attr - class EKGIntentNodeSchema(EKGNodeSchema): - # path: str = '' - - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return { - **{ - "name": self.name, - "description": self.description, - "teamids": self.teamids, - "path": self.path - }, - **extra_attr - } + pass + class EKGScheduleNodeSchema(EKGNodeSchema): # do action or not enable: bool - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return { - **{ - "name": self.name, - "description": self.description, - "teamids": self.teamids, - "enable": self.enable - }, - **extra_attr - } class EKGTaskNodeSchema(EKGNodeSchema): # tool: str @@ -114,20 +101,6 @@ class EKGTaskNodeSchema(EKGNodeSchema): executetype: str # # owner: str - - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return { - **{ - "name": self.name, - "description": self.description, - "teamids": self.teamids, - "accesscriteria": self.accesscriteria, - "executetype": self.executetype, - # "tool": self.tool - }, - **extra_attr - } class EKGAnalysisNodeSchema(EKGNodeSchema): @@ -138,33 +111,9 @@ class EKGAnalysisNodeSchema(EKGNodeSchema): # summary template dsltemplate: str - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return { - **{ - "name": self.name, - "description": self.description, - "teamids": self.teamids, - "accesscriteria": self.accesscriteria, - "summaryswtich": self.summaryswtich, - "dsltemplate": self.dsltemplate - }, - **extra_attr - } - class EKGPhenomenonNodeSchema(EKGNodeSchema): - - def attrbutes(self, ): - extra_attr = json.loads(self.extra) - return { - **{ - "name": self.name, - "description": self.description, - "teamids": self.teamids, - }, - **extra_attr - } + pass # Ekg Tool Schemas @@ -196,13 +145,17 @@ class EKGGraphSlsSchema(BaseModel): # ADD/DELETE operation_type: str = '' # {tool_id},{tool_id},{tool_id} - tool: str = '' - access_criteria: str = '' + executetype: str = '' + accesscriteria: str = '' teamids: str = '' extra: str = '' enable: bool = False - dslTemplate: str = '' + dsltemplate: str = '' + summaryswtich: bool = False + gdb_timestamp: int + original_src_id1__: str = "" + original_dst_id2__: str = "" class EKGNodeTbaseSchema(BaseModel): node_id: str diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index da30637..a493d4c 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -527,8 +527,8 @@ def create_ekg( if intent_nodes: ancestor_list = intent_nodes - elif intent_text: - ancestor_list, all_intent_list = self.get_intents(intent_text) + elif intent_text or text: + ancestor_list, all_intent_list = self.get_intents(intent_text or text) else: raise Exception(f"must have intent infomation") @@ -539,9 +539,8 @@ def create_ekg( result = self.text2graph(text, ancestor_list, all_intent_list, teamid) # do write - if do_save: - self.write2kg(result) - + graph = self.write2kg(result["sls_graph"], result["tbase_graph"], teamid, do_save=do_save) + result["graph"] = graph return result def alarm2graph( @@ -608,29 +607,42 @@ def dsl2graph(self, ) -> dict: # dsl2graph => write2kg pass - def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData): + def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData, teamid, do_save: bool=False) -> Graph: # everytimes, it will add new nodes and edges - nodes = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] - nodes = [GNode(id=node.id, type=node.type, attributes=node.attrbutes()) for node in nodes] - gb_result = self.gb.add_nodes(nodes) + gbase_nodes = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] + gbase_nodes = [GNode(id=node.id, type=node.type, attributes=node.attributes()) for node in gbase_nodes] - edges = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] - edges = [GEdge(start_id=edge.start_id, end_id=edge.end_id, type=edge.type, attributes=edge.attrbutes()) for edge in edges] - gb_result = self.gb.add_edges(edges) + gbase_edges = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] + gbase_edges = [ + GEdge(start_id=edge.original_src_id1__, end_id=edge.original_dst_id2__, + type="opsgptkg_"+edge.type.split("_")[2] + "_route_" + "opsgptkg_"+edge.type.split("_")[3], + attributes=edge.attributes()) + for edge in gbase_edges + ] - nodes = [node.dict() for node in ekg_tbase_data.nodes] - tb_result = self.tb.insert_data_hash(nodes) - - edges = [edge.dict() for edge in ekg_tbase_data.edges] - tb_result = self.tb.insert_data_hash(edges) + tbase_nodes = [ + { + k: np.array(v).astype(dtype=np.float32).tobytes() if k in ["name_vector", "desc_vector"] else v + for k, v in node.dict().items() + } + for node in ekg_tbase_data.nodes + ] + tbase_edges = [edge.dict() for edge in ekg_tbase_data.edges] - # dsl2graph => write2kg - ## delete tbase/graph by graph_id - ### diff the tabse within newest by graph_id - ### diff the graph within newest by graph_id - ## update tbase/graph by graph_id - pass + if do_save: + # gb_node_result = self.gb.add_nodes(gbase_nodes) + # gb_node_result = [self.gb.add_node(node) for node in gbase_nodes] + node_result = self.add_nodes(gbase_nodes, teamid) + edge_result = self.add_edges(gbase_edges, teamid) + logger.info(f"{node_result}\n{edge_result}") + # gb_edge_result = self.gb.add_edges(gbase_edges) + # gb_edge_result = [self.gb.add_edge(edge) for edge in gbase_edges] + # tb_node_result = self.tb.insert_data_hash(tbase_nodes, key="node_id", need_etime=False) + # tb_edge_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) + # logger.info(f"{gb_node_result}\n{gb_edge_result}\n{tb_node_result}\n{tb_edge_result}") + + return Graph(nodes=gbase_nodes, edges=gbase_edges, paths=[]) def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: # 返回值需要返回 dsl 结构的数据用于展示，这里稍微做下数据处理，但主要就需要 dsl 对应的值 @@ -703,13 +715,13 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= ekg_slsdata = EKGGraphSlsSchema( id=node_id, - type='node_' + node_type, + type='opsgptkg_' + node_type, name=node_info['content'], description=node_info['content'], - tool='', need_check='false', operation_type='ADD', - teamids=teamid + teamids=teamid, + gdb_timestamp=getCurrentTimestap(), ) sls_nodes.append(ekg_slsdata) @@ -722,7 +734,10 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= type=f'edge_route_intent_{node_type}', # 需要注意与老逻辑的兼容 end_id=node_id, operation_type='ADD', - teamids=teamid + teamids=teamid, + original_src_id1__=pid, + original_dst_id2__=node_id, + gdb_timestamp=getCurrentTimestap(), ) ) # edges @@ -741,7 +756,10 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= type=edge_type, end_id=end_id, operation_type='ADD', - teamids=teamid + teamids=teamid, + original_src_id1__=start_id, + original_dst_id2__=end_id, + gdb_timestamp=getCurrentTimestap(), ) ) return EKGSlsData(nodes=sls_nodes, edges=sls_edges) @@ -761,15 +779,16 @@ def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData node_str=f'graph_id={teamid}', name_keyword=" | ".join(extract_tags(name, topK=None)), desc_keyword=" | ".join(extract_tags(description, topK=None)), - name_vector=np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), - desc_vector= np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), + name_vector= name_vector[name], # np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), + desc_vector= desc_vector[description], # np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), ) ) for edge in ekg_sls_data.edges: + edge_type = "opsgptkg_"+edge.type.split("_")[2] + "_route_" + "opsgptkg_"+edge.type.split("_")[3] tbase_edges.append( EKGEdgeTbaseSchema( - edge_id=edge.id, - edge_type=edge.type, + edge_id=f"{edge.start_id}__{edge.end_id}", + edge_type=edge_type, edge_source=edge.start_id, edge_target=edge.end_id, edge_str=f'graph_id={teamid}', @@ -799,10 +818,10 @@ def get_md5(s): for node in ekg_sls_data.nodes: # 需要注意下 dsl的id md编码 nodes.append( - YuqueDslNodeData(id=node.id, type=type_dict.get(node.type.split("node_")[-1]), label=node.description) + YuqueDslNodeData(id=node.id, type=type_dict.get(node.type.split("opsgptkg_")[-1]), label=node.description) ) # 记录 schedule id 用于添加意图节点的边 - if node.type.split("node_")[-1] == 'schedule': + if node.type.split("opsgptkg_")[-1] == 'schedule': schedule_id = node.id # 添加意图节点 @@ -1001,7 +1020,7 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by flag = any([ field not in node.attributes for field in fields - if field not in ["start_id", "end_id", "ID", "id", "extra"] + if field not in ["type", "start_id", "end_id", "ID", "id", "extra"] ]) if flag and do_check: raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") @@ -1045,7 +1064,7 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by flag = any([ field not in edge.attributes for field in fields - if field not in ["dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"]]) + if field not in ["type", "dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"]]) if flag and do_check: raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") @@ -1063,3 +1082,48 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by edge.attributes.pop("extra") return edges + + + def get_intent_by_alarm(self, alarm: dict, ) -> EKGIntentResp: + '''according content search intent''' + import requests + error_type = alarm.get('errorType', '') + title = alarm.get('title', '') + content = alarm.get('content', '') + biz_code = alarm.get('bizCode', '') + + if not error_type or not title or not content or not biz_code: + return None, None + + alarm = { + 'type': 'ANTEMC_DINGTALK', + 'user_input': { + 'bizCode': biz_code, + 'title': title, + 'content': content, + 'execute_type': 'gql', + 'errorType': error_type + } + } + + body = { + 'features': { + 'query': alarm + } + } + intent_url = 'https://paiplusinferencepre.alipay.com/inference/ff998e48456308a9_EKG_route/0.1' + headers = { + 'Content-Type': 'application/json;charset=utf-8', + 'MPS-app-name': 'test', + 'MPS-http-version': '1.0' + } + ans = requests.post(intent_url, json=body, headers=headers) + + ans_json = ans.json() + output = ans_json.get('resultMap').get('output') + logger.debug(f"{body}") + logger.debug(f"{output}") + output_json = json.loads(output) + res = output_json[-1] + all_intent = output_json + return res, all_intent \ No newline at end of file From 1972658368a4217426a2dc555ec062a6ca897a14 Mon Sep 17 00:00:00 2001 From: lightislost Date: Thu, 15 Aug 2024 20:16:11 +0800 Subject: [PATCH 021/128] update alarm2graph --- .../ekg_construct/ekg_construct_base.py | 116 ++++-------------- 1 file changed, 21 insertions(+), 95 deletions(-) diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index a493d4c..d72c56e 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -621,26 +621,10 @@ def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData, teami for edge in gbase_edges ] - tbase_nodes = [ - { - k: np.array(v).astype(dtype=np.float32).tobytes() if k in ["name_vector", "desc_vector"] else v - for k, v in node.dict().items() - } - for node in ekg_tbase_data.nodes - ] - tbase_edges = [edge.dict() for edge in ekg_tbase_data.edges] - if do_save: - # gb_node_result = self.gb.add_nodes(gbase_nodes) - # gb_node_result = [self.gb.add_node(node) for node in gbase_nodes] node_result = self.add_nodes(gbase_nodes, teamid) edge_result = self.add_edges(gbase_edges, teamid) logger.info(f"{node_result}\n{edge_result}") - # gb_edge_result = self.gb.add_edges(gbase_edges) - # gb_edge_result = [self.gb.add_edge(edge) for edge in gbase_edges] - # tb_node_result = self.tb.insert_data_hash(tbase_nodes, key="node_id", need_etime=False) - # tb_edge_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) - # logger.info(f"{gb_node_result}\n{gb_edge_result}\n{tb_node_result}\n{tb_edge_result}") return Graph(nodes=gbase_nodes, edges=gbase_edges, paths=[]) @@ -649,17 +633,26 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: res = {'dsl': '', 'details': {}, 'intent_node_list': intents} merge_dsl_nodes, merge_dsl_edges = [], [] + merge_gbase_nodes, merge_gbase_edges = [], [] id_sets = set() + gid_sets = set() for path_id, graph_datas in graph_datas_by_path.items(): res['details'][path_id] = { 'dsl': graph_datas["dsl_graph"], - 'sls': graph_datas["sls_graph"] + 'sls': graph_datas["sls_graph"], } merge_dsl_nodes.extend([node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets]) id_sets.update([i.id for i in graph_datas["dsl_graph"].nodes]) merge_dsl_edges.extend([edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets]) id_sets.update([i.id for i in graph_datas["dsl_graph"].edges]) + + merge_gbase_nodes.extend([node for node in graph_datas["graph"].nodes if node.id not in gid_sets]) + gid_sets.update([i.id for i in graph_datas["graph"].nodes]) + merge_gbase_edges.extend([edge for edge in graph_datas["graph"].edges if f"{edge.start_id}__{edge.end_id}" not in gid_sets]) + gid_sets.update([f"{i.start_id}__{i.end_id}" for i in graph_datas["graph"].edges]) + res["dsl"] = {"nodes": merge_dsl_nodes, "edges": merge_dsl_edges} + res["graph"] = Graph(nodes=merge_gbase_nodes, edges=merge_gbase_edges, paths=[]) return res def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: @@ -814,15 +807,12 @@ def get_md5(s): } nodes, edges = [], [] - schedule_id = '' + # schedule_id = '' for node in ekg_sls_data.nodes: # 需要注意下 dsl的id md编码 nodes.append( - YuqueDslNodeData(id=node.id, type=type_dict.get(node.type.split("opsgptkg_")[-1]), label=node.description) + YuqueDslNodeData(id=f"ekg_node:{node.type}:{node.id}", type=type_dict.get(node.type.split("opsgptkg_")[-1]), label=node.description) ) - # 记录 schedule id 用于添加意图节点的边 - if node.type.split("opsgptkg_")[-1] == 'schedule': - schedule_id = node.id # 添加意图节点 # 需要记录哪些是被添加过的 @@ -838,7 +828,7 @@ def get_md5(s): nodes.append( YuqueDslNodeData( - id=node.id, type='display', + id=dsl_pid, type='display', label=intent_names_dict.get(dsl_pid, pid),) ) added_intent.add(dsl_pid) @@ -865,35 +855,17 @@ def get_md5(s): added_intent.add(intent_id) for edge in ekg_sls_data.edges: + start_type, end_type = edge.type.split("_")[2:] edges.append( YuqueDslEdgeData( - id=f'{edge.start_id}___{edge.end_id}', - source=edge.start_id, - target=edge.end_id, + id=f'{start_type}:{edge.start_id}___{end_type}:{edge.end_id}', + source=f"{start_type}:{edge.start_id}", + target=f"{end_type}:{edge.end_id}", label='' ) ) - - # 添加意图边 + # # 添加意图边 added_edges = set() - for pid in pnode_ids: - # 处理意图节点展示样式 - dsl_pid = get_md5(pid) - dsl_pid = f'ekg_node:{teamid}:intent:{dsl_pid}' - - # 处理意图节点展示样式 - edge_id = f'{dsl_pid}___{schedule_id}' - if edge_id not in added_edges: - edges.append( - YuqueDslEdgeData( - id=edge_id, - source=dsl_pid, - target=schedule_id, - label='' - ) - ) - added_edges.add(edge_id) - for intent_list in all_intents: for idx in range(len(intent_list[0:-1])): if 'SRE_Agent' in intent_list[idx]: @@ -907,14 +879,14 @@ def get_md5(s): end_id = get_md5(intent_list[idx+1]) end_id = f'ekg_node:{teamid}:intent:{end_id}' - edge_id = f'{start_id}___{end_id}' + edge_id = f'intent:{start_id}___intent:{end_id}' if edge_id not in added_edges: edges.append( YuqueDslEdgeData( id=edge_id, - source=start_id, - target=end_id, + source=f"intent:{start_id}", + target=f"intent:{end_id}", label='' ) ) @@ -1081,49 +1053,3 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by edge.attributes.pop("@timestamp") edge.attributes.pop("extra") return edges - - - - def get_intent_by_alarm(self, alarm: dict, ) -> EKGIntentResp: - '''according content search intent''' - import requests - error_type = alarm.get('errorType', '') - title = alarm.get('title', '') - content = alarm.get('content', '') - biz_code = alarm.get('bizCode', '') - - if not error_type or not title or not content or not biz_code: - return None, None - - alarm = { - 'type': 'ANTEMC_DINGTALK', - 'user_input': { - 'bizCode': biz_code, - 'title': title, - 'content': content, - 'execute_type': 'gql', - 'errorType': error_type - } - } - - body = { - 'features': { - 'query': alarm - } - } - intent_url = 'https://paiplusinferencepre.alipay.com/inference/ff998e48456308a9_EKG_route/0.1' - headers = { - 'Content-Type': 'application/json;charset=utf-8', - 'MPS-app-name': 'test', - 'MPS-http-version': '1.0' - } - ans = requests.post(intent_url, json=body, headers=headers) - - ans_json = ans.json() - output = ans_json.get('resultMap').get('output') - logger.debug(f"{body}") - logger.debug(f"{output}") - output_json = json.loads(output) - res = output_json[-1] - all_intent = output_json - return res, all_intent \ No newline at end of file From 793c6ba8d6ede5278890892c2c5096e40e2a21f7 Mon Sep 17 00:00:00 2001 From: lightislost Date: Mon, 19 Aug 2024 15:36:55 +0800 Subject: [PATCH 022/128] [feat][add local fastapi server] --- .gitignore | 3 +- examples/ekg_examples/ekg.yaml | 43 +++ examples/ekg_examples/start.py | 248 ++++++++++++++++++ .../graph_db_handler/geabase_handler.py | 7 +- muagent/httpapis/__init__.py | 0 muagent/httpapis/ekg_construct/__init__.py | 6 + muagent/httpapis/ekg_construct/api.py | 218 +++++++++++++++ muagent/llm_models/openai_embedding.py | 9 +- muagent/schemas/apis/__init__.py | 0 muagent/schemas/apis/ekg_api_schema.py | 94 +++++++ .../ekg_construct/ekg_construct_base.py | 109 +------- .../service/ekg_construct/ekg_db_service.py | 243 ----------------- requirements.txt | 2 +- tests/httpapis/fastapi_test.py | 28 ++ 14 files changed, 652 insertions(+), 358 deletions(-) create mode 100644 examples/ekg_examples/ekg.yaml create mode 100644 examples/ekg_examples/start.py create mode 100644 muagent/httpapis/__init__.py create mode 100644 muagent/httpapis/ekg_construct/__init__.py create mode 100644 muagent/httpapis/ekg_construct/api.py create mode 100644 muagent/schemas/apis/__init__.py create mode 100644 muagent/schemas/apis/ekg_api_schema.py delete mode 100644 muagent/service/ekg_construct/ekg_db_service.py create mode 100644 tests/httpapis/fastapi_test.py diff --git a/.gitignore b/.gitignore index 73e0fc3..9dd8a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ build *egg-info dist .ipynb_checkpoints -zdatafront* \ No newline at end of file +zdatafront* +*antgroup* \ No newline at end of file diff --git a/examples/ekg_examples/ekg.yaml b/examples/ekg_examples/ekg.yaml new file mode 100644 index 0000000..62a26db --- /dev/null +++ b/examples/ekg_examples/ekg.yaml @@ -0,0 +1,43 @@ +# geabase config +geabase_config: + metaserver_address: 'deafault' + project: 'deafault' + city: 'deafault' + lib_path: 'deafault' + + +# nebula config +nebula_config: + host: 'localhost' + port: 7070 + username: 'default' + password: 'default' + space_name: 'default' + +# tbase config +tbase_config: + host: 'localhost' + port: 321321 + username: 'default' + password: '' + definition_value: 'opsgptkg' + + +# model +llm: + model_type: 'openai' + model_name: 'default' + stop: '' + temperature: 0.3 + top_p: 0.95 + top_k: 50 + url: '' + token: '' + + +# embedding +embedding: + embedding_type: 'openai' + model_name: 'default' + url: '' + token: '' diff --git a/examples/ekg_examples/start.py b/examples/ekg_examples/start.py new file mode 100644 index 0000000..18f4158 --- /dev/null +++ b/examples/ekg_examples/start.py @@ -0,0 +1,248 @@ +import time, sys +st = time.time() +import os +import yaml +import requests +from typing import List +from loguru import logger +import tqdm +from concurrent.futures import ThreadPoolExecutor + +print(time.time()-st) +from langchain.llms.base import LLM +from langchain.embeddings.base import Embeddings +print(time.time()-st) +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +print(src_dir) +sys.path.append(src_dir) + +from muagent.schemas.db import * +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService + +from pydantic import BaseModel + +# llm config +class CustomLLM(LLM, BaseModel): + url: str = "http://localhost:11434/api/generate" + model_name: str = "qwen2:1b" + model_type: str = "ollama" + api_key: str = "" + stop: str = "" + temperature: float = 0.3 + top_k: int = 50 + top_p: float = 0.95 + + def params(self): + keys = ["url", "model_name", "model_type", "api_key", "stop", "temperature", "top_k", "top_p"] + return { + k:v + for k,v in self.__dict__.items() + if k in keys} + + def update_params(self, **kwargs): + logger.debug(f"{kwargs}") + # 更新属性 + for key, value in kwargs.items(): + logger.debug(f"{key}, {value}") + setattr(self, key, value) + + def _llm_type(self, *args): + return "" + + def predict(self, prompt: str, stop = None) -> str: + return self._call(prompt, stop) + + def _call(self, prompt: str, + stop = None) -> str: + """_call + """ + return_str = "" + stop = stop or self.stop + + if self.model_type == "ollama": + data = { + "model": self.model_name, + "prompt": prompt + } + r = requests.post(self.url, json=data, ) + return r.json() + elif self.model_type == "openai": + from muagent.llm_models.openai_model import getChatModelFromConfig + llm_config = LLMConfig( + model_name=self.model_name, + model_engine="openai", + api_key=self.api_key, + api_base_url=self.url, + temperature=self.temperature, + stop=self.stop + ) + model = getChatModelFromConfig(llm_config) + return model.predict(prompt, stop=self.stop) + elif self.model_type == "lingyiwangwu": + from muagent.llm_models.openai_model import getChatModelFromConfig + llm_config = LLMConfig( + model_name=self.model_name, + model_engine="lingyiwangwu", + api_key=self.api_key, + api_base_url=self.url, + temperature=self.temperature, + stop=self.stop + ) + model = getChatModelFromConfig(llm_config) + return model.predict(prompt, stop=self.stop) + else: + pass + + return return_str + + +class CustomEmbeddings(Embeddings): + # ollama embeddings + url = "http://localhost:11434/api/embeddings" + # + embedding_type = "ollama" + model_name = "" + api_key = "" + + def params(self): + return { + "url": self.url, "model_name": self.model_name, + "embedding_type": self.embedding_type, "api_key": self.api_key + } + + def update_params(self, **kwargs): + logger.debug(f"{kwargs}") + # 更新属性 + for key, value in kwargs.items(): + logger.debug(f"{key}, {value}") + setattr(self, key, value) + + def _get_sentence_emb(self, sentence: str) -> dict: + """ + 调用句子向量提取服务 + """ + if self.embedding_type == "ollama": + data = { + "model": self.model_name, + "prompt": sentence + } + r = requests.post(self.url, json=data, ) + return r.json() + elif self.embedding_type == "openai": + from muagent.llm_models.get_embedding import get_embedding + embed_config = EmbedConfig( + embed_engine="openai", + api_key=self.api_key, + api_base_url=self.url, + ) + text2vector_dict = get_embedding("openai", [sentence], embed_config=embed_config) + return text2vector_dict[sentence] + else: + pass + + return [] + + def embed_documents(self, texts: List[str]) -> List[List[float]]: + embeddings = [] + + def process_text(text): + # print("分句：" + str(text) + "\n") + emb_str = self._get_sentence_emb(text) + # print("向量：" + str(emb_str) + "\n") + return emb_str + + with ThreadPoolExecutor() as executor: + results = list(tqdm(executor.map(process_text, texts), total=len(texts), desc="Embedding documents")) + + embeddings.extend(results) + print("向量个数" + str(len(embeddings))) + return embeddings + + def embed_query(self, text: str) -> List[float]: + """Compute query embeddings using a HuggingFace transformer model. + + Args: + text: The text to embed. + + Returns: + Embeddings for the text. + """ + logger.info("提问query: " + str(text)) + embedding = self._get_sentence_emb(text) + logger.info("提问向量:" + str(embedding)) + return embedding + + + +cur_dir = os.path.dirname(__file__) +print(cur_dir) + +# 要打开的YAML文件路径 +file_path = 'ekg.yaml' + +# 使用 'with' 语句确保文件正确关闭 +with open(os.path.join(cur_dir, file_path), 'r') as file: + # 加载YAML文件内容 + config_data = yaml.safe_load(file) + + + +# gb_config = GBConfig( +# gb_type="GeaBaseHandler", +# extra_kwargs={ +# 'metaserver_address': config_data["gbase_config"]['metaserver_address'], +# 'project': config_data["gbase_config"]['project'], +# 'city': config_data["gbase_config"]['city'], +# 'lib_path': config_data["gbase_config"]['lib_path'], +# } +# ) + + +# gb_config = GBConfig( +# gb_type="NebulaHandler", +# extra_kwargs={} +# ) + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="muagent_test", + host=config_data["tbase_config"]["host"], + port=config_data["tbase_config"]['port'], + username=config_data["tbase_config"]['username'], + password=config_data["tbase_config"]['password'], + extra_kwargs={ + 'host': config_data["tbase_config"]['host'], + 'port': config_data["tbase_config"]['port'], + 'username': config_data["tbase_config"]['username'] , + 'password': config_data["tbase_config"]['password'], + 'definition_value': config_data["tbase_config"]['definition_value'] + } +) + +llm = CustomLLM() +llm_config = LLMConfig( + llm=llm +) + + +embeddings = CustomEmbeddings() +embed_config = EmbedConfig( + embed_model="default", + langchain_embeddings=embeddings +) + + +# ekg_construct_service = EKGConstructService( +# embed_config=embed_config, +# llm_config=llm_config, +# tb_config=tb_config, +# gb_config=gb_config, +# ) + +from muagent.httpapis.ekg_construct import create_api +create_api(llm, embeddings) \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 42d2788..9f93444 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -3,8 +3,11 @@ from loguru import logger import json -from gdbc2.geabase_client import GeaBaseClient, Node, Edge, MutateBatchOperation, GeaBaseUtil -from gdbc2.geabase_env import GeaBaseEnv +try: + from gdbc2.geabase_client import GeaBaseClient, Node, Edge, MutateBatchOperation, GeaBaseUtil + from gdbc2.geabase_env import GeaBaseEnv +except: + logger.error("ignore this sdk") from .base_gb_handler import GBHandler from muagent.db_handler.utils import deduplicate_dict diff --git a/muagent/httpapis/__init__.py b/muagent/httpapis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/muagent/httpapis/ekg_construct/__init__.py b/muagent/httpapis/ekg_construct/__init__.py new file mode 100644 index 0000000..fd9ac78 --- /dev/null +++ b/muagent/httpapis/ekg_construct/__init__.py @@ -0,0 +1,6 @@ +from .api import create_api + + +__all__ = [ + "create_api" +] \ No newline at end of file diff --git a/muagent/httpapis/ekg_construct/api.py b/muagent/httpapis/ekg_construct/api.py new file mode 100644 index 0000000..4fc2ef6 --- /dev/null +++ b/muagent/httpapis/ekg_construct/api.py @@ -0,0 +1,218 @@ +from fastapi import FastAPI +from typing import Dict +import asyncio +import uvicorn + +from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService +from muagent.schemas.apis.ekg_api_schema import * + + +# +# def init_app(llm, embeddings, ekg_construct_service: EKGConstructService): +def init_app(llm, embeddings): + + app = FastAPI() + + # ~/llm/params + @app.get("/llm/params", response_model=LLMParamsResponse) + async def llm_params(): + return llm.params() + + # ~/llm/params/update + @app.post("/llm/params/update", response_model=EKGResponse) + async def update_llm_params(kwargs: Dict): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + llm.update_params(**kwargs) + except Exception as e: + errorMessage = str(e) + successCode = False + + return EKGResponse( + successCode=successCode, errorMessage=errorMessage, + ) + + # ~/embeddings/params + @app.get("/embeddings/params", response_model=EmbeddingsParamsResponse) + async def embedding_params(): + return embeddings.params() + + # ~/embeddings/params/update + @app.post("/embeddings/params/update", response_model=EKGResponse) + async def update_embedding_params(kwargs: Dict): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + embeddings.update_params(**kwargs) + except Exception as e: + errorMessage = str(e) + successCode = False + + return EKGResponse( + successCode=successCode, errorMessage=errorMessage, + ) + + # # ~/ekg/text2graph + # @app.post("/ekg/text2graph", response_model=EKGGraphResponse) + # async def text2graph(request: EKGT2GRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # result = ekg_construct_service.create_ekg( + # text=request.text, teamid=request.teamid, + # service_name="text2graph", + # intent_text=request.intentText, + # intent_nodes=request.intentNodeids, + # all_intent_list=request.intentPath, + # do_save=request.write2kg + # ) + # graph = result["graph"] + # nodes = [node.dict() for node in graph.nodes] + # edges = [edge.dict() for edge in graph.edges] + # except Exception as e: + # errorMessage = str(e) + # successCode = False + # nodes = [] + # edges = [] + + # return EKGGraphResponse( + # successCode=successCode, errorMessage=errorMessage, + # nodes=nodes, edges=edges + # ) + + + # # ~/ekg/graph/update + # @app.post("/ekg/graph/update", response_model=EKGResponse) + # async def update_graph(request: UpdateGraphRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # result = ekg_construct_service.update_graph( + # origin_nodes=request.originNodes, + # origin_edges=request.originEdges, + # nodes=request.nodes, + # edges=request.edges, + # teamid=request.teamid + # ) + # except Exception as e: + # errorMessage = str(e) + # successCode = False + + # return EKGResponse( + # successCode=successCode, errorMessage=errorMessage, + # ) + + + + # # ~/ekg/node/search + # @app.get("/ekg/node", response_model=GetNodeResponse) + # def get_node(request: GetNodeRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # node = ekg_construct_service.get_node_by_id( + # request.node_id, request.node_type + # ) + # node = node.dict() + # except Exception as e: + # errorMessage = str(e) + # successCode = False + # node = {} + + # return GetNodeResponse( + # successCode=successCode, errorMessage=errorMessage, + # node=node + # ) + + + # # ~/ekg/node/search + # @app.get("/ekg/graph", response_model=EKGGraphResponse) + # def get_graph(request: GetGraphRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # if request.layer == "first": + # graph = ekg_construct_service.get_graph_by_nodeid( + # nodeid=request.nodeid, node_type=request.nodeType, + # hop=8, block_attributes={"type": "opsgptkg_task"}) + # else: + # graph = ekg_construct_service.get_graph_by_nodeid( + # nodeid=request.nodeid, node_type=request.nodeType, + # hop=request.hop + # ) + # nodes = graph.nodes.dict() + # edges = graph.edges.dict() + # except Exception as e: + # errorMessage = str(e) + # successCode = False + # nodes, edges = {}, {} + + # return EKGGraphResponse( + # successCode=successCode, errorMessage=errorMessage, + # nodes=nodes, edges=edges + # ) + + + + # # ~/ekg/node/search + # @app.post("/ekg/node/search", response_model=GetNodesResponse) + # def search_node(request: SearchNodesRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # nodes = ekg_construct_service.search_nodes_by_text( + # request.text, teamid=request.teamid + # ) + # nodes = [node.dict() for node in nodes] + # except Exception as e: + # errorMessage = str(e) + # successCode = False + # nodes = [] + + # return GetNodesResponse( + # successCode=successCode, errorMessage=errorMessage, + # nodes=nodes + # ) + + # # ~/ekg/graph/ancestor + # @app.get("/ekg/graph/ancestor", response_model=EKGGraphResponse) + # def get_ancestor(request: SearchAncestorRequest): + # # 添加预测逻辑的代码 + # errorMessage = "ok" + # successCode = True + # try: + # graph = ekg_construct_service.search_rootpath_by_nodeid( + # nodeid=request.nodeid, node_type=request.nodeType, + # rootid=request.rootid + # ) + # nodes = graph.nodes.dict() + # edges = graph.edges.dict() + # except Exception as e: + # errorMessage = str(e) + # successCode = False + # nodes, edges = {}, {} + + + # return EKGGraphResponse( + # successCode=successCode, errorMessage=errorMessage, + # nodes=nodes, edges=edges + # ) + + return app + + +def create_api(llm, embeddings): + app = init_app(llm, embeddings) + uvicorn.run(app, host="localhost", port=3737) + +# def create_api(ekg_construct_service: EKGConstructService): +# app = init_app(ekg_construct_service) +# uvicorn.run(app, host="localhost", port=3737) diff --git a/muagent/llm_models/openai_embedding.py b/muagent/llm_models/openai_embedding.py index 17c70da..052540e 100644 --- a/muagent/llm_models/openai_embedding.py +++ b/muagent/llm_models/openai_embedding.py @@ -48,12 +48,11 @@ def get_emb(self, text_list): res = {} # logger.debug(emb_all_result) - logger.debug(f'len of result={len(emb_all_result["data"])}') - for emb_result in emb_all_result['data']: - index = emb_result['index'] - # logger.debug(index) + logger.debug(f'len of result={len(emb_all_result.data)}') + for emb_result in emb_all_result.data: + index = emb_result.index text = text_list[index] - emb = emb_result['embedding'] + emb = emb_result.embedding res[text] = emb return res diff --git a/muagent/schemas/apis/__init__.py b/muagent/schemas/apis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/muagent/schemas/apis/ekg_api_schema.py b/muagent/schemas/apis/ekg_api_schema.py new file mode 100644 index 0000000..0b680ae --- /dev/null +++ b/muagent/schemas/apis/ekg_api_schema.py @@ -0,0 +1,94 @@ +from pydantic import BaseModel +from typing import List, Dict +from enum import Enum + +from muagent.schemas.common import GNode, GEdge + + + + +class EKGResponse(BaseModel): + successCode: int + errorMessage: str + + +# text2graph +class EKGT2GRequest(BaseModel): + text: str + intentText: str = "" + intentNodeids: List[str] = [] + intentPath: List[str] = [] + teamid: str + write2kg: bool = False + +class EKGGraphResponse(EKGResponse): + nodes: List[GNode] + edges: List[GEdge] + + + +# update graph by compare +class UpdateGraphRequest(BaseModel): + originNodes: List[GNode] + originEdges: List[GEdge] + nodes: List[GNode] + edges: List[GEdge] + teamid: str + + + +# get node by nodeid and nodetype +class GetNodeRequest(BaseModel): + nodeid: str + nodeType: str + +class GetNodeResponse(EKGResponse): + node: GNode + + + +# get graph by nodeid\nodetpye\hop +class GetGraphRequest(BaseModel): + nodeid: str + nodeType: str + hop: int = 10 + layer: str + + + +# get node by nodeid and nodetype +class SearchNodesRequest(BaseModel): + text: str + nodeType: str + teamid: str + topK: int = 5 + +class GetNodesResponse(EKGResponse): + nodes: List[GNode] + + + +# get node by nodeid and nodetype +class SearchAncestorRequest(BaseModel): + nodeid: str + nodeType: str + rootid: str + hop: int = 10 + + +class LLMParamsResponse(BaseModel): + url: str + model_name: str + model_type: str + api_key: str + stop: str + temperature: float + top_k: int + top_p: float + +class EmbeddingsParamsResponse(BaseModel): + # ollama embeddings + url: str + embedding_type: str + model_name: str + api_key: str diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index d72c56e..6bcbc40 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -543,53 +543,6 @@ def create_ekg( result["graph"] = graph return result - def alarm2graph( - self, - alarms: List[dict], - alarm_analyse_content: dict, - teamid: str, - do_save: bool = False - ): - ancestor_list, all_intent_list = self.get_intents_by_alarms(alarms) - - graph_datas_by_pathid = {} - for path_id, diagnose_path in enumerate(alarm_analyse_content["diagnose_path"]): - - content = f"路径:{diagnose_path['name']}\n" - cur_count = 1 - for idx, step in enumerate(diagnose_path['diagnose_step']): - step_text = step['content'] if type(step['content']) == str else \ - step['content']['textInfo']['text'] - step_text = step_text.strip('[').strip(']') - step_text_no_time = EKGConstructService.remove_time(step_text) - # continue update - content += f'''{cur_count}. {step_text_no_time}\n''' - cur_count += 1 - - result = self.create_ekg( - content, teamid, service_name="text2graph", - intent_nodes=ancestor_list, all_intent_list=all_intent_list, - do_save= do_save - ) - graph_datas_by_pathid[path_id] = result - - return self.returndsl(graph_datas_by_pathid, intents=ancestor_list) - - def yuque2graph(self, **kwargs): - # get yuque loader - - # - self.create_ekg() - # yuque_url, write2kg, intent_node - - # get_graph(yuque_url) - # graph_id from md(yuque_content) - - # yuque_dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict - - # dsl2graph => write2kg - pass - def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], teamid: str) -> dict: # generate graph by llm result = self.get_graph_by_text(text, ) @@ -599,14 +552,6 @@ def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], dsl_graph = self.transform2dsl(sls_graph, intents, all_intent_list, teamid=teamid) return {"tbase_graph": tbase_graph, "sls_graph": sls_graph, "dsl_graph": dsl_graph} - def dsl2graph(self, ) -> dict: - # dsl, write2kg, intent_node, graph_id - - # dsl ==|code2graph|==> node_dict, edge_list, abnormal_dict - - # dsl2graph => write2kg - pass - def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData, teamid, do_save: bool=False) -> Graph: # everytimes, it will add new nodes and edges @@ -655,6 +600,9 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: res["graph"] = Graph(nodes=merge_gbase_nodes, edges=merge_gbase_edges, paths=[]) return res + def get_intent_by_alarm(self, ): + pass + def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: '''according contents search intents''' ancestor_list = set() @@ -906,57 +854,6 @@ def _get_embedding(self, text): text_vector = {text: [random.random() for _ in range(768)]} return text_vector - @staticmethod - def preprocess_json_contingency(content_dict, remove_time_flag=True): - # 专门处理告警数据合并成文本 - # 将对应的 content json 预处理为需要的样子，由于可能含有多个 path，用 dict 存储结果 - diagnose_path_list = content_dict.get('diagnose_path', []) - res = {} - if diagnose_path_list: - for idx, diagnose_path in enumerate(diagnose_path_list): - path_id = idx - path_name = diagnose_path['name'] - content = f'路径:{path_name}\n' - cur_count = 1 - - for idx, step in enumerate(diagnose_path['diagnose_step']): - step_name = step['name'] - - if type(step['content']) == str: - step_text = step['content'] - else: - step_text = step['content']['textInfo']['text'] - - step_text = step_text.strip('[') - step_text = step_text.strip(']') - - # step_text = step['step_summary'] - - step_text_no_time = EKGConstructService.remove_time(step_text) - to_append = f'''{cur_count}. {step_text_no_time}\n''' - cur_count += 1 - - content += to_append - - res[path_id] = { - 'path_id': path_id, - 'path_name': path_name, - 'content': content - } - return res - - @staticmethod - def remove_time(text): - re_pat = '''\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}(:\d{2})*''' - text_res = re.split(re_pat, text) - res = '' - for i in text_res: - if i: - i_strip = i.strip('，。\n') - i_strip = f'{i_strip}' - res += i_strip - return res - def _update_tbase_attr_for_nodes(self, attrs): tbase_attrs = {} for k in ["name", "description"]: diff --git a/muagent/service/ekg_construct/ekg_db_service.py b/muagent/service/ekg_construct/ekg_db_service.py deleted file mode 100644 index 6793c8b..0000000 --- a/muagent/service/ekg_construct/ekg_db_service.py +++ /dev/null @@ -1,243 +0,0 @@ -from typing import List, Dict -import numpy as np -import random - -from .ekg_construct_base import * -from muagent.schemas.common import * - -from muagent.llm_models.get_embedding import get_embedding -from muagent.utils.common_utils import getCurrentDatetime, getCurrentTimestap - - -def getClassFields(model): - # 收集所有字段，包括继承自父类的字段 - all_fields = set(model.__annotations__.keys()) - for base in model.__bases__: - if hasattr(base, '__annotations__'): - all_fields.update(getClassFields(base)) - return all_fields - - -class EKGDBService(EKGConstructService): - - def __init__( - self, - embed_config: EmbedConfig, - llm_config: LLMConfig, - db_config: DBConfig = None, - vb_config: VBConfig = None, - gb_config: GBConfig = None, - tb_config: TBConfig = None, - sls_config: SLSConfig = None, - do_init: bool = False, - kb_root_path: str = KB_ROOT_PATH - ): - super().__init__(embed_config, llm_config, db_config, vb_config, gb_config, tb_config, sls_config, do_init, kb_root_path) - - def add_nodes(self, nodes: List[GNode], teamid: str): - nodetype2fields_dict = {} - for node in nodes: - node_type = node.type - node.attributes["teamid"] = teamid - node.attributes["gdb_timestamp"] = getCurrentTimestap() - node.attributes["version"] = getCurrentDatetime() - node.attributes.setdefault("extra", '{}') - - # todo 根据节点类型进行数据校验 - schema = node_type - if node_type in nodetype2fields_dict: - fields = nodetype2fields_dict[node_type] - else: - fields = list(getClassFields(schema)) - nodetype2fields_dict[node_type] = fields - - flag = any([ - field not in node.attributes - for field in fields - if field not in ["start_id", "end_id", "id"] - ]) - if flag: - raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") - - tbase_nodes = [{ - "node_id": f'''ekg_node:{teamid}:{node.id}''', - "node_type": node.type, - "node_str": f"graph_id={teamid}", - "node_vector": np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() - } - for node in nodes - ] - - try: - gb_result = self.gb.add_nodes(nodes) - tb_result = self.tb.insert_data_hash(tbase_nodes, key_name='node_id', need_etime=False) - except Exception as e: - pass - - return gb_result or tb_result - - def add_edges(self, edges: List[GEdge], teamid: str): - edgetype2fields_dict = {} - for edge in edges: - edge_type = edge.type - edge.attributes["teamid"] = teamid - edge.attributes["@timestamp"] = getCurrentTimestap() - edge.attributes["gdb_timestamp"] = getCurrentTimestap() - edge.attributes["version"] = getCurrentDatetime() - edge.attributes["extra"] = '{}' - - # todo 根据边类型进行数据校验 - schema = edge_type - if edge_type in edgetype2fields_dict: - fields = edgetype2fields_dict[edge_type] - else: - fields = list(getClassFields(schema)) - edgetype2fields_dict[edge_type] = fields - - flag = any([field not in edge.attributes for field in fields if field not in ["start_id", "end_id", "id"]]) - if flag: - raise Exception(f"edge is wrong, type is {edge_type}, data is {edge.attributes}") - - tbase_edges = [{ - 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", - 'edge_type': edge.type, - 'edge_source': edge.start_id, - 'edge_target': edge.end_id, - 'edge_str': f'graph_id={teamid}' - } - for edge in edges - ] - - try: - gb_result = self.gb.add_edges(edges) - tb_result = self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) - except Exception as e: - pass - - return gb_result or tb_result - - def delete_nodes(self, nodes: List[GNode], teamid: str): - # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') - tbase_nodeids = [data['id'] for data in r.docs] # 存疑 - delete_nodeids = [node.id for node in nodes] - tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] - delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] - - if len(tbase_missing_nodeids) > 0: - logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - - # delete the nodeids in tbase - tb_result = [] - for nodeid in delete_tbase_nodeids: - self.tb.delete(nodeid) - resp = self.tb.delete(nodeid) - tb_result.append(resp) - # logger.info(f'id={nodeid}, delete resp={resp}') - - # delete the nodeids in geabase - gb_result = self.gb.delete_nodes(delete_tbase_nodeids) - - return gb_result or tb_result - - def delete_edges(self, edges: List[GEdge], teamid: str): - # delete tbase nodes - r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name='ekg_edge') - tbase_edgeids = [data['id'] for data in r.docs] # 存疑 - delete_edgeids = [f"edge:{edge.start_id}:{edge.end_id}" for edge in edges] - tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] - delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] - - if len(tbase_missing_edgeids) > 0: - logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") - - # delete the edgeids in tbase - tb_result = [] - for edgeid in delete_tbase_edgeids: - self.tb.delete(edgeid) - resp = self.tb.delete(edgeid) - tb_result.append(resp) - # logger.info(f'id={edgeid}, delete resp={resp}') - - # delete the nodeids in geabase - gb_result = self.gb.delete_edges(delete_tbase_edgeids) - - def update_nodes(self, nodes: List[GNode], teamid: str): - # delete tbase nodes - r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name='ekg_node') - tbase_nodeids = [data['id'] for data in r.docs] # 存疑 - update_nodeids = [node.id for node in nodes] - tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] - update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] - - if len(tbase_missing_nodeids) > 0: - logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - - # delete the nodeids in tbase - tb_result = [] - for node in nodes: - if node.id not in update_tbase_nodeids: continue - data = node.attributes - data.update({"node_id": node.id}) - resp = self.tb.insert_data_hash(data, key="node_id", need_etime=False) - tb_result.append(resp) - # logger.info(f'id={nodeid}, delete resp={resp}') - - # update the nodeids in geabase - gb_result = [] - for node in nodes: - if node.id not in update_tbase_nodeids: continue - resp = self.gb.update_node({}, node.attributes, node_type=node.type, ID=node.id) - gb_result.append(resp) - return gb_result or tb_result - - def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: - result = self.gb.get_current_node({'id': nodeid}, node_type=node_type) - nodeid = result.pop("id") - node_type = result.pop("node_type") - return GNode(id=nodeid, type=node_type, attributes=result) - - def get_graph_by_nodeid(self, nodeid: str, node_type: str, teamid: str, hop: int = 10) -> Graph: - if hop >= 15: - raise Exception(f"hop can't be larger than 15, now hop is {hop}") - # filter the node which dont match teamid - result = self.gb.get_hop_infos({'id': nodeid}, node_type=node_type, hop=hop, select_attributes={"teamid": teamid}) - return result - - def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: - - if text is None: return [] - - # if self.embed_config: - # raise Exception(f"can't use vector search, because there is no {self.embed_config}") - - # 直接检索文本 - # r = self.tb.search(text) - # nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) - # nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) - - if self.embed_config: - vector_dict = get_embedding( - self.embed_config.embed_engine, [text], - self.embed_config.embed_model_path, self.embed_config.model_device, - self.embed_config - ) - query_embedding = np.array(vector_dict[text]).astype(dtype=np.float32).tobytes() - base_query = f'(@teamid:{teamid})=>[KNN {top_k} @vector $vector AS distance]' - query_params = {"vector": query_embedding} - else: - query_embedding = np.array([random.random() for _ in range(768)]).astype(dtype=np.float32).tobytes() - base_query = f'(@teamid:{teamid})' - query_params = {} - - r = self.tb.vector_search(base_query, query_params=query_params) - - return - - def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, teamid: str): - rootid = f"{teamid}" - result = self.gb.get_hop_infos({"@ID": nodeid}, node_type=node_type, hop=15) - - # 根据nodeid和teamid来检索path - - diff --git a/requirements.txt b/requirements.txt index 1adf00a..cc31f80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ SQLAlchemy==2.0.19 docker redis==5.0.1 pydantic<=1.10.14 -# aliyun-log-python-sdk==0.9.0 +aliyun-log-python-sdk==0.9.0 # pydantic # duckduckgo-search urllib3==1.26.6 diff --git a/tests/httpapis/fastapi_test.py b/tests/httpapis/fastapi_test.py new file mode 100644 index 0000000..de7eb85 --- /dev/null +++ b/tests/httpapis/fastapi_test.py @@ -0,0 +1,28 @@ +from fastapi import FastAPI +import asyncio +import uvicorn +import time + +app = FastAPI() + +# 一个异步路由 +@app.get("/") +async def read_root(): + await asyncio.sleep(1) # 模拟一个异步操作 + return {"message": "Hello, World!"} + +# 另一个异步路由 +@app.get("/items/{item_id}") +async def read_item(item_id: int, q: str = None): + await asyncio.sleep(5) # 模拟延迟 + return {"item_id": item_id, "q": q} + +# 另一个异步路由 +@app.get("/itemstest/{item_id}") +def read_item(item_id: int, q: str = None): + time.sleep(5) # 模拟延迟 + return {"item_id": item_id, "q": q} + + +# 均能并发触发，好像对于io操作会默认管理 +uvicorn.run(app, host="localhost", port=3737) From 206cac2900abbf328af2efe346d8e15ce9077083 Mon Sep 17 00:00:00 2001 From: lightislost Date: Mon, 19 Aug 2024 20:55:07 +0800 Subject: [PATCH 023/128] [feat][add intent mudule] --- .../prompts/intention_template_prompt.py | 77 ++++ .../vector_db_handler/tbase_handler.py | 4 +- .../common/auto_extract_graph_schema.py | 3 + .../ekg_construct/ekg_construct_base.py | 48 +- .../ekg_inference/intention_match_rule.py | 41 ++ .../service/ekg_inference/intention_router.py | 436 ++++++++++++++++++ requirements.txt | 1 + 7 files changed, 579 insertions(+), 31 deletions(-) create mode 100644 muagent/service/ekg_inference/intention_match_rule.py create mode 100644 muagent/service/ekg_inference/intention_router.py diff --git a/muagent/base_configs/prompts/intention_template_prompt.py b/muagent/base_configs/prompts/intention_template_prompt.py index 5c20913..b177253 100644 --- a/muagent/base_configs/prompts/intention_template_prompt.py +++ b/muagent/base_configs/prompts/intention_template_prompt.py @@ -1,3 +1,6 @@ +from typing import Union, Optional + + RECOGNIZE_INTENTION_PROMPT = """你是一个任务决策助手，能够将理解用户意图并决策采取最合适的行动，尽可能地以有帮助和准确的方式回应人类， 使用 JSON Blob 来指定一个返回的内容，提供一个 action（行动）。 有效的 'action' 值为：'planning'(需要先进行拆解计划) or 'only_answer' （不需要拆解问题即可直接回答问题）or "tool_using" (使用工具来回答问题) or 'coding'(生成可执行的代码)。 @@ -59,3 +62,77 @@ ### Ouput Response ''' + + +def get_intention_prompt( + background: str, intentions: Union[list, tuple], examples: Optional[dict]=None +) -> str: + nums_zh = ('一', '两', '三', '四', '五', '六', '七', '八', '九', '十') + + intention_num = len(intentions) + num_zh = nums_zh[intention_num - 1] if intention_num <= 10 else intention_num + prompt = f'##背景##\n{background}\n\n##任务##\n辨别用户的询问意图，包括以下{num_zh}类：\n' + + for i, val in enumerate(intentions): + if isinstance(val, (list, tuple)): + k, v = val + cur_intention = f'{i + 1}. {k}：{v}\n' + else: + cur_intention = f'{i + 1}. {val}\n' + prompt += cur_intention + + prompt += '\n##注意事项##\n' + num_range_str = '、'.join(map(str, range(1, intention_num + 1))) + prompt += f'回答：数字{num_range_str}中的一个来表示用户的询问意图，对应上述{num_zh}种分类。避免提供额外解释或其他信息。\n\n' + + if examples: + prompt += '##示例##\n' + intention_idx_map = {k[0]: idx + 1 for idx, k in enumerate(intentions)} + for query, ans in examples.items(): + ans = intention_idx_map[ans] + prompt += f'询问：{query}\n回答：{ans}\n\n' + + prompt += '##用户询问##\n询问：{query}\n回答：' + return prompt + + +INTENTIONS_CONSULT_WHICH = [ + ('整体计划查询', '用户询问关于某个解决方案的完整流程或步骤，包含但不限于“整个流程”、“步骤”、“流程图”等词汇或概念。'), + ('下一步任务查询', '用户询问在某个解决方案的特定步骤中应如何操作或处理，通常会提及“下一步”、“具体操作”、“如何做”等，且明确指向解决方案中的某个特定环节。'), + ('闲聊', '用户询问的内容与当前的技术问题或解决方案无关，更多是出于兴趣或社交性质的交流。') +] +CONSULT_WHICH_PROMPT = get_intention_prompt( + '作为运维领域的客服，您的职责是根据用户询问的内容，精准判断其背后的意图，以便提供最恰当的服务和支持。', + INTENTIONS_CONSULT_WHICH, + { + '系统升级的整个流程是怎样的？': '整体计划查询', + '听说你们采用了新工具，能讲讲它的特点吗？': '闲聊' + } +) + +INTENTIONS_WHETHER_EXEC = [ + ('执行', '当用户声明自己在使用某平台、服务、产品或功能时遇到具体问题，且明确表示不知道如何解决时，其意图应被分类为“执行”。'), + ('询问', '当用户明确询问某些解决方案的背景、流程、方式方法等信息，或只是出于好奇想要了解更多信息，或只是简单闲聊时，其意图应被分类为“询问”。') +] +WHETHER_EXECUTE_PROMPT = get_intention_prompt( + '作为运维领域的客服，您需要根据用户询问判断其主要意图，以确定接下来的运维流程。', + INTENTIONS_WHETHER_EXEC, + { + '为什么我的优惠券使用失败？': '执行', + '我想知道如何才能更好地优化我的服务器性能，你们有什么建议吗？': '询问' + } +) + +DIRECT_CHAT_PROMPT = """##背景## +作为运维领域的客服，您的职责是根据自身专业知识回答用户询问，以便提供最恰当的服务和支持。 + +##任务## +基于您所掌握的领域知识，对用户的提问进行回答。 + +##注意事项## +请尽量从客观的角度来回答问题，内容符合事实、有理有据。 + +##用户询问## +询问：{query} +回答： +""" diff --git a/muagent/db_handler/vector_db_handler/tbase_handler.py b/muagent/db_handler/vector_db_handler/tbase_handler.py index 24b05d8..1f46ce2 100644 --- a/muagent/db_handler/vector_db_handler/tbase_handler.py +++ b/muagent/db_handler/vector_db_handler/tbase_handler.py @@ -101,7 +101,7 @@ def search(self, query, index_name: str = None, query_params: dict = {}, limit=1 res = index.search(query, query_params=query_params) return res - def vector_search(self, base_query: str, index_name: str = None, query_params: dict={}): + def vector_search(self, base_query: str, index_name: str = None, query_params: dict={}, limit=10): ''' vector_search :param base_query: @@ -114,7 +114,7 @@ def vector_search(self, base_query: str, index_name: str = None, query_params: d .sort_by('distance') .dialect(2) ) - r = self.search(query, index_name, query_params) + r = self.search(query, index_name, query_params, limit=limit) return r def delete(self, content): diff --git a/muagent/schemas/common/auto_extract_graph_schema.py b/muagent/schemas/common/auto_extract_graph_schema.py index 4514dd9..68175e4 100644 --- a/muagent/schemas/common/auto_extract_graph_schema.py +++ b/muagent/schemas/common/auto_extract_graph_schema.py @@ -33,6 +33,9 @@ class GNode(BaseModel): type: str attributes: Dict + def __getattr__(self, name: str): + return self.attributes.get(name) + class GEdge(BaseModel): start_id: str diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 6bcbc40..029f562 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -28,6 +28,7 @@ from muagent.base_configs.env_config import KB_ROOT_PATH +from muagent.service.ekg_inference.intention_router import IntentionRouter from muagent.llm_models.get_embedding import get_embedding from muagent.utils.common_utils import getCurrentDatetime, getCurrentTimestap from muagent.utils.common_utils import double_hashing @@ -53,6 +54,7 @@ def __init__( gb_config: GBConfig = None, tb_config: TBConfig = None, sls_config: SLSConfig = None, + intention_router: IntentionRouter = None, do_init: bool = False, kb_root_path: str = KB_ROOT_PATH, ): @@ -71,6 +73,7 @@ def __init__( # get llm model self.model = getChatModelFromConfig(self.llm_config) + self.intention_router = IntentionRouter(self.model, embed_config=self.embed_config) # init db handler self.init_handler() @@ -519,16 +522,17 @@ def create_ekg( text: str, teamid: str, service_name: str, + rootid: str, intent_text: str = None, intent_nodes: List[str] = [], all_intent_list: List=[], - do_save: bool = False + do_save: bool = False, ): if intent_nodes: ancestor_list = intent_nodes elif intent_text or text: - ancestor_list, all_intent_list = self.get_intents(intent_text or text) + ancestor_list, all_intent_list = self.get_intents(rootid, intent_text or text) else: raise Exception(f"must have intent infomation") @@ -542,6 +546,9 @@ def create_ekg( graph = self.write2kg(result["sls_graph"], result["tbase_graph"], teamid, do_save=do_save) result["graph"] = graph return result + + def dsl2graph(self, ): + pass def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], teamid: str) -> dict: # generate graph by llm @@ -600,34 +607,17 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: res["graph"] = Graph(nodes=merge_gbase_nodes, edges=merge_gbase_edges, paths=[]) return res - def get_intent_by_alarm(self, ): - pass - - def get_intents(self, alarm_list: list[dict], ) -> EKGIntentResp: + def get_intents(self, rootid, text: str): '''according contents search intents''' - ancestor_list = set() - all_intent_list = [] - for alarm in alarm_list: - ancestor, all_intent = self.get_intent_by_alarm(alarm) - if ancestor is None: - continue - ancestor_list.add(ancestor) - all_intent_list.append(all_intent) - - return list(ancestor_list), all_intent_list - - def get_intents_by_alarms(self, alarm_list: list[dict], ) -> EKGIntentResp: - '''according contents search intents''' - ancestor_list = set() - all_intent_list = [] - for alarm in alarm_list: - ancestor, all_intent = self.get_intent_by_alarm(alarm) - if ancestor is None: - continue - ancestor_list.add(ancestor) - all_intent_list.append(all_intent) - - return list(ancestor_list), all_intent_list + result = self.intention_router.get_intention_by_node_info_nlp( + root_node_id=rootid, + query=text, + start_from_root=True, + gb_handler=self.gb, + tb_handler=self.tb, + agent=self.model + ) + return result.get("node_id", ), [] def get_graph_by_text(self, text: str) -> EKGSlsData: '''according text generate ekg's raw datas''' diff --git a/muagent/service/ekg_inference/intention_match_rule.py b/muagent/service/ekg_inference/intention_match_rule.py new file mode 100644 index 0000000..17293ec --- /dev/null +++ b/muagent/service/ekg_inference/intention_match_rule.py @@ -0,0 +1,41 @@ +import re +import Levenshtein +from muagent.schemas.common import GNode + + +class MatchRule: + @classmethod + def edit_distance(cls, node: GNode, pattern=None, **kwargs): + if len(kwargs) == 0: + return -float('inf') + + s = list(kwargs.values())[0] + desc: str = node.attributes.get('description', '') + + if pattern is None: + return -Levenshtein.distance(desc, s) + + desc_list = re.findall(pattern, desc) + if not desc_list: + return -float('inf') + + return max([-Levenshtein.distance(x, s) for x in desc_list]) + + @classmethod + def edit_distance_integer(cls, node: GNode, **kwargs): + return cls.edit_distance(node, pattern='\d+', **kwargs) + + +class RuleDict(dict): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def save(self): + """save the rules""" + raise NotImplementedError + + def load(self, **kwargs): + """load the rules""" + raise NotImplementedError + +rule_dict = RuleDict() diff --git a/muagent/service/ekg_inference/intention_router.py b/muagent/service/ekg_inference/intention_router.py new file mode 100644 index 0000000..4f38a34 --- /dev/null +++ b/muagent/service/ekg_inference/intention_router.py @@ -0,0 +1,436 @@ +import re +import numpy as np +import muagent.base_configs.prompts.intention_template_prompt as itp +from loguru import logger +from dataclasses import dataclass, field, asdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Callable, Union, Optional, Any +from muagent.db_handler.graph_db_handler.base_gb_handler import GBHandler +from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler +from muagent.schemas.ekg.ekg_graph import NodeTypesEnum, TYPE2SCHEMA +from muagent.schemas.common import GNode +from muagent.llm_models.get_embedding import get_embedding +from .intention_match_rule import rule_dict, MatchRule + + +@dataclass +class NLPRetInfo: + node_id: str + is_leaf: bool = False + nodes_to_choose: Optional[list[dict]] = field(default=None) + answer: str = None + error_msg: str = '' + + +@dataclass +class RuleRetInfo: + node_id: str + is_leaf: bool = False + error_msg: str = '' + + +class IntentionRouter: + Rule_type = Optional[str] + def __init__(self, agent=None, gb_handler: GBHandler=None, tb_handler: TbaseHandler=None, embed_config=None): + self.agent = agent + self.gb_handler = gb_handler + self.tb_handler = tb_handler + self.embed_config = embed_config + self._node_type = NodeTypesEnum.INTENT.value + self._max_num_tb_retrieval = 10 + self._filter_max_depth = 5 + + def add_rule(self, node_id2rule: dict[str, str], gb_handler: Optional[GBHandler] = None): + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + ori_len = len(rule_dict) + fail_nodes, fail_rules = [], [] + for node_id, rule in node_id2rule.items(): + try: + gb_handler.get_current_node({'id': node_id}, node_type=self._node_type) + except IndexError: + fail_nodes.append(node_id) + continue + + rule_func = _get_rule_by_str(rule) + if rule_func is None: + fail_rules.append(node_id) + continue + + rule_dict[node_id] = rule + + if len(rule_dict) > ori_len: + rule_dict.save_to_odps() + + error_msg = '' + if fail_nodes: + error_msg += ', '.join([ + f'Node(id={node_id}, node_type={self._node_type})' + for node_id in fail_nodes + ]) + error_msg += ' do not exist! ' + if fail_rules: + error_msg += 'Rule of ' + error_msg += ', '.join([ + f'Node(id={node_id}, node_type={self._node_type})' + for node_id in fail_rules + ]) + error_msg += ' is not valid!' + return error_msg + + def get_intention_by_info2id(self, gb_handler: GBHandler, rule: Union[str, Callable]=':', **kwargs) -> str: + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + node_id = rule.join(kwargs.values()) if isinstance(rule, str) else rule(**kwargs) + try: + gb_handler.get_current_node({'id': node_id}, node_type=self._node_type) + except IndexError: + logger.error(f'Node(id={node_id}, node_type={self._node_type}) does not exist!') + return None + return node_id + + def _get_intention_by_node_info_match( + self, gb_handler: GBHandler, root_node_id: str, + rule: Rule_type = None, **kwargs + ) -> str: + def _func(node: GNode, rule: Callable): + return rule(node, **kwargs), node.id + + error_msg, rule_str = '', rule + if rule is None: + if root_node_id in rule_dict: + rule = rule_dict(root_node_id) + else: + rule = MatchRule.edit_distance + rule_str = 'edit_distance' + if isinstance(rule, str): + rule = _get_rule_by_str(rule) + if isinstance(rule, str): + try: + rule = getattr(MatchRule, rule) + except AttributeError: + rule = None + if rule is None: + error_msg = f'Rule {rule_str} is not valid!' + return None, error_msg + + intention_nodes = gb_handler.get_neighbor_nodes({'id': root_node_id}, self._node_type) + intention_nodes = [node for node in intention_nodes if node.type == self._node_type] + if len(intention_nodes) == 0: + return root_node_id, error_msg + elif len(intention_nodes) == 1: + return intention_nodes[0].id, error_msg + elif len(intention_nodes) < 20: + scores = [_func(node, rule) for node in intention_nodes] + else: + params = [(node, rule) for node in intention_nodes] + scores = _execute_func_distributed(_func, params) + + return max(scores)[-1], error_msg + + def get_intention_by_node_info_match( + self, root_node_id: str, filter_attribute: Optional[dict]=None, gb_handler: Optional[GBHandler] = None, + rule: Union[Rule_type, list[Rule_type]]=None, **kwargs + ) -> dict[str, Any]: + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + root_node_id = self._filter_from_root_node(gb_handler, root_node_id, filter_attribute) + + is_leaf = False + if len(kwargs) == 0: + error_msg = 'No information in query to be matched.' + return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + + args_list = _parse_kwargs(**kwargs) + if not isinstance(rule, (list, tuple)): + rule = [rule] * len(args_list) + if len(rule) != len(args_list): + error_msg = 'Length of rule should be equal to the length of Arguments.' + return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + + if not root_node_id: + error_msg = f'No node matches attribute {filter_attribute}.' + return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + + for cur_kw_arg, cur_rule in zip(args_list, rule): + next_node_id, error_msg = self._get_intention_by_node_info_match( + gb_handler, root_node_id, cur_rule, **cur_kw_arg + ) + if next_node_id is None or next_node_id == root_node_id: + is_leaf = True if next_node_id else False + return asdict(RuleRetInfo(node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg)) + root_node_id = next_node_id + + next_node_id = root_node_id + while next_node_id: + root_node_id = next_node_id + intention_nodes = gb_handler.get_neighbor_nodes({'id': next_node_id}, self._node_type) + intention_nodes = [ + node for node in intention_nodes + if node.type == self._node_type + ] + if len(intention_nodes) == 1: + next_node_id = intention_nodes[0].id + else: + next_node_id = None + if len(intention_nodes) == 0: + is_leaf = True + + if not is_leaf: + error_msg = 'Not enough to arrive the leaf node.' + return asdict(RuleRetInfo(node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg)) + + def get_intention_by_node_info_nlp( + self, root_node_id: str, query: str, start_from_root: bool = False, + gb_handler: Optional[GBHandler] = None, tb_handler: Optional[TbaseHandler] = None, agent=None, + ) -> dict[str, Any]: + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + tb_handler = tb_handler if tb_handler is not None else self.tb_handler + agent = agent if agent is not None else self.agent + + if start_from_root: + return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + + nodes_tb = self._tb_match(tb_handler, query, self._node_type) + filter_nodes_tb = self._filter_ancestors_hop(gb_handler, set(nodes_tb), root_node_id) + + filter_nodes_tb = { + k: v for k, v in filter_nodes_tb.items() + if self.is_node_valid(gb_handler, k) + } + + if len(filter_nodes_tb) == 0: + error_msg = 'No intention matched after tb_handler retrieval.' + ans = self._get_agent_ans_no_ekg(agent, query) + return asdict(NLPRetInfo(root_node_id, answer=ans, error_msg=error_msg)) + elif len(filter_nodes_tb) > 1: + error_msg = 'More than one intention matched after tb_handler retrieval.' + desc_list = [] + for k, v in filter_nodes_tb.items(): + node_desc = gb_handler.get_current_node({'id': k}, node_type=self._node_type) + node_desc = node_desc.attributes.get('description', '') + desc_list.append({'description': node_desc, 'path': ' -> '.join(v)}) + return asdict(NLPRetInfo(root_node_id, nodes_to_choose=desc_list, error_msg=error_msg)) + + root_node_id = list(filter_nodes_tb.keys())[0] + return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + + def _get_intention_by_nlp_from_root( + self, gb_handler: GBHandler, agent, root_node_id: str, query: str, + ) -> dict[str, Any]: + canditates = gb_handler.get_neighbor_nodes({'id': root_node_id}, self._node_type) + canditates = [n for n in canditates if n.type == self._node_type] + if len(canditates) == 0: + return asdict(NLPRetInfo(root_node_id, True)) + elif len(canditates) == 1: + root_node_id = canditates[0].id + return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + + desc_list = [x.attributes.get('description', '') for x in canditates] + desc_list.append('与上述意图都不匹配，属于其他类型的询问意图。') + query_intention = itp.get_intention_prompt( + '作为运维领域的客服，您需要根据用户询问判断其主要意图，以确定接下来的运维流程。', desc_list + ).format(query=query) + + ans = agent.predict(query_intention).strip() + ans = re.search('\d+', ans) + if ans: + ans = int(ans.group(0)) - 1 + if ans < len(desc_list) - 1: + root_node_id = canditates[ans].id + return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + + error_msg = f'No intention matched at Node(id={root_node_id}).' + ans = self._get_agent_ans_no_ekg(agent, query) + return asdict(NLPRetInfo(root_node_id, answer=ans, error_msg=error_msg)) + + def get_intention_whether_execute(self, query: str, agent=None) -> bool: + agent = agent if agent else self.agent + query = itp.WHETHER_EXECUTE_PROMPT.format(query=query) + ans = agent.predict(query).strip() + ans = re.search('\d+', ans) + if ans: + ans = int(ans.group(0)) - 1 + if ans < len(itp.INTENTIONS_WHETHER_EXEC): + return itp.INTENTIONS_WHETHER_EXEC[ans][0] == '执行' + + return False + + def get_intention_consult_which(self, query: str, agent=None) -> str: + agent = agent if agent else self.agent + query = itp.CONSULT_WHICH_PROMPT.format(query=query) + ans = agent.predict(query).strip() + ans = re.search('\d+', ans) + if ans: + ans = int(ans.group(0)) - 1 + if ans < len(itp.INTENTIONS_CONSULT_WHICH): + return itp.INTENTIONS_CONSULT_WHICH[ans][0] + + return itp.INTENTIONS_CONSULT_WHICH[-1][0] + + def _filter_from_root_node( + self, gb_handler: GBHandler, root_node_id: str, attribute: Optional[dict] = None + ) -> Optional[str]: + if attribute is None or len(attribute) == 0: + return root_node_id + canditates = gb_handler.get_hop_infos( + {'id': root_node_id}, self._node_type, hop=self._filter_max_depth, + ).nodes + canditates = [node for node in canditates if node.type == self._node_type] + + for node in canditates: + count = len(attribute) + for k, v in attribute.items(): + if v in getattr(node, k): + count -= 1 + if not count: + return node.id + + return None + + def _tb_match(self, tb_handler: TbaseHandler, query: str, node_type: str) -> list: + base_query = f'(*)=>[KNN {self._max_num_tb_retrieval} @desc_vector $query AS distance]' + + query_vector = get_embedding( + self.embed_config.embed_engine, [query], + self.embed_config.embed_model_path, self.embed_config.model_device, + self.embed_config + )[query] + + query_params = {'query': np.array(query_vector).astype(dtype=np.float32).tobytes()} + + canditates = tb_handler.vector_search( + base_query, limit=self._max_num_tb_retrieval, query_params=query_params + ).docs + canditates = [ + node.node_id for node in canditates + if node.node_type == node_type + ] + return canditates + + def is_node_valid(self, node_id: str, gb_handler: Optional[GBHandler] = None) -> bool: + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + canditates = gb_handler.get_neighbor_nodes({'id': node_id}, self._node_type) + if len(canditates) == 0: + return False + canditates = [n.id for n in canditates if n.type == self._node_type] + if len(canditates) == 0: + return True + return self.is_node_valid(gb_handler, canditates[0]) + + def _get_agent_ans_no_ekg(self, agent, query: str) -> str: + query = itp.DIRECT_CHAT_PROMPT.format(query=query) + ans = agent.predict(query).strip() + ans += f'\n\n以上内容由语言模型生成，仅供参考。' + return ans + + def _filter_ancestors_hop( + self, gb_handler: GBHandler, nodes: set, root_node: str + ) -> dict[str, list[str]]: + gb_ret = gb_handler.get_hop_infos( + {'id': root_node}, self._node_type, hop=self._filter_max_depth, + ) + paths, nodes = gb_ret.paths, gb_ret.nodes + if len(paths) == 0: + return dict() + + visited, ret_dict = set(), dict() + for path_list in paths: + ancestor, pos = path_list[0], 0 + for i, node_id in enumerate(path_list): + if node_id in ret_dict: + ret_dict.pop(node_id) + if node_id in nodes: + ancestor, pos = node_id, i + if ancestor not in visited and ancestor in nodes: + ret_dict[ancestor] = path_list[:pos + 1] + for j in range(i + 1): + visited.add(path_list[j]) + + id2name = {node.id: node.attributes.get('name', '') for node in nodes} + for k, v in ret_dict.items(): + ret_dict[k] = [id2name[x] for x in v] + + return ret_dict + + def _filter_ancestors( + self, gb_handler: GBHandler, nodes: set, root_node: str + ) -> dict[str, list[str]]: + split = '<->' + + def _dfs(s: str, ancestor: str, path: str, out: dict, visited: set): + childs = gb_handler.get_neighbor_nodes({'id': s}, self._node_type) + childs = [child.id for child in childs if child.type == self._node_type] + + if len(childs) == 0: + if ancestor not in visited and ancestor in nodes: + index = path.index(ancestor) + out[ancestor] = path[:index + len(ancestor)] + return + + for child in childs: + if child in nodes: + if ancestor in out: + out.pop(ancestor) + temp_ancestor = child + else: + temp_ancestor = ancestor + child_path = split.join((path, child)) + _dfs(child, temp_ancestor, child_path, out, visited) + if s in nodes: + visited.add(s) + + if len(nodes) == 0: + return dict() + filter_nodes = dict() + _dfs(root_node, root_node, root_node, filter_nodes, set()) + + for k, v in filter_nodes.items(): + filter_nodes[k] = v.split(split) + return filter_nodes + + +def _parse_kwargs(**kwargs) -> list: + keys = list(kwargs.keys()) + if not isinstance(kwargs[keys[0]], (list, tuple)): + return [kwargs] + + ret_list = [] + max_len = max([len(v) for v in kwargs.values()]) + for i in range(max_len): + cur_kwargs = dict() + for k, v in kwargs.items(): + if i < len(v): + cur_kwargs[k] = v[i] + ret_list.append(cur_kwargs) + + return ret_list + + +def _execute_func_distributed(func: Callable, params: list[Union[tuple, dict]]): + tasks = [] + with ThreadPoolExecutor(max_workers=8) as exec: + for param in params: + if isinstance(param, tuple): + task = exec.submit(func, *param) + else: + task = exec.submit(func, **param) + tasks.append(task) + + results = [] + for task in as_completed(tasks): + results.append(task.result()) + return results + + +def _get_rule_by_str(rule: str) -> Union[str, Callable]: + has_func = re.search('def ([a-zA-Z0-9_]+)\(.*\):', rule) + if not has_func: + return getattr(MatchRule, rule.strip()) + + func_name = has_func.group(1) + try: + exec(rule) + except Exception as e: + logger.info(f'Rule {rule} cannot be executed!') + logger.error(e) + return None + + return locals().get(func_name, None) diff --git a/requirements.txt b/requirements.txt index cc31f80..7361a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ Pyarrow python-magic-bin; sys_platform == 'win32' SQLAlchemy==2.0.19 docker +Levenshtein redis==5.0.1 pydantic<=1.10.14 aliyun-log-python-sdk==0.9.0 From b0845f792567ed23250d6a93f59b340d3746d09a Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 20 Aug 2024 15:51:54 +0800 Subject: [PATCH 024/128] [bugfix][hop_infos speed up] --- .../base_configs/prompts/simple_prompts.py | 194 ++++++++++++++++-- .../graph_db_handler/geabase_handler.py | 26 ++- .../ekg_construct/ekg_construct_base.py | 3 + muagent/utils/common_utils.py | 6 +- 4 files changed, 193 insertions(+), 36 deletions(-) diff --git a/muagent/base_configs/prompts/simple_prompts.py b/muagent/base_configs/prompts/simple_prompts.py index e2b7588..d804f21 100644 --- a/muagent/base_configs/prompts/simple_prompts.py +++ b/muagent/base_configs/prompts/simple_prompts.py @@ -207,32 +207,28 @@ -text2EKG_prompt_en = '''你是一个结构化信息抽取的专家，你需要根据输入的文档，抽取其中的关键节点及节点间的连接顺序。请用json结构返回。 - -json结构定义如下: +text2EKG_prompt_en = '''# 上下文 # +给定一个关于某个描述流程或者步骤的输入文本，我需要根据给定的输入文本，得到输入文本中流程或者操作步骤的结构化表示，之后可以用来在其它程序中绘制流程图。 +你是一个结构化信息抽取和总结的专家，你可以根据输入的流程相关描述文本，抽取其中的关键节点及节点间的连接顺序，生成流程或者步骤的结构化json表示。 +############# +# 目标 # +我希望你根据输入文本，提供一个输入文本中流程、操作的结构化json表示。可以参考以下步骤思考，但是不要输出每个步骤中间结果，只输出最后的流程图json： +1. 确定流程图节点: 根据输入文本内容，确定流程图的各个节点。节点可以用如下结构表示： { "nodes": { "节点序号": { "type": "节点类型", "content": "节点内容" - } - }, - "edges": [ - { - "start": "起始节点序号", - "end": "终止节点序号" - } - ] + }, + } } -其中 nodes 用来存放抽取的节点，每个 node 的 key 通过从0开始对递增序列表示，value 是一个字典，包含 type 和 content 两个属性， type 对应下面定义的三种节点类型，content 为抽取的节点内容。 -edges 用来存放节点间的连接顺序，它是一个列表，每个元素是一个字典，包含 start 和 end 两个属性， start 为起始 node 的 节点序号, end 为结束 node 的 节点序号。 - +其中 nodes 用来存放抽取的节点，每个 node 的 key 通过从0开始对递增序列表示，value 是一个字典，包含 type 和 content 两个属性， type 对应下面定义的四种节点类型，content 为抽取的节点内容。 节点类型定义如下: Schedule: - 表示整篇输入文档所做的事情，是对整篇输入文档的总结； + 表示输入文本中流程和操作要完成的事情和任务，是对输入文本的意图的总结； 第一个节点永远是Schedule节点。 Task: - 表示需要执行的任务。 + 表示该节点需要执行的任务。 Phenomenon: 表示依据Task节点的执行结果，得到的事实结论。 Phenomenon节点只能连接在Task节点之后。 @@ -240,12 +236,54 @@ 表示依据Phenomenon节点的事实进行推断的过程； Analysis节点只能连接在Phenomenon节点之后。 -以下是一个例子： -input: 路径：排查网络问题 +2. 连接流程图节点: 根据输入文本内容，确定流程图的各个节点的连接关系。节点之间的连接关系可以用如下结构表示： +{ + "edges": [ + { + "start": "起始节点序号", + "end": "终止节点序号" + } + ] +} +edges 用来存放节点间的连接顺序，它是一个列表，每个元素是一个字典，包含 start 和 end 两个属性， start 为起始 node 的 节点序号, end 为结束 node 的 节点序号。 + +3. 生成表示流程图的完整json: 将上面[确定流程图节点]和[连接流程图节点]步骤中的结果放到一个json，检查生成的流程图是否符合给定输入文本的内容，优化流程图的结构，合并相邻同类型节点，返回最终的json。 +############# +# 风格 # +流程图节点数尽可能少，保持流程图结构简洁，相邻同类型节点可以合并。流程图节点中的节点内容content要准确、详细、充分。 +############# +# 语气 # +专业，技术性 +############# +# 受众 # +面向提供流程文本的人员，让他们确信你生成的流程图准确表示了文本中的流程步骤。 +############# +# 响应 # +返回json结构定义如下: +{ + "nodes": { + "节点序号": { + "type": "节点类型", + "content": "节点内容" + } + }, + "edges": [ + { + "start": "起始节点序号", + "end": "终止节点序号" + } + ] +} +############# +# 例子 # +以下是几个例子： + +# 例子1 # +输入文本:路径：排查网络问题 1. 通过观察sofagw网关监控发现，BOLT失败数突增 2. 且失败曲线与退保成功率曲线相关性较高，判定是网络问题。 -output: { +输出:{ "nodes": { "0": { "type": "Schedule", @@ -296,10 +334,122 @@ ] } -请根据上述说明和例子来对以下的输入文档抽取结构化信息: +# 例子2 # +输入文本:二、使用模版创建选品集 +STEP1：创建选品集 +注：因为只能选择同类型模版，必须先选择数据类型，才能选择模版创建 +STEP2：按需选择模版后，点击确认 + +- 我的收藏：个人选择收藏的模版 +- 我的创建：个人创建的模版 +- 模版广场：公开的模版，可以通过名称/创建人搜索到需要的模版并选择使用 + +STEP3：按需调整指标模版内的值，完成选品集创建 + +输出:{ + "nodes": { + "0": { + "type": "Schedule", + "content": "使用模版创建选品集" + }, + "1": { + "type": "Task", + "content": "创建选品集\n\n注：因为只能选择同类型模版，必须先选择数据类型，才能选择模版创建" + }, + "2": { + "type": "Task", + "content": "按需选择模版后，点击确认\n\n - 我的收藏：个人选择收藏的模版 \n\n - 我的创建：个人创建的模版 \n\n - 模版广场：公开的模版，可以通过名称/创建人搜索到需要的模版并选择使用" + }, + "3": { + "type": "Task", + "content": "按需调整指标模版内的值，完成选品集创建" + } + }, + "edges": [ + { + "start": "0", + "end": "1" + }, + { + "start": "1", + "end": "2" + }, + { + "start": "2", + "end": "3" + } + ] +} + +# 例子3 # +输入文本:Step1 + +- 点击右侧的左右切换箭头，找到自己所在的站点或业务模块; +Step2 + +- 查询对应一级场景，若没有所需一级场景则联系 [@小明][@小红]添加：具体操作如下： +- 邮件模板 +| 项目背景：
场景名称：
场景描述：
数据类型：商家/商品/营销商家/营销商品/权益商家/权益选品（必要的才选）
业务管理员：（花名） | +| --- | + +- 发送邮件[@小明][@小红]抄送 [@小白] +Step3 + +- 查询对应二级场景，若没有所需二级场景则联系一级场景管理员添加，支持通过搜索二级场景名称和ID快速查询二级场景； +- 一级管理员为下图蓝色框①所在位置查看 +Step4 + +- 申请二级场景数据权限，由对应二级场景管理员审批。若二级场景管理员为@小花@小映，按一级场景走申请流程。 +- 二级管理员为下图蓝色框②所在位置查看 + +输出:{ + "nodes": { + "0": { + "type": "Schedule", + "content": "场景权限申请" + }, + "1": { + "type": "Task", + "content": "点击右侧的左右切换箭头，找到自己所在的站点或业务模块" + }, + "2": { + "type": "Task", + "content": "查询对应一级场景，若没有所需一级场景则联系 [@小明][@小红]添加：具体操作如下：\n 发送邮件给小明和小红，抄送小白，邮件内容包括项目背景，场景名称，场景描述，数据类型和业务管理员" + }, + "3": { + "type": "Task", + "content": "查询对应二级场景，若没有所需二级场景则联系一级场景管理员添加，支持通过搜索二级场景名称和ID快速查询二级场景" + }, + "4": { + "type": "Task", + "content": "申请二级场景数据权限，由对应二级场景管理员审批。若二级场景管理员为@小花@小映，按一级场景走申请流程" + } + }, + "edges": [ + { + "start": "0", + "end": "1" + }, + { + "start": "1", + "end": "2" + }, + { + "start": "2", + "end": "3" + }, + { + "start": "3", + "end": "4" + } + ] +} +############# +# 开始抽取 # +请根据上述说明和例子来对以下的输入文本抽取结构化流程json: -input: {text} +输入文本:{text} -output:''' +输出:''' text2EKG_prompt_zh = text2EKG_prompt_en \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 9f93444..9f2371a 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -13,7 +13,7 @@ from muagent.db_handler.utils import deduplicate_dict from muagent.schemas.db import GBConfig from muagent.schemas.common import * -from muagent.utils.common_utils import double_hashing +from muagent.utils.common_utils import double_hashing, func_timer class GeaBaseHandler(GBHandler): @@ -142,7 +142,6 @@ def get_current_nodeID(self, attributes: dict, node_type: str) -> int: def get_current_edgeID(self, src_id, dst_id, edeg_type:str = None): if not isinstance(src_id, int) or not isinstance(dst_id, int): result = self.get_current_edge(src_id, dst_id, edeg_type) - logger.debug(f"{result}") return result.attributes.get("SRCID"), result.attributes.get("DSTID"), result.attributes.get("timestamp") else: return src_id, dst_id, 1 @@ -225,7 +224,7 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b ''' hop >= 2， 表面需要至少两跳 ''' - hop_max = 10 + hop_max = 8 # where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) if reverse: @@ -235,6 +234,7 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b last_node_ids, last_node_types = [], [] result = {} + iter_index = 0 while hop > 1: if last_node_ids == []: # @@ -247,11 +247,13 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b # _result = self.execute(gql) _result = self.decode_result(_result, gql) + # logger.info(f"p_lens: {len(_result['p'])}") result = self.merge_hotinfos(result, _result) # - last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes) + last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes, hop=min(hop, hop_max)+iter_index*hop_max) hop -= hop_max + iter_index += 1 nodes = self.convert2GNodes(result.get("n1", [])) edges = self.convert2GEdges(result.get("e", [])) @@ -274,7 +276,7 @@ def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, b result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.paths - def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}): + def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}, hop:int=None): # 获取数据 n0, n1, e, p = result["n0"], result["n1"], result["e"], result["p"] block_node_ids = [ @@ -308,24 +310,26 @@ def deduplicate_paths(self, result, block_attributes: dict = {}, select_attribut # 根据保留路径进行合并 nodeid2type = {i["id"]: i["type"] for i in n0+n1} unique_node_ids = [j for i in new_p for j in i] - last_node_ids = [i[-1] for i in new_p] + last_node_ids = list(set([i[-1] for i in new_p if len(i)>=hop])) last_node_types = [nodeid2type[i] for i in last_node_ids] new_n0 = deduplicate_dict([i for i in n0 if i["id"] in unique_node_ids]) new_n1 = deduplicate_dict([i for i in n1 if i["id"] in unique_node_ids]) new_e = deduplicate_dict([i for i in e if i["start_id"] in unique_node_ids and i["end_id"] in unique_node_ids]) return last_node_ids, last_node_types, {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} - + def merge_hotinfos(self, result1, result2) -> Dict: - new_n0 = result1["n0"] + result2["n0"] - new_n1 = result1["n1"] + result2["n1"] + old_n0_sets = set([n["id"] for n in result1["n0"]]) + old_n1_sets = set([n["id"] for n in result1["n1"]]) + new_n0 = result1["n0"] + [n for n in result2["n0"] if n["id"] not in old_n0_sets] + new_n1 = result1["n1"] + [n for n in result2["n1"] if n["id"] not in old_n1_sets] new_e = result1["e"] + result2["e"] - new_p = result1["p"] + result2["p"] + [ + new_p = result1["p"] + [ p_old_1 + p_old_2[1:] for p_old_1 in result1["p"] for p_old_2 in result2["p"] if p_old_2[0] == p_old_1[-1] - ] + ] # + result2["p"] new_result = {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} return new_result diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 029f562..dbfcd24 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -609,6 +609,9 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: def get_intents(self, rootid, text: str): '''according contents search intents''' + if rootid is None or rootid=="": + raise Exception(f"rootid={rootid}, it is empty") + result = self.intention_router.get_intention_by_node_info_nlp( root_node_id=rootid, query=text, diff --git a/muagent/utils/common_utils.py b/muagent/utils/common_utils.py index 73cf6f4..377a41a 100644 --- a/muagent/utils/common_utils.py +++ b/muagent/utils/common_utils.py @@ -38,7 +38,7 @@ def datefromatToTimestamp(dt, interval=1000, dateformat=DATE_FORMAT): return int(datetime.strptime(dt, dateformat).timestamp()*interval) -def func_timer(): +def func_timer(function): ''' 用装饰器实现函数计时 :param function: 需要计时的函数 @@ -46,13 +46,13 @@ def func_timer(): ''' @wraps(function) def function_timer(*args, **kwargs): + # logger.info('[Function: {name} start...]'.format(name=function.__name__)) t0 = time.time() result = function(*args, **kwargs) t1 = time.time() logger.info('[Function: {name} finished, spent time: {time:.3f}s]'.format( name=function.__name__, - time=t1 - t0 - )) + time=t1 - t0)) return result return function_timer From d5e223239169577cc3de5fdcf0063d180b0a93f6 Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 21 Aug 2024 11:31:33 +0800 Subject: [PATCH 025/128] [bugfix][update_graph: data decode error][vector_search: index_name is empty] --- .../ekg_construct/ekg_construct_base.py | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index dbfcd24..9240fa8 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -181,28 +181,29 @@ def update_graph( origin_nodes: List[GNode], origin_edges: List[GEdge], new_nodes: List[GNode], new_edges: List[GEdge], teamid: str ): - # - origin_nodeids = set([node["id"] for node in origin_nodes]) - origin_edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in origin_edges]) - nodeids = set([node["id"] for node in new_nodes]) - edgeids = set([f"{edge['start_id']}__{edge['end_id']}" for edge in new_edges]) + + origin_nodeids = set([node.id for node in origin_nodes]) + origin_edgeids = set([f"{edge.start_id}__{edge.end_id}" for edge in origin_edges]) + nodeids = set([node.id for node in new_nodes]) + edgeids = set([f"{edge.start_id}__{edge.end_id}" for edge in new_edges]) + unique_nodeids = origin_nodeids&nodeids unique_edgeids = origin_edgeids&edgeids nodeid2nodes_dict = {} for node in origin_nodes + new_nodes: - nodeid2nodes_dict.setdefault(node["id"], []).append(node) + nodeid2nodes_dict.setdefault(node.id, []).append(node) edgeid2edges_dict = {} for edge in origin_edges + new_edges: - edgeid2edges_dict.setdefault(f"{edge['start_id']}__{edge['end_id']}", []).append(edge) + edgeid2edges_dict.setdefault(f"{edge.start_id}__{edge.end_id}", []).append(edge) # get add nodes & edges - add_nodes = [node for node in new_nodes if node["id"] not in origin_nodeids] - add_edges = [edge for edge in new_edges if f"{edge['start_id']}__{edge['end_id']}" not in origin_edgeids] + add_nodes = [node for node in new_nodes if node.id not in origin_nodeids] + add_edges = [edge for edge in new_edges if f"{edge.start_id}__{edge.end_id}" not in origin_edgeids] # get delete nodes & edges - delete_nodes = [node for node in origin_nodes if node["id"] not in nodeids] - delete_edges = [edge for edge in origin_edges if f"{edge['start_id']}__{edge['end_id']}" not in edgeids] + delete_nodes = [node for node in origin_nodes if node.id not in nodeids] + delete_edges = [edge for edge in origin_edges if f"{edge.start_id}__{edge.end_id}" not in edgeids] # get update nodes & edges update_nodes = [ @@ -215,16 +216,16 @@ def update_graph( for edgeid in unique_edgeids if edgeid2edges_dict[edgeid][0]!=edgeid2edges_dict[edgeid][1] ] - + # - add_node_result = self.add_nodes([GNode(**n) for n in add_nodes], teamid) - add_edge_result = self.add_edges([GEdge(**e) for e in add_edges], teamid) + add_node_result = self.add_nodes(add_nodes, teamid) + add_edge_result = self.add_edges(add_edges, teamid) - delete_edge_result = self.delete_nodes([GNode(**n) for n in delete_nodes], teamid) - delete_node_result = self.delete_edges([GEdge(**e) for e in delete_edges], teamid) + delete_edge_result = self.delete_nodes(delete_nodes, teamid) + delete_node_result = self.delete_edges(delete_edges, teamid) - update_node_result = self.update_nodes([GNode(**n) for n in update_nodes], teamid) - update_edge_result = self.update_edges([GEdge(**e) for e in update_edges], teamid) + update_node_result = self.update_nodes(update_nodes, teamid) + update_edge_result = self.update_edges(update_edges, teamid) return [add_node_result, add_edge_result, delete_edge_result, delete_node_result, update_node_result, update_edge_result] @@ -275,15 +276,11 @@ def add_edges(self, edges: List[GEdge], teamid: str): def delete_nodes(self, nodes: List[GNode], teamid: str=''): # delete tbase nodes - # r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name=self.node_indexname) r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 - # tbase_nodeids_dict = {data["node_id"]:data['id'] for data in r.docs} # 附带了definition信息 delete_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] - # delete_tbase_nodeids = [nodeid for nodeid in delete_nodeids if nodeid in tbase_nodeids] - # delete_tbase_nodeids = [tbase_nodeids_dict[nodeid] for nodeid in delete_nodeids if nodeid in tbase_nodeids] if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") @@ -313,15 +310,11 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): def delete_edges(self, edges: List[GEdge], teamid: str): # delete tbase nodes - # r = self.tb.search(f"@edge_str: 'graph_id={teamid}'", index_name=self.edge_indexname) r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.edge_indexname, limit=len(edges)) tbase_edgeids = [data['edge_id'] for data in r.docs] - # tbase_edgeids_dict = {data["edge_id"]:data['id'] for data in r.docs} # id附带了definition信息 delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] - # delete_tbase_edgeids = [edgeid for edgeid in delete_edgeids if edgeid in tbase_edgeids] - # delete_tbase_edgeids = [tbase_edgeids_dict[edgeid] for edgeid in delete_edgeids if edgeid in tbase_edgeids] if len(tbase_missing_edgeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") @@ -343,14 +336,12 @@ def delete_edges(self, edges: List[GEdge], teamid: str): def update_nodes(self, nodes: List[GNode], teamid: str): # delete tbase nodes - # r = self.tb.search(f"@node_str: 'graph_id={teamid}'", index_name=self.node_indexname) r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) teamids_by_nodeid = {data['node_id']: data["node_str"] for data in r.docs} tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 update_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] - # update_tbase_nodeids = [nodeid for nodeid in update_nodeids if nodeid in tbase_nodeids] if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") @@ -464,10 +455,12 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N # base_query = f'(@node_str: graph_id={teamid})=>[KNN {top_k} @{key} $vector AS distance]' base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' query_params = {"vector": query_embedding} - r = self.tb.vector_search(base_query, query_params=query_params) + r = self.tb.vector_search(base_query, index_name=self.node_indexname, query_params=query_params) for i in r.docs: - nodeid_with_dist.append((i["ID"], float(i["distance"]))) + data_dict = i.__dict__ + if "ID" not in data_dict: continue # filter data + nodeid_with_dist.append((data_dict["ID"], float(data_dict["distance"]))) nodeid_with_dist = sorted(nodeid_with_dist, key=lambda x:x[1], reverse=False) for nodeid, dis in nodeid_with_dist: @@ -523,6 +516,7 @@ def create_ekg( teamid: str, service_name: str, rootid: str, + graphid: str = "", intent_text: str = None, intent_nodes: List[str] = [], all_intent_list: List=[], @@ -543,7 +537,7 @@ def create_ekg( result = self.text2graph(text, ancestor_list, all_intent_list, teamid) # do write - graph = self.write2kg(result["sls_graph"], result["tbase_graph"], teamid, do_save=do_save) + graph = self.write2kg(result["sls_graph"], teamid, graphid, do_save=do_save) result["graph"] = graph return result @@ -559,13 +553,20 @@ def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], dsl_graph = self.transform2dsl(sls_graph, intents, all_intent_list, teamid=teamid) return {"tbase_graph": tbase_graph, "sls_graph": sls_graph, "dsl_graph": dsl_graph} - def write2kg(self, ekg_sls_data: EKGSlsData, ekg_tbase_data: EKGTbaseData, teamid, do_save: bool=False) -> Graph: + def write2kg(self, ekg_sls_data: EKGSlsData, teamid: str, graphid: str="", do_save: bool=False) -> Graph: + ''' + :param graphid: str, use for record the new path + ''' # everytimes, it will add new nodes and edges - gbase_nodes = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] - gbase_nodes = [GNode(id=node.id, type=node.type, attributes=node.attributes()) for node in gbase_nodes] + gbase_nodes: List[EKGNodeSchema] = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] + gbase_nodes: List[GNode] = [ + GNode( + id=node.id, type=node.type, + attributes=node.attributes() if graphid else {**node.attributes(), **{"graphid": f"{graphid}"}} + ) for node in gbase_nodes] - gbase_edges = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] + gbase_edges: List[EKGEdgeSchema] = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] gbase_edges = [ GEdge(start_id=edge.original_src_id1__, end_id=edge.original_dst_id2__, type="opsgptkg_"+edge.type.split("_")[2] + "_route_" + "opsgptkg_"+edge.type.split("_")[3], From 17fb8ecacad5482e21eec0a52b8aa6f65b2a822f Mon Sep 17 00:00:00 2001 From: lightislost Date: Mon, 26 Aug 2024 11:40:18 +0800 Subject: [PATCH 026/128] [bugfix][server bound match] --- .../codeGenTest_example_copy.py | 247 ++++++++++++++++++ muagent/connector/memory_manager.py | 6 +- muagent/connector/schema/message.py | 1 + .../graph_db_handler/geabase_handler.py | 16 +- muagent/schemas/ekg/ekg_graph.py | 6 +- .../ekg_construct/ekg_construct_base.py | 201 ++++++++++---- 6 files changed, 414 insertions(+), 63 deletions(-) create mode 100644 examples/muagent_examples/codeGenTest_example_copy.py diff --git a/examples/muagent_examples/codeGenTest_example_copy.py b/examples/muagent_examples/codeGenTest_example_copy.py new file mode 100644 index 0000000..85192d3 --- /dev/null +++ b/examples/muagent_examples/codeGenTest_example_copy.py @@ -0,0 +1,247 @@ +import os +import json +from loguru import logger + +try: + import os, sys + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +from muagent.base_configs.env_config import CB_ROOT_PATH +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +from muagent.connector.phase import BasePhase +from muagent.connector.agents import BaseAgent, SelectorAgent +from muagent.connector.chains import BaseChain +from muagent.connector.schema import Message, Role, ChainConfig +from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler + +from muagent.tools import CodeRetrievalSingle + + + +# # 下面给定了一份代码片段，以及来自于它的依赖类、依赖方法相关的代码片段，你需要判断是否为这段指定代码片段生成测例。 +# # 理论上所有代码都需要写测例，但是受限于人的精力不可能覆盖所有代码 +# # 考虑以下因素进行裁剪： +# # 功能性: 如果它实现的是一个具体的功能或逻辑，则通常需要编写测试用例以验证其正确性。 +# # 复杂性: 如果代码较为，尤其是包含多个条件判断、循环、异常处理等的代码，更可能隐藏bug，因此应该编写测试用例。如果代码涉及复杂的算法或者逻辑，那么编写测试用例可以帮助确保逻辑的正确性，并在未来的重构中防止引入错误。 +# # 关键性: 如果它是关键路径的一部分或影响到核心功能，那么它就需要被测试。对于核心业务逻辑或者系统的关键组件，应当编写全面的测试用例来确保功能的正确性和稳定性。 +# # 依赖性: 如果代码有外部依赖，可能需要编写集成测试或模拟这些依赖进行单元测试。 +# # 用户输入: 如果代码处理用户输入，尤其是来自外部的、非受控的输入，那么创建测试用例来检查输入验证和处理是很重要的。 +# # 频繁更改：对于经常需要更新或修改的代码，有相应的测试用例可以确保更改不会破坏现有功能。 + + +# # 代码公开或重用：如果代码将被公开或用于其他项目，编写测试用例可以提高代码的可信度和易用性。 + + +# update new agent configs +judgeGenerateTests_PROMPT = """#### Agent Profile +When determining the necessity of writing test cases for a given code snippet, +it's essential to evaluate its interactions with dependent classes and methods (retrieved code snippets), +in addition to considering these critical factors: +1. Functionality: If it implements a concrete function or logic, test cases are typically necessary to verify its correctness. +2. Complexity: If the code is complex, especially if it contains multiple conditional statements, loops, exceptions handling, etc., +it's more likely to harbor bugs, and thus test cases should be written. +If the code involves complex algorithms or logic, then writing test cases can help ensure the accuracy of the logic and prevent errors during future refactoring. +3. Criticality: If it's part of the critical path or affects core functionalities, then it needs to be tested. +Comprehensive test cases should be written for core business logic or key components of the system to ensure the correctness and stability of the functionality. +4. Dependencies: If the code has external dependencies, integration testing may be necessary, or mocking these dependencies during unit testing might be required. +5. User Input: If the code handles user input, especially from unregulated external sources, creating test cases to check input validation and handling is important. +6. Frequent Changes: For code that requires regular updates or modifications, having the appropriate test cases ensures that changes do not break existing functionalities. + +#### Input Format + +**Code Snippet:** the initial Code or objective that the user wanted to achieve + +**Retrieval Code Snippets:** These are the associated code segments that the main Code Snippet depends on. +Examine these snippets to understand how they interact with the main snippet and to determine how they might affect the overall functionality. + +#### Response Output Format +**Action Status:** Set to 'finished' or 'continued'. +If set to 'finished', the code snippet does not warrant the generation of a test case. +If set to 'continued', the code snippet necessitates the creation of a test case. + +**REASON:** Justify the selection of 'finished' or 'continued', contemplating the decision through a step-by-step rationale. +""" + +generateTests_PROMPT = """#### Agent Profile +As an agent specializing in software quality assurance, +your mission is to craft comprehensive test cases that bolster the functionality, reliability, and robustness of a specified Code Snippet. +This task is to be carried out with a keen understanding of the snippet's interactions with its dependent classes and methods—collectively referred to as Retrieval Code Snippets. +Analyze the details given below to grasp the code's intended purpose, its inherent complexity, and the context within which it operates. +Your constructed test cases must thoroughly examine the various factors influencing the code's quality and performance. + +ATTENTION: response carefully referenced "Response Output Format" in format. + +Each test case should include: +1. clear description of the test purpose. +2. The input values or conditions for the test. +3. The expected outcome or assertion for the test. +4. Appropriate tags (e.g., 'functional', 'integration', 'regression') that classify the type of test case. +5. these test code should have package and import + +#### Input Format + +**Code Snippet:** the initial Code or objective that the user wanted to achieve + +**Retrieval Code Snippets:** These are the interrelated pieces of code sourced from the codebase, which support or influence the primary Code Snippet. + +#### Response Output Format +**SaveFileName:** construct a local file name based on Question and Context, such as + +```java +package/class.java +``` + +**Test Code:** generate the test code for the current Code Snippet. +```java +... +``` + +""" + +from muagent.tools import CodeRetrievalSingle, RelatedVerticesRetrival, Vertex2Code + +# 定义一个新的agent类 +class CodeRetrieval(BaseAgent): + + def start_action_step(self, message: Message) -> Message: + '''do action before agent predict ''' + # 根据问题获取代码片段和节点信息 + action_json = CodeRetrievalSingle.run(message.code_engine_name, message.input_query, llm_config=self.llm_config, embed_config=self.embed_config, search_type="tag", + local_graph_path=message.local_graph_path, use_nh=message.use_nh) + current_vertex = action_json['vertex'] + message.customed_kargs["Code Snippet"] = action_json["code"] + message.customed_kargs['Current_Vertex'] = current_vertex + + # 获取邻近节点 + action_json = RelatedVerticesRetrival.run(message.code_engine_name, message.customed_kargs['Current_Vertex']) + # 获取邻近节点所有代码 + relative_vertex = [] + retrieval_Codes = [] + for vertex in action_json["vertices"]: + # 由于代码是文件级别，所以相同文件代码不再获取 + # logger.debug(f"{current_vertex}, {vertex}") + current_vertex_name = current_vertex.replace("#", "").replace(".java", "" ) if current_vertex.endswith(".java") else current_vertex + if current_vertex_name.split("#")[0] == vertex.split("#")[0]: continue + + action_json = Vertex2Code.run(message.code_engine_name, vertex) + if action_json["code"]: + retrieval_Codes.append(action_json["code"]) + relative_vertex.append(vertex) + # + message.customed_kargs["Retrieval_Codes"] = retrieval_Codes + message.customed_kargs["Relative_vertex"] = relative_vertex + + code_snippet = message.customed_kargs.get("Code Snippet", "") + current_vertex = message.customed_kargs.get("Current_Vertex", "") + message.customed_kargs["Code Snippet"] = f"name: {current_vertex}\n{code_snippet}" + + Retrieval_Codes = message.customed_kargs["Retrieval_Codes"] + Relative_vertex = message.customed_kargs["Relative_vertex"] + message.customed_kargs["Retrieval Code Snippets"] = "\n".join([ + f"name: {vertext}\n{code}" for vertext, code in zip(Relative_vertex, Retrieval_Codes) + ]) + + return message + + +llm_config = LLMConfig( + model_name="gpt-4", api_key=api_key, api_base_url=api_base_url, temperature=0.3 +) +embed_config = EmbedConfig( + embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +) + + +# initialize codebase +# delete codebase +codebase_name = 'client_local' +code_path = "D://chromeDownloads/devopschat-bot/client_v2/client" +use_nh = True +do_interpret = False +# cbh = CodeBaseHandler(codebase_name, code_path, crawl_type='dir', use_nh=use_nh, local_graph_path=CB_ROOT_PATH, +# llm_config=llm_config, embed_config=embed_config) +# cbh.delete_codebase(codebase_name=codebase_name) + +# # load codebase +# cbh = CodeBaseHandler(codebase_name, code_path, crawl_type='dir', use_nh=use_nh, local_graph_path=CB_ROOT_PATH, +# llm_config=llm_config, embed_config=embed_config) +# cbh.import_code(do_interpret=do_interpret) + + +# log-level，print prompt和llm predict +os.environ["log_verbose"] = "1" + +CodeJudger_role = Role(role_type="assistant", role_name="CodeJudger_role", prompt=judgeGenerateTests_PROMPT) +CodeJudger = CodeRetrieval( + role=CodeJudger_role, + chat_turn=1, + llm_config=llm_config, embed_config=embed_config, +) + + +generateTests_role = Role(role_type="assistant", role_name="generateTests_role", prompt=generateTests_PROMPT) +generateTests = CodeRetrieval( + role=generateTests_role, + chat_turn=1, + llm_config=llm_config, embed_config=embed_config, +) + +chain_config = ChainConfig( + chain_name="code2test_chain", + agents=[CodeJudger_role.role_name, generateTests_role.role_name,], + chat_turn=1) + +chain = BaseChain( + chainConfig=chain_config, agents=[CodeJudger, generateTests], + llm_config=llm_config, embed_config=embed_config, +) + +phase = BasePhase( + phase_name="code2test_phase", chains=[chain], + embed_config=embed_config, llm_config=llm_config +) + +# round-1 +# 根据前面的load过程进行初始化 +cbh = CodeBaseHandler(codebase_name, code_path, crawl_type='dir', use_nh=use_nh, local_graph_path=CB_ROOT_PATH, + llm_config=llm_config, embed_config=embed_config) +vertexes = cbh.search_vertices(vertex_type="class") +logger.debug(vertexes) + +test_cases = [] +for vertex in vertexes: + query_content = f"为{vertex}生成可执行的测例 " + query = Message( + role_name="human", role_type="user", input_query=query_content, + code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="tag", + use_nh=use_nh, local_graph_path=CB_ROOT_PATH, + ) + output_message, output_memory = phase.step(query, reinit_memory=True) + # print(output_memory.to_str_messages(return_all=True, content_key="parsed_output_list")) + print(output_memory.get_spec_parserd_output()) + values = output_memory.get_spec_parserd_output() + test_code = {k:v for i in values for k,v in i.items() if k in ["SaveFileName", "Test Code"]} + test_cases.append(test_code) + + os.makedirs(f"{CB_ROOT_PATH}/tests", exist_ok=True) + if "SaveFileName" in test_code: + with open(f"{CB_ROOT_PATH}/tests/{test_code['SaveFileName']}", "w") as f: + f.write(test_code["Test Code"]) + break \ No newline at end of file diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index 3c4c0a1..9c0cebd 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -521,6 +521,7 @@ def get_vbname_from_chatindex(self, chat_index: str) -> str: TextField("role_type", ), TextField('input_query'), TextField("role_content", ), + TextField("role_tags"), TextField("parsed_output"), TextField("customed_kargs",), TextField("db_docs",), @@ -561,7 +562,7 @@ def __init__( self.save_message_keys = [ 'chat_index', 'message_index', 'user_name', 'role_name', 'role_type', 'input_query', 'role_content', 'step_content', 'parsed_output', 'parsed_output_list', 'customed_kargs', "db_docs", "code_docs", "search_docs", 'start_datetime', 'end_datetime', - "keyword", "vector", + "keyword", "vector", "role_tags" ] self.use_vector = use_vector self.init_tb() @@ -624,6 +625,9 @@ def append_tools(self, tool_information: dict, chat_index: str, nodeid: str, use pass self.append(message) + def get_memory_by_tag(self, tag: str) -> Memory: + return self.get_memory_pool_by_key_content(key='tag', content=f'*{tag}*') + def get_memory_pool(self, chat_index: str = "") -> Memory: return self.get_memory_pool_by_all({"chat_index": chat_index}) diff --git a/muagent/connector/schema/message.py b/muagent/connector/schema/message.py index 819b292..7ccfc83 100644 --- a/muagent/connector/schema/message.py +++ b/muagent/connector/schema/message.py @@ -15,6 +15,7 @@ class Message(BaseModel): input_query: str = "" start_datetime: str = None end_datetime: str = None + role_tags: str = "" # llm output role_content: str = "" diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 9f2371a..6da989a 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -170,7 +170,6 @@ def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys nodes = result.get("n0", []) or result.get("n0.attr", []) return self.convert2GNodes(nodes) - return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = []) -> GEdge: # todo 业务逻辑 @@ -183,7 +182,6 @@ def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: li edges = result.get("e", []) or result.get("e.attr", []) return self.convert2GEdges(edges)[0] - return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges][0] def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = [], reverse=False) -> List[GNode]: # @@ -213,7 +211,6 @@ def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_key edges = result.get("e", []) or result.get("e.attr", []) return self.convert2GEdges(edges) - return [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in edges] def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}) -> bool: result = self.get_neighbor_nodes(attributes, node_type,) @@ -255,10 +252,8 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b hop -= hop_max iter_index += 1 - nodes = self.convert2GNodes(result.get("n1", [])) + nodes = self.convert2GNodes(result.get("n1", [])+result.get("n0", [])) edges = self.convert2GEdges(result.get("e", [])) - # nodes = [GNode(id=node["id"], type=node["type"], attributes=node) for node in result.get("n1", [])] - # edges = [GEdge(start_id=edge["start_id"], end_id=edge["end_id"], type=edge["type"], attributes=edge) for edge in result.get("e", [])] return Graph(nodes=nodes, edges=edges, paths=result.get("p", [])) def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: @@ -385,10 +380,6 @@ def decode_path(self, col_data, k) -> List: connections = {} for step in steps: props = step["props"] - # if path == []: - # path.append(props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1)) - # path.append(props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1)) - start = props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1) end = props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1) connections[start] = end @@ -449,10 +440,13 @@ def get_nodetypes_by_edgetype(self, edge_type: str): def convert2GNodes(self, raw_nodes: List[Dict]) -> List[GNode]: nodes = [] + nodeids_set = set() for node in raw_nodes: node_id = node.pop("id") node_type = node.pop("type") - nodes.append(GNode(id=node_id, type=node_type, attributes=node)) + if node_id not in nodeids_set: + nodes.append(GNode(id=node_id, type=node_type, attributes=node)) + nodeids_set.add(node_id) return nodes def convert2GEdges(self, raw_edges: List[Dict]) -> List[GEdge]: diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 6f57802..2ba052f 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -31,7 +31,8 @@ class NodeSchema(BaseModel): def attributes(self, ): attrs = copy.deepcopy(vars(self)) - for k in ["ID", "type", "id"]: + # for k in ["ID", "type", "id"]: + for k in ["type", "id"]: attrs.pop(k) attrs.update(json.loads(attrs.pop("extra", '{}') or '{}')) return attrs @@ -51,7 +52,8 @@ class EdgeSchema(BaseModel): def attributes(self, ): attrs = copy.deepcopy(vars(self)) - for k in ["SRCID", "DSTID", "type", "timestamp", "original_src_id1__", "original_dst_id2__"]: + # for k in ["SRCID", "DSTID", "type", "timestamp", "original_src_id1__", "original_dst_id2__"]: + for k in ["type", "original_src_id1__", "original_dst_id2__"]: attrs.pop(k) attrs.update(json.loads(attrs.pop("extra", '{}') or '{}')) return attrs diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 9240fa8..24b6b78 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -179,9 +179,32 @@ def init_sls(self, do_init: bool=None): def update_graph( self, origin_nodes: List[GNode], origin_edges: List[GEdge], - new_nodes: List[GNode], new_edges: List[GEdge], teamid: str + new_nodes: List[GNode], new_edges: List[GEdge], + teamid: str, rootid: str ): + # search unconnect nodes and edges + connections = {} + for edge in new_edges: + connections.setdefault(edge.start_id, []).append(edge.end_id) + + if rootid not in connections: + raise Exception(f"Error: rootid not in this graph") + + # dfs for those nodes from rootid + visited = set() + rootid_can_arrive_nodeids = [] + def _dfs(node): + if node not in visited: + visited.add(node) + rootid_can_arrive_nodeids.append(node) + for neighbor in connections.get(node, []): + _dfs(neighbor) + _dfs(rootid) + + # rootid_can_arrive_node = [n for n in new_nodes if n.id in rootid_can_arrive_nodeids] + # rootid_can_arrive_edge = [e for e in new_edges if (e.start_id in rootid_can_arrive_nodeids) and (e.end_id in rootid_can_arrive_nodeids)] + origin_nodeids = set([node.id for node in origin_nodes]) origin_edgeids = set([f"{edge.start_id}__{edge.end_id}" for edge in origin_edges]) nodeids = set([node.id for node in new_nodes]) @@ -197,9 +220,11 @@ def update_graph( for edge in origin_edges + new_edges: edgeid2edges_dict.setdefault(f"{edge.start_id}__{edge.end_id}", []).append(edge) - # get add nodes & edges + # get add nodes & edges and filter those nodes/edges cant be arrived from rootid add_nodes = [node for node in new_nodes if node.id not in origin_nodeids] + add_nodes = [n for n in add_nodes if n.id in rootid_can_arrive_nodeids] add_edges = [edge for edge in new_edges if f"{edge.start_id}__{edge.end_id}" not in origin_edgeids] + add_edges = [edge for e in add_edges if (e.start_id in rootid_can_arrive_nodeids) and (e.end_id in rootid_can_arrive_nodeids)] # get delete nodes & edges delete_nodes = [node for node in origin_nodes if node.id not in nodeids] @@ -254,7 +279,7 @@ def add_nodes(self, nodes: List[GNode], teamid: str): return gb_result + tb_result def add_edges(self, edges: List[GEdge], teamid: str): - edges = self._update_new_attr_for_edges(edges, teamid) + edges = self._update_new_attr_for_edges(edges) tbase_edges = [{ # 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", 'edge_id': f"{edge.start_id}__{edge.end_id}", @@ -288,12 +313,39 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): node_neighbor_lens = [ len([ n.id # reverse neighbor nodes which are not in delete nodes - for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=False) + for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=True) if n.id not in delete_nodeids]) for node in nodes ] - # delete the nodeids in tbase + + extra_delete_nodes, extra_delete_edges = [], [] + extra_delete_nodeids = set() + for node in nodes: + if node.id in extra_delete_nodeids: continue + + extra_delete_nodeids.add(node.id) + # + graph = self.gb.get_hop_infos(attributes={"id": node.id,}, node_type=node.type, hop=30) + extra_delete_nodes.extend(graph.nodes) + extra_delete_edges.extend(graph.edges) + for _node in graph.nodes: + extra_delete_nodeids.add(_node.id) + + + # directly delete extra_delete_nodes in tbase + tb_result = [] + for edge in extra_delete_edges: + resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") + tb_result.append(resp) + + # directly delete extra_delete_edges in tbase tb_result = [] + for edge in extra_delete_edges: + resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") + tb_result.append(resp) + + # delete the nodeids in tbase + # tb_result = [] for node, node_len in zip(nodes, node_neighbor_lens): if node_len >= 1: continue resp = self.tb.delete(node.id) @@ -355,7 +407,9 @@ def update_nodes(self, nodes: List[GNode], teamid: str): tbase_data = {} tbase_data["node_id"] = node.id if teamid not in teamids_by_nodeid[node.id]: - tbase_data["teamids"] = teamids_by_nodeid[node.id] + f", {teamid}" + teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()])) + tbase_data["teamids"] = ", ".join(teamids+[teamid]) # teamids_by_nodeid[node.id] + f", {teamid}" + # tbase_data["teamids"] = teamids_by_nodeid[node.id] + f", {teamid}" tbase_data.update(self._update_tbase_attr_for_nodes(node.attributes)) # resp = self.tb.insert_data_hash(tbase_data, key="node_id", need_etime=False) @@ -375,7 +429,7 @@ def update_nodes(self, nodes: List[GNode], teamid: str): def update_edges(self, edges: List[GEdge], teamid: str): r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.node_indexname, limit=len(edges)) - teamids_by_edgeid = {data['edge_id']: data["edge_str"] for data in r.docs} + # teamids_by_edgeid = {data['edge_id']: data["edge_str"] for data in r.docs} tbase_edgeids = [data['edge_id'] for data in r.docs] delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] @@ -383,12 +437,12 @@ def update_edges(self, edges: List[GEdge], teamid: str): if len(tbase_missing_edgeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") - for edge in edges: - r = self.tb.search(f"@edge_id: {edge.start_id}__{edge.end_id}", index_name=self.node_indexname) - teamids_by_edgeid.update({data['edge_id']: data["node_str"] for data in r.docs}) + # for edge in edges: + # r = self.tb.search(f"@edge_id: {edge.start_id}__{edge.end_id}", index_name=self.node_indexname) + # teamids_by_edgeid.update({data['edge_id']: data["node_str"] for data in r.docs}) # update the nodeids in geabase - edges = self._update_new_attr_for_edges(edges, teamid, teamids_by_edgeid, do_check=False, do_update=True) + edges = self._update_new_attr_for_edges(edges, do_check=False, do_update=True) gb_result = [] for edge in edges: # if node.id not in update_tbase_nodeids: continue @@ -402,6 +456,7 @@ def update_edges(self, edges: List[GEdge], teamid: str): def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) + return self._normalized_nodes_type(nodes=[node])[0] extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") node.attributes.update(extra_attrs) return node @@ -422,22 +477,31 @@ def get_graph_by_nodeid( {'id': nodeid}, node_type=node_type, hop=hop, block_attributes=block_attributes ) + if result.nodes == []: + current_node = self.gb.get_current_node({"id": nodeid}, node_type=node_type) + current_node = self._normalized_nodes_type([current_node])[0] + result.nodes.append(current_node) if block_attributes: leaf_nodeids = [node.id for node in result.nodes if node.type=="opsgptkg_schedule"] else: leaf_nodeids = [path[-1] for path in result.paths if len(path)==hop+1] - for node in result.nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - node.attributes.update(extra_attrs) + nodes = self._normalized_nodes_type(result.nodes) + # for node in result.nodes: + # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") + # node.attributes.update(extra_attrs) + for node in nodes: if node.id in leaf_nodeids: neighbor_nodes = self.gb.get_neighbor_nodes({"id": node.id}, node_type=node.type) node.attributes["cnode_nums"] = len(neighbor_nodes) - - for edge in result.edges: - extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") - edge.attributes.update(extra_attrs) + + edges = self._normalized_edges_type(result.edges) + result.nodes = nodes + result.edges = edges + # for edge in result.edges: + # extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") + # edge.attributes.update(extra_attrs) return result def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: @@ -452,8 +516,8 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N nodeid_with_dist = [] for key in ["name_vector", "desc_vector"]: - # base_query = f'(@node_str: graph_id={teamid})=>[KNN {top_k} @{key} $vector AS distance]' - base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' + base_query = f'(@node_str: *{teamid}*)=>[KNN {top_k} @{key} $vector AS distance]' + # base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' query_params = {"vector": query_embedding} r = self.tb.vector_search(base_query, index_name=self.node_indexname, query_params=query_params) @@ -479,20 +543,27 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) nodes = self.gb.get_nodes_by_ids(nodeids) - for node in nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - node.attributes.update(extra_attrs) - return nodes_by_name + nodes_by_desc + nodes + # for node in nodes: + # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") + # node.attributes.update(extra_attrs) + + nodes = self._normalized_nodes_type(nodes) + nodes = nodes_by_name + nodes_by_desc + nodes + # tmp iead to filter by teamid + nodes = [node for node in nodes if teamid in str(node.attributes)] + return nodes def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: # rootid = f"{teamid}" # todo check the rootid result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) - for node in result.nodes: - extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - node.attributes.update(extra_attrs) - for edge in result.edges: - extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") - edge.attributes.update(extra_attrs) + + # for node in result.nodes: + # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") + # node.attributes.update(extra_attrs) + + # for edge in result.edges: + # extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") + # edge.attributes.update(extra_attrs) # paths must be ordered from start to end paths = result.paths @@ -508,6 +579,9 @@ def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> nodeid_set = set([nodeid for path in paths for nodeid in path]) new_nodes = [node for node in result.nodes if node.id in nodeid_set] new_edges = [edge for edge in result.edges if edge.start_id in nodeid_set and edge.end_id in nodeid_set] + + new_nodes = self._normalized_nodes_type(new_nodes) + new_edges = self._normalized_edges_type(new_edges) return Graph(nodes=new_nodes, edges=new_edges, paths=new_paths) def create_ekg( @@ -865,7 +939,9 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by node_type = node.type if node.id in teamids_by_nodeid: - node.attributes["teamids"] = teamids_by_nodeid.get(node.id, "").split("=")[1] + f", {teamid}" + teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()])) + node.attributes["teamids"] = ", ".join(teamids+[teamid]) + # node.attributes["teamids"] = teamids_by_nodeid.get(node.id, "").split("=")[1] + f", {teamid}" else: node.attributes["teamids"] = f"{teamid}" @@ -880,13 +956,14 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by fields = list(getClassFields(schema)) nodetype2fields_dict[node_type] = fields - flag = any([ - field not in node.attributes + missing_fields = [ + field for field in fields - if field not in ["type", "start_id", "end_id", "ID", "id", "extra"] - ]) - if flag and do_check: - raise Exception(f"node is wrong, type is {node_type}, fields is {fields}, data is {node.attributes}") + if field not in ["type", "start_id", "end_id", "ID", "id", "extra"] + and field not in node.attributes + ] + if len(missing_fields)>0 and do_check: + raise Exception(f"node is wrong, type is {node_type}, missing_fields is {missing_fields}, fields is {fields}, data is {node.attributes}") # update extra infomations to extra extra_fields = [k for k in node.attributes.keys() if k not in fields] @@ -899,16 +976,16 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by ) return nodes - def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by_edgeid={}, do_check=True, do_update=False): + def _update_new_attr_for_edges(self, edges: List[GEdge], do_check=True, do_update=False): '''update new attributes for nodes''' edgetype2fields_dict = {} for edge in edges: edge_type = edge.type - edge_id = f"{edge.start_id}__{edge.end_id}" - if edge_id in teamids_by_edgeid: - edge.attributes["teamids"] = teamids_by_edgeid.get(edge_id, "").split("=")[1] + f", {teamid}" - else: - edge.attributes["teamids"] = f"{teamid}" + # edge_id = f"{edge.start_id}__{edge.end_id}" + # if edge_id in teamids_by_edgeid: + # edge.attributes["teamids"] = teamids_by_edgeid.get(edge_id, "").split("=")[1] + f", {teamid}" + # else: + # edge.attributes["teamids"] = f"{teamid}" edge.attributes["@timestamp"] = edge.attributes.pop("timestamp", 0) or 1 # getCurrentTimestap() edge.attributes["gdb_timestamp"] = getCurrentTimestap() @@ -925,11 +1002,14 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by fields = list(getClassFields(schema)) edgetype2fields_dict[edge_type] = fields - flag = any([ - field not in edge.attributes for field in fields - if field not in ["type", "dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"]]) - if flag and do_check: - raise Exception(f"edge is wrong, type is {edge_type}, fields is {fields}, data is {edge.attributes}") + missing_fields = [ + field + for field in fields + if field not in ["type", "dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"] + and field not in edge.attributes + ] + if len(missing_fields)>0 and do_check: + raise Exception(f"edge is wrong, type is {edge_type}, missing_fields is {missing_fields}, fields is {fields}, data is {edge.attributes}") # update extra infomations to extra extra_fields = [k for k in edge.attributes.keys() if k not in fields+["@timestamp"]] @@ -942,5 +1022,28 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], teamid: str, teamids_by ) if do_update: edge.attributes.pop("@timestamp") - edge.attributes.pop("extra") + # edge.attributes.pop("extra") return edges + + def _normalized_nodes_type(self, nodes: List[GNode]) -> List[GNode]: + '''将数据进行格式转换''' + valid_nodes = [] + for node in nodes: + node_type = node.type + node_data: EKGNodeSchema = TYPE2SCHEMA[node_type](**{**{"id": node.id, "type": node_type}, **node.attributes}) + valid_node = GNode(id=node.id, type=node_type, attributes=node_data.attributes()) + valid_nodes.append(valid_node) + return valid_nodes + + def _normalized_edges_type(self, edges: List[GEdge]) -> GEdge: + valid_edges = [] + for edge in edges: + edge_data: EKGEdgeSchema = TYPE2SCHEMA["edge"]( + **{**{"original_src_id1__": edge.start_id, "original_dst_id2__": edge.end_id, "type": edge.type}, **edge.attributes} + ) + valid_edge = GEdge( + start_id=edge_data.original_src_id1__, end_id=edge_data.original_dst_id2__, + type=edge.type, attributes=edge_data.attributes() + ) + valid_edges.append(valid_edge) + return valid_edges \ No newline at end of file From cc8e44a2bfed63d4bd5b67ae3637f20692f6b18a Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 27 Aug 2024 16:26:27 +0800 Subject: [PATCH 027/128] update from internal --- muagent/connector/memory_manager.py | 16 +- .../graph_db_handler/geabase_handler.py | 66 ++++---- .../ekg_construct/ekg_construct_base.py | 146 +++++++++--------- tests/httpapis/fastapi_test.py | 56 +++++-- 4 files changed, 167 insertions(+), 117 deletions(-) diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index 9c0cebd..d50746e 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -625,8 +625,20 @@ def append_tools(self, tool_information: dict, chat_index: str, nodeid: str, use pass self.append(message) - def get_memory_by_tag(self, tag: str) -> Memory: - return self.get_memory_pool_by_key_content(key='tag', content=f'*{tag}*') + def get_memory_by_chatindex_tags(self, chat_index: str, tags: List[str], limit: int = 10) -> Memory: + ''' + :param chat_index: str, + :param tags: List[str], search message by any tag (match or) + ''' + tags_str = '|'.join([f"*{tag}*" for tag in tags]) + querys = [ + f"@chat_index:{chat_index}", + f"@role_tags:{tags_str}", + ] + query = f"({')('.join(querys)})" if len(querys) >=2 else "".join(querys) + logger.debug(f"{query}") + r = self.th.search(query, limit=limit) + return self.tbasedoc2Memory(r) def get_memory_pool(self, chat_index: str = "") -> Memory: return self.get_memory_pool_by_all({"chat_index": chat_index}) diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 6da989a..5a2fd41 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -81,7 +81,11 @@ def add_edges(self, edges: List[GEdge]) -> dict: def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None) -> dict: # demo: "MATCH (n:opsgptkg_employee {@ID: xxxx}) SET n.originname = 'xxx', n.description = 'xxx'" - set_str = ", ".join([f"n.{k}='{v}'" if isinstance(v, (str, bool)) else f"n.{k}={v}" for k, v in set_attributes.items()]) + set_str = ", ".join([ + f"n.{k}='{v}'" if isinstance(v, (str, bool)) else f"n.{k}={v}" + for k, v in set_attributes.items() + if k not in ["ID"] + ]) if (ID is None) or (not isinstance(ID, int)): ID = self.get_current_nodeID(attributes, node_type) @@ -94,7 +98,10 @@ def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = Non src_id, dst_id, timestamp = self.get_current_edgeID(src_id, dst_id, edge_type) src_type, dst_type = self.get_nodetypes_by_edgetype(edge_type) # src_id, dst_id = double_hashing(src_id), double_hashing(dst_id) - set_str = ", ".join([f"e.{k}='{v}'" if isinstance(v, (str, bool)) else f"e.{k}={v}" for k, v in set_attributes.items()]) + set_str = ", ".join([ + f"e.{k}='{v}'" if isinstance(v, (str, bool)) else f"e.{k}={v}" + for k, v in set_attributes.items() + ]) # demo： MATCH ()-[r:PlayFor{@src_id:1, @dst_id:100, @timestamp:0}]->() SET r.contract = 0; # gql = f"MATCH ()-[e:{edge_type}{{@src_id:{src_id}, @dst_id:{dst_id}, timestamp:{timestamp}}}]->() SET {set_str}" gql = f"MATCH (n0:{src_type} {{@id: {src_id}}})-[e]->(n1:{dst_type} {{@id:{dst_id}}}) SET {set_str}" @@ -197,7 +204,6 @@ def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_key result = self.decode_result(result, gql) nodes = result.get("n1", []) or result.get("n1.attr", []) return self.convert2GNodes(nodes) - return [GNode(id=node["id"], type=node["type"], attributes=node) for node in nodes] def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: # @@ -233,22 +239,27 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b result = {} iter_index = 0 while hop > 1: - if last_node_ids == []: + if last_node_ids == [] and iter_index==0: # result = self.execute(gql) result = self.decode_result(result, gql) + elif last_node_ids == []: + pass else: for _node_id, _node_type in zip(last_node_ids, last_node_types): where_str = f"n0.id='{_node_id}'" - gql = f"MATCH p = (n0:{_node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + if reverse: + gql = f"MATCH p = (n0:{_node_type} WHERE {where_str})<-[e]-{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" + else: + gql = f"MATCH p = (n0:{_node_type} WHERE {where_str})-[e]->{{1,{min(hop, hop_max)}}}(n1) RETURN n0, n1, e, p" # _result = self.execute(gql) _result = self.decode_result(_result, gql) - # logger.info(f"p_lens: {len(_result['p'])}") + # logger.info(f"p_lens: {_result['p']}") - result = self.merge_hotinfos(result, _result) + result = self.merge_hotinfos(result, _result, reverse=reverse) # - last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes, hop=min(hop, hop_max)+iter_index*hop_max) + last_node_ids, last_node_types, result = self.deduplicate_paths(result, block_attributes, select_attributes, hop=min(hop, hop_max)+iter_index*hop_max, reverse=reverse) hop -= hop_max iter_index += 1 @@ -271,7 +282,7 @@ def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, b result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.paths - def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}, hop:int=None): + def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}, hop:int=None, reverse=False): # 获取数据 n0, n1, e, p = result["n0"], result["n1"], result["e"], result["p"] block_node_ids = [ @@ -292,20 +303,15 @@ def deduplicate_paths(self, result, block_attributes: dict = {}, select_attribut for path_str, _p in zip(path_strs, p): if not any(path_str in other for other in path_strs if path_str != other): new_p.append(_p) - # # 路径去重 - # path_strs = ["&&".join(_p) for _p in p] - # new_p = [] - # new_path_strs_set = set() - # for path_str, _p in zip(path_strs, p): - # if not any(path_str in other for other in path_strs if path_str != other): - # if path_str not in new_path_strs_set and all([_pid not in block_node_ids for _pid in _p]): - # new_p.append(_p) - # new_path_strs_set.add(path_str) # 根据保留路径进行合并 nodeid2type = {i["id"]: i["type"] for i in n0+n1} unique_node_ids = [j for i in new_p for j in i] - last_node_ids = list(set([i[-1] for i in new_p if len(i)>=hop])) + if reverse: + last_node_ids = list(set([i[0] for i in new_p if len(i)>=hop])) + else: + last_node_ids = list(set([i[-1] for i in new_p if len(i)>=hop])) + last_node_types = [nodeid2type[i] for i in last_node_ids] new_n0 = deduplicate_dict([i for i in n0 if i["id"] in unique_node_ids]) new_n1 = deduplicate_dict([i for i in n1 if i["id"] in unique_node_ids]) @@ -313,18 +319,26 @@ def deduplicate_paths(self, result, block_attributes: dict = {}, select_attribut return last_node_ids, last_node_types, {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} - def merge_hotinfos(self, result1, result2) -> Dict: + def merge_hotinfos(self, result1, result2, reverse=False) -> Dict: old_n0_sets = set([n["id"] for n in result1["n0"]]) old_n1_sets = set([n["id"] for n in result1["n1"]]) new_n0 = result1["n0"] + [n for n in result2["n0"] if n["id"] not in old_n0_sets] new_n1 = result1["n1"] + [n for n in result2["n1"] if n["id"] not in old_n1_sets] new_e = result1["e"] + result2["e"] - new_p = result1["p"] + [ - p_old_1 + p_old_2[1:] - for p_old_1 in result1["p"] - for p_old_2 in result2["p"] - if p_old_2[0] == p_old_1[-1] - ] # + result2["p"] + if reverse: + new_p = result1["p"] + [ + p_old_2[:-1] + p_old_1 + for p_old_1 in result1["p"] + for p_old_2 in result2["p"] + if p_old_2[-1] == p_old_1[0] + ] # + result2["p"] + else: + new_p = result1["p"] + [ + p_old_1 + p_old_2[1:] + for p_old_1 in result1["p"] + for p_old_2 in result2["p"] + if p_old_2[0] == p_old_1[-1] + ] # + result2["p"] new_result = {"n0": new_n0, "n1": new_n1, "e": new_e, "p": new_p} return new_result diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 24b6b78..a8a7454 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -187,8 +187,11 @@ def update_graph( connections = {} for edge in new_edges: connections.setdefault(edge.start_id, []).append(edge.end_id) + + if len(new_nodes)==1: + connections.setdefault(new_nodes[0].id, []) - if rootid not in connections: + if rootid not in connections and len(new_nodes)>0: raise Exception(f"Error: rootid not in this graph") # dfs for those nodes from rootid @@ -224,12 +227,24 @@ def _dfs(node): add_nodes = [node for node in new_nodes if node.id not in origin_nodeids] add_nodes = [n for n in add_nodes if n.id in rootid_can_arrive_nodeids] add_edges = [edge for edge in new_edges if f"{edge.start_id}__{edge.end_id}" not in origin_edgeids] - add_edges = [edge for e in add_edges if (e.start_id in rootid_can_arrive_nodeids) and (e.end_id in rootid_can_arrive_nodeids)] + add_edges = [edge for edge in add_edges if (edge.start_id in rootid_can_arrive_nodeids) and (edge.end_id in rootid_can_arrive_nodeids)] # get delete nodes & edges delete_nodes = [node for node in origin_nodes if node.id not in nodeids] delete_edges = [edge for edge in origin_edges if f"{edge.start_id}__{edge.end_id}" not in edgeids] + delete_nodeids = [node.id for node in delete_nodes] + node_neighbor_lens = [ + len([ + n.id # reverse neighbor nodes which are not in delete nodes + for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=True) + if n.id not in delete_nodeids]) + for node in delete_nodes + ] + delete_nodes = [n for n, n_len in zip(delete_nodes, node_neighbor_lens) if n_len==0] + undelete_nodeids = [n.id for n, n_len in zip(delete_nodes, node_neighbor_lens) if n_len>0] + delete_edges = [e for e in delete_edges if e.start_id not in undelete_nodeids] + # get update nodes & edges update_nodes = [ nodeid2nodes_dict[nodeid][1] @@ -310,51 +325,58 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - node_neighbor_lens = [ - len([ - n.id # reverse neighbor nodes which are not in delete nodes - for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=True) - if n.id not in delete_nodeids]) - for node in nodes - ] + # node_neighbor_lens = [ + # len([ + # n.id # reverse neighbor nodes which are not in delete nodes + # for n in self.gb.get_neighbor_nodes({"id": node.id}, node.type, reverse=True) + # if n.id not in delete_nodeids]) + # for node in nodes + # ] - extra_delete_nodes, extra_delete_edges = [], [] - extra_delete_nodeids = set() - for node in nodes: - if node.id in extra_delete_nodeids: continue + # extra_delete_nodes, extra_delete_edges = [], [] + # extra_delete_nodeids = set() + # for node in nodes: + # if node.id in extra_delete_nodeids: continue - extra_delete_nodeids.add(node.id) - # - graph = self.gb.get_hop_infos(attributes={"id": node.id,}, node_type=node.type, hop=30) - extra_delete_nodes.extend(graph.nodes) - extra_delete_edges.extend(graph.edges) - for _node in graph.nodes: - extra_delete_nodeids.add(_node.id) + # extra_delete_nodeids.add(node.id) + # # + # graph = self.gb.get_hop_infos(attributes={"id": node.id,}, node_type=node.type, hop=30) + # extra_delete_nodes.extend(graph.nodes) + # extra_delete_edges.extend(graph.edges) + # for _node in graph.nodes: + # extra_delete_nodeids.add(_node.id) - # directly delete extra_delete_nodes in tbase - tb_result = [] - for edge in extra_delete_edges: - resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") - tb_result.append(resp) + # # directly delete extra_delete_nodes in tbase + # tb_result = [] + # for edge in extra_delete_edges: + # resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") + # tb_result.append(resp) - # directly delete extra_delete_edges in tbase - tb_result = [] - for edge in extra_delete_edges: - resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") - tb_result.append(resp) + # # directly delete extra_delete_edges in tbase + # tb_result = [] + # for edge in extra_delete_edges: + # resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") + # tb_result.append(resp) # delete the nodeids in tbase # tb_result = [] - for node, node_len in zip(nodes, node_neighbor_lens): - if node_len >= 1: continue + # for node, node_len in zip(nodes, node_neighbor_lens): + # if node_len >= 1: continue + # resp = self.tb.delete(node.id) + # tb_result.append(resp) + + # delete the nodeids in tbase + tb_result = [] + for node in nodes: resp = self.tb.delete(node.id) tb_result.append(resp) - # # delete the nodeids in geabase + # delete the nodeids in geabase gb_result = [] - for node, node_len in zip(nodes, node_neighbor_lens): - if node_len >= 1: continue + # for node, node_len in zip(nodes, node_neighbor_lens): + # if node_len >= 1: continue + for node in nodes: gb_result.append(self.gb.delete_node( {"id": node.id}, node.type, ID=node.attributes.get("ID") or double_hashing(node.id) )) @@ -407,8 +429,8 @@ def update_nodes(self, nodes: List[GNode], teamid: str): tbase_data = {} tbase_data["node_id"] = node.id if teamid not in teamids_by_nodeid[node.id]: - teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()])) - tbase_data["teamids"] = ", ".join(teamids+[teamid]) # teamids_by_nodeid[node.id] + f", {teamid}" + teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()]+[teamid])) + tbase_data["teamids"] = ", ".join(teamids) # teamids_by_nodeid[node.id] + f", {teamid}" # tbase_data["teamids"] = teamids_by_nodeid[node.id] + f", {teamid}" tbase_data.update(self._update_tbase_attr_for_nodes(node.attributes)) # @@ -420,9 +442,10 @@ def update_nodes(self, nodes: List[GNode], teamid: str): gb_result = [] for node in nodes: # if node.id not in update_tbase_nodeids: continue + ID = node.attributes.pop("ID", None) or double_hashing(node.id) resp = self.gb.update_node( {}, node.attributes, node_type=node.type, - ID=node.attributes.get("ID") or double_hashing(node.id) + ID=ID ) gb_result.append(resp) return gb_result or tb_result @@ -446,9 +469,10 @@ def update_edges(self, edges: List[GEdge], teamid: str): gb_result = [] for edge in edges: # if node.id not in update_tbase_nodeids: continue + SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) + DSTID = edge.attributes.pop("DSTID", None) or double_hashing(edge.start_id) resp = self.gb.update_edge( - edge.attributes.get("SRCID") or double_hashing(edge.start_id), - edge.attributes.get("DSTID") or double_hashing(edge.end_id), + SRCID, DSTID, edge.attributes, edge_type=edge.type ) gb_result.append(resp) @@ -457,9 +481,6 @@ def update_edges(self, edges: List[GEdge], teamid: str): def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) return self._normalized_nodes_type(nodes=[node])[0] - extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - node.attributes.update(extra_attrs) - return node def get_graph_by_nodeid( self, @@ -479,7 +500,6 @@ def get_graph_by_nodeid( ) if result.nodes == []: current_node = self.gb.get_current_node({"id": nodeid}, node_type=node_type) - current_node = self._normalized_nodes_type([current_node])[0] result.nodes.append(current_node) if block_attributes: @@ -488,9 +508,6 @@ def get_graph_by_nodeid( leaf_nodeids = [path[-1] for path in result.paths if len(path)==hop+1] nodes = self._normalized_nodes_type(result.nodes) - # for node in result.nodes: - # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - # node.attributes.update(extra_attrs) for node in nodes: if node.id in leaf_nodeids: neighbor_nodes = self.gb.get_neighbor_nodes({"id": node.id}, node_type=node.type) @@ -499,9 +516,6 @@ def get_graph_by_nodeid( edges = self._normalized_edges_type(result.edges) result.nodes = nodes result.edges = edges - # for edge in result.edges: - # extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") - # edge.attributes.update(extra_attrs) return result def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: @@ -543,28 +557,19 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) nodes = self.gb.get_nodes_by_ids(nodeids) - # for node in nodes: - # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - # node.attributes.update(extra_attrs) nodes = self._normalized_nodes_type(nodes) nodes = nodes_by_name + nodes_by_desc + nodes # tmp iead to filter by teamid nodes = [node for node in nodes if teamid in str(node.attributes)] + + # select the node which can connect the rootid + nodes = [node for node in nodes if len(self.search_rootpath_by_nodeid(node.id, node.type, f"ekg_team:{teamid}").paths)>0] return nodes def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: - # rootid = f"{teamid}" # todo check the rootid result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) - # for node in result.nodes: - # extra_attrs = json.loads(node.attributes.pop("extra", "{}") or "{}") - # node.attributes.update(extra_attrs) - - # for edge in result.edges: - # extra_attrs = json.loads(edge.attributes.pop("extra", "{}") or "{}") - # edge.attributes.update(extra_attrs) - # paths must be ordered from start to end paths = result.paths new_paths = [] @@ -788,8 +793,8 @@ def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData node_str=f'graph_id={teamid}', name_keyword=" | ".join(extract_tags(name, topK=None)), desc_keyword=" | ".join(extract_tags(description, topK=None)), - name_vector= name_vector[name], # np.array(name_vector[name]).astype(dtype=np.float32).tobytes(), - desc_vector= desc_vector[description], # np.array(desc_vector[description]).astype(dtype=np.float32).tobytes(), + name_vector= name_vector[name], + desc_vector= desc_vector[description], ) ) for edge in ekg_sls_data.edges: @@ -939,8 +944,8 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by node_type = node.type if node.id in teamids_by_nodeid: - teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()])) - node.attributes["teamids"] = ", ".join(teamids+[teamid]) + teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()]+ [teamid])) + node.attributes["teamids"] = ", ".join(teamids) # node.attributes["teamids"] = teamids_by_nodeid.get(node.id, "").split("=")[1] + f", {teamid}" else: node.attributes["teamids"] = f"{teamid}" @@ -981,11 +986,6 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], do_check=True, do_updat edgetype2fields_dict = {} for edge in edges: edge_type = edge.type - # edge_id = f"{edge.start_id}__{edge.end_id}" - # if edge_id in teamids_by_edgeid: - # edge.attributes["teamids"] = teamids_by_edgeid.get(edge_id, "").split("=")[1] + f", {teamid}" - # else: - # edge.attributes["teamids"] = f"{teamid}" edge.attributes["@timestamp"] = edge.attributes.pop("timestamp", 0) or 1 # getCurrentTimestap() edge.attributes["gdb_timestamp"] = getCurrentTimestap() @@ -1030,7 +1030,9 @@ def _normalized_nodes_type(self, nodes: List[GNode]) -> List[GNode]: valid_nodes = [] for node in nodes: node_type = node.type - node_data: EKGNodeSchema = TYPE2SCHEMA[node_type](**{**{"id": node.id, "type": node_type}, **node.attributes}) + node_data_dict = {**{"id": node.id, "type": node_type}, **node.attributes} + node_data_dict = {k: 'False' if k in ["enable", "summaryswtich"] and v=="" else v for k,v in node_data_dict.items()} + node_data: EKGNodeSchema = TYPE2SCHEMA[node_type](**node_data_dict) valid_node = GNode(id=node.id, type=node_type, attributes=node_data.attributes()) valid_nodes.append(valid_node) return valid_nodes diff --git a/tests/httpapis/fastapi_test.py b/tests/httpapis/fastapi_test.py index de7eb85..eed19a1 100644 --- a/tests/httpapis/fastapi_test.py +++ b/tests/httpapis/fastapi_test.py @@ -3,25 +3,47 @@ import uvicorn import time +from pydantic import BaseModel + + app = FastAPI() -# 一个异步路由 -@app.get("/") -async def read_root(): - await asyncio.sleep(1) # 模拟一个异步操作 - return {"message": "Hello, World!"} - -# 另一个异步路由 -@app.get("/items/{item_id}") -async def read_item(item_id: int, q: str = None): - await asyncio.sleep(5) # 模拟延迟 - return {"item_id": item_id, "q": q} - -# 另一个异步路由 -@app.get("/itemstest/{item_id}") -def read_item(item_id: int, q: str = None): - time.sleep(5) # 模拟延迟 - return {"item_id": item_id, "q": q} +# get node by nodeid and nodetype +class GetNodeRequest(BaseModel): + nodeid: str + nodeType: str + + +# get node by nodeid and nodetype +class GetNodeResponse(BaseModel): + test: str + + +# ~/ekg/node/search +@app.get("/ekg/node", response_model=GetNodeResponse) +def get_node(request: GetNodeRequest): + # 添加预测逻辑的代码 + print(request) + return GetNodeResponse( + test="test" + ) +# # 一个异步路由 +# @app.get("/") +# async def read_root(): +# await asyncio.sleep(1) # 模拟一个异步操作 +# return {"message": "Hello, World!"} + +# # 另一个异步路由 +# @app.get("/items/{item_id}") +# async def read_item(item_id: int, q: str = None): +# await asyncio.sleep(5) # 模拟延迟 +# return {"item_id": item_id, "q": q} + +# # 另一个异步路由 +# @app.get("/itemstest/{item_id}") +# def read_item(item_id: int, q: str = None): +# time.sleep(5) # 模拟延迟 +# return {"item_id": item_id, "q": q} # 均能并发触发，好像对于io操作会默认管理 From aa4b0cd87a1005d753eb7a39255873a61f9d45da Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 27 Aug 2024 19:30:41 +0800 Subject: [PATCH 028/128] [add test][test llm and embedding api] --- examples/ekg_examples/start.py | 25 +++++++++++------- muagent/httpapis/ekg_construct/api.py | 36 ++++++++++++++++++++++++++ muagent/schemas/apis/ekg_api_schema.py | 22 ++++++++++++++-- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/examples/ekg_examples/start.py b/examples/ekg_examples/start.py index 18f4158..026a0d6 100644 --- a/examples/ekg_examples/start.py +++ b/examples/ekg_examples/start.py @@ -1,23 +1,30 @@ import time, sys -st = time.time() import os import yaml import requests from typing import List from loguru import logger -import tqdm +from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor -print(time.time()-st) from langchain.llms.base import LLM from langchain.embeddings.base import Embeddings -print(time.time()-st) src_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) -print(src_dir) sys.path.append(src_dir) +try: + import os, sys + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config +except Exception as e: + # set your config + logger.error(f"{e}") + from muagent.schemas.db import * from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService @@ -30,7 +37,7 @@ class CustomLLM(LLM, BaseModel): model_name: str = "qwen2:1b" model_type: str = "ollama" api_key: str = "" - stop: str = "" + stop: str = None temperature: float = 0.3 top_k: int = 50 top_p: float = 0.95 @@ -43,10 +50,8 @@ def params(self): if k in keys} def update_params(self, **kwargs): - logger.debug(f"{kwargs}") # 更新属性 for key, value in kwargs.items(): - logger.debug(f"{key}, {value}") setattr(self, key, value) def _llm_type(self, *args): @@ -114,10 +119,8 @@ def params(self): } def update_params(self, **kwargs): - logger.debug(f"{kwargs}") # 更新属性 for key, value in kwargs.items(): - logger.debug(f"{key}, {value}") setattr(self, key, value) def _get_sentence_emb(self, sentence: str) -> dict: @@ -133,6 +136,8 @@ def _get_sentence_emb(self, sentence: str) -> dict: return r.json() elif self.embedding_type == "openai": from muagent.llm_models.get_embedding import get_embedding + os.environ["OPENAI_API_KEY"] = self.api_key + os.environ["API_BASE_URL"] = self.url embed_config = EmbedConfig( embed_engine="openai", api_key=self.api_key, diff --git a/muagent/httpapis/ekg_construct/api.py b/muagent/httpapis/ekg_construct/api.py index 4fc2ef6..0533d5e 100644 --- a/muagent/httpapis/ekg_construct/api.py +++ b/muagent/httpapis/ekg_construct/api.py @@ -2,6 +2,7 @@ from typing import Dict import asyncio import uvicorn +from loguru import logger from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService from muagent.schemas.apis.ekg_api_schema import * @@ -18,6 +19,24 @@ def init_app(llm, embeddings): async def llm_params(): return llm.params() + # ~/llm/params + @app.post("/llm/generate", response_model=LLMResponse) + async def llm_predict(request: LLMRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + answer = llm.predict(request.text, request.stop) + except Exception as e: + errorMessage = str(e) + successCode = False + answer = "error" + + return LLMResponse( + successCode=successCode, errorMessage=errorMessage, + answer=answer + ) + # ~/llm/params/update @app.post("/llm/params/update", response_model=EKGResponse) async def update_llm_params(kwargs: Dict): @@ -55,6 +74,23 @@ async def update_embedding_params(kwargs: Dict): successCode=successCode, errorMessage=errorMessage, ) + @app.post("/embeddings/generate", response_model=EmbeddingsResponse) + async def embedding_predict(request: EmbeddingsRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + embeddings_list = embeddings.embed_documents(request.texts) + except Exception as e: + logger.exception(e) + errorMessage = str(e) + successCode = False + embeddings_list = [] + + return EmbeddingsResponse( + successCode=successCode, errorMessage=errorMessage, + embeddings=embeddings_list + ) # # ~/ekg/text2graph # @app.post("/ekg/text2graph", response_model=EKGGraphResponse) # async def text2graph(request: EKGT2GRequest): diff --git a/muagent/schemas/apis/ekg_api_schema.py b/muagent/schemas/apis/ekg_api_schema.py index 0b680ae..c74faa3 100644 --- a/muagent/schemas/apis/ekg_api_schema.py +++ b/muagent/schemas/apis/ekg_api_schema.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from typing import List, Dict +from typing import List, Dict, Optional from enum import Enum from muagent.schemas.common import GNode, GEdge @@ -12,6 +12,24 @@ class EKGResponse(BaseModel): errorMessage: str +# embeddings +class EmbeddingsResponse(EKGResponse): + successCode: int + errorMessage: str + embeddings: List[List[float]] + +class EmbeddingsRequest(BaseModel): + texts: List[str] + +class LLMRequest(BaseModel): + text: str + stop: Optional[str] + +class LLMResponse(EKGResponse): + successCode: int + errorMessage: str + answer: str + # text2graph class EKGT2GRequest(BaseModel): text: str @@ -81,7 +99,7 @@ class LLMParamsResponse(BaseModel): model_name: str model_type: str api_key: str - stop: str + stop: Optional[str] = None temperature: float top_k: int top_p: float From fae1ae09be0692acf08861829069bf16e287203c Mon Sep 17 00:00:00 2001 From: lightislost Date: Sat, 31 Aug 2024 09:50:35 +0800 Subject: [PATCH 029/128] [bugfix][update_graph] --- Dockerfile | 8 +- docker-compose.yaml | 170 ++++++++++++++++++ muagent/connector/memory_manager.py | 20 ++- muagent/connector/schema/message.py | 6 +- .../graph_db_handler/base_gb_handler.py | 2 +- .../graph_db_handler/geabase_handler.py | 13 +- muagent/httpapis/ekg_construct/api.py | 2 +- muagent/schemas/ekg/ekg_graph.py | 32 ++-- .../ekg_construct/ekg_construct_base.py | 148 +++++++++++---- .../service/ekg_inference/intention_router.py | 8 +- muagent/utils/common_utils.py | 8 +- tests/httpapis/docker_test.py | 48 +++++ tests/httpapis/fastapi_test.py | 32 ++-- 13 files changed, 399 insertions(+), 98 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 tests/httpapis/docker_test.py diff --git a/Dockerfile b/Dockerfile index 2ac6041..500261b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ From python:3.9.18-bookworm +# FROM python:3.9-slim-bookworm WORKDIR /home/user @@ -11,10 +12,11 @@ COPY ./requirements.txt /home/user/docker_requirements.txt # RUN service inetutils-inetd start # service inetutils-inetd status -RUN wget https://oss-cdn.nebula-graph.com.cn/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb -RUN dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb +# RUN wget https://oss-cdn.nebula-graph.com.cn/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb +# RUN dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install -r /home/user/docker_requirements.txt +RUN pip install fastapi uvicorn notebook +# RUN pip install -r /home/user/docker_requirements.txt CMD ["bash"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..6760a0f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,170 @@ +version: '0.1' + +services: + metad0: + image: vesoft/nebula-metad:v3.8.0 + container_name: metad0 + environment: + USER: root + command: + - --meta_server_addrs=metad0:9559 + - --local_ip=metad0 + - --ws_ip=metad0 + - --port=9559 + - --ws_http_port=19559 + - --data_path=/data/meta + - --log_dir=/logs + - --v=0 + - --minloglevel=0 + healthcheck: + test: ["CMD", "curl", "-sf", "http://metad0:19559/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + ports: + - 9559:9559 + - 19559:19559 + - 19560 + volumes: + - ./data/meta0:/data/meta + - ./logs/meta0:/logs + networks: + - ekg-net + restart: on-failure + cap_add: + - SYS_PTRACE + + storaged0: + image: vesoft/nebula-storaged:v3.8.0 + container_name: storaged0 + environment: + USER: root + TZ: "${TZ}" + command: + - --meta_server_addrs=metad0:9559 + - --local_ip=storaged0 + - --ws_ip=storaged0 + - --port=9779 + - --ws_http_port=19779 + - --data_path=/data/storage + - --log_dir=/logs + - --v=0 + - --minloglevel=0 + depends_on: + - metad0 + healthcheck: + test: ["CMD", "curl", "-sf", "http://storaged0:19779/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + ports: + - 9779:9779 + - 19779:19779 + - 19780 + volumes: + - ./data/storage0:/data/storage + - ./logs/storage0:/logs + networks: + - ekg-net + restart: on-failure + cap_add: + - SYS_PTRACE + + graphd: + image: vesoft/nebula-graphd:v3.8.0 + container_name: graphd + environment: + USER: root + TZ: "${TZ}" + command: + - --meta_server_addrs=metad0:9559 + - --port=9669 + - --local_ip=graphd + - --ws_ip=graphd + - --ws_http_port=19669 + - --log_dir=/logs + - --v=0 + - --minloglevel=0 + depends_on: + - storaged0 + healthcheck: + test: ["CMD", "curl", "-sf", "http://graphd:19669/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + ports: + - 9669:9669 + - 19669:19669 + - 19670 + volumes: + - ./logs/graph:/logs + networks: + - ekg-net + restart: on-failure + cap_add: + - SYS_PTRACE + + redis-stack: + image: redis/redis-stack:7.4.0-v0 + container_name: redis + ports: + - "6379:6379" + - "8001:8001" + volumes: + - ./logs/redis:/var/lib/redis/logs + networks: + - ekg-net + restart: always + + + ollama: + image: ollama/ollama:0.3.6 + container_name: ollama + environment: + USER: root + TZ: "${TZ}" + ports: + - 11434:11434 + volumes: + - //d/models/ollama:/root/.ollama # windows path + # - /User/models:/root/.ollama # linux/mac path + networks: + - ekg-net + restart: on-failure + cap_add: + - SYS_PTRACE + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all # 或者您想要的数量，例如 1 + # capabilities: [gpu] + + ekgservice: + build: + context: . + dockerfile: Dockerfile + container_name: ekgservice + image: muagent:test + environment: + USER: root + TZ: "${TZ}" + ports: + - 5050:3737 + # - 8080:8888 + volumes: + - ./examples:/home/user/muagent/examples + - ./muagent:/home/user/muagent/muagent + - ./tests:/home/user/muagent/tests + restart: on-failure + networks: + - ekg-net + # command: ["python", "/home/user/muagent/examples/ekg_examples/start.py"] # 指定要执行的脚本 + command: ["python", "/home/user/muagent/tests/httpapis/fastapi_test.py"] # 指定要执行的脚本 + +networks: + ekg-net: \ No newline at end of file diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index d50746e..22018de 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -655,7 +655,7 @@ def get_memory_pool_by_key_content(self, key: str, content: str, ): r = self.th.search(content) return self.tbasedoc2Memory(r) - def get_memory_pool_by_all(self, search_key_contents: dict): + def get_memory_pool_by_all(self, search_key_contents: dict, limit: int =10): ''' search_key_contents: - key: str, key must in message keys @@ -666,11 +666,17 @@ def get_memory_pool_by_all(self, search_key_contents: dict): if not v: continue if k == "keyword": querys.append(f"@{k}:{{{v}}}") + elif k == "role_tags": + tags_str = '|'.join([f"*{tag}*" for tag in v]) if isinstance(v, list) else f"{v}" + querys.append(f"@role_tags:{tags_str}") + elif k == "start_datetime": + query = f"(@start_datetime:[{v[0]} {v[1]}])" + querys.append(query) else: querys.append(f"@{k}:{v}") query = f"({')('.join(querys)})" if len(querys) >=2 else "".join(querys) - r = self.th.search(query) + r = self.th.search(query, limit=limit) return self.tbasedoc2Memory(r) def embedding_retrieval(self, text: str, top_k=1, score_threshold=1.0, chat_index: str = "default", **kwargs) -> List[Message]: @@ -707,7 +713,7 @@ def text_retrieval(self, text: str, chat_index: str = "default", **kwargs) -> L return self._text_retrieval_from_cache(memory.messages, text) def datetime_retrieval(self, chat_index: str, datetime: str, text: str = None, n: int = 5, key: str = "start_datetime", **kwargs) -> List[Message]: - intput_timestamp = datefromatToTimestamp(datetime, 1) + intput_timestamp = dateformatToTimestamp(datetime, 1000, "%Y-%m-%d %H:%M:%S.%f") query = f"(@chat_index:{chat_index})(@{key}:[{intput_timestamp-n*60} {intput_timestamp+n*60}])" # logger.debug(f"datetime_retrieval query: {query}") r = self.th.search(query) @@ -787,8 +793,8 @@ def localMessage2TbaseMessage(self, message: Message): # if content is not None: # tbase_message["customed_kargs"][key] = content - tbase_message["start_datetime"] = datefromatToTimestamp(message.start_datetime, 1) - tbase_message["end_datetime"] = datefromatToTimestamp(message.end_datetime, 1) + tbase_message["start_datetime"] = dateformatToTimestamp(message.start_datetime, 1000, "%Y-%m-%d %H:%M:%S.%f") + tbase_message["end_datetime"] = dateformatToTimestamp(message.end_datetime, 1000, "%Y-%m-%d %H:%M:%S.%f") if self.use_vector and self.embed_config: vector_dict = get_embedding( @@ -830,8 +836,8 @@ def tbasedoc2Memory(self, r_docs) -> Memory: memory.append(message) for message in memory.messages: - message.start_datetime = timestampToDateformat(int(message.start_datetime), 1) - message.end_datetime = timestampToDateformat(int(message.end_datetime), 1) + message.start_datetime = timestampToDateformat(int(message.start_datetime), 1000, "%Y-%m-%d %H:%M:%S.%f") + message.end_datetime = timestampToDateformat(int(message.end_datetime), 1000, "%Y-%m-%d %H:%M:%S.%f") memory.sort_by_key("end_datetime") # for message in memory.message: diff --git a/muagent/connector/schema/message.py b/muagent/connector/schema/message.py index 7ccfc83..e2fa0ce 100644 --- a/muagent/connector/schema/message.py +++ b/muagent/connector/schema/message.py @@ -79,9 +79,9 @@ def check_datetime(cls, values): start_datetime = values.get("start_datetime") end_datetime = values.get("end_datetime") if start_datetime is None: - values["start_datetime"] = getCurrentDatetime() + values["start_datetime"] = getCurrentDatetime("%Y-%m-%d %H:%M:%S.%f") if end_datetime is None: - values["end_datetime"] = getCurrentDatetime() + values["end_datetime"] = getCurrentDatetime("%Y-%m-%d %H:%M:%S.%f") return values @root_validator(pre=True) @@ -99,7 +99,7 @@ def check_message_index(cls, values): def update_attribute(self, key: str, value): if hasattr(self, key): setattr(self, key, value) - self.end_datetime = getCurrentDatetime() + self.end_datetime = getCurrentDatetime("%Y-%m-%d %H:%M:%S.%f") else: raise AttributeError(f"{key} is not a valid property of {self.__class__.__name__}") diff --git a/muagent/db_handler/graph_db_handler/base_gb_handler.py b/muagent/db_handler/graph_db_handler/base_gb_handler.py index e21b0eb..280edd9 100644 --- a/muagent/db_handler/graph_db_handler/base_gb_handler.py +++ b/muagent/db_handler/graph_db_handler/base_gb_handler.py @@ -64,5 +64,5 @@ def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_key def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: pass - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse=False) -> Graph: + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: List[dict] = {}, select_attributes: dict = {}, reverse=False) -> Graph: pass diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 5a2fd41..cfcd2f0 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -223,7 +223,7 @@ def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_at filter_result = [i for i in result if all([item in i.attributes.items() for item in check_attributes.items()])] return len(filter_result) > 0 - def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse=False) -> Graph: + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: List[dict] = [], select_attributes: dict = {}, reverse=False) -> Graph: ''' hop >= 2， 表面需要至少两跳 ''' @@ -267,30 +267,31 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b edges = self.convert2GEdges(result.get("e", [])) return Graph(nodes=nodes, edges=edges, paths=result.get("p", [])) - def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: + def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: List[dict] = []) -> List[GNode]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.nodes - def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GEdge]: + def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: List[dict] = []) -> List[GEdge]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.edges - def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[str]: + def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: List[dict] = []) -> List[str]: # result = self.get_hop_infos(attributes, node_type, hop, block_attributes) return result.paths - def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}, hop:int=None, reverse=False): + def deduplicate_paths(self, result, block_attributes: List[dict] = {}, select_attributes: dict = {}, hop:int=None, reverse=False): # 获取数据 n0, n1, e, p = result["n0"], result["n1"], result["e"], result["p"] block_node_ids = [ i["id"] for i in n0+n1 + for block_attribute in block_attributes # 这里block为空时也会生效，属于合理情况 # if block_attributes=={} or all(item in i.items() for item in block_attributes.items()) - if block_attributes and all(item in i.items() for item in block_attributes.items()) + if block_attribute and all(item in i.items() for item in block_attribute.items()) ] + [ i["id"] for i in n0+n1 diff --git a/muagent/httpapis/ekg_construct/api.py b/muagent/httpapis/ekg_construct/api.py index 0533d5e..b0f3f0a 100644 --- a/muagent/httpapis/ekg_construct/api.py +++ b/muagent/httpapis/ekg_construct/api.py @@ -247,7 +247,7 @@ async def embedding_predict(request: EmbeddingsRequest): def create_api(llm, embeddings): app = init_app(llm, embeddings) - uvicorn.run(app, host="localhost", port=3737) + uvicorn.run(app, host="127.0.0.1", port=3737) # def create_api(ekg_construct_service: EKGConstructService): # app = init_app(ekg_construct_service) diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 2ba052f..0761405 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from typing import List, Dict +from typing import List, Dict, Optional from enum import Enum import copy import json @@ -21,13 +21,13 @@ ##################################################################### class NodeSchema(BaseModel): type: str - ID: int = None # depend on id-str + ID: Optional[int] = None # depend on id-str id: str # depend on user's difine name: str # depend on user's difine description: str - gdb_timestamp: int + gdb_timestamp: Optional[int] = None def attributes(self, ): attrs = copy.deepcopy(vars(self)) @@ -41,14 +41,14 @@ def attributes(self, ): class EdgeSchema(BaseModel): type: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - SRCID: int = None + SRCID: Optional[int] = None original_src_id1__: str # entity_id, ekg_node:{graph_id}:{node_type}:{content_md5} - DSTID: int = None + DSTID: Optional[int] = None original_dst_id2__: str # - timestamp: int = None - gdb_timestamp: int + timestamp: Optional[int] = None + gdb_timestamp: Optional[int] = None def attributes(self, ): attrs = copy.deepcopy(vars(self)) @@ -75,15 +75,15 @@ class NodeTypesEnum(Enum): # EKG Node and Edge Schemas class EKGNodeSchema(NodeSchema): - teamids: str + teamids: str = '' # version:str # yyyy-mm-dd HH:MM:SS - extra: str = '' + extra: str = '{}' class EKGEdgeSchema(EdgeSchema): # teamids: str # version:str # yyyy-mm-dd HH:MM:SS - extra: str = '' + extra: str = '{}' class EKGIntentNodeSchema(EKGNodeSchema): @@ -92,26 +92,26 @@ class EKGIntentNodeSchema(EKGNodeSchema): class EKGScheduleNodeSchema(EKGNodeSchema): # do action or not - enable: bool + enable: bool = False class EKGTaskNodeSchema(EKGNodeSchema): # tool: str # needcheck: bool # when to access - accesscriteria: str - executetype: str + accesscriteria: str = '{}' + executetype: str = '' # # owner: str class EKGAnalysisNodeSchema(EKGNodeSchema): # when to access - accesscriteria: str + accesscriteria: str = '{}' # do summary or not - summaryswtich: bool + summaryswtich: bool = False # summary template - dsltemplate: str + dsltemplate: str = '' class EKGPhenomenonNodeSchema(EKGNodeSchema): diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index a8a7454..b5956bd 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -176,34 +176,58 @@ def init_sls(self, do_init: bool=None): else: self.sls = None - def update_graph( - self, - origin_nodes: List[GNode], origin_edges: List[GEdge], - new_nodes: List[GNode], new_edges: List[GEdge], - teamid: str, rootid: str - ): - - # search unconnect nodes and edges + def _get_local_graph(self, nodes: List[GNode], edges: List[GEdge], rootid): + # search and delete unconnect nodes and edges connections = {} - for edge in new_edges: + for edge in edges: connections.setdefault(edge.start_id, []).append(edge.end_id) - if len(new_nodes)==1: - connections.setdefault(new_nodes[0].id, []) + for node in nodes: + connections.setdefault(node.id, []) - if rootid not in connections and len(new_nodes)>0: + if rootid not in connections and len(nodes)>0: raise Exception(f"Error: rootid not in this graph") # dfs for those nodes from rootid visited = set() rootid_can_arrive_nodeids = [] - def _dfs(node): + paths = [] + def _dfs(node, current_path: List): if node not in visited: visited.add(node) + current_path.append(node) rootid_can_arrive_nodeids.append(node) - for neighbor in connections.get(node, []): - _dfs(neighbor) - _dfs(rootid) + # 假设终止条件是没有更多的邻居 + if not connections.get(node, []): + # 到达终止节点，保存当前路径的副本 + paths.append(list(current_path)) + else: + for neighbor in connections.get(node, []): + _dfs(neighbor, current_path) + + # 回溯：移除最后一个节点 + current_path.pop() + + # 初始化 DFS + _dfs(rootid, []) + logger.info(f"graph paths, {paths}") + logger.info(f"rootid can not arrive nodeids, {[n for n in nodes if n.id not in rootid_can_arrive_nodeids]}") + + graph = Graph( + nodes=[n for n in nodes if n.id in rootid_can_arrive_nodeids], + edges=[e for e in edges if e.start_id in rootid_can_arrive_nodeids and e.end_id in rootid_can_arrive_nodeids], + paths=paths + ) + return rootid_can_arrive_nodeids, graph + + def update_graph( + self, + origin_nodes: List[GNode], origin_edges: List[GEdge], + new_nodes: List[GNode], new_edges: List[GEdge], + teamid: str, rootid: str + ): + rootid_can_arrive_nodeids, _ = self._get_local_graph(new_nodes, new_edges, rootid=rootid) + _, _ = self._get_local_graph(origin_nodes, origin_edges, rootid=rootid) # rootid_can_arrive_node = [n for n in new_nodes if n.id in rootid_can_arrive_nodeids] # rootid_can_arrive_edge = [e for e in new_edges if (e.start_id in rootid_can_arrive_nodeids) and (e.end_id in rootid_can_arrive_nodeids)] @@ -257,17 +281,71 @@ def _dfs(node): if edgeid2edges_dict[edgeid][0]!=edgeid2edges_dict[edgeid][1] ] - # + # execute action add_node_result = self.add_nodes(add_nodes, teamid) add_edge_result = self.add_edges(add_edges, teamid) - delete_edge_result = self.delete_nodes(delete_nodes, teamid) - delete_node_result = self.delete_edges(delete_edges, teamid) + delete_node_result = self.delete_nodes(delete_nodes, teamid) + delete_edge_result = self.delete_edges(delete_edges, teamid) update_node_result = self.update_nodes(update_nodes, teamid) update_edge_result = self.update_edges(update_edges, teamid) - return [add_node_result, add_edge_result, delete_edge_result, delete_node_result, update_node_result, update_edge_result] + # 返回明确更新的graph + add_fail_edge_ids = [] + add_edge_dict = {} + for add_edge, gb_res in zip(add_edges, add_edge_result["gb_result"]): + add_edge_dict["__".join([add_edge.start_id, add_edge.end_id])] = add_edge + if gb_res["status"]["errorMessage"] not in ["GDB_SUCCEED", "GDB_ENGINE_PRIMARY_KEY_DUPLICATE"]: + add_fail_edge_ids.append("__".join([add_edge.start_id, add_edge.end_id])) + + delete_fail_edge_ids = [] + for delete_edge, gb_res in zip(delete_edges, delete_edge_result["gb_result"]): + if gb_res["status"]["errorMessage"] not in ["GDB_SUCCEED"]: + delete_fail_edge_ids.append("__".join([delete_edge.start_id, delete_edge.end_id])) + + add_fail_node_ids = [] + add_node_dict = {} + for add_node, gb_res in zip(add_nodes, add_node_result["gb_result"]): + add_node_dict[add_node.id] = add_node + if gb_res["status"]["errorMessage"] not in ["GDB_SUCCEED", "GDB_ENGINE_PRIMARY_KEY_DUPLICATE"]: + add_fail_node_ids.append(add_node.id) + + delete_fail_node_ids = [] + for delete_node, gb_res in zip(delete_nodes, delete_node_result["gb_result"]): + if gb_res["status"]["errorMessage"] not in ["GDB_SUCCEED"]: + delete_fail_node_ids.append(delete_node.id) + + new_nodes_copy = [] + for n in new_nodes: + if n.id not in add_fail_node_ids: + new_nodes_copy.append(add_node_dict.get(n.id, n)) + + for n in delete_nodes: + if n.id in delete_fail_node_ids: + new_nodes_copy.append(n) + + new_edges_copy = [] + for e in new_edges: + if "__".join([e.start_id, e.end_id]) not in add_fail_edge_ids: + new_edges_copy.append(add_edge_dict.get("__".join([e.start_id, e.end_id]), e)) + for e in delete_edges: + if "__".join([e.start_id, e.end_id]) in delete_fail_edge_ids: + new_edges_copy.append(e) + + _, old_graph = self._get_local_graph( + self._normalized_nodes_type(new_nodes_copy), + self._normalized_edges_type(new_edges_copy), rootid=rootid + ) + # old_graph = Graph(nodes=new_nodes_copy, edges=new_edges_copy, paths=[]) + return old_graph, { + "add_node_result": add_node_result, + "add_edge_result": add_edge_result, + "delete_edge_result": delete_edge_result, + "delete_node_result": delete_node_result, + "update_node_result": update_node_result, + "update_edge_result": update_edge_result + } def add_nodes(self, nodes: List[GNode], teamid: str): @@ -291,7 +369,7 @@ def add_nodes(self, nodes: List[GNode], teamid: str): tb_result.append(self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False)) except Exception as e: logger.error(e) - return gb_result + tb_result + return {"gb_result": gb_result, "tb_result": tb_result} def add_edges(self, edges: List[GEdge], teamid: str): edges = self._update_new_attr_for_edges(edges) @@ -312,7 +390,7 @@ def add_edges(self, edges: List[GEdge], teamid: str): tb_result.append(self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False)) except Exception as e: logger.error(e) - return gb_result + tb_result + return {"gb_result": gb_result, "tb_result": tb_result} def delete_nodes(self, nodes: List[GNode], teamid: str=''): # delete tbase nodes @@ -380,7 +458,7 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): gb_result.append(self.gb.delete_node( {"id": node.id}, node.type, ID=node.attributes.get("ID") or double_hashing(node.id) )) - return gb_result + tb_result + return {"gb_result": gb_result, "tb_result": tb_result} def delete_edges(self, edges: List[GEdge], teamid: str): # delete tbase nodes @@ -406,7 +484,7 @@ def delete_edges(self, edges: List[GEdge], teamid: str): edge.attributes.get("DSTID") or double_hashing(edge.end_id), edge.type )) - return gb_result + tb_result + return {"gb_result": gb_result, "tb_result": tb_result} def update_nodes(self, nodes: List[GNode], teamid: str): # delete tbase nodes @@ -419,8 +497,8 @@ def update_nodes(self, nodes: List[GNode], teamid: str): if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") - for node in nodes: - r = self.tb.search(f"@node_id: {node.id}", index_name=self.node_indexname) + for nodeid in tbase_missing_nodeids: + r = self.tb.search(f"@node_id: {nodeid}", index_name=self.node_indexname) teamids_by_nodeid.update({data['node_id']: data["node_str"] for data in r.docs}) tb_result = [] @@ -448,7 +526,7 @@ def update_nodes(self, nodes: List[GNode], teamid: str): ID=ID ) gb_result.append(resp) - return gb_result or tb_result + return {"gb_result": gb_result, "tb_result": tb_result} def update_edges(self, edges: List[GEdge], teamid: str): r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.node_indexname, limit=len(edges)) @@ -470,13 +548,13 @@ def update_edges(self, edges: List[GEdge], teamid: str): for edge in edges: # if node.id not in update_tbase_nodeids: continue SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) - DSTID = edge.attributes.pop("DSTID", None) or double_hashing(edge.start_id) + DSTID = edge.attributes.pop("DSTID", None) or double_hashing(edge.end_id) resp = self.gb.update_edge( SRCID, DSTID, edge.attributes, edge_type=edge.type ) gb_result.append(resp) - return gb_result + return {"gb_result": gb_result, "tb_result": []} def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) @@ -487,7 +565,7 @@ def get_graph_by_nodeid( nodeid: str, node_type: str, hop: int = 10, - block_attributes: dict = {} + block_attributes: List[dict] = {} ) -> Graph: if hop<2: raise Exception(f"hop must be smaller than 2, now hop is {hop}") @@ -549,20 +627,16 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N keywords = extract_tags(text) keyword = "|".join(keywords) for key in ["name_keyword", "desc_keyword"]: - r = self.tb.search(f"(@{key}:{{{keyword}}})", index_name=self.node_indexname, limit=30) + query = f"(@node_str: *{teamid}*)(@{key}:{{{keyword}}})" + r = self.tb.search(query, index_name=self.node_indexname, limit=30) for i in r.docs: if i["ID"] not in nodeids: nodeids.append(i["ID"]) - nodes_by_name = self.gb.get_current_nodes({"name": text}, node_type=node_type) - nodes_by_desc = self.gb.get_current_nodes({"description": text}, node_type=node_type) nodes = self.gb.get_nodes_by_ids(nodeids) - nodes = self._normalized_nodes_type(nodes) - nodes = nodes_by_name + nodes_by_desc + nodes # tmp iead to filter by teamid - nodes = [node for node in nodes if teamid in str(node.attributes)] - + nodes = [node for node in nodes if str(teamid) in str(node.attributes)] # select the node which can connect the rootid nodes = [node for node in nodes if len(self.search_rootpath_by_nodeid(node.id, node.type, f"ekg_team:{teamid}").paths)>0] return nodes diff --git a/muagent/service/ekg_inference/intention_router.py b/muagent/service/ekg_inference/intention_router.py index 4f38a34..c6c2aab 100644 --- a/muagent/service/ekg_inference/intention_router.py +++ b/muagent/service/ekg_inference/intention_router.py @@ -59,7 +59,7 @@ def add_rule(self, node_id2rule: dict[str, str], gb_handler: Optional[GBHandler] rule_dict[node_id] = rule if len(rule_dict) > ori_len: - rule_dict.save_to_odps() + rule_dict.save() error_msg = '' if fail_nodes: @@ -189,11 +189,11 @@ def get_intention_by_node_info_nlp( return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) nodes_tb = self._tb_match(tb_handler, query, self._node_type) - filter_nodes_tb = self._filter_ancestors_hop(gb_handler, set(nodes_tb), root_node_id) + filter_nodes_tb = self._filter_ancestors(gb_handler, set(nodes_tb), root_node_id) filter_nodes_tb = { k: v for k, v in filter_nodes_tb.items() - if self.is_node_valid(gb_handler, k) + if self.is_node_valid(k, gb_handler) } if len(filter_nodes_tb) == 0: @@ -313,7 +313,7 @@ def is_node_valid(self, node_id: str, gb_handler: Optional[GBHandler] = None) -> canditates = [n.id for n in canditates if n.type == self._node_type] if len(canditates) == 0: return True - return self.is_node_valid(gb_handler, canditates[0]) + return self.is_node_valid(canditates[0], gb_handler) def _get_agent_ans_no_ekg(self, agent, query: str) -> str: query = itp.DIRECT_CHAT_PROMPT.format(query=query) diff --git a/muagent/utils/common_utils.py b/muagent/utils/common_utils.py index 377a41a..0dea3d4 100644 --- a/muagent/utils/common_utils.py +++ b/muagent/utils/common_utils.py @@ -13,8 +13,8 @@ DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -def getCurrentDatetime(): - return datetime.now().strftime("%Y-%m-%d %H:%M:%S") +def getCurrentDatetime(dateformat=DATE_FORMAT): + return datetime.now().strftime(dateformat) def getCurrentTimestap(): return int(datetime.now().timestamp()) @@ -30,10 +30,10 @@ def addMinutesToTime(input_time: str, n: int = 5, dateformat=DATE_FORMAT): def timestampToDateformat(ts, interval=1000, dateformat=DATE_FORMAT): '''将标准时间戳转换标准指定时间格式''' - return datetime.fromtimestamp(ts//interval).strftime(dateformat) + return datetime.fromtimestamp(ts/interval).strftime(dateformat) -def datefromatToTimestamp(dt, interval=1000, dateformat=DATE_FORMAT): +def dateformatToTimestamp(dt, interval=1000, dateformat=DATE_FORMAT): '''将标准时间格式转换未标准时间戳''' return int(datetime.strptime(dt, dateformat).timestamp()*interval) diff --git a/tests/httpapis/docker_test.py b/tests/httpapis/docker_test.py new file mode 100644 index 0000000..b648e33 --- /dev/null +++ b/tests/httpapis/docker_test.py @@ -0,0 +1,48 @@ + +import docker, sys, os, time, requests, psutil +import subprocess +from docker.types import Mount, DeviceRequest + +import platform +system_name = platform.system() +USE_TTY = system_name in ["Windows"] + + +client = docker.from_env() + +mount = Mount( + type='bind', + source="/d/project/gitlab/ant_code/muagent", + target='/home/user/muagent', + read_only=False # 如果需要只读访问，将此选项设置为True + ) + +ports={ + f"8080/tcp": f"8888/tcp", + f"3737/tcp": f"3737/tcp", +} + + +print( client.networks.list()) + +network_name ='my_network' +networks = client.networks.list() +if any([network_name==i.attrs["Name"] for i in networks]): + network = client.networks.get(network_name) +else: + network = client.networks.create('my_network', driver='bridge') +container = client.containers.run( + image="muagent:test", + command="bash", + mounts=[mount], + name="test", + mem_limit="8g", + # device_requests=[DeviceRequest(count=-1, capabilities=[['gpu']])], + # network_mode="host", + ports=ports, + stdin_open=True, + detach=True, + tty=USE_TTY, + network='my_network', + ) + diff --git a/tests/httpapis/fastapi_test.py b/tests/httpapis/fastapi_test.py index eed19a1..e7104a3 100644 --- a/tests/httpapis/fastapi_test.py +++ b/tests/httpapis/fastapi_test.py @@ -27,24 +27,24 @@ def get_node(request: GetNodeRequest): return GetNodeResponse( test="test" ) -# # 一个异步路由 -# @app.get("/") -# async def read_root(): -# await asyncio.sleep(1) # 模拟一个异步操作 -# return {"message": "Hello, World!"} +# 一个异步路由 +@app.get("/") +async def read_root(): + await asyncio.sleep(1) # 模拟一个异步操作 + return {"message": "Hello, World!"} -# # 另一个异步路由 -# @app.get("/items/{item_id}") -# async def read_item(item_id: int, q: str = None): -# await asyncio.sleep(5) # 模拟延迟 -# return {"item_id": item_id, "q": q} +# 另一个异步路由 +@app.get("/items/{item_id}") +async def read_item(item_id: int, q: str = None): + await asyncio.sleep(5) # 模拟延迟 + return {"item_id": item_id, "q": q} -# # 另一个异步路由 -# @app.get("/itemstest/{item_id}") -# def read_item(item_id: int, q: str = None): -# time.sleep(5) # 模拟延迟 -# return {"item_id": item_id, "q": q} +# 另一个异步路由 +@app.get("/itemstest/{item_id}") +def read_item(item_id: int, q: str = None): + time.sleep(5) # 模拟延迟 + return {"item_id": item_id, "q": q} # 均能并发触发，好像对于io操作会默认管理 -uvicorn.run(app, host="localhost", port=3737) +uvicorn.run(app, host="127.0.0.1", port=3737) From ff7aff7faffc12d243c837293898e43f63e6e668 Mon Sep 17 00:00:00 2001 From: lightislost Date: Tue, 3 Sep 2024 10:39:02 +0800 Subject: [PATCH 030/128] [bugfix][update analysis node attribute's name error] --- muagent/schemas/ekg/ekg_graph.py | 4 ++-- muagent/service/ekg_construct/ekg_construct_base.py | 2 +- tests/service/ekg_construct_test_2.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 0761405..17b831f 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -109,7 +109,7 @@ class EKGAnalysisNodeSchema(EKGNodeSchema): # when to access accesscriteria: str = '{}' # do summary or not - summaryswtich: bool = False + summaryswitch: bool = False # summary template dsltemplate: str = '' @@ -153,7 +153,7 @@ class EKGGraphSlsSchema(BaseModel): extra: str = '' enable: bool = False dsltemplate: str = '' - summaryswtich: bool = False + summaryswitch: bool = False gdb_timestamp: int original_src_id1__: str = "" diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index b5956bd..66079c9 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -1105,7 +1105,7 @@ def _normalized_nodes_type(self, nodes: List[GNode]) -> List[GNode]: for node in nodes: node_type = node.type node_data_dict = {**{"id": node.id, "type": node_type}, **node.attributes} - node_data_dict = {k: 'False' if k in ["enable", "summaryswtich"] and v=="" else v for k,v in node_data_dict.items()} + node_data_dict = {k: 'False' if k in ["enable", "summaryswitch"] and v=="" else v for k,v in node_data_dict.items()} node_data: EKGNodeSchema = TYPE2SCHEMA[node_type](**node_data_dict) valid_node = GNode(id=node.id, type=node_type, attributes=node_data.attributes()) valid_nodes.append(valid_node) diff --git a/tests/service/ekg_construct_test_2.py b/tests/service/ekg_construct_test_2.py index a14be26..9a55b47 100644 --- a/tests/service/ekg_construct_test_2.py +++ b/tests/service/ekg_construct_test_2.py @@ -108,7 +108,7 @@ def generate_node(id, type): if type == "opsgptkg_analysis": extra_attr["accesscriteria"] = "hello" - extra_attr["summaryswtich"] = False + extra_attr["summaryswitch"] = False extra_attr['dsltemplate'] = "hello" return GNode(**{ From 335aa70bdbb2f105ebc73fc1ba3cac3012588c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E7=8E=96?= Date: Wed, 4 Sep 2024 11:14:39 +0800 Subject: [PATCH 031/128] update nebulaGraph service and config docker yaml --- .gitignore | 3 +- Dockerfile | 6 +- docker-compose.yaml | 33 +- examples/ekg_examples/ekg.yaml | 15 +- examples/ekg_examples/start.py | 34 +- .../graph_db_handler/nebula_handler.py | 736 ++++++++++++++---- muagent/httpapis/ekg_construct/api.py | 328 ++++---- muagent/schemas/apis/ekg_api_schema.py | 3 +- .../ekg_construct/ekg_construct_base.py | 63 ++ requirements.txt | 2 +- tests/db_handler/redis_test.py | 39 + tests/httpapis/fastapi_connect_test.py | 13 + tests/service/ekg_construct_test_2_nebula.py | 308 ++++++++ 13 files changed, 1232 insertions(+), 351 deletions(-) create mode 100644 tests/db_handler/redis_test.py create mode 100644 tests/httpapis/fastapi_connect_test.py create mode 100644 tests/service/ekg_construct_test_2_nebula.py diff --git a/.gitignore b/.gitignore index 9dd8a3e..63afe5c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ build dist .ipynb_checkpoints zdatafront* -*antgroup* \ No newline at end of file +*antgroup* +*ipynb \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 500261b..fdf707a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -From python:3.9.18-bookworm -# FROM python:3.9-slim-bookworm +FROM python:3.9-bookworm WORKDIR /home/user @@ -16,7 +15,6 @@ COPY ./requirements.txt /home/user/docker_requirements.txt # RUN dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install fastapi uvicorn notebook -# RUN pip install -r /home/user/docker_requirements.txt +RUN pip install -r /home/user/docker_requirements.txt --retries 5 --timeout 120 CMD ["bash"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 6760a0f..66ad9a0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,9 +1,8 @@ -version: '0.1' - +version: '3.4' services: metad0: + # image: docker.io/vesoft/nebula-metad:v3.8.0 image: vesoft/nebula-metad:v3.8.0 - container_name: metad0 environment: USER: root command: @@ -36,8 +35,8 @@ services: - SYS_PTRACE storaged0: + # image: docker.io/vesoft/nebula-storaged:v3.8.0 image: vesoft/nebula-storaged:v3.8.0 - container_name: storaged0 environment: USER: root TZ: "${TZ}" @@ -73,8 +72,8 @@ services: - SYS_PTRACE graphd: + # image: docker.io/vesoft/nebula-graphd:v3.8.0 image: vesoft/nebula-graphd:v3.8.0 - container_name: graphd environment: USER: root TZ: "${TZ}" @@ -109,7 +108,7 @@ services: redis-stack: image: redis/redis-stack:7.4.0-v0 - container_name: redis + container_name: redis-stack ports: - "6379:6379" - "8001:8001" @@ -118,7 +117,7 @@ services: networks: - ekg-net restart: always - + ollama: image: ollama/ollama:0.3.6 @@ -129,8 +128,8 @@ services: ports: - 11434:11434 volumes: - - //d/models/ollama:/root/.ollama # windows path - # - /User/models:/root/.ollama # linux/mac path + # - //d/models/ollama:/root/.ollama # windows path + - /Users/yunjiu/ant/models:/root/.ollama # linux/mac path networks: - ekg-net restart: on-failure @@ -149,13 +148,14 @@ services: context: . dockerfile: Dockerfile container_name: ekgservice - image: muagent:test + image: muagent:0.1.0 + # image: muagent:0.2.0 environment: USER: root TZ: "${TZ}" ports: - - 5050:3737 - # - 8080:8888 + # - 5050:3737 + - 3737:3737 volumes: - ./examples:/home/user/muagent/examples - ./muagent:/home/user/muagent/muagent @@ -163,8 +163,11 @@ services: restart: on-failure networks: - ekg-net - # command: ["python", "/home/user/muagent/examples/ekg_examples/start.py"] # 指定要执行的脚本 - command: ["python", "/home/user/muagent/tests/httpapis/fastapi_test.py"] # 指定要执行的脚本 + command: ["python", "/home/user/muagent/examples/ekg_examples/start.py"] # 指定要执行的脚本 + # command: ["python", "/home/user/muagent/tests/httpapis/fastapi_test.py"] # 指定要执行的脚本 + networks: - ekg-net: \ No newline at end of file + ekg-net: + # driver: bridge + external: true diff --git a/examples/ekg_examples/ekg.yaml b/examples/ekg_examples/ekg.yaml index 62a26db..e7df07c 100644 --- a/examples/ekg_examples/ekg.yaml +++ b/examples/ekg_examples/ekg.yaml @@ -8,16 +8,17 @@ geabase_config: # nebula config nebula_config: - host: 'localhost' - port: 7070 - username: 'default' - password: 'default' - space_name: 'default' + host: 'graphd' + port: '9669' + username: 'root' + password: 'nebula' + space_name: 'client' # tbase config tbase_config: - host: 'localhost' - port: 321321 + # host: 'localhost' + host: 'redis-stack' + port: '6379' username: 'default' password: '' definition_value: 'opsgptkg' diff --git a/examples/ekg_examples/start.py b/examples/ekg_examples/start.py index 026a0d6..de9a287 100644 --- a/examples/ekg_examples/start.py +++ b/examples/ekg_examples/start.py @@ -211,6 +211,17 @@ def embed_query(self, text: str) -> List[float]: # extra_kwargs={} # ) +# 初始化 NebulaHandler 实例 +gb_config = GBConfig( + gb_type="NebulaHandler", + extra_kwargs={ + 'host': config_data["nebula_config"]['host'], + 'port': config_data["nebula_config"]['port'], + 'username': config_data["nebula_config"]['username'] , + 'password': config_data["nebula_config"]['password'], + "space": config_data["nebula_config"]['space_name'], + } +) # 初始化 TbaseHandler 实例 tb_config = TBConfig( @@ -236,18 +247,19 @@ def embed_query(self, text: str) -> List[float]: embeddings = CustomEmbeddings() -embed_config = EmbedConfig( - embed_model="default", - langchain_embeddings=embeddings -) +# embed_config = EmbedConfig( +# embed_model="default", +# langchain_embeddings=embeddings +# ) +embed_config = None -# ekg_construct_service = EKGConstructService( -# embed_config=embed_config, -# llm_config=llm_config, -# tb_config=tb_config, -# gb_config=gb_config, -# ) +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) from muagent.httpapis.ekg_construct import create_api -create_api(llm, embeddings) \ No newline at end of file +create_api(llm, embeddings, ekg_construct_service) \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/nebula_handler.py b/muagent/db_handler/graph_db_handler/nebula_handler.py index b16fafa..0731da6 100644 --- a/muagent/db_handler/graph_db_handler/nebula_handler.py +++ b/muagent/db_handler/graph_db_handler/nebula_handler.py @@ -1,19 +1,31 @@ # encoding: utf-8 ''' -@author: 温进 +@author: 云玖 @file: nebula_handler.py -@time: 2023/11/16 下午3:15 +@time: 2024/8/9 下午14:56 @desc: ''' import time from loguru import logger +from typing import List, Dict, Any + +from muagent.schemas.common import GNode, GEdge, Graph +from muagent.schemas.db import GBConfig +from .base_gb_handler import GBHandler +from muagent.schemas.common import * +from muagent.utils.common_utils import double_hashing + from nebula3.gclient.net import ConnectionPool from nebula3.Config import Config +from nebula3.data.DataObject import ValueWrapper +from nebula3.common.ttypes import * + + class NebulaHandler: - def __init__(self, host: str, port: int, username: str, password: str = '', space_name: str = ''): + def __init__(self,gb_config : GBConfig = None): ''' init nebula connection_pool @param host: host @@ -24,14 +36,13 @@ def __init__(self, host: str, port: int, username: str, password: str = '', spac config = Config() self.connection_pool = ConnectionPool() - self.connection_pool.init([(host, port)], config) - self.username = username - self.password = password - self.space_name = space_name + self.connection_pool.init([(gb_config.extra_kwargs.get("host"), gb_config.extra_kwargs.get("port"))], config) + self.username = gb_config.extra_kwargs.get("username") + self.password = gb_config.extra_kwargs.get("password") + self.space_name = gb_config.extra_kwargs.get("space") - def execute_cypher(self, cypher: str, space_name: str = '', format_res: bool = False, use_space_name: bool = True): + def execute_cypher(self, cypher: str, space_name: str = '', format_res: str = 'as_primitive', use_space_name: bool = True): ''' - @param space_name: space_name, if provided, will execute use space_name first @param cypher: @return: @@ -46,14 +57,24 @@ def execute_cypher(self, cypher: str, space_name: str = '', format_res: bool = F # logger.debug(cypher) resp = session.execute(cypher) - if format_res: - resp = self.result_to_dict(resp) + if resp.is_succeeded(): + logger.info(f"Successfully executed Cypher query: {cypher}") + + else: + logger.error(f"Failed to execute Cypher query: {cypher}") + print(resp.error_msg()) + + + if format_res == 'as_primitive': + resp = resp.as_primitive() + elif format_res == 'dict_for_vis': + resp = resp.dict_for_vis() return resp def close_connection(self): self.connection_pool.close() - def create_space(self, space_name: str, vid_type: str, comment: str = ''): + def create_space(self, space_name: str, vid_type: str = 'FIXED_STRING(32)', comment: str = ''): ''' create space @param space_name: cannot startwith number @@ -102,54 +123,37 @@ def show_tags(self): resp = self.execute_cypher(cypher, self.space_name) return resp - def insert_vertex(self, tag_name: str, value_dict: dict): - ''' - insert vertex - @param tag_name: - @param value_dict: {'properties_name': [], values: {'vid':[]}} order should be the same in properties_name and values - @return: - ''' - cypher = f'INSERT VERTEX {tag_name} (' - - properties_name = value_dict['properties_name'] - - for property_name in properties_name: - cypher += f'{property_name},' - cypher = cypher.rstrip(',') - - cypher += ') VALUES ' - - for vid, properties in value_dict['values'].items(): - cypher += f'"{vid}":(' - for property in properties: - if type(property) == str: - cypher += f'"{property}",' - else: - cypher += f'{property}' - cypher = cypher.rstrip(',') - cypher += '),' - cypher = cypher.rstrip(',') - cypher += ';' - - res = self.execute_cypher(cypher, self.space_name) - return res - def create_edge_type(self, edge_type_name: str, prop_dict: dict = {}): ''' - 创建 tag - @param edge_type_name: tag 名称 - @param prop_dict: 属性字典 {'prop 名字': 'prop 类型'} - @return: + 创建边标签 + @param edge_info: 边的信息字典，包括标签名称和属性字典 + @return: 执行结果 ''' + + # 构建 Cypher 查询 cypher = f'CREATE EDGE IF NOT EXISTS {edge_type_name}' - cypher += '(' - for k, v in prop_dict.items(): - cypher += f'{k} {v},' - cypher = cypher.rstrip(',') - cypher += ')' + if prop_dict: + cypher += '(' + for k, v in prop_dict.items(): + # 数据库系统命名字段，需要加`` + if k.upper() == 'TIMESTAMP': + k = f'`{k}`' + # 需根据输入类型进一步补充 + # if isinstance(v, int): + # prop_type = 'INT' + # elif isinstance(v, float): + # prop_type = 'FLOAT' + # elif isinstance(v, str): + # prop_type = 'STRING' + # else: + # prop_type = 'UNKNOWN' + cypher += f'{k} {v},' + cypher = cypher.rstrip(',') + cypher += ')' cypher += ';' + # 执行查询 res = self.execute_cypher(cypher, self.space_name) return res @@ -162,124 +166,574 @@ def show_edge_type(self): resp = self.execute_cypher(cypher, self.space_name) return resp - def drop_edge_type(self, edge_type_name: str): + def delete_edge_type(self, edge_type_name: str): cypher = f'DROP EDGE {edge_type_name}' return self.execute_cypher(cypher, self.space_name) + + def add_node(self, node: GNode) -> dict: + ''' + Insert vertex into the graph. + + @param node: Dictionary containing node information + @return: Result of Cypher query execution + ''' + # 从GNode实例中提取信息 + vid = node.id + tag_name = node.type # 使用type作为tag名称 + # attributes = node.attributes + + # 初始化节点属性字典，并将节点的ID属性添加进去 + node_attributes = {"id": node.id} + node_attributes["ID"] = node.attributes.pop("ID", "") or double_hashing(node.id) + node_attributes.update(node.attributes) + + # 构建 Cypher 查询 + properties_name = list(node_attributes.keys()) + + # 构建查询字符串 + cypher = f'INSERT VERTEX {tag_name} (' + cypher += ','.join(properties_name) + cypher += ') VALUES ' + + cypher += f'"{vid}":(' + for prop_name in properties_name: + value = node_attributes.get(prop_name) + if isinstance(value, str): + if prop_name == 'extra': + # 转义双引号 + value = value.replace('"', '\\"') + cypher += f'"{value}",' + else: + cypher += f'"{value}",' + #cypher += f'"{value}",' + else: + cypher += f'{value},' + cypher = cypher.rstrip(',') + cypher += ');' + + # 执行 Cypher 查询 + res = self.execute_cypher(cypher, self.space_name) + return res + + def add_nodes(self, nodes: List[GNode]) -> dict: + for node in nodes: + self.add_node(node) - def insert_edge(self, edge_type_name: str, value_dict: dict): + def add_edge(self, edge: GEdge) -> dict: ''' - insert edge - @param edge_type_name: - @param value_dict: value_dict: {'properties_name': [], values: {(src_vid, dst_vid):[]}} order should be the - same in properties_name and values - @return: + 插入边 + @param edge: 边的信息字典，包括标签名称、起始节点 ID、结束节点 ID 和属性字典 + @return: 执行结果 ''' + edge_type_name = edge.type + src_vid = edge.start_id + dst_vid = edge.end_id + attributes = edge.attributes + + # edge_attributes = { + # "`@src_id`": edge.attributes.pop("SRCID", 0) or double_hashing(edge.start_id), + # "`@dst_id`": edge.attributes.pop("DSTID", 0) or double_hashing(edge.end_id), + # } + # edge_attributes.update(edge.attributes) + + # 构建 Cypher 查询 cypher = f'INSERT EDGE {edge_type_name} (' - properties_name = value_dict['properties_name'] - + # 获取属性名称 + properties_name = list(attributes.keys()) for property_name in properties_name: - cypher += f'{property_name},' + # 处理 @timestamp 字段 + if property_name == '@timestamp': + cypher += f'`TIMESTAMP`,' + else: + cypher += f'{property_name},' + #cypher += f'{property_name},' cypher = cypher.rstrip(',') cypher += ') VALUES ' - - for (src_vid, dst_vid), properties in value_dict['values'].items(): - cypher += f'"{src_vid}"->"{dst_vid}":(' - for property in properties: - if type(property) == str: - cypher += f'"{property}",' + cypher += f'"{src_vid}"->"{dst_vid}":(' + + # 添加属性值 + for attr_name in properties_name: + attr_value = attributes[attr_name] + if isinstance(attr_value, str): + if attr_name == 'extra': + # 处理 extra 字段中的 JSON 字符串，转义双引号 + attr_value = attr_value.replace('"', '\\"') + cypher += f'"{attr_value}",' else: - cypher += f'{property}' - cypher = cypher.rstrip(',') - cypher += '),' + cypher += f'"{attr_value}",' + else: + cypher += f'{attr_value},' cypher = cypher.rstrip(',') - cypher += ';' + cypher += ');' + # 执行查询 + res = self.execute_cypher(cypher, self.space_name) + return res + + def add_edges(self, edges: List[GEdge]) -> dict: + for edge in edges: + self.add_edge(edge) + + def update_node(self, attributes: dict, set_attributes: dict, node_type: str = None, ID: int = None) -> dict: + # 添加引号并构造 SET 子句 + # set_clause = ', '.join([f'{k} = "{v}"' if isinstance(v, str) else f'{k} = {v}' for k, v in set_attributes.items()]) + + set_clause_parts = [] + for k, v in set_attributes.items(): + if k == 'extra' and isinstance(v, str): + # 转义 extra 字段中的 JSON 字符串中的双引号 + v = v.replace('"', '\\"') + set_clause_parts.append(f'{k} = "{v}"') + elif isinstance(v, str): + set_clause_parts.append(f'{k} = "{v}"') + else: + set_clause_parts.append(f'{k} = {v}') + + set_clause = ', '.join(set_clause_parts) + + #ngql里字段str需要用""括起来, 语句最好采用拼接的方式 + cypher = f'UPDATE VERTEX ON {node_type} "{ID}" ' + + cypher += f'SET {set_clause} ' + + cypher += f'YIELD {", ".join(set_attributes.keys())};' + + # 执行查询 res = self.execute_cypher(cypher, self.space_name) return res - def set_space_name(self, space_name): - self.space_name = space_name + def update_edge(self, src_id, dst_id, set_attributes: dict, edge_type: str = None) -> dict: + # set_clause = ', '.join([f'{k} = "{v}"' if isinstance(v, str) else f'{k} = {v}' for k, v in set_attributes.items()]) - def add_host(self, host: str, port: str): - ''' - add host - @return: - ''' - cypher = f'ADD HOSTS {host}:{port}' - res = self.execute_cypher(cypher, use_space_name=False) + set_clause_parts = [] + for k, v in set_attributes.items(): + if k == 'extra' and isinstance(v, str): + # 转义 extra 字段中的 JSON 字符串中的双引号 + v = v.replace('"', '\\"') + set_clause_parts.append(f'{k} = "{v}"') + elif isinstance(v, str): + set_clause_parts.append(f'{k} = "{v}"') + else: + set_clause_parts.append(f'{k} = {v}') + + set_clause = ', '.join(set_clause_parts) + + cypher = f'UPDATE EDGE ON {edge_type} "{src_id}" -> "{dst_id}" ' + + cypher += f'SET {set_clause} ' + + # 执行查询 + res = self.execute_cypher(cypher, self.space_name) return res + + def delete_node(self, attributes: dict = None, node_type: str = None, ID: int = None) -> dict: + cypher = f'DELETE VERTEX "{ID}" WITH EDGE;' - def get_stat(self): - ''' + # 执行查询 + res = self.execute_cypher(cypher, self.space_name) + return res - @return: - ''' - submit_cypher = 'SUBMIT JOB STATS;' - self.execute_cypher(cypher=submit_cypher, space_name=self.space_name) - time.sleep(2) - - stats_cypher = 'SHOW STATS;' - stats_res = self.execute_cypher(cypher=stats_cypher, space_name=self.space_name) - time.sleep(2) - res = {'vertices': -1, 'edges': -1} - - stats_res_dict = self.result_to_dict(stats_res) - logger.info(f"{self.space_name}: {stats_res_dict}") - for idx in range(len(stats_res_dict['Type'])): - t = stats_res_dict['Type'][idx].as_string() - name = stats_res_dict['Name'][idx].as_string() - count = stats_res_dict['Count'][idx].as_int() - - if t == 'Space' and name in res: - res[name] = count + def delete_nodes(self, attributes: dict, node_type: str = None, IDs: List[int] = None) -> dict: + for id in IDs: + self.delete_node(id) + + def delete_edge(self, src_id, dst_id, edge_type: str = None) -> dict: + cypher = f'DELETE EDGE {edge_type} "{src_id}" -> "{dst_id}"@0;' + + # 执行查询 + res = self.execute_cypher(cypher, self.space_name) return res - def get_vertices(self, tag_name: str = '', limit: int = 10000): - ''' - get all vertices - @return: - ''' - if tag_name: - cypher = f'''MATCH (v:{tag_name}) RETURN v LIMIT {limit};''' - else: - cypher = f'MATCH (v) RETURN v LIMIT {limit};' + def delete_edges(self, id_pairs: List, edge_type: str = None): + # 构建DELETE EDGE语句 + edge_deletions = [] + for src_id, dst_id in id_pairs: + edge_deletions.append(f'"{src_id}" -> "{dst_id}"') + # 将所有边的删除语句拼接成一条完整的Cypher查询 + cypher = f'DELETE EDGE {edge_type} {", ".join(edge_deletions)};' + + # 执行查询 res = self.execute_cypher(cypher, self.space_name) - return self.result_to_dict(res) + return res + + def get_nodeIDs(self, attributes: dict, node_type: str): + result = self.get_current_nodes(attributes, node_type) + return [i.attributes.get("ID") for i in result] - def get_all_vertices(self,): - ''' - get all vertices - @return: - ''' - cypher = "MATCH (v) RETURN v;" + # 输出待确认，是自带id还是属性里的ID + def get_current_nodeID(self, attributes: dict, node_type: str) -> int: + result = self.get_current_node(attributes, node_type) + + return result.id + #return result.attributes.get("ID") + + # 输出待确认 + def get_current_edgeID(self, src_id, dst_id, edge_type:str = None): + result = self.get_current_edge(src_id, dst_id, edge_type) + return result.start_id, result.end_id, 1 + + def get_current_node(self, attributes: dict, node_type: str = None, return_keys: list = []) -> GNode: + return self.get_current_nodes(attributes, node_type, return_keys)[0] + + def get_nodes_by_ids(self, ids: List[int] = []) -> List[GNode]: + # 生成MATCH子句 + match_clause = 'MATCH (n0) WITH n0, properties(n0) as props, keys(properties(n0)) as kk ' + + list_clause = ', '.join(f'{id}' for id in ids) + # 生成WHERE子句 + where_clause = f'WHERE props["@id"] IN [{list_clause}]' + + # 生成RETURN子句 + return_clause = 'RETURN n0' + + # 拼接最终的Cypher查询语句 + cypher = f'{match_clause} {where_clause} {return_clause}' + + # 执行查询 res = self.execute_cypher(cypher, self.space_name) - return self.result_to_dict(res) + + decode_resp = self.decode_result(res,'n0') + + return [item.get('n0') for item in decode_resp if 'n0' in item] - def get_relative_vertices(self, vertice): + def get_current_nodes(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GNode]: + + # 生成MATCH子句 + match_clause = f'MATCH (n0{":" + node_type if node_type else ""}) WITH n0, properties(n0) as props, keys(properties(n0)) as kk ' + + # 生成WHERE子句 + where_clause = ' AND '.join( + f'props["{key}"] == "{value}"' if isinstance(value, str) else f'props["{key}"] == {value}' + for key, value in attributes.items() + ) + + # 将WHERE子句包装成正确的格式 + where_clause = f'WHERE [i IN kk WHERE {where_clause}]' + + # 生成RETURN子句 + return_clause = 'RETURN n0' if not return_keys else f'RETURN n0, {", ".join(return_keys)}' + + # 拼接最终的Cypher查询语句 + cypher = f'{match_clause} {where_clause} {return_clause}' + + # 执行查询 + resp = self.execute_cypher(cypher, self.space_name) + + decode_resp = self.decode_result(resp,['n0']) + + # print('====================get_current_nodes==============') + # print([item.get('n0') for item in decode_resp if 'n0' in item]) + # test_node = [item.get('n0') for item in decode_resp if 'n0' in item] + # print(type(test_node[0].id)) + # print(type(test_node[0].id)) + + + return [item.get('n0') for item in decode_resp if 'n0' in item] + + def get_current_edge(self, src_id, dst_id, edge_type:str = None, return_keys: list = [], limits: int = 100) -> GEdge: + cypher = f''' + MATCH (n0)-[e:{edge_type}]->(n1) \ + WHERE id(n0) == "{src_id}" AND id(n1) == "{dst_id}" \ + RETURN e \ + LIMIT {limits}; ''' - get all vertices - @return: + + # 执行查询 + resp = self.execute_cypher(cypher, self.space_name) + + decode_resp = self.decode_result(resp,['e']) + + resp = [item.get('e') for item in decode_resp if 'e' in item] + + return resp[0] + + # 结果待去重 + def get_neighbor_nodes(self, attributes: dict, node_type: str = None, return_keys: list = [],reverse: bool = False) -> List[GNode]: + # 先通过get_current_nodes搜索到节点 + # 如果属性字典里直接没有包含id，先根据其他属性查询node id + if not attributes['id']: + result = self.get_current_nodes(attributes, node_type) + id_list = [item.id for item in result] + else: + id_list = [attributes['id']] + + #print(result) + + #print(id_list) # ['yunjiu_3', 'yunjiu_4', 'yunjiu_2'] + id_list_str = '", "'.join(id_list) + + # 再根据搜索到的节点id查找附近的nodes + if reverse: + cypher = f'''MATCH (n0)<--(n1) \ + WHERE id(n0) in ["{id_list_str}"] \ + RETURN n1''' + else: + cypher = f'''MATCH (n0)-->(n1) \ + WHERE id(n0) in ["{id_list_str}"] \ + RETURN n1''' + + # 执行查询 + resp = self.execute_cypher(cypher, self.space_name) + + decode_resp = self.decode_result(resp,['n1']) + + return [item.get('n1') for item in decode_resp if 'n1' in item] + + # 结果待去重 + def get_neighbor_edges(self, attributes: dict, node_type: str = None, return_keys: list = []) -> List[GEdge]: + # 先通过get_current_nodes搜索到节点 + result = self.get_current_nodes(attributes, node_type) + + edges_id_list = [item.id for item in result] + + edges_id_list_str = '", "'.join(edges_id_list) + + # 再根据搜索到的节点id查找附近的edges，示例： + cypher = f'''MATCH (n0)-[e]-(n1) \ + WHERE id(n0) in ["{edges_id_list_str}"] \ + RETURN e''' + + # 执行查询 + resp = self.execute_cypher(cypher, self.space_name) + + decode_resp = self.decode_result(resp,['e']) + + return [item.get('e') for item in decode_resp if 'e' in item] + + def check_neighbor_exist(self, attributes: dict, node_type: str = None, check_attributes: dict = {}) -> bool: + # 判断是否有邻居nodes + result = self.get_neighbor_nodes(attributes, node_type) + return len(result) > 0 + + # 结果待去重 + def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = {}, select_attributes: dict = {}, reverse=False) -> Graph: ''' - cypher = f'''MATCH (v1)--(v2) WHERE id(v1) == '{vertice}' RETURN id(v2) as id;''' - res = self.execute_cypher(cypher, self.space_name) - return self.result_to_dict(res) - - def result_to_dict(self, result) -> dict: - """ - build list for each column, and transform to dataframe - """ - # logger.info(result.error_msg()) - assert result.is_succeeded() - columns = result.keys() - d = {} - for col_num in range(result.col_size()): - col_name = columns[col_num] - col_list = result.column_values(col_name) - d[col_name] = [x for x in col_list] - return d + hop >= 2， 表面需要至少两跳 + ''' + hop_max = 10 + + hop_num = max(min(hop, hop_max), 2) # 2 <= hop_num <= 10 + + result = self.get_current_nodes(attributes, node_type) + + nodes_id_list = [item.id for item in result] + + nodes_id_list_str = '", "'.join(nodes_id_list) + + if reverse: + cypher = f'''MATCH p=(n0)<-[es*1..{hop_num}]-(n1) \ + WHERE id(n0) in ["{nodes_id_list_str}"] \ + RETURN n0, es, n1, p''' + else: + cypher = f'''MATCH p=(n0)-[es*1..{hop_num}]->(n1) \ + WHERE id(n0) in ["{nodes_id_list_str}"] \ + RETURN n0, es, n1, p''' + + # 执行查询 + resp = self.execute_cypher(cypher, self.space_name) + + decode_resp = self.decode_result(resp,['n0','es','n1','p']) + + #print(decode_resp) + + # 筛选 + decode_resp = self.deduplicate_paths(decode_resp, block_attributes, select_attributes) + + nodes = [item.get('n1') for item in decode_resp if 'n1' in item] + + edges = [item.get('es') for item in decode_resp if 'es' in item] + edges = [item for sublist in edges for item in sublist] + + path = [item.get('p') for item in decode_resp if 'p' in item] + + return Graph(nodes=nodes, edges=edges, paths=path) + + def get_hop_nodes(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GNode]: + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.nodes + + def get_hop_edges(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[GEdge]: + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.edges + + def get_hop_paths(self, attributes: dict, node_type: str = None, hop: int = 2, block_attributes: dict = []) -> List[str]: + # + result = self.get_hop_infos(attributes, node_type, hop, block_attributes) + return result.paths + + def deduplicate_paths(self, result, block_attributes: dict = {}, select_attributes: dict = {}): + + if not block_attributes and not select_attributes: + return result + + # 筛选1 - 根据block_attributes查找的nodeid列表: 如果path中的node出现在block列表里，删除 + if block_attributes: + block_result = self.get_current_nodes(block_attributes) + block_nodes_id_list = [item.id for item in block_result] + block_filtered_result = [ + path for path in result + if not any(node in block_nodes_id_list for node in path.get('p', {})) + ] + result = block_filtered_result + + # 筛选2 - 根据select_attributes筛选:如果path中有node没有出现在select列表里，删除 + # （注意如果select列表为空，不执行该筛选条件） + if select_attributes: + select_result = self.get_current_nodes(select_attributes) + select_nodes_id_list = [item.id for item in select_result] + select_filtered_result = [ + path for path in result + if all(node in select_nodes_id_list for node in path.get('p', {})) + ] + result = select_filtered_result + + + return result + + def decode_result(self, data: List[Dict[str, Any]], return_keys: List[str]) -> List[Dict[str, Any]]: + #return_keys = [var.strip() for var in cypher.split("RETURN")[-1].split(',') if var.strip()] + + #print(return_keys) + + extracted_info = [] + + for item in data: + + info = {} + + if 'n0' in return_keys: + n0 = item.get('n0', {}) + info['n0'] = self.decode_node(n0) + + if 'e' in return_keys: + e = item.get('e', {}) + info['e'] = self.decode_edge(e) + + if 'es' in return_keys: + es = item.get('es', {}) + info['es'] = self.decode_edges(es) + + if 'n1' in return_keys: + n1 = item.get('n1', {}) + info['n1'] = self.decode_node(n1) + + if 'p' in return_keys: + p = item.get('p', {}) + info['p'] = self.decode_path(p) + + # 将字典添加到结果列表中 + extracted_info.append(info) + + return extracted_info # format: [{n0/n1/e/p:,},{},{}......] + + def decode_node(self, node_data: Dict[str, Any]) -> GNode: + #nodes = [] + + vid = node_data.get('vid', '') + tags = node_data.get('tags', {}) + + # 只处理第一个 tag + for type_name, attributes in tags.items(): + # 使用 convert_value 处理 attributes 中的每一个值 + processed_attributes = {k: self.convert_value(v) for k, v in attributes.items()} + + node_instance = GNode( + id=vid, + type=type_name, + # attributes=attributes + attributes=processed_attributes + ) + return node_instance + + #nodes.append(node_instance) + + return None + + def decode_edge(self, edge_data: List[Dict[str, Any]]) -> GEdge: + if not edge_data: + raise ValueError("The edge_data list is empty") + + #print(edge_data) + + #edge = edge_data[0] # Since input data only contains one edge + + start_id = edge_data.get('src', '') + end_id = edge_data.get('dst', '') + edge_type = edge_data.get('type', '') + attributes = edge_data.get('props', {}) + + # 使用 convert_value 处理 attributes 中的每一个值 + processed_attributes = {k: self.convert_value(v) for k, v in attributes.items()} + + return GEdge( + start_id=start_id, + end_id=end_id, + type=edge_type, + # attributes=attributes + attributes=processed_attributes + ) + + def decode_edges(self, edge_data: List[Dict[str, Any]]) -> List[GEdge]: + if not edge_data: + raise ValueError("The edge_data list is empty") + + edges = [] + + for edge in edge_data: + start_id = edge.get('src', '') + end_id = edge.get('dst', '') + edge_type = edge.get('type', '') + attributes = edge.get('props', {}) + + # 使用 convert_value 处理 attributes 中的每一个值 + processed_attributes = {k: self.convert_value(v) for k, v in attributes.items()} + + edges.append(GEdge( + start_id=start_id, + end_id=end_id, + type=edge_type, + attributes=processed_attributes + )) + + return edges + + def decode_path(self, data: Dict[str, Any]) -> List[str]: + nodes = data.get("nodes", []) + path = [node.get("vid") for node in nodes if "vid" in node] + return path + + # nebula对于加了/符号的str类型会转换成ValueWrapper类型输出，需要进行格式转换 + def convert_value(self, value_wrapper): + if isinstance(value_wrapper, ValueWrapper): + + value_type = value_wrapper._get_type_name() + if value_type == 'string': + return value_wrapper.as_string() + elif value_type == 'int': + return value_wrapper.as_int() + elif value_type == 'double': + return value_wrapper.as_double() + elif value_type == 'bool': + return value_wrapper.as_bool() + elif value_type == 'list': + return value_wrapper.as_list() + elif value_type == 'map': + return value_wrapper.as_map() + else: + return None # 未知类型处理 + return value_wrapper + + + + + + + + + diff --git a/muagent/httpapis/ekg_construct/api.py b/muagent/httpapis/ekg_construct/api.py index b0f3f0a..13a7d61 100644 --- a/muagent/httpapis/ekg_construct/api.py +++ b/muagent/httpapis/ekg_construct/api.py @@ -3,14 +3,15 @@ import asyncio import uvicorn from loguru import logger +import tqdm from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService from muagent.schemas.apis.ekg_api_schema import * # -# def init_app(llm, embeddings, ekg_construct_service: EKGConstructService): -def init_app(llm, embeddings): +def init_app(llm, embeddings, ekg_construct_service: EKGConstructService): +# def init_app(llm, embeddings): app = FastAPI() @@ -19,24 +20,6 @@ def init_app(llm, embeddings): async def llm_params(): return llm.params() - # ~/llm/params - @app.post("/llm/generate", response_model=LLMResponse) - async def llm_predict(request: LLMRequest): - # 添加预测逻辑的代码 - errorMessage = "ok" - successCode = True - try: - answer = llm.predict(request.text, request.stop) - except Exception as e: - errorMessage = str(e) - successCode = False - answer = "error" - - return LLMResponse( - successCode=successCode, errorMessage=errorMessage, - answer=answer - ) - # ~/llm/params/update @app.post("/llm/params/update", response_model=EKGResponse) async def update_llm_params(kwargs: Dict): @@ -91,164 +74,169 @@ async def embedding_predict(request: EmbeddingsRequest): successCode=successCode, errorMessage=errorMessage, embeddings=embeddings_list ) - # # ~/ekg/text2graph - # @app.post("/ekg/text2graph", response_model=EKGGraphResponse) - # async def text2graph(request: EKGT2GRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # result = ekg_construct_service.create_ekg( - # text=request.text, teamid=request.teamid, - # service_name="text2graph", - # intent_text=request.intentText, - # intent_nodes=request.intentNodeids, - # all_intent_list=request.intentPath, - # do_save=request.write2kg - # ) - # graph = result["graph"] - # nodes = [node.dict() for node in graph.nodes] - # edges = [edge.dict() for edge in graph.edges] - # except Exception as e: - # errorMessage = str(e) - # successCode = False - # nodes = [] - # edges = [] + + # + # ~/ekg/text2graph + @app.post("/ekg/text2graph", response_model=EKGGraphResponse) + async def text2graph(request: EKGT2GRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + result = ekg_construct_service.create_ekg( + text=request.text, teamid=request.teamid,rootid=request.rootid, + service_name="text2graph", + intent_text=request.intentText, + intent_nodes=request.intentNodeids, + all_intent_list=request.intentPath, + do_save=request.write2kg + ) + graph = result["graph"] + nodes = [node.dict() for node in graph.nodes] + edges = [edge.dict() for edge in graph.edges] + except Exception as e: + errorMessage = str(e) + successCode = False + nodes = [] + edges = [] - # return EKGGraphResponse( - # successCode=successCode, errorMessage=errorMessage, - # nodes=nodes, edges=edges - # ) - - - # # ~/ekg/graph/update - # @app.post("/ekg/graph/update", response_model=EKGResponse) - # async def update_graph(request: UpdateGraphRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # result = ekg_construct_service.update_graph( - # origin_nodes=request.originNodes, - # origin_edges=request.originEdges, - # nodes=request.nodes, - # edges=request.edges, - # teamid=request.teamid - # ) - # except Exception as e: - # errorMessage = str(e) - # successCode = False + return EKGGraphResponse( + successCode=successCode, errorMessage=errorMessage, + nodes=nodes, edges=edges + ) + + # done + # ~/ekg/graph/update + @app.post("/ekg/graph/update", response_model=EKGResponse) + async def update_graph(request: UpdateGraphRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + result = ekg_construct_service.update_graph( + origin_nodes=request.originNodes, + origin_edges=request.originEdges, + new_nodes=request.nodes, + new_edges=request.edges, + teamid=request.teamid + ) + except Exception as e: + logger.exception(e) + errorMessage = str(e) + successCode = False - # return EKGResponse( - # successCode=successCode, errorMessage=errorMessage, - # ) - - - - # # ~/ekg/node/search - # @app.get("/ekg/node", response_model=GetNodeResponse) - # def get_node(request: GetNodeRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # node = ekg_construct_service.get_node_by_id( - # request.node_id, request.node_type - # ) - # node = node.dict() - # except Exception as e: - # errorMessage = str(e) - # successCode = False - # node = {} + return EKGResponse( + successCode=successCode, errorMessage=errorMessage, + ) + + + # done + # ~/ekg/node/search + @app.get("/ekg/node", response_model=GetNodeResponse) + def get_node(request: GetNodeRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + node = ekg_construct_service.get_node_by_id( + request.nodeid, request.nodeType + ) + # node = node.dict() + except Exception as e: + errorMessage = str(e) + successCode = False + node = None - # return GetNodeResponse( - # successCode=successCode, errorMessage=errorMessage, - # node=node - # ) - - - # # ~/ekg/node/search - # @app.get("/ekg/graph", response_model=EKGGraphResponse) - # def get_graph(request: GetGraphRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # if request.layer == "first": - # graph = ekg_construct_service.get_graph_by_nodeid( - # nodeid=request.nodeid, node_type=request.nodeType, - # hop=8, block_attributes={"type": "opsgptkg_task"}) - # else: - # graph = ekg_construct_service.get_graph_by_nodeid( - # nodeid=request.nodeid, node_type=request.nodeType, - # hop=request.hop - # ) - # nodes = graph.nodes.dict() - # edges = graph.edges.dict() - # except Exception as e: - # errorMessage = str(e) - # successCode = False - # nodes, edges = {}, {} + return GetNodeResponse( + successCode=successCode, errorMessage=errorMessage, + node=node + ) + + # done + # ~/ekg/node/search + @app.get("/ekg/graph", response_model=EKGGraphResponse) + def get_graph(request: GetGraphRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + if request.layer == "first": + graph = ekg_construct_service.get_graph_by_nodeid( + nodeid=request.nodeid, node_type=request.nodeType, + hop=8, block_attributes={"type": "opsgptkg_task"}) + else: + graph = ekg_construct_service.get_graph_by_nodeid( + nodeid=request.nodeid, node_type=request.nodeType, + hop=request.hop + ) + + # nodes = graph.nodes.dict() + # edges = graph.edges.dict() + nodes = graph.nodes + edges = graph.edges + except Exception as e: + errorMessage = str(e) + successCode = False + nodes, edges = {}, {} - # return EKGGraphResponse( - # successCode=successCode, errorMessage=errorMessage, - # nodes=nodes, edges=edges - # ) - - - - # # ~/ekg/node/search - # @app.post("/ekg/node/search", response_model=GetNodesResponse) - # def search_node(request: SearchNodesRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # nodes = ekg_construct_service.search_nodes_by_text( - # request.text, teamid=request.teamid - # ) - # nodes = [node.dict() for node in nodes] - # except Exception as e: - # errorMessage = str(e) - # successCode = False - # nodes = [] + return EKGGraphResponse( + successCode=successCode, errorMessage=errorMessage, + nodes=nodes, edges=edges + ) + + # done + # ~/ekg/node/search + @app.post("/ekg/node/search", response_model=GetNodesResponse) + def search_node(request: SearchNodesRequest): + print(f"search_nodes_by_text function: {ekg_construct_service.search_nodes_by_text}") + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + nodes = ekg_construct_service.search_nodes_by_text( + request.text, teamid=request.teamid + ) + nodes = [node.dict() for node in nodes] + except Exception as e: + errorMessage = str(e) + successCode = False + nodes = [] - # return GetNodesResponse( - # successCode=successCode, errorMessage=errorMessage, - # nodes=nodes - # ) - - # # ~/ekg/graph/ancestor - # @app.get("/ekg/graph/ancestor", response_model=EKGGraphResponse) - # def get_ancestor(request: SearchAncestorRequest): - # # 添加预测逻辑的代码 - # errorMessage = "ok" - # successCode = True - # try: - # graph = ekg_construct_service.search_rootpath_by_nodeid( - # nodeid=request.nodeid, node_type=request.nodeType, - # rootid=request.rootid - # ) - # nodes = graph.nodes.dict() - # edges = graph.edges.dict() - # except Exception as e: - # errorMessage = str(e) - # successCode = False - # nodes, edges = {}, {} + return GetNodesResponse( + successCode=successCode, errorMessage=errorMessage, + nodes=nodes + ) + + # done + # ~/ekg/graph/ancestor + @app.get("/ekg/graph/ancestor", response_model=EKGGraphResponse) + def get_ancestor(request: SearchAncestorRequest): + # 添加预测逻辑的代码 + errorMessage = "ok" + successCode = True + try: + graph = ekg_construct_service.search_rootpath_by_nodeid( + nodeid=request.nodeid, node_type=request.nodeType, + rootid=request.rootid + ) + # nodes = graph.nodes.dict() + # edges = graph.edges.dict() + nodes = graph.nodes + edges = graph.edges + except Exception as e: + errorMessage = str(e) + successCode = False + nodes, edges = {}, {} - # return EKGGraphResponse( - # successCode=successCode, errorMessage=errorMessage, - # nodes=nodes, edges=edges - # ) + return EKGGraphResponse( + successCode=successCode, errorMessage=errorMessage, + nodes=nodes, edges=edges + ) return app -def create_api(llm, embeddings): - app = init_app(llm, embeddings) - uvicorn.run(app, host="127.0.0.1", port=3737) - -# def create_api(ekg_construct_service: EKGConstructService): -# app = init_app(ekg_construct_service) -# uvicorn.run(app, host="localhost", port=3737) +def create_api(llm, embeddings,ekg_construct_service: EKGConstructService): + app = init_app(llm, embeddings,ekg_construct_service) + uvicorn.run(app, host="0.0.0.0", port=3737) diff --git a/muagent/schemas/apis/ekg_api_schema.py b/muagent/schemas/apis/ekg_api_schema.py index c74faa3..9dbd6e9 100644 --- a/muagent/schemas/apis/ekg_api_schema.py +++ b/muagent/schemas/apis/ekg_api_schema.py @@ -37,6 +37,7 @@ class EKGT2GRequest(BaseModel): intentNodeids: List[str] = [] intentPath: List[str] = [] teamid: str + rootid: str write2kg: bool = False class EKGGraphResponse(EKGResponse): @@ -61,7 +62,7 @@ class GetNodeRequest(BaseModel): nodeType: str class GetNodeResponse(EKGResponse): - node: GNode + node: GNode = None diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 66079c9..664ef6e 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -13,6 +13,9 @@ ) from jieba.analyse import extract_tags +import time + + from muagent.schemas.ekg import * from muagent.schemas.db import * @@ -148,6 +151,19 @@ def init_gb(self, do_init: bool=None): gb_dict = {"NebulaHandler": NebulaHandler, "NetworkxHandler": NetworkxHandler, "GeaBaseHandler": GeaBaseHandler,} gb_class = gb_dict.get(self.gb_config.gb_type, NebulaHandler) self.gb: GBHandler = gb_class(self.gb_config) + + initialize_space = True # True or False + if initialize_space: + # 初始化space + # self.gb.drop_space('client') + self.gb.create_space('client') + + # 创建node tags和edge types + self.create_gb_tags_and_edgetypes() + + print('Node Tags和Edge Types初始化中，等待20秒......') + time.sleep(20) + else: self.gb = None @@ -219,6 +235,53 @@ def _dfs(node, current_path: List): paths=paths ) return rootid_can_arrive_nodeids, graph + def create_gb_tags_and_edgetypes(self): + #print('create_gb_tags_and_edgetypes') + + # 节点标签名 + node_tag_list = ['opsgptkg_intent', 'opsgptkg_schedule','opsgptkg_task', + 'opsgptkg_analysis','opsgptkg_phenomenon','opsgptkg_tool', + 'opsgptkg_toolType'] + + # 每个标签的属性 + node_attributes_dic = [{'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','teamids': 'string','version': 'string','extra': 'string'}, + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','teamids': 'string','version': 'string','extra': 'string','enable': 'bool'}, + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','teamids': 'string','version': 'string','extra': 'string','accesscriteria': 'string','executetype': 'string','communication': 'string'}, + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','teamids': 'string','version': 'string','extra': 'string','accesscriteria': 'string','summaryswitch': 'bool', 'dsltemplate': 'string'},# dsltemplate + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','teamids': 'string','version': 'string','extra': 'string'}, + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int','extra': 'string','input': 'string','output': 'string'}, # opsgptkg_tool + {'ID': 'int', 'id': 'string', 'description': 'string','name': 'string','gdb_timestamp': 'int'}] #opsgptkg_tool + + edge_type_list = ['opsgptkg_intent_extend_opsgptkg_intent', + 'opsgptkg_intent_route_opsgptkg_schedule', + 'opsgptkg_intent_route_opsgptkg_phenomenon', + 'opsgptkg_intent_route_opsgptkg_task', + 'opsgptkg_schedule_route_opsgptkg_task', + 'opsgptkg_schedule_opsgptkg_phenomenon', + 'opsgptkg_task_route_opsgptkg_task', + 'opsgptkg_task_route_opsgptkg_analysis', + 'opsgptkg_task_route_opsgptkg_phenomenon', + 'opsgptkg_phenomenon_route_opsgptkg_task', + 'opsgptkg_phenomenon_route_opsgptkg_analysis', + 'opsgptkg_analysis_route_opsgptkg_task', + 'opsgptkg_schedule_route_opsgptkg_analysis' # 测试集独有 + ] + + edge_attributes_dic = {'SRCID': 'int', + 'original_src_id1__': 'string', + 'DSTID': 'int', + 'original_dst_id2__': 'string', + 'TIMESTAMP': 'int', + 'gdb_timestamp': 'int', + 'version': 'string', + 'extra': 'string'} + + for i in range(len(node_tag_list)): + self.gb.create_tag(node_tag_list[i],node_attributes_dic[i]) + + for i in range(len(edge_type_list)): + self.gb.create_edge_type(edge_type_list[i],edge_attributes_dic) + def update_graph( self, diff --git a/requirements.txt b/requirements.txt index 7361a63..d7f0233 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ faiss-cpu notebook chromadb==0.4.17 javalang==0.13.0 -nebula3-python==3.1.0 +nebula3-python==3.8.2 Pyarrow python-magic-bin; sys_platform == 'win32' SQLAlchemy==2.0.19 diff --git a/tests/db_handler/redis_test.py b/tests/db_handler/redis_test.py new file mode 100644 index 0000000..dc67afe --- /dev/null +++ b/tests/db_handler/redis_test.py @@ -0,0 +1,39 @@ +import redis + +# 连接到Redis +client = redis.Redis(host='redis-stack', port=6379) + +# print('host:', '172.18.0.3') +# 检查连接是否成功 +try: + pong = client.ping() + if pong: + print("Connected to Redis") + else: + print("Failed to connect to Redis") +except redis.ConnectionError as e: + print(f"Connection error: {e}") + + + +# from nebula3.gclient.net import ConnectionPool +# from nebula3.Config import Config + +# from loguru import logger + +# # 配置 +# config = Config() +# config.max_connection_pool_size = 10 + +# # 初始化连接池 +# connection_pool = ConnectionPool() + + +# # 连接到NebulaGraph，假设NebulaGraph服务运行在本地 + +# connection_pool.init([('127.0.0.1', 9669)], config) + +# # 创建会话 +# username = 'root' +# password = 'nebula' +# session = connection_pool.get_session(username, password) \ No newline at end of file diff --git a/tests/httpapis/fastapi_connect_test.py b/tests/httpapis/fastapi_connect_test.py new file mode 100644 index 0000000..cc484a8 --- /dev/null +++ b/tests/httpapis/fastapi_connect_test.py @@ -0,0 +1,13 @@ +# 导入必要的库 +import requests +import json + +#from muagent.schemas.apis.ekg_api_schema import * + +# 定义 API 基本 URL +base_url = "http://localhost:3737" + +# 测试获取嵌入模型参数 +response = requests.get(f"{base_url}/embeddings/params") +print(response.json()) + diff --git a/tests/service/ekg_construct_test_2_nebula.py b/tests/service/ekg_construct_test_2_nebula.py new file mode 100644 index 0000000..af6d973 --- /dev/null +++ b/tests/service/ekg_construct_test_2_nebula.py @@ -0,0 +1,308 @@ +import time +import sys, os +from loguru import logger + +import pdb + +# 在需要设置断点的地方 +# pdb.set_trace() + + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + model_engine = "" + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) + +import os, sys +from loguru import logger + +sys.path.append("/ossfs/workspace/muagent") +from muagent.schemas.common import GNode, GEdge +from muagent.schemas.db import GBConfig, TBConfig +from muagent.service.ekg_construct import EKGConstructService +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig + + + +# 初始化 GeaBaseHandler 实例 +# gb_config = GBConfig( +# gb_type="GeaBaseHandler", +# extra_kwargs={ +# 'metaserver_address': os.environ['metaserver_address'], +# 'project': os.environ['project'], +# 'city': os.environ['city'], +# 'lib_path': os.environ['lib_path'], +# } +# ) + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="muagent_test", + host=os.environ['host'], + port=os.environ['port'], + username=os.environ['username'], + password=os.environ['password'], + extra_kwargs={ + 'host': os.environ['host'], + 'port': os.environ['port'], + 'username': os.environ['username'] , + 'password': os.environ['password'], + 'definition_value': os.environ['definition_value'] + } +) + +# 初始化 NebulaHandler 实例 +gb_config = GBConfig( + gb_type="NebulaHandler", + extra_kwargs={ + 'host': os.environ['nb_host'], + 'port': os.environ['nb_port'], + 'username': os.environ['nb_username'] , + 'password': os.environ['nb_password'], + "space": os.environ['nb_space'], + 'definition_value': os.environ['nb_definition_value'] + + } +) + + + + +# llm config +llm_config = LLMConfig( + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +) + + +# emebdding config +# embed_config = EmbedConfig( +# embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +# ) + +# embed_config = EmbedConfig( +# embed_model="default", +# langchain_embeddings=embeddings +# ) +embed_config = None + +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) + + + +def generate_node(id, type): + extra_attr = {"tmp": "hello"} + if type == "opsgptkg_schedule": + extra_attr["enable"] = False + + if type == "opsgptkg_task": + extra_attr["accesscriteria"] = "hello" + extra_attr["executetype"] = "hello" + + if type == "opsgptkg_analysis": + extra_attr["accesscriteria"] = "hello" + extra_attr["summaryswtich"] = False + extra_attr['dsltemplate'] = "hello" + + return GNode(**{ + "id": id, + "type": type, + "attributes": {**{ + "path": id, + "name": id, + "description": id, + }, **extra_attr} + }) + + +def generate_edge(node1, node2): + type_connect = "extend" if node1.type == "opsgptkg_intent" and node2.type == "opsgptkg_intent" else "route" + return GEdge(**{ + "start_id": node1.id, + "end_id": node2.id, + "type": f"{node1.type}_{type_connect}_{node2.type}", + "attributes": { + "lat": "hello", + "attr": "hello" + } + }) + + +nodetypes = [ + 'opsgptkg_intent', 'opsgptkg_schedule', 'opsgptkg_task', + 'opsgptkg_phenomenon', 'opsgptkg_analysis' +] + +nodes_dict = {} +for nodetype in nodetypes: + for i in range(8): + # print(f"shanshi_{nodetype}_{i}") + nodes_dict[f"shanshi_{nodetype}_{i}"] = generate_node(f"shanshi_{nodetype}_{i}", nodetype) + +edge_ids = [ + ["shanshi_opsgptkg_intent_0", "shanshi_opsgptkg_intent_1"], + ["shanshi_opsgptkg_intent_1", "shanshi_opsgptkg_intent_2"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_0"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_1"], + ["shanshi_opsgptkg_schedule_1", "shanshi_opsgptkg_analysis_3"], + ["shanshi_opsgptkg_schedule_0", "shanshi_opsgptkg_task_0"], + ["shanshi_opsgptkg_task_0", "shanshi_opsgptkg_task_1"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_analysis_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_1"], + ["shanshi_opsgptkg_phenomenon_0", "shanshi_opsgptkg_task_2"], + ["shanshi_opsgptkg_phenomenon_1", "shanshi_opsgptkg_task_3"], + ["shanshi_opsgptkg_task_2", "shanshi_opsgptkg_analysis_1"], + ["shanshi_opsgptkg_task_3", "shanshi_opsgptkg_analysis_2"], +] + +nodeid_set = set() +origin_edges = [] +origin_nodes = [] +for src_id, dst_id in edge_ids: + origin_edges.append(generate_edge(nodes_dict[src_id], nodes_dict[dst_id])) + if src_id not in nodeid_set: + nodeid_set.add(src_id) + origin_nodes.append(nodes_dict[src_id]) + if dst_id not in nodeid_set: + nodeid_set.add(dst_id) + origin_nodes.append(nodes_dict[dst_id]) + + + + +new_edge_ids = [ + ["shanshi_opsgptkg_intent_0", "shanshi_opsgptkg_intent_1"], + ["shanshi_opsgptkg_intent_1", "shanshi_opsgptkg_intent_2"], + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_0"], + # 新增 + ["shanshi_opsgptkg_intent_2", "shanshi_opsgptkg_schedule_2"], + ["shanshi_opsgptkg_schedule_2", "shanshi_opsgptkg_analysis_4"], + # + ["shanshi_opsgptkg_schedule_0", "shanshi_opsgptkg_task_0"], + ["shanshi_opsgptkg_task_0", "shanshi_opsgptkg_task_1"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_analysis_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_0"], + ["shanshi_opsgptkg_task_1", "shanshi_opsgptkg_phenomenon_1"], + ["shanshi_opsgptkg_phenomenon_0", "shanshi_opsgptkg_task_2"], + ["shanshi_opsgptkg_phenomenon_1", "shanshi_opsgptkg_task_3"], + ["shanshi_opsgptkg_task_2", "shanshi_opsgptkg_analysis_1"], + ["shanshi_opsgptkg_task_3", "shanshi_opsgptkg_analysis_2"], +] + +nodeid_set = set() +edges = [] +nodes = [] +for src_id, dst_id in new_edge_ids: + edges.append(generate_edge(nodes_dict[src_id], nodes_dict[dst_id])) + if src_id not in nodeid_set: + nodeid_set.add(src_id) + nodes.append(nodes_dict[src_id]) + if dst_id not in nodeid_set: + nodeid_set.add(dst_id) + nodes.append(nodes_dict[dst_id]) + +for node in nodes: + if node.type == "opsgptkg_task": + node.attributes["name"] += "_update" + node.attributes["tmp"] += "_update" + node.attributes["description"] += "_update" + +for edge in edges: + if edge.type == "opsgptkg_task_route_opsgptkg_task": + edge.attributes["lat"] += "_update" + + +# +teamid = "shanshi_test" + +# logger.info(origin_nodes[0]) + +# origin_nodes = [GNode(**n) for n in origin_nodes] +# origin_edges = [GEdge(**e) for e in origin_edges] + +# logger.info(origin_edges[0]) + +ekg_construct_service.add_nodes(origin_nodes, teamid) +ekg_construct_service.add_edges(origin_edges, teamid) + +#print(len(origin_edges)) + +#print(origin_edges) + +# done +teamid = "shanshi_test_2" +rootid="shanshi_opsgptkg_intent_0" +ekg_construct_service.update_graph(origin_nodes, origin_edges, nodes, edges, teamid, rootid) +# ekg_construct_service.update_graph(origin_nodes, origin_edges, nodes, edges, teamid) + + + +# do search +# node = ekg_construct_service.get_node_by_id(nodeid="shanshi_opsgptkg_task_3", node_type="opsgptkg_task") +# print(node) +# graph = ekg_construct_service.get_graph_by_nodeid(nodeid="shanshi_opsgptkg_intent_0", node_type="opsgptkg_intent") +# print(len(graph.nodes), len(graph.edges), len(graph.paths)) +# logger.info(graph.paths[1]) + + +# # search nodes by text +text = 'shanshi_test' +teamid = "shanshi_test" +# nodes = ekg_construct_service.search_nodes_by_text(text, teamid=teamid) +# print(len(nodes)) + +# # search path by node and rootid +# graph = ekg_construct_service.search_rootpath_by_nodeid(nodeid="shanshi_opsgptkg_analysis_2", node_type="opsgptkg_analysis", rootid="shanshi_opsgptkg_intent_0") +# graph = ekg_construct_service.search_rootpath_by_nodeid(nodeid="shanshi_opsgptkg_analysis_2", node_type="opsgptkg_analysis", rootid="shanshi_opsgptkg_analysis_2") + +# print(len(graph.nodes), len(graph.edges)) +# print(len(graph.paths)) +# print(graph.paths) +# print(graph) + + +# create_ekg +# params = { +# "text": '测试文本', +# "teamid": "shanshi_test_2", +# "intentNodeids": ["shanshi_opsgptkg_intent_1"], +# "rootid": "shanshi_opsgptkg_intent_0", +# "intent_text": None, +# "all_intent_list": [], +# "do_save": False +# } +# result = ekg_construct_service.create_ekg( +# text=params["text"], teamid=params["teamid"],rootid=params["rootid"], +# service_name="text2graph" +# ) +# print(result) \ No newline at end of file From a0751b8e0bac83f332bdd32c8df721202b38d988 Mon Sep 17 00:00:00 2001 From: lightislost Date: Wed, 4 Sep 2024 15:18:40 +0800 Subject: [PATCH 032/128] [req update][update nebula version] --- docker_build.sh | 3 --- muagent/httpapis/ekg_construct/api.py | 1 - muagent/service/ekg_construct/ekg_construct_base.py | 4 +--- setup.py | 2 +- 4 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 docker_build.sh diff --git a/docker_build.sh b/docker_build.sh deleted file mode 100644 index ac6dfc1..0000000 --- a/docker_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker build -t muagent:0.0.1 . \ No newline at end of file diff --git a/muagent/httpapis/ekg_construct/api.py b/muagent/httpapis/ekg_construct/api.py index 13a7d61..517e35e 100644 --- a/muagent/httpapis/ekg_construct/api.py +++ b/muagent/httpapis/ekg_construct/api.py @@ -11,7 +11,6 @@ # def init_app(llm, embeddings, ekg_construct_service: EKGConstructService): -# def init_app(llm, embeddings): app = FastAPI() diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 664ef6e..ce42900 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -15,8 +15,6 @@ import time - - from muagent.schemas.ekg import * from muagent.schemas.db import * from muagent.schemas.common import * @@ -153,7 +151,7 @@ def init_gb(self, do_init: bool=None): self.gb: GBHandler = gb_class(self.gb_config) initialize_space = True # True or False - if initialize_space: + if initialize_space and self.gb_config.gb_type=="NebulaHandler": # 初始化space # self.gb.drop_space('client') self.gb.create_space('client') diff --git a/setup.py b/setup.py index 1ef0b2c..4dc32d1 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ # "chromadb==0.4.17", "javalang==0.13.0", - "nebula3-python==3.1.0", + "nebula3-python==3.8.2", "SQLAlchemy==2.0.19", "redis==5.0.1", "pydantic<=1.10.14", From 4b4e32105e4d9ff39b842f548fa91beb5eab8c79 Mon Sep 17 00:00:00 2001 From: lightislost Date: Thu, 5 Sep 2024 11:44:33 +0800 Subject: [PATCH 033/128] [update readme][update readme and fix tbase bug by uuid] --- README.md | 115 ++++------ README_zh.md | 100 +++------ docs/resources/ekg-arch-en.webp | Bin 0 -> 32922 bytes docs/resources/ekg-arch-zh.webp | Bin 0 -> 30570 bytes .../base_configs/prompts/simple_prompts.py | 205 ++++-------------- muagent/connector/schema/message.py | 4 +- .../ekg_construct/ekg_construct_base.py | 4 +- 7 files changed, 126 insertions(+), 302 deletions(-) create mode 100644 docs/resources/ekg-arch-en.webp create mode 100644 docs/resources/ekg-arch-zh.webp diff --git a/README.md b/README.md index 929f475..4f86f98 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 中文  ｜  English 

-#

CodeFuse-muAgent: A Multi-Agent FrameWork For Faster Build Agents

+#

CodeFuse-muAgent: An Innovative Agent Framework Driven By KG Engine

ZH doc @@ -17,98 +17,71 @@ ## 🔔 News -- [2024.04.01] codefuse-muagent is now open source, featuring functionalities such as knowledge base, code library, tool usage, code interpreter, and more +- [2024.04.01] codefuse-muAgent is now open source, featuring functionalities such as knowledge base, code library, tool usage, code interpreter, and more +- [2024.09.05] we release muAgent v2.0 about EKG (An Innovative Agent Framework Driven By KG Engine). + + ## 📜 Contents - [🤝 Introduction](#-Introduction) - [🚀 QuickStart](#-QuickStart) -- [🧭 Key Technologies](#-Key-Technologies) +- [🧭 Features](#-Features) +- [🤗 Contribution](#-Contribution) - [🗂 Miscellaneous](#-Miscellaneous) - [📱 Contact Us](#-Contact-Us) ## 🤝 Introduction -Developed by the Ant CodeFuse Team, CodeFuse-muAgent is a Multi-Agent framework whose primary goal is to streamline the Standard Operating Procedure (SOP) orchestration for agents. muagent integrates a rich collection of toolkits, code libraries, knowledge bases, and sandbox environments, enabling users to rapidly construct complex Multi-Agent interactive applications in any field. This framework allows for the efficient execution and handling of multi-layered and multi-dimensional complex tasks. -![](docs/resources/agent_runtime.png) +A brand new Agent Framework driven by LLM and EKG(Eventic Knowledge Graph, Industry Knowledge Carrier)，collaboratively utilizing MultiAgent, FunctionCall, CodeInterpreter, etc. Through canvas-based drag-and-drop and simple text writing, the large language model can assists you in executing various complex SOP under human guidance. It is compatbile with existing frameworks on the market and can achieve four core differentiating technical functions: Complex Reasoning, Online Collaboration, Human Interaction, Knowledge On-demand. +This framework has been validated in multiple complex DevOps scenarios within Ant Group. At the sametime, come and experience the Undercover game we quickly built! -## 🚀 快速使用 -For complete documentation, see: [CodeFuse-muAgent](https://codefuse-ai.github.io/docs/api-docs/MuAgent/overview/multi-agent) -For more [demos](https://codefuse-ai.github.io/docs/api-docs/MuAgent/connector/customed_examples) +![](docs/resources/ekg-arch-en.webp) -1. Installation -``` -pip install codefuse-muagent -``` +## 🚀 Quick Start +For complete documentation, see: [CodeFuse-muAgent](https://codefuse.ai/docs/api-docs/MuAgent/overview/multi-agent) +For more [demos](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_examples) -2. Code answer Prepare related llm and embedding model configurations -``` -import os - -# set your config -api_key = "" -api_base_url= "" -model_name = "" -embed_model = "" -embed_model_path = "" - -from muagent.llm_models.llm_config import EmbedConfig, LLMConfig -from muagent.connector.phase import BasePhase -from muagent.connector.schema import Message, Memory -from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler - -llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 -) - -embed_config = EmbedConfig( - embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path -) -``` -
+### EKG Services -Initialize the codebase -``` -from muagent.base_configs.env_config import CB_ROOT_PATH -codebase_name = 'client_local' -code_path = "D://chromeDownloads/devopschat-bot/client_v2/client" - -cbh = CodeBaseHandler( - codebase_name, code_path, crawl_type='dir', use_nh=use_nh,local_graph_path=CB_ROOT_PATH, - llm_config=llm_config, embed_config=embed_config -) -cbh.import_code(do_interpret=do_interpret) +```bash +# use ekg services only three steps +# step1. git clone +git clone https://github.com/codefuse-ai/CodeFuse-muAgent.git + +# step2. +cd CodeFuse-muAgent + +# step3. start all container services, it might cost some time +docker-compose up -d ``` -
+The current image version includes only the basic EKG service. We expect to provide a front-end interface and back-end interaction services in September. + +To Be Continued! + + -Start codebase Q&A +### SDK +We also provide a version of the SDK for using muagent. +1. Installation ``` -# -phase_name = "codeChatPhase" -phase = BasePhase( - phase_name, embed_config=embed_config, llm_config=llm_config, -) -# -query_content = "what does the remove' function?" -query = Message( - role_name="user", role_type="human", input_query=query_content, - code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="tag", - local_graph_path=CB_ROOT_PATH, use_nh=False - ) -output_message3, output_memory3 = phase.step(query) -print(output_memory3.to_str_messages(return_all=True, content_key="parsed_output_list")) +pip install codefuse-muagent ``` -## Key Technologies +2. Code answer Prepare related llm and embedding model configurations +you can see [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_examples) and [~/examples](https://github.com/codefuse-ai/CodeFuse-muAgent/tree/main/examples) + + -- Agent Base：Four fundamental Agent types are constructed – BaseAgent, ReactAgent, ExecutorAgent, SelectorAgent, supporting basic activities across various scenarios -- Communication: Information transmission between Agents is accomplished through Message and Parse Message entities, interacting with Memory Manager and managing memories in the Memory Pool -- Prompt Manager: Customized Agent Prompts are automatically assembled with the aid of Role Handler, Doc/Tool Handler, Session Handler, and Customized Handler -- Memory Manager: Facilitates the management of chat history storage, information compression, and memory retrieval, culminating in storage within databases, local systems, and vector databases via the Memory Pool -- Component: Auxiliary ecosystem components to construct Agents, including Retrieval, Tool, Action, Sandbox, etc. -- Customized Model: Supports the integration of private LLM and Embedding models +## Features +- EKG Builder：Through the design of virtual teams, scene intentions, and semantic nodes, you can experience the differences between online and local documentation, or annotated versus unannotated code handover. For a vast amount of existing documents (text, diagrams, etc.), we support intelligent parsing, which is available for one-click import. +- EKG Assets：Through comprehensive KG Schema design—including Intention Nodes, Workflow Nodes, Tool Nodes, and Character Nodes—we can meet various SOP Automation requirements. The inclusion of Tool Nodes in the KG enhances the accuracy of tool selection and parameter filling. Additionally, the incorporation of Characters (whether human or agents) in the KG allows for human-involved process advancement, making it flexible for use in multiplayer text-based games. +- EKG Reasoning：Compared to purely model-based or entirely fix-flow Reasoning, our framework allows LLM to operate under human guidance-flexibility, control, and enabling exploration in unknown scenarios. Additionally, successful exploration experiences can be summarized and documented into KG, minimizing detours for similar issues. +- Diagnose：After KG editing, visual interface allows for quick debugging, and successful Execution path configurations will be automatically documented, which reduces model interactions, accelerates inference, and minimizes LLM Token costs. Additionally, during online execution, we provide comprehensive end-to-end visual monitoring. +- Memory：Unified message pooling design supports categorized message delivery and subscription based on the needs of different scenarios, like multi-agent. Additionally, through message retrievel, rerank and distillation, it facilitates long-context handling, improving the overall question-answer quality. +- ActionSpace：Adhering to Swagger protocol, we provide tool registration, tool categorization, and permission management, facilitating LLM Function Calling. We offer a secure and trustworthy code execution environment, and ensuring precise code generation to meet the demands of various scenarios, including visual plot, numerical calculations, and table editing. ## Contribution We are deeply grateful for your interest in the Codefuse project and warmly welcome any suggestions, opinions (including criticism), comments, and contributions. diff --git a/README_zh.md b/README_zh.md index 498902e..4c6f60d 100644 --- a/README_zh.md +++ b/README_zh.md @@ -2,7 +2,7 @@ 中文  ｜  English 

-#

CodeFuse-muAgent: A Multi-Agent FrameWork For Faster Build Agents

+#

CodeFuse-muAgent: An Innovative Agent Framework Driven By KG Engine

ZH doc @@ -18,102 +18,64 @@ ## 🔔 更新 - [2024.04.01] CodeFuse-muAgent 开源，支持知识库、代码库、工具使用、代码解释器等功能 +- [2024.09.05] muAgent v2.0 全新版本, 实现了由知识图谱引擎驱动的创新Agent框架 ## 📜 目录 - [🤝 介绍](#-介绍) - [🚀 快速使用](#-快速使用) - [🧭 关键技术](#-关键技术) +- [🤗 贡献](#-贡献) - [🗂 其他](#-其他) - [📱 联系我们](#-联系我们) ## 🤝 介绍 -CodeFuse-muAgent 是蚂蚁CodeFuse团队开发的Mulit Agent框架，其核心宗旨在于简化agents的标准操作程序（SOP）编排流程。muagent整合了一系列丰富的工具库、代码库、知识库以及沙盒环境，可支撑用户在任何领域场景都能依托muagent迅速搭建起复杂的多Agent交互应用。通过这一框架，用户能够高效地执行和处理多层次、多维度的复杂任务。 +全新体验的 Agent 框架，将KG从知识获取来源直接升级为Agent编排引擎！基于 LLM+ EKG（Eventic Knowledge Graph 行业知识承载）驱动，协同 MultiAgent、FunctionCall、CodeInterpreter等技术，通过画布式拖拽、轻文字编写，让大模型在人的经验指导下帮助你实现各类复杂 SOP 流程。兼容现有市面各类 Agent 框架，同时可实现复杂推理、在线协同、人工交互、知识即用四大核心差异技术功能。这套框架目前在蚂蚁集团内多个复杂DevOps场景落地验证，同时来体验下我们快速搭建的谁是卧底游戏吧。 -![](docs/resources/agent_runtime.png) + +![](docs/resources/ekg-arch-zh.webp) ## 🚀 快速使用 -完整文档见：[CodeFuse-muAgent](https://codefuse-ai.github.io/zh-CN/docs/api-docs/MuAgent/overview/multi-agent) -更多[demo](https://codefuse-ai.github.io/zh-CN/docs/api-docs/MuAgent/connector/customed_examples) +完整文档见：[CodeFuse-muAgent](https://codefuse.ai/zh-CN/docs/api-docs/MuAgent/overview/multi-agent) +更多[demo](https://codefuse.ai/zh-CN/docs/api-docs/MuAgent/connector/customed_examples) -1. 安装 -``` -pip install codefuse-muagent -``` +### EKG服务 -2. code answer +```bash +# 使用我们的EKG服务只需要三步！（beta版本，需要将本地代码打包到容器中） -准备相关llm 和embedding model 配置 -``` -import os - -# set your config -api_key = "" -api_base_url= "" -model_name = "" -embed_model = "" -embed_model_path = "" - -from muagent.llm_models.llm_config import EmbedConfig, LLMConfig -from muagent.connector.phase import BasePhase -from muagent.connector.schema import Message, Memory -from muagent.codechat.codebase_handler.codebase_handler import CodeBaseHandler - -llm_config = LLMConfig( - model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3 -) - -embed_config = EmbedConfig( - embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path -) -``` +# 第一步. 加载代码 +git clone https://github.com/codefuse-ai/CodeFuse-muAgent.git -
+# 第二步. +cd CodeFuse-muAgent -初始化代码库 +# 第三步. 启动所有容器服务，EKG基础镜像构建需要花费点时间 +docker-compose up -d ``` -# initialize codebase -from muagent.base_configs.env_config import CB_ROOT_PATH -codebase_name = 'client_local' -code_path = "D://chromeDownloads/devopschat-bot/client_v2/client" +当前镜像版本仅包含了EKG基础服务。我们将会在9月底提供前端交互和后端交互的镜像服务。 -cbh = CodeBaseHandler( - codebase_name, code_path, crawl_type='dir', use_nh=use_nh,local_graph_path=CB_ROOT_PATH, - llm_config=llm_config, embed_config=embed_config -) -cbh.import_code(do_interpret=do_interpret) -``` - -
+敬请期待! -开始代码库问答 +### SKD版本 +1. 安装 ``` -# -phase_name = "codeChatPhase" -phase = BasePhase( - phase_name, embed_config=embed_config, llm_config=llm_config, -) - -query_content = "remove 可以做什么？" -query = Message( - role_name="user", role_type="human", input_query=query_content, - code_engine_name=codebase_name, score_threshold=1.0, top_k=3, cb_search_type="tag", - local_graph_path=CB_ROOT_PATH, use_nh=False - ) -output_message3, output_memory3 = phase.step(query) -print(output_memory3.to_str_messages(return_all=True, content_key="parsed_output_list")) +pip install codefuse-muagent ``` +2. 代码问答和相关配置，可以看 [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_examples) 和代码示例 [examples](https://github.com/codefuse-ai/CodeFuse-muAgent/tree/main/examples) + + ## 🧭 关键技术 -- Agent Base：构建了四种基本的Agent类型BaseAgent、ReactAgent、ExecutorAgent、SelectorAgent，支撑各种场景的基础活动 -- Communication：通过Message和Parse Message 实体完成Agent间的信息传递，并与Memory Manager交互再Memory Pool完成记忆管理 -- Prompt Manager：通过Role Handler、Doc/Tool Handler、Session Handler、Customized Handler，来自动化组装Customized 的Agent Prompt -- Memory Manager： 用于支撑 chat history 的存储管理、信息压缩、记忆检索等管理，最后通过Memory Pool在数据库、本地、向量数据库中完成存储 -- Component：用于构建Agent的辅助生态组件，包括Retrieval、Tool、Action、Sandbox等 -- Customized Model：支持私有化的LLM和Embedding的接入 +- 图谱构建：通过虚拟团队构建、场景意图划分，让你体验在线文档VS本地文档的差别；同时，文本语义输入的节点使用方式，让你感受有注释代码VS无注释代码的差别，充分体现在线协同的优势；面向海量存量文档（通用文本、流程画板等），支持文本智能解析、一键导入 +- 图谱资产：通过场景意图、事件流程、统一工具、组织人物四部分的统一图谱设计，满足各类SOP场景所需知识承载；工具在图谱的纳入进一步提升工具选择、参数填充的准确性，人物/智能体在图谱的纳入，让人可加入流程的推进，可灵活应用于多人文本游戏 +- 图谱推理：相比其他Agent框架纯模型推理、纯人工编排的推理模式，让大模型在人的经验/设计指导下做事，灵活、可控，同时面向未知局面，可自由探索，同时将成功探索经验总结、图谱沉淀，面向相似问题，少走弯路；整体流程唤起支持平台对接（规则配置）、语言触发，满足各类诉求 +- 调试运行：图谱编辑完成后，可视调试，快速发现流程错误、修改优化，同时面向调试成功路径，关联配置自动沉淀，减少模型交互、模型开销，加速推理流程；此外，在线运行中，我们提供全链路可视化监控 +- 记忆管理：统一消息池设计，支持各类场景所需分门别类消息投递、订阅，隔离且互通，便于多Agent场景消息管理使用；同时面向超长上下文，支持消息检索、排序、蒸馏，提升整体问答质量 +- 操作空间：遵循Swagger协议，提供工具注册、权限管理、统一分类，方便LLM在工具调用中接入使用；提供安全可信代码执行环境，同时确保代码精准生成，满足可视绘图、数值计算、图表编辑等各类场景诉求 ## 贡献指南 非常感谢您对 Codefuse 项目感兴趣，我们非常欢迎您对 Codefuse 项目的各种建议、意见（包括批评）、评论和贡献。 diff --git a/docs/resources/ekg-arch-en.webp b/docs/resources/ekg-arch-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..d2475aefa63d7e23a185cf7e3d97d6ad124a9f76 GIT binary patch literal 32922 zcmagEW0Ypi)-9N}ZQHhO+qSb(m3F0V+qPMmS!vt0t>@Nz&bi&EzrH;>;!li-y(0Er zbH!XUVr&&@35mo2ARtX~5hV>JE~2o%Z6$7y9AFxAuzU~!d$v?5GE$NPQr5`X0Qd-V zhfmoHHm2i{sP3)4^M*BL^Ew1)`*;S%N3%!^+?;SOI1zqVJ% zSMQS-ApE20`sAbTlh7#cChywUQSi~5)ISE0003+Ehxa}NfsD`a568Q{!>_P!$>+SA zolXBgfadR~kD)up_nF(g&TYqU;}6dFnVY^YLZH4Bf4c9Nuc7Ca56WtSv8%MN@XtCW zLp{H!@0m}`H^I-m;~p455kUG|@|*Am@SRr(Q2$>0PWte8BYXjz_5lE|U%=n0U$CEL z&vjdX+dKf^?d#-w<@+Y8Z3nOfSO8$(f`3kYaDMpT37-4E_$LGYmb>uxy3f8-|JCo! zkAN@K&$_Rh2meb!JHJ8ybbtXM>Fef;@O$TR2LSl?y#oMVct0loHeZG~uLS{sPq0=@ zY^(MO@ev&cmO1CtsOV-r^TGrA|Iu08j5fo8xdGj(=C~BoEy_}|>N0MokE(PTm9{j- zHe29WxAkHZ7DX*KVf=qy*VWGJ1A-|1AVpsqEF;|!yrZ#mmKA&*!IlVYaE_DIByb+c zuQCzieI|iM_o)4HsIuula2W1g(3F^G;_~Q zFAj)v?0aHZ?b!(LxeU=oy%}N!RdDS@ zPypZ>vrIsavqqzcf%cz1T)?-UEF?RN8FM? zTRSY41;>H3Eta|P&Hf*Ds_+L4kN*h`R#%qRJuc}!%ftWj8?g%R^IxMr&#^>K40k;S z-T!zreW;3iS+&|;)S`1lXn{8!iK0Ewq`-PfcF<^$de?cfv z;TOnC^uoCE2s7j8wj%j_h6|$qScnvG%#Czbru`lXVw^D=BUDj_)c^14+*Vm27n2T& zA2;bJ4I-^`_yO~8YX<)^2u?U@puwMS&|66%S=>^#ET)abXWnXGDT~;qRxK8oJBZaOWIYbUwHg$m-v}Nu9wP!Tpp*yUb%NKui7`Da2UFY^N zHcfY|@rAm3Bk?uvq46RDfPF8xLubaoT9&`JLWJ8NvYt+>g}SORhpspxzuP7hU;p2J z`!|ArajH)L3(t%bqV&)}q<6|e!SO5*)Bt2WFO+7Id%P`UymWflmLu*+2^J=DPRYx` zLq#14{SKPS{}tN*h1xw&bH?FDRk?jpw(&DjK68IrR7T5D19#~STZQ&7IZTmTPAfnb}^96^KK+I9(dH$^n z8&x~;<>p%XCLl9n0hzi+_EUg%Z7{{fadp=$Se%}OsWG@_Llw6ym0Ehfj|L)MP3*7x z*T0nBuuFUbn|W4)%Et+mH#$^Zj-M@5JI4_-ehF01(W_NsoLj6b!sD{H&Lp$)5`G5-zD75{Y zxpVtc71TDVy|%uY#`E_}!1w1hy{zj8;SuwO06(3Fu-DAw-HSJ(>BJIy(210}b}&y; zoN7ly-+;Lw#DaLBAoaG%um{WQ^=JOsJ`{M~=92Ht%h~+UOH01uPv*Kr2wxBc3@k~eRSb3^QW7FGVLcewR(bmP=;nOlV zTU$R>m!O9WO0`{Kr>zEx$t+3QrW?7-w3*l&-yG?>itV$XA91mJO%Ba zb5n3#jI1nBmoY@27cM$Tmt)51Js zGBF2gu)H}kPd2*c&03TEpSFIo@XG%eD{!8eTqdwQI}$w&cQa|TB2X*cbWsI;n^CF# zpm3`6!V=UtqXd^ua;myUY}uH%{TUwn|IPxGV@yYedO<6=xERCFx+jI%YBGR^{HTY2 zSm$3$6VL)q{5(Rvgd5493ZUq2ob5&)coq@38tKaSql%)6YE=we z3WxgExuLQb-$}H5Tjd}|MQtik_5~bzs2lRy4YJ>uW(_ec2BxOQ+`(I_({a}00oTcr zN`h}b?yk>2{@_2s)$Infy7<+gsgo0xxq0)6n-N7P4yAv_D z;CpyqDmL0Sl04^$jnm$L!GyR*)ak!xd0u9PaUJY%Z{({ZYCim6$| zRCK!jaC2QCelvfZIO>y!>L5&@g3yDseqUZe%m?SB55hV6pdmdT@p+*{V3OyKm@#LU#goB|EOdR)f`sWQMDbPgng)H{P8 zl&!ehXNPa5Q*l-*PsaLQzMNU|Ur*vcw~gu4PX0C@^5gkk5UfH*?7_Igi6mp9fqaFa zmvLUH6qf7dswKb8!TUbQUoepBsC`|aSK zOHpg^$g|C_G}~7ed}sdHty}rhkv{=1879wUvJzoJt3X;xdenc@p`VM>@4xA|ZF>T! z(G+z<7nsgNC#>fYp@=|-jRj*n;@#b1FbkIgOv=x^%Wn0rD8snmfd>lL+I34zEAXg} zW#i1acmpHBH{(&}kck!frTVr9=82fpWXZ#)J;ccoj(P#b?^@|e4UK{?0oQEA$1Z#- zK9LB9FM^a%Q$(3{Y_~r0HbPg`olBSY)#h`qmei65W=fjrmiVre<=iZf6Wk5Okc(xu zZ=Lj{R+erapv@d{RTGfPTO(@l4yhbRkAp5Fd?ehHeMl|kuFADhqV8A9qUM4sTF9)9 z7}@|?G!4Atr6k?{Xd($j(q%nvJkTqT8Jc3ArRmZ6_3ICnabIQq zp)DV7o0sVMj!KMM$E5Omz$@P}gCg4kgy1XtjL_VIukhdyrxH zJU=)}&r+^o=JHLgxr&-kvtxd6dzDfe+2P*|>aG_u<3Ak~+hhgCgjqb)V3+lqDhc=b z=vM)4`ZiXL`*qdm|h#d;T!W#@T|)o{22pbe1;qvsloTre534 zvqE2AaGgU9hN;A@(gr`;J?E$g{=AFL$xmxFg&6G5B<|Rkx0nDv>_dH;~I2F$Uj-(EMM`PkfTkU;qC?3cK}3Sgyd?E zj8N-3?D4^^w;PO8>T%L>ihXu}WVI#9yYJv1&H7iEVsJ%@6CE=BOS9JSe`6)Vk|O-+ z<}ci6Zwu)8k3{Nunfc!mDQGbDb+?|A|LF22kEqiGS4uFtEAMY6kTYL<%liSJ;GEj% z(I+Pgie|GtQUel9G@Z+1*e`$S@IUm>pe-J+bIJT_b?@fg{3WWhdlHpbu?ifH>gu8w zb&xRBi?fSCds-$<0!LZyj#c0~Jy2x{a1k|R8!TygB3P`)BG= zVKEIQA_x#P_2858y+gDs(9q^*Uo#(HGBI00CUG97#_$hzXOtD5O5I!o4kDyJ^ZCVw zvZmB^6_2$R4_yrip{2{XQjM<3)SIdJ@=KSlK#lRA0lGFaM%&AYu21Gy2c)RG#@1fkA>%Xb6j#p0y1``i=CRFB_#f=+)j zc^?e0Q0q$-DILMmcIt6G+zC$BI4_L}7IUF&qQCJsmfMpSb5e4q!n#+YW>AQfb?BbdsE_K|aTL=w?@U2i)?w^-s5z{ zzrqgvD`IR@{M4NPfJzoFFt$sK4rE6(oPQa0nTgy@{UCb zf^IPKP=r;{*TN7NkTzN5=*5Bd+_6r)k!ZhP8{ zov?2|+66A0$1OD~Dz_b1o6WPK zTx6h8PoygP7b9B@5aG}68W_(UagmcRtI(|~`s?LhTDNq(m^SWa22ePA)4*jc3tW3` z#3JdTD*^*4Vx0O3Dsqd4THvwxel|uS@y{Ni&O16LfN9`_R`m^++_}ja3kr{_Ue+y{ zq%(dc%!Tni#FNISf!FTl_SAB^f$*ci2YCc>4fU72y&5@RlEj3sruc6gm~t&2mRrxl zyt<>LDK24u_IwxTByC31vxLBAgF9b|l9m;9b6X&?LTCg=(*i5Yqm6hd&9I(BkOg7N z^Ms$yZ%2ldntu<*zY_mBAe;ZsgYsVLzb+1}fPlUMCHX-AE|~8k9^6aN=qX$0zDk9d zPP_(wt~DM0lp0Y{wJD@U!9lkzPgl>dgU(P>XoccLqR)}qd$Q52gN$_$Z zuc7b-bGTald1!0t+$HgD!+s1A#lT+T=8GTJUrBKpde-lEp{5W;qm>E-*3SZj+ZC*g zce<+}7ZV`4I%G=JB^4m5v_jDR))B(CpwouPT|)DPcY_-=r`kBv}qKqX?egOP<%$v|9>;2sYP9s z4Vez{^$krCHawo$4>)_^FHbw3M{wV#SbG*CEN=8{Sk={NE%qP;Z{dsXDGk&+mJxNv zL=-ZcgP9hrxUnOzu`D`@#HB+(gkDpwsUg_lWhjhibViFI8Z&QiRbFxC#ng?zl16x*&^^(7DE#=^<#=>?2 z$cCtWO<$>iDWnzfyU7AZeqo{xO6aQMWi7a?r$6?t#t|;mZ$IQnHYfx~jnB$jIvvr( zHI5`cDTp3mRP+;8{<>Jq+tI_sw^YM7btv4O?YejMVl)lPD*nD*@0SN}tf3B1IWzw_ z9X)#cv;SkuD$X%oR@>MsxmcOMkH0j9Z6aJhRvL`{GB|2wXux*nK5bnjA?pVzFOjixYApS~h63-$(6vg&lY`o*n}N+g|){=Fm#4 zD;`U@Q1P(L=pLOu0R3i5i@Ja46}f%b99ejodTBcDZ!Rgk4091^wMO)+KMN05QKO}( z*y)UTd#%3;7y}rQ*d3J-`vhcpZ^6LDWZJ46eOjBlktI%^MuFJRGy^8`N%M@j2l@qhP{v{Rpv`+GgkyU$l)pu*~@fMv1^? zYFVY>%gF#pPa}QCPcMz$I!dv8M>s{>+_ZRVwLO}LhRkwjOV+PNJ?m#orf^i(whEZU zeEP7KyKax(G?oo)Q!be5xQVKa(!aQi)JL&<>-KhMMt*ZuS z?csBXNcceFm{DJ@M9GXMxxImU^|zv<*=|_umZyH7IYZ%e9A;R|(#313ETs`>q}3=B zmg!FN;tQ3Uo@E^-+#ZW033jEFL-G-Z#!S+-38{^X z#oz>?m19VwKY=ZmDt`(nvU=wFEmIiY))Q6sbu^b`m9bo{m&Z1EO@c?P$d z5b{(;(qHNh;$Qs}j*dEA`Aspmsk{ncL8u5PS#NZMR4AgL_T!@QKv0Hxa`(WljRN&eVq|0O1-QipLASDbsDw=>I(Er8HFUh$hi;&}a`R*s;jM1R%6twJ`=sF{R-==CUFrqmHKbHRz<|MZ379`Ty!*)REJU5+CFBR-p7o$?N zqHMo?>^wM49!DWFr@wvzA+_+HW#eLXXgR_k0+2gItbY2mkA8FAwTbWpW~2V<5kLB- z8cvK16MwWL2W@sOj`$&5oSeq|x^1-i{Lq(ab4+AAUpakgZ-?TXZycv0H27Lxk3 zus|Dpj&na%MuVy{@Uh3SP<*ITHG55&5G)WwkIC+sja<+=ASRQ9V-P#?tsTjQdGBD# zNT=54`tA7QUrS+{7jP4{T|%JOsQr8Ny775C9h^*YJCI$(%eS#vT@0o>QQB4V-j%e1 zo9F(U$%Re5rLGg&DKCr~6I=QNDJqe5YfP7TDjqr$n$RF*=~7CE${z`?2W?I&aw}0_ zl8V?gUm_L+yvw%iXYO$=ADSdn_`fbrzAuJaWFm;!o=>vnmim!ze&zRu!U6CpAUNT5 zYSaPn%t8|uILvq)9k#V>ejotwi+U&JwYjA!JvtJi3>@66rub$LcD)yrfN6VO!Cm2O zZ{Zjiz-IFYGRkb;L~*J;u2t4a=rD85FOPR87;h;j8h(&@ST$g+5q~6DBkp>=l68`> z19HKMk-ny=?RqK4xJ;)Tzi{$p|3UOwaqs=E$kicCp?{bJS_(57)d%NLV)E0N=#5v% zcc21^&|%bHpMo$ZPs3*%y|*C-=TIBAimS#`U8gB>B~<OL@sf#$<~wGr=~( z%ZRDelbljuq3AjCJK(t&`wM;*!Jq85VZ5pWRYiqiX75oSe)P3;KST6}^Wj0xyNlfEQcJl^-Z|Dirp6ou>sApeH;zR5 zIO(LM4u%@*GwQKnKA&W8i;kwKuaeyHcYV7Xjs`NROt=vfjAiCHO8KS4y4#`r0UbTp z^v$xb)-=d759KEGic4^YwWZBBp?k8w#$BAesrTeP7l|Kj1DjJ|PWr#z zfe&p^-W@RP5Ooj;!=0EAMfkfUu#58O=N)XTG9=K?(OxSb9op=cp&tgS_k_0>2NmAa z9#!CIKl*5VSd~tH8Xpk!>}>C4%v7}=_Mtf}Ul7Y9ouTr9_4W0B^|qL$mQv02RCtNv z$*20;n%mq@RRT9!Bk)O^ckyyR5cUE`!d03dy!68wHc6t^ zA9=qjW`747HgU>~eS!2@IHYn2=4+aNH+};H{+brJju){G?r?uoM$olc9J`%bV(GHH zee8nQPQ2#w2LoJRhJdj0$4k+HkY}t#;G?V2k%ug9HRuTEz6_%nJs2!vnj~lKdDq95 zWqTN6)=e!~lGiaCP~TIiphdQ{OjDFUM#|y{%n%T?Bbk4>f0iCw9-!bTZNwE;H1rHiDTIhsF7@=lXQ5s&PcMXtkrp25{Ey0CI<@?tx@Z}- zhFqlgMp3_c;bqg#v^hnRsavY*{+(^D(uX65me_%o5Y|2@@YzV2_g>qQo0KCXnzP}k zs(gm`47UK?UE&f7-66;oP>X>OtGV^F(I{p{cVTnMTkEl+<`4bSo^k_^?-!j|lZR`5 zFn7m_sOGRD4UvzD(Gq+MFOY}CW&-}JfihIOc`0sBMXl(V*x+&cWPBX}0C)2Qo!!2j zy6xiY{+jgtGV1osXg4gg)W3Sf#UE_p$XBX7AN(+l%AP6o?3FN@(maqwy+`2-f~(FH zdjsWOM|#lb>1FzL9t=M?Eh1kcqKOaIHe%90@QJ^YwA~a6U-MBFlT;I#s18+aq~r14 zW-(gjE~9;}%K)Tci-h}2B_YTRahqJ;Cd%K;a-11m>b~KqR)B$JE4$71Q>X$hyU9IG zJo_FLP&Y|ousy0(J=j+qcrqz~^~D?N)j9C`nLbUby+c!+#C##DuY*MvbC)BlT6Jn? z`F!i&<#*hI-XP8goFUTWTL4Os&-`cl5bdl2EVMGw(!w`#vlqLe<<*{2?w@@3PXYF4 z#_NF^nEV~9@_lO{00t5UZL5ALmsBI8Rh^=`7BFwDyg!psYOFBc??&-0E&g1DJvy=zW~2 zVrYfD{$XGWTCu(>GASk%+;;wGD+Bx{=dahz0As2iOc5+XDju0ATWAfkH8PcZ4e+@7};O=8G-JYIb( zzUQ9WA&8{3$Hl1?N8PCYXmb$OS|~%_F4fRyFgluH*fhA-`ieH;em*6CP{=#bkQ zx0#1$;<|2V5OPmb;AdWtzBwG;&Y5^29ji)D^K(t$s){M^m8ICnX8qF;Z327|K@YlN z{wvM}dpW%gs9Oq`UfWCgp|KvBM>nt*%^pM19GKgwwraC|2nf5{xTxz3>yq5K+K7Ut}7$ejib_qp17?xcBY#=XikgOo~$LZQo`!XUvqBRxs3w>9o z)sesyzUV1YC7pjFp`xzhjhi-qNpYP#iMi3)`%BMerQZDJp5DHJyxA>rFb@R+-JS7T z*w|2g6Q-P5DuD_nA-<*s(7&fb68R)3 zYK9puxQ~&Fskcf9hJ|A$t)teYt*HqW%F*lx34%&ft2xoK4xRXW-1b-Z{+C{o=*~G# z#K@z8ZA=cb>z zTrwe?I)~Ut7``5kiMQ1|{~sz%^vgS!-rB!~kV%)9?QN$=(#>D^Y?7B<)q#74XD_9# zvtqMp-bfa1{}3=+jZbM#)q}j2?c!(4uyP4aQ!M>lySA%nm}U&L4#V^7)4qjS?CW+qu;^g8V3HSUsA2?V!8dkQCx&6R~8@A0BNB4eq3yGzD^<6J&GL(5Ym zlS%av_M9|&j$cDa$||+_6qiSGuy=|PF#|&8w1f2Ga(R#zU@Z=nort}_BNXCC3^B{} zw7ZXWkfX%@q}Q-uF(lb>nISIkG;O~(rg?%|?RUa;6|Vl!2a2#gh;_&T=G$9qC@0MT z0^6V)608%it^5<{SzCarDzv?+8|dQ3x@-b5B3~YZm!GfHUvjh)99i| z>0}1tcTQ2t5|lNa8JcOM|J*KjI>VhsK6HO4cC$~hyX>`XH{^kArwYz`-eqI>B~wzm zS{`K+9g8KoeZys+bk`Q3)w#b-BE=^QM1Qr~#7COB?XXa#xTsw*(vTP~H)1?0SEy$M za?JAH${gJe%jscX)B1{22|XM@P;Kj%%4r=et~YBAo3ll-^Rc&=E}Hbc8ImTU*psx68F5Y&oc=HBe>d1T%WjB{w^=r=^6 z3?Wpk?yh8cT)K!VAIwY+fJ)#R?P_W8ic84sLIo`qJB=EYb*4)2ExVWQV2Oe#DsOLS zv+g^`15@7*6KGlAh?U9-Z`5$>EY2ydzOqo0L647g(or2krU#Yo#!RWHqSUBF_mG4h zuh?oXew`gC4b9z$dHT>VF9uUbD|@Ksf>G_-DMchT;~e}7(ByD;lrgydOWxtcmUX3u zmSCtd;kp$Cs$pMrBRVdi7!AFzLa#;$_=03J^V7cO+s<=KWL(t6JrupW)PdQAuEpIU zSDqQMzRv*Lna-4Hpm@&(3by+E@C~By+z544*7=V|NM<7?7zOdy$7DLZ>Vw8FC!)Ho z6R@A)KcN7hYR6CzCi4-#7F9?O-lyKhI3`#wQ(2#e*;{&;+czbsK)T0<@y3ev$dSw= zFj2-NI<9r4T>Zf-ChFRCi_70=wxmCq;Zw*(Dzv4Ow=M$)iC*VNT3ZOePdA0DCB{lj zKwladd-xesdXHneSo-$@uy|(*+|!C&ezckRbhmt17ucrjgEe8yFJ2jq=Cd8SZ7?fg z5AqXi7|AB-3%}T7_ZPPhs!oKE86W)kvFYfaPJ^a=W{}ae6k8oU6>nE{; zTe6lXyw+L=)joA=Ris8W&%^0@qi10@7pzd%DIk*)>Xpdj((`hOIa412X0$bhN_8%* zck?phpt(U~?H7>+&{!8qmr<+i!Z5=6JY!X=sP$+(Dx!{Lo8W@5ag&F``aJSA>lb95 z6(3S?ij+n{DkgFDo`(wvAq-->5!ewVp#-6 zsT6L@w|h6i1%P~N-;d)FFUu;khW;|n(zog3znp5CEDHJP!1$95Cv8ZiUZGgk-O5OH z$4T-8kLFqk`UWgVPMvS~@zfeqVW|eYcaTpWYGrrp;e3gz2ee1P9^LpLj>vN5}^a zb9MQwr%Y2+mo{mYyzRx`@CP=u>6azPr=IB-?VwY%`i}V}Ae7-Id7;gvM`%RQPirM- zV)Q0cEuIntn*Iv74#E>>$WCr{-A8S}{2KF$!uh>tXoM7E2~~}xdFGiN*GL0B)zYO2 z9zIW_C1ZDT9cxwVy7&qB(Lv>aBO4nk?atXpi>vn-58q{Cs4paRed?PSM9Yk1_eD+EGQgwt#sl9( zFasPd$pgNQ9Uq<94;n(_k(P?MT8;6yJRXGkLjBjWx^S6%Y0)Q)d=PZ%3#fH|oUaCw zCfo>fYHb(ftnL86PjZ}=0D&ahI#D@8F_zJ9!VNDMm|fBJdtmb~^wVl(3a{&T?^91` zC`oU(uvkAwr!2MN(;JqIS)prih0JMzH3R(w63hC-TELQ} zQWs1ZB21nYNHX?~ahSnhkPb%-`BNOO_&Zae7Q8)Q1m{lf=q|?g)Uug~aVOLqsBiiz zoF$82BvDixV_-y`E7T_QSt9zj`12BbUQX996 zh(mLkucu^(BvUvv38rMFtZVB;+%3M*x6i$~RMTkP^>979YQPnLBIS9$c}JEDmr5iV zdyj0|%DM`Gf_#Bbmo~UWc#SJdX=kZc$_Bj>C-g{UX{y7(FKbImnHg)f3UkiBX=0F> z(b0!X5P#C{8DER}4sUgQD=%)92WEef6RJ|SnZf(Fpfp%85XjbwUcV`JU@L$c!Uj(= zrJh5)gwK_s``}-JlLu(CFmL+VRi>;bnZX60CuS=o_EW;JzsHvK4P;v=a;ni-a+#GOEoZ{uqPhVbk&bb z7wcr2+=u8Osy9R{KAMsc%91eYU1nMs17&CTYQ;FwsEit?K;iX0&ZRE}c)Ax`iO|0` zDPJP#yX9=aU;PJg;3xrH%DH+qg9`ke$mYzaXc?1i%cKA|Q$o4Bre1Pr1V-5FZ5UY* z`%kQ&f3LSPP>h@W9IJl}7DwmJ&?|g6Lv&fjx-&V9g7pE}Jg`4zo5m?UY|-+&P%bK| zVoSV^C&;@&agu2IEnKL^YKiWTKu@saUt7t6Uux!^o3^DT8AiwFYGCj$=!z19gdIAc zHm8vYWmwiz0y-{PsDSYK39*$W0l3C@^2OtFv!~egZ9*nLC*^PW5GV-LZ9a>xLM1S< z#tgk#&={JEk?H)fY%LLgN90DOE5q~POS6#<1-)ShG*AiAm%21^uE2Lu`B>bDSmd~6 zUg$v~OoDKADMeH3r69oynBI!H>ZU$8pOJm7>?R&+IItnFVs-sVN%*Z-sZEmh)MQb& z{+F$q+i)qLkArcd+qe#n1yB4{^h+pZmVa$qOAeITuGp}`v|c_GJ#1QehJ~0Q1%pXv zzsy=F-YaB0VcecJ1Q<53n-{vzpeWHjh#gQ`7cKd&dU4S{5%VoL9FZ_Tkf-hKD*GQ} zEzIYV0%;3--J86EVqiI@&7i^5qd*{`tW8`CEd-X-L4NiwnaQR`a{=$)gnPqe1vsv? zG2oEU)8!P_H1Hc7o%Xc)EA}+yK$nc9|K_W%nQwo&k-Id&(lszZj3w6m-bb@Ui0-pe z?cFSj*~6gekO|fty%wG=Vm?{|HI?DX7fhYl z3Vpwmc(a>y&;dru{M`G=sF6~Bx=ds#rtgvfvdJi_03CigZ;JInWCU7rMiOPNWSwZN z#tx8&leca!0vAm$Wo^34IzRUUHqkSCwj)#L?n;pc=d2JU1smbwNYQSRE7+hB03>7~#Rpf?=+qz*Q zEBMd*#fjW){w{ms*E;&e^XA}f&BigQPz3!zibD=Vo3Lwmgmorw7(#9Eny|UgSPwdB z5wg4s99fOv+2rB;G6^l3r4HGF-Y=KnFtnn`FA=y!L$c{%D=umm2Hf7zXxEEHcOZ}R z)EjMUlHZ^Uz8PLD{TB{E`_GP)KSXfdwrC|WQn~ip79FRZ%sQRKmS^)<-E4-)<9`(Y zNU=l(dOIeXFYT(JFO0q^{CLJb4sjV4zyN_~-tW(#2m{GIFd^-I;OTzcimYwh3zffa z6h+9xeWpa3QJ3405^HY-pMbY!ABXf2#FGq43~{Iw`xEz&;#NeX+BNm>iz*-lDJqnN z1sD&$O}+e)RcU+_()&9fzhg7z7>I5#G5-PR7|8_4J4^f<4@_f7Pp3+uyNGSb3tOdd z2N57>T21n$Sk7%KV-!(=d-Cqxvtyd(LdutW){ROP{MLqNM{Qkyy}yO+Y{71a6?Rd{sD_1syVI4 zqxAFB=9r91!dbXA3umiGVEY)q>_B&Q7({SiuD{Kbt+sjQ#zR=bb=ehtWbvUr_~EVJ z^L0E-^=U6g;jCAXyywK7dLFk2c`~Rbm(=zD)XfsbOlus6ot2Fe8lMGe5W3A$)~M-3 zYRH;fU-#!C4ih>Di&<*C3a{Y8sqxHMqfB#r-J69Hq8^ugW%13oj~Yy(9r5~E`X)<> zX=DaP=YCz^%AxScP5Os5@rRI)!Nu9JOIqOb6(+eD{*?13SS7E?_Y6lg-V$)NV8LAq2|#Alo~~P>^W!w7tY%Ls52^x1U5T zOCHq8DiL?t(u5lOC~oV|79yRdtdcaHFT|Us8ATj^kov>B`0?-*E*oPqa#67C9F_Et z@jtS{$1gB(cH@~JF%|D}AV6wujQkM1Lkh}V%2_7aU>?8vZ3=0V-GUj#!?z}oX&!ds z4ApYbO6ib_D&&91`=dS+6cDjd5;9BpnFX)-*7H{CKXo(3`ZZZ7RomUglFrs80m-TH zM!g$qwibP}c6f;T5$<4PjZdk%Epk>t6IVkgMj zdd|7c2nc=lv}}}VokqD10!gp#4>5|jZblD}`*NwI_JPW)rQ)!4m%Y{$3U|XTdey^o zu1@0pDzCid6H+TY9sOE!lS}-f<;r7%e{Mcy|LxMj!$dpGjdHE$JmDPs@+EAsscMz7 zB7lD?FJ=-W>N(V33q@r2Rw?VJp52En>eqGsnK;lx-VlY3hbTxDe=;Jjt}xlfZl@oH zF>K+;N}L7qBKI8UdFK4^{#9eR-4T5;ppuNZ6Yk(#?S;0oZZVX9{JiRkSka}k@++7R zRmb1n2#0dit9J*k-3C#vm8s+v@^KDq4gK{`!tdfSP9fUdqF+H-PbOxIIEILpiccSD8sI2Of>qrH46l*L3+%4Jgc zp@t}=J6Qyg6Cz*T55Ds)2H>8IwZJF<7a83hH!S3>|1rV5!G6tO{VB4XV+r4xGGoFX_-mwD z0Eqh9q(b>6+|t+~}Zk+-vEn zRW_%kal)4%alK{;yRfAWlF;x;l+yN}-iT4wZp2HqCJ0qLD5k?g9Q*6^Ft`0BArqYo z^;5ngbxfd$Buhw&P-DoG$d0x+p4@^FT2iPr4B>T+mND%)R?`M$m@;473`C`661KU~ z;}r#xjB_xDTD|gbzmcrEt^Gm2R{vnb>9~qZPaERgxIt$(GZH4;w5$QkKs2e@6 zyjOCx`BXH5(7&zYQ2&)Sqdp|a9j0k*MSpN@{&C|exEb>(w<%&T%+!rV*#7(hA-IZE zp}5dTTaiC~_>38;m}Ip@coQ1LnGdYjguZK)7-`@o)CbPzLTORx-t3z<@yg}?HqXNB z7b)H!%5v3AO3E{eL=P6cTiSq#*C9p{UuMLQWHI@rS}bhlGgUVXi3O=v5Sk0bSE^J>c|USY43S zwn$E?nl2=tVD+?%899k~Q^ykgWN1m|=Zm^@BIiWdunadLm8Eg*nGz#sH^8J;84p>d z_S7I-Pj^@~5K2q9;NS%9f|F+H2Y(EHZ_l2c0)9YJTtAn$3jvj=1F*5?(gCEDt2Q{} zqzai`5E~ihmZ#HULAsRHDoS>X*)`;*4nBTN54{wSeTvt8@d0R@lzuwoNvJWFhzy98 zQPm&FYlya6a#fM6=E|%Nn24Vj@^C8~iahg(Zy?wPP$r5g!^+96;UJ;Q%ha;feW+d7 z<~i|@E+rD7*RG?(nqJoux7jXkK4&cubT>|M^Jbu;`FL;jUJ$JVzjBo*k!~s?&7Ce1 znLS2o>!Lxr3MDkrubzM-*&JaaxmD4M34`s~SswhW1X zb(fEuiyei(7s^_cpMr{QH-%VEV@u*@nodeMOaC~-xpMX=1i&2s zxY~q|<%D=8yJNcy#8e7ua2Z0wEgwJBt{02f-p75CMGMJIm82D8rX9YbYqlI!W!4)2MtFGZl=NsIO;y<9c|wTE*j(vGPVgdb^@xS-DFNN2ljwlN<~{6b+lCz z6<&U34iydfgT*GN)R zj%yuo+%r~QI9E^^o%GtumNsE>`^jTX^Ex{Y&1OCka*d;YGy;E0*f5vSURMsJ+*D+V1TR%d;(HFGk7KY1}wn zWY9(N$a}cNEW)agjiyOJrkV*5S9eAc7L>i=cs*U1hS+m_|Km5#Nj|(%+e`_r*2Izd zZS%+5GEH1YLr9yL&b_oj5^4@cT*gUBNm&5~S&#sWP6ldybnvu;JP9k;7NY47u@&P9 z(#kvT$O(eD!-cZ(>Wq4ztfRH3nMM^Dy*2m*E@OqhZlSSVfJ34Vjo^}0GNpZrmaCoO-MvjXig4q^106Gsp&y<5qg)DMkVKBEV3TNqRHaTQKOPt z(NKs6S7E~5z$o*u;u%pXA+#v8?dT1E_;CW1UrPHvsJ3E&$o`}7r*`mgr$A1BQ`M=R z=(zZtH?c%`20F^LJ$e_iOWP1n9KwpLW(6*|cLr}Ax}5ljo=wBJl*QT(A2J|NCb5WO z8{mo<#T?_XIBfv&k-zf_n>sSW6@!RhZV~Gbd1~Iapgsnz5WqV9Oj7ptB6ybx1r${J zBb7x~NZJ1i*FGr0D0-~sRtZV~6Sxjl`m|z$IZ!&bpAsNzA3fQn7XSC&t7NQ<_XupK z?v=lJ^(;ds5307lehF@FfbFyd>e#O*Fl_-$$QwZ(aqZ>pi=22?9@X&bSL#0GOioNi ztqC7VWSRq2bKrgV8|_h{&mq-djg5>l=13gjG2VU&UxcLTy>s!1O0WSON3rNS5viR; zICox2d(QtHQnbe?le@ z(drTU`_JR}gqwV#Hj1ZVM2!!~q|>$AndRJ6nuS8D=kQuj_h~y|H2IAcq7rs4b6#Kg=j> zV-QrFJH$!zFyS*071X#d`fG|#WJ#tOZZ4`XQD^}Tc;CAm-Ftn`Hm*wJ>JSaoJSAR- zpyxmw!f{mn9n(+WLp2d<<@(AXGHvATs$c|ll*z*L!c%yYNB)hzAc2x+I)<*gvvey_%PrI<@h6+Khv( zYVrzNYWn@QwnbFm#iy^V;bGgn+lq0ma&(M}-IXTDsPzEJY!<6U@lznv%9@g^;)#0t zNOt90Cg*m}jRTdj_<;=nvX=AvVixl;o~bY>3^AY}wR;-?Jwx9T>b8Wi7}k?0@tzj* zP)hn5M4eD56&+eoN7KfT=u~LUVI`xMMKBVJ zmx^tuJd)LMu|URr)#Y%ABcw@qLvv&LZmh@n+E8oL{(p{iKew97ssX;A<8!$0{5Fpd zNnGkJmnuZ#=_*aZG;=m4Dde~9rv&4fMFDSG-=_b?T)+b8kaE8~Lys zRZLl(ZF{K7NW7*v-9#&EbM`u|RA;Q#%3Pua)lDMT@;5U`mKH>%D2tU?C|5q1v?A)h^>ch%IJSeDS7?kk}K*x zohr+oPltlNaN8-0TjBdG|Kty?Gcx|g^tA2<3(VD3tmzu&@XZu(U$u!y*z)MFn<*g> z?J2O)%Z(*9_9FzziIkqDiO&B(Rl8s|)C>$mncvlMKYbKGt zt_*S8!tah|_L>rh2vmf9?m3}G-vx;`5SX#A1M&q7MsjjWX!mOYjc7pSnwZSa@_)0^ zW;C!V_z&V!awArgLz`h#L3Z@805IIvQe%wr(JsKFd-=D@Jdpn@1$lvq%^>o8w;a}K z%RbUdZEZgbj~h{=+QoF@W`mPQ-ERvnAk5c|XX}&g!RL#Ny(uoB$HDCZqmN!hR}1XO zm29o_d(B%=X0K+JM8&cUtHXjfd`-)1bV8hOwy5wIQebB@!PIx#bVxL&s=bG%8<|BR zbn(nWK(9<+@n9^jMVASKq~!+Kj=(z(E8o0mM{u}J5o<=Ltn=Q5m*t6b$sPf(kw=aU z{*+>W=Mf6^gZQnEQ~@(wSyI}tfQXHfrAQ=l+)7ltEIc+k+GxUBojCMbW|GBi3j&l@ zB#x$GSBcFDTb?#s!x4d7!7Ht+deRo?vbGkk?)5ta?f0<`Nqz{A{osWWI>pRR8TJdZ z^u}K&{*^azHZXI*Y8%fP0h(GEblY!o1#V+WhknhrS6tUh7)@~)O^nAkgMvj_PfGS`9YoKphxgInqcivCh zRI**!zlcU&Q_RaQB~Aqv5h`yN5Rv4oTntQZObb~%k!5<Wa!?^ ze@k6vgB-86-@@?HZ52v{{`4FFfIzx|F?cjoE<#uXRImU;E1h=!Sf%uLCgqj>E!~XI zhQ~JYaAD98`h0W*Cu_v1c%F{BtxOVkw1MrQ&&Q%^>gLOqa|}ANcL|Al`&JFi+1Y-m z#Z;izv&^6(*`pJY2Tq5C6!h&{#yyoa-E{E!eeW7;$1}KKiBq){c^C?8EL^}hGGKA~ zkhMP!p{WGpQ4NWvl4^Rnsh)^mKRJe&!&{h65jLno#{pSEGw!vTepuY)k@h=gI9ndR z1#pJ4C`bn@!O1tuqw~K=wB?r{&;uXpH3?|vT{f?reeN4sKDdtjZcW1aXON?kugN=SZzedaxOra zMyuEo74We;#}&69XILV}r6^%fy=)t;aLLLYZt(?1Q`Y{GcXGUiNZA}=1FOHjc;1JE z&G1g2aNfr-nW>?w?RmSgtC}}#9MC_gqmM|gqaCbaj{EHd*lRv-mZ^}~-W%SJqM~Z& znd8&~xIWS4pN0EnW(W@GW za*4QzL;J(0b${%`V(4BafDNqL3Pd#QL6tAC^;HmDB{&&t2R}_uKxQI2SL^yV;&ml* zLp!YbMQwM5Gv#|7L8pw7`Ni4~2Lt#f#7&)bIdA1w6;Q!m*6eSw z7oGXo$CCc=xfV3jf(ZEKhN!iWQ*g!I7Z+ixB7TIB8v>hpLstd*QgvZioe3QXe28rg5sj9`yw}QMkq>4b?T04Gg6L&52{*`!zFk;>X z&*t0yvc9!ebNEufA|S`QN=&+3qdBQzivZsbHu>Aze?-Y(=ue1p_tLr`{)E3hcnZ!6 zr1*R1zZ}2(?d_)*Y8VCdpg4y0dO1e?#4v#Yx3q(KGrowSlp(Cdq)iqL43RHLOEm;c zY|TDF%NZ1|0{XS+*r+!8qfn|-2VG+7&z|lj!8n7Zvo!}rF5RJAW+5BvDxb$k3Ithv zbQPh1(4h|yf!En0AitA`z}uc$t+LdC?z zGg1a!_`1TUwkd`3fezyeSui@(Y!(`iO^38X*uh{bp48?)PZ)W>Tsrs8 z#Kn+=<+>;e}K0uaIG*1mb@E44vj0QBe~F{@gf;kyY74wj8fuu$lL2do+a zDC(?`V80XPZJ-H z7Dh0z6*-vb0fcg{m;>)K{x11C8=S!=eFGSMZ?3f1OJHbEuvqZ@Qm9 z^aZa^&j8yC{JYR%Oe>7SU+_-6d$OHm03qwx%T|vic&40>jfq%c+!J&vzZI)%_Fd5@ z;r=LRMAw3q@vXP2?z(Pxvq{+5+>Wp&KsN#`4E2Sq7U*NjKh1&U+VT^^>O7`@do-HzJY9L&DX&7naWkuQM zBIMpG7Jdbu!}NBtLztVKyo1|0f_t?JyWqMu6VOz9sXPARZZvSNS3tgA(Z7SerdE0^X&eBs zEkjN#71noY_aAaw__XdQ*7M4__pl#7bHwhQ2A za%3`}`?~m1he%5Pna&mU-2VIXwv;n(8{y!$vodpt^<$X6Sn;bYZ3bLzv!Q?gcULP% zQ~Oei*TI|0-Tr-yL3Q3Acg?V6WRtgRKND;3z~@k>rY)LV8;S;5>p0jkNEb}Jowxun z=^GY(9-aTFum2d8oN4X?55EP}+*Wdl2bKYPJ6eLXmi`e1m$cnmY6t`|~0h zbBG8a$`(lp=6=h_#P_L)M@+1daRmn8MySyu$O@v-VW=U3> zDGf|qkQ2KtMsBM`oerDDh08lkh8}}Zb2nskQnrl)?Hv(JVZj?SVk%AJbPd4H{^ENViQf{CY(sEhi-b&!z)VX5#NX|o=@VUt+{wUS`i?u5X4jr9r~PRZDjKT4;Qkly zEv0fZka6Byu>1t|r!6-FVF&z4Nql9b3^uQjjtIqjUBgnq>K0Sl;2hh$X_= zGWPaex8`pC!j0k^>v}in%q~^ql0E1Jg6~Afr2Z6SRQ9_UDIRx6{iDm`N?bacF7f-g zR;H*}T#iSkBBwUvLHzWcFkt*?=!44Z>n^hXt3}oHeiU?)v`@AD570 zegg{Yo?h$FZm|~MwT6#d+V6(u^52#3mn3Yv3k%c^e+BB4U;%4C3`*`=ZSg)`OhCW1;unYqf!G}~hrwery2m}yXeiw)T( ziT8tV{Xo=+7C`kC+PkoPv?{p|w(?R>7@#d)B|4f^RTyDRor!wmq{Y(S;H)ATbZEGA zt6hAw$D+2qn?`xb%S$8D-VoWZOKI&{H-YXnk&x$qMjX$tj}e(dpRZTvv$i#r?_$9% zsk#fxlV$1Pt8ZKDRpXZ({PX=dlx|B);+5B#8;KPJlXoukTV(_utDmtm5jL5kzj#L6 zuEafIu6w;(Zx$B#rQ+4r(~I!*;}I9xo%NeUd&N*-X4;nd=J&{-CM(QfYtX(yeTDoc z*(q0A)nQ~a{|0RQQu#3nb@0x$*a)8q{SW>GLx?9rPwq^ ze;b2eu1(5(qrc%bYkHpOWQ|2cR9BmCAO0Z-epI-h$!wmowBNJiXH}=Pv{f<66~liW zJg`e?i$M7;BMPhdh-v$6laxwGbb2ilkr$Qu=se@p|D*tx4K3QrJA{=)Bul4q({0r3 za&Qbb$6rD`&@VDFT{3f$Q z7r&PVe&9hsgfK$kP=6@c>6h(=1PAPl-PyOSdf=m-_Tf5CVeoSZty)(3jUh@%9#ty0 z-YP!yVEpnl^iv1FSlepgD1g}>nX;zpaCHhdKD<)1H!crQo%sGb=(s10cpf>|b=X;+ z2|yxfgHTq_rQm?7o&iZ=r^)S;jF7@REr!I^`z6M!G44j()c~X?1yw+FQA$a=h12@C zu+_dkMdb#0%l@rTM+Bmr1a_l-B8S9{Y#jM855c-q7jJtUdkF!IF=!*b1)+dg;Q1pr zvndbvxIY%B=0u^41f1^|qeoJQEbua;x6Dw+#MjKWUjUtXo<%so z8}X|Lidf!FCgQ+lbcsV~?jP-+7b@ZE$dVhg6Icvp){M(y=S__Yuyk{$`S7Ge4<4z~kb08h_(_-3I=6(724VmIA3*{JCiOU?ieb?aa5Ml@w`oWqgIuPn zGqbAZL947%5$^$>QF_n0H}QzZ<(;V+`G@(sRPlxuJolJW@i+?%!05sZio^&KKbzw? zVe#V`ERUb8BLSw8H7BlVsG5Jil@l%mHdEUApGN;Pm2RP*RXaN|;@qmRe7)MJ{C~cQ zA9h9mLs^y{LQ{FCW-;_c?HIziqw!(5^4Gm%yK!4#%&&QkK-ANc4yxky(uN9+NDN)& zwX?tJj4Fn1#aaf0Au-|oE&D}5py=J8&J35_RJ@}jmwT4MU+iY2EfrG zGUN6(vaEo}EZ}iuPlUMuDKH|1=|aahU)DIoJG6xVEIhbNe?NEalPOIRh3Sx#+Vk?# zHAOiE~s@?-tVhm^WBt2EY?tYl%$cWh#?$QfOxIdl)}z)|sy24Q zq0d0;LP!Sr1HM0fY}?md^6?k|RIs3rNm!1vA)fPES7Qq7`%6++D}%Y3L>ufGn$%sY4DEa)~J!%h9t0M~SJiDCJJ2oDKd?1iAS3OY{%j#IQ& z>G)#BE@0;1b$^E0C@C|_k(=3(%|#jFHiI`Ry9592s7Tfq{9Ke5j7#jFu+V~f9d`Ir zVGSnPTsKhGEb7bo;?=;Bnoq^^K7np2%+jq)+PTMIPD%h55Z74{c>^DkW~ALlB-!^Ub{-nJB5s5ABB67MqgDolgns z_2)L<7IUr^cGlOGG8kWioRF60lzwH$-b)(n znz#hGp-=#dP@MCK)FZ03n}jJ_(EB1;LN|{dtq>P*xF2=>l=iCuh~F-jVn#j!;sG-0 z;mKE--5Y-&hXu!TuVJ)F)rhSqkn8n+j>>l-7u4pM<~bAMlfz+`I9AW>0WJn7D1*{Y z`wk`ZCOtFyT@v%pt%f1{R2da|wdF9y;m!KJh^y{Skv4{|o5Xxhlwp2r__*#`H#Z~a zfT-^?h26kXQ2uj0fC2h{8txq-lU9OdVdOPXYI7M?z*@jWG=+mLO&`ix7lIQ}Hk-L- zwrQ(otK~4IKj61*0V9~h7amVB?=b2lCmvyT9DX>h3f)fh&x_4X62(L46*6+aLDGaZk z@+rc;QJ}CjdtW8b;dW|11uD^-$(0k~$)m~}h`Gd`7d8)|O;9z>c%!2)XvGlfH;sj; zuely9remKKGhlPa)S_0#5)T- z)}#K0Lt9-&E0lzBECf;LRE#iDM|MNTP0M+q^l(Vkhj+>uyW12TMmoY$s;y#r(@5NY zs*&N*1EJvNb2wW!+=+6vUL|i;s5uGM#O@MTFlVm$M^17r`;_Z*;W3FKI-=_v|0*fj zO+t;WwBUkQsApZ=uZc>r=Z_uWxt+2K9HndBpfwYEx;98T~Il+{3ES>f9(O=nSf{uIdJRV-bB&AVKH|1WVxh* z?$Ok3O?7ug1R)tlD1Gpgzc-bU$0{@rgW?^&0BX>DqQ4q3-gsU5BophS+ zbIQQ=YmlSCm@gnyEpxRkinbry0<1yfoiCj>8|$PJl&?dgWD&{t%;<&<1fPOBQT0Q_ zA{g}Likf~I3TZ1)Bv@G5mm$|0JZ?ENuhDX4_H=_k*UuE8^ku$=0D228Qg5|;)O+Ig zE;hMp|NPgZ^QNxeyXP7h@ilXo1mXM+&+eOetoV^~Q2Eh?kwTaN3GXTu>&^QU!gDej z9Y=5OR9zr_Pgg6jN)>jZm>H}lZVYOmhk&gK`KIKRCWUitB~^w1ygo=HXmH2{wgs7% zGYRuUypHN8Aj>rTIK>}N@=_~MSTqhKwB}`S!cX4@e;oS4Pf)wlDDhrztn0mHvB7Gk z%Rv{*-3oKm3{xvgJFi%|X={5W>q;GASy<0|ll#2&_QWlLSSrt~|8x2jA@PH6n-66{ z@d}Z4pb)%rYKxv{Wr2zVo zWK+?SAZ~g?_YGnK1{|KoUUsU@r-wB)5}r5~fu53rkd`1ACe;1JkDKwBRT}vhua06^ zd3eqR5c8>Zf#y8O*Za?E1skXfm)NjLDj|-f#rDv4-0iye?twpS01G%82+Wt)^u~qN zHsJ`|tsWRQAhg4r%bECy{%358n{dpN%ZjzOcj~M}++T!TP{v0H;cs4L)kFDc$Gy~` z+SodR;%(Ow5sX^-k_v9C-OwB8%00J zh*j|b3jlrG%>_3o^2JIcWJoS)UFec)`ck<3AelbFhGanK)Aq#d_B9ZcGzKv|)f)!Y zeGSIiPXP@vm3!xZi1}d+o`|WtLsuN)>8%Um8fMLWhF4u$>_-r{N?27?hueZbR z|7!O>ZH{)Yw_7ekwO1{*)SaYqq_W;^g&br0yBVmI0gdjrRW(Y|KIzR)i0H}kC_wXG zFh5bG6d&N?PX~JI|B!_xdECK{2CN53@lO5=`7I^VR9|sI>a8o!-Jy&1# z&S7m8mnRd9AL5C!xoS!uIDN_)t)I}(Ixb$K!TL$Qf2q%|&g%A+lrx2rEiZjkbtP{U zG<I*sxT*2+Gm}3NI8?~SBe6-m=O)EKyVfBMgVU&cXOube!Xn(KM1)-`*zdtHf<#R{3S!u zByu3Jqx;L~0jeb7dAi!^A$7DtrCV&6#Lc*xj}#ZZEU7Eu--i9B7N+Ng1dlgsX4ROr z3bUJl3St;vRs z!#jC}WlsM6LefqgNU+U&T=Mhnj3n04dOOzb+A+LTH689Mvvj6ftZQb@Gd+(8eV+HS zXakQlyG25|^s#MJ!0=XxRKE?|=<{uK3nUa3Ek97}@qEv*o+%hk6!(CecPpt_Qlko+ zm^D=+Prei!0YsNck7?4S?UjGfYvWmR1m}kh!#Vojkqs`80!nJBayO;?o!TU#-WE-^ ztnF@6o=TaLd{5n?uHf>pH|OYx))6eaKrfR@UuN5AI~jw44J@=!z)%-2G^BtY(S6k9 z;T!Z3|8L&g2xut*zmW@}iV2u)`p1zUev{GUQwy5#r@ne_4CaM33(Iygn1I)6 zh1Z}2K{XfqQ5ByO?`Mnqk80p}I5LgMRTd(A5V$wk3H~g1q3@_ry5Rotd5nZ@IsBrN z)0Ct$5`nA>7|dI}XtePW2k3&5z#0HEq6~9sE>FuCJi_mFNl5coxS+t4#NjtHHRJn; zLx^5mSI(J<2~D5-#YenvozC`Qlth_%N*8Nj$m);s-|Uah7@~fyQFirKN|8zPWT3+b zq+O+}Vl52vgl0m1zF*aF$W}jf+Q@z8+cF^7E#sLYd5e)b>jTc}qJ`c3<*r;{EIX zJ8P=t%ymJ=9PbO9bE=bMp4K_g==vEAw7w4`HL|X7%V=zdpXk7pcvQy`M%JkO!=H!> zN)_ynbb*{$zL11-2=o6!zcavZNPn#o7*ucBMa$GbS!`Cu@%N?3Z3_1Y5HhCryc_Ia9T;7L* zD_=Ty!=S0N`T_A|ji}H3DnVB0>#@tU@`;}Dg0d0uJ6$;dMn!g$5WaIwJClF@)JW$1!K#_dJGQxI?ypW_T~8?I^~5^ zQrJvh1%J0cANw5S(#qUoy<|?2M=CpqX;K4m{K5h);M1n3&vxBgprCVU7+8uNExHv}W zu1UGK=oiKtgb79J`VwjP;zRr4tZrSQOZj3YTB#0x17kSgfh}TPyzpJhhJV*Ub7=uD zo365aPpoV{g{rnS&Rf@HJFn9CQ%%C(Ei;@{h@4*3^KVcH$F-6)m=_ulC_)YmH_}`O z4Jr(584;t^k87&?_cBu!yP_Po%?ayo+8NX}V}05GLylzq=CjFqMnF4y@iqp zJR~ELI)xJ3NOIQEP0Dp`w32rB7eyO1!v&IB^*`!ikoHnFsz}UU4q?io!b%)Bgv_r& zj;{hJGI8su3}pu;dFYCIRvk9Q@nhX+fHj~D09V~~4(UpB)0nWh*|P%R++Z3!F-D|U zby8}vlT~pHyso|DUTdbQOY7ZYu|zEeoMgR&YrdBOx(Ao`ygxozt2R!j?k?xfbwa!_jY}RNbqb7w6o)CMW0QMg=MoC zf!kKYBzgqh(_enL!UO*vRi|~HY+)0yvl@*tcGJM8<5EBLfl5R7Ggxbm)o+lw{u;`` z@rk3wAba)DUX3U882$CLr3SGs&L7_XCeJhfJ-a=Ob)>xre<28PTLW{`5n?L$%=1<{ zWk0vxco?MSx$&A6+NvEZx#T!E>>@A2Y+eidOaH+gv)mZcKOfABSEKarGNL!hi@P>` z&xiZf^Sd}PB4fZtvqxi6q;BL}l?mJ2F0Yp|SRfqlYL(9_S-&0!nl;Z7tR10%nm{dR zkjngQ5>0Css`s~p^^_YXjup9f3Ul9DO1#V~%PhR90S$$EeEUu*z3oHzv=0$+knhbE zHq3w73r9e}A85x}0kSQYMaV!N`~Mnk1OFCVD3SI_L&yz|a9Q1yAQ3VWSLj=&g_uRS zS2#SAA$R12ogBl+!oYfN8S)AZD!He-O0Ob6{*&F4&y(Q=)AcX_z$HXkoi>OBPGW-r>wA2wJ5crm+0$E(;h2b z0JkG!W<5&&yk;<28?=?}&*cA1y8S3QudSWz4I$WulsG2hJh)RO^GNIVj@@aEb78*) zkSDPpm~=WCLspmPlk~VaNCrsigVOyA7S|qgo9o&u(%joHa2EXOMKy2b*k&3y!$7@DrS6n`` z62QrEpMmKrO8wo%3ezl1NP*F5rNkR+Qax5QtSKZ5T8TcWL2Sa|(+o0(sq}+Gx}+LM z()OO97_NP}zUqk7R%8mw6_CCzZ@I&j=ilvbOAE*bvjll9E&jG3+QP8425m4lx7@Mb zD?!Zkhoguen=0LjL^Y-Sk(u;yd-OG(t7Zpt4F8-<GCrTriqY1>8_M_al&Whw#6C zLCE`(c&{ZepUHUngA3FPYkfg!%`sqY^Mur!XAt;DD8zsS{Tg&TK&&qes$wh&&z9qkrHI3Fg8fL>DqdY&xy?j9!17KbhkDfggpuVR>sC<|Je%)`;Z~fjqGm@*8jx4 zM|}4%9r@+)68ihNcXMkx>V#!uJ=V+Nmpm{TCXj?`dKPs_#b|I?ceIn01#J$I=Agjn zWWt58n$wII`E&A0HxP`BRv}6jYwM;qxw3&pQl@KkTpsYDpr%Xpz2%BUsJuUTmou5& z9;J&{(tLAlC$XF`2P}L?i@3v2v>7f9w$Uf|yactuPM)Pb9V45Hl6UNRjJR%fmHsz! znZqDUUbtn4$_!fSFAj4&T9Ch%f@V7LLRDwMz;{-zN{#@Aw57+O-t}ko2_#+V1L@z$j=T_K!O-DCDD&#E(-Inz8xFDqL`A@L0e8RJ zF$yU8yc~oevO$YC^;?&jgx%v(&a+kJIQg8gZPk@?r;`V1UZFEPK;JO!`k?*WkO>(Z z%m5&qk@q6sJdvP3M(?U|P_3EAn&0C4aB)Hg-VWAupsJ$ZBp^vTKb@P;ZdDveQo<;; zX;OTa8FP$9_ES2y-Na${R~20IU4GLiUogA-qy@Z#@1>Y_Wd6^fBG??J5&vsk$V&4ZOdZs$@?;*nOI3 zVjp{N02RkFA9POjy_#9!T2l4~giz~*QzMx6&BMl^iFmlE9P$$&9#q_Bd?U*+RyRaD z-4i9J;|=qRP)_t$ZmuV_NE3$tzAJDXYE`9hr_v9J$XiCWaoF0R_T{fx{sSY9fv}9C z2KB9}2B-NYIF9V9W{2TFbBUcgV1lH{PvxINs%a@TM9FwWj7%`i~N^Lya#TTeda`VigDV^xzu z!sF+!BU;E$R8Dz!mQ(Fwq|4xhC==cOsqInF!j}_vz{4Z8#-KE2gfi=E7U7`!bLY~zEe4%0J_ogOmn$4xl%r`l`kX{oh0ZB1 z-T6NEe!YeX!KBus-iW)yT2cD!dM*F||MOQt_saaNG=QpJC0b%O%$m(J+IeT)E-zwK zAPlS0-apbc`9c75Jo z7z$*-mej*?SxG{>D^ULmy4sTp`IMbA$baN45aB#P;yjAK& zbLO#!@ujqG1lQ%4SZ6SSjS~`Wn2$dPNV9|?-NMv>EnX$zHB0Tj&zlyHUHfn8cj_|@ z@^5%Kb2NS3r80pWHVUR_s?n3oY_-QyT#zgR6JJdxOVbgh)jTS1Hr{2oQc``&7b&F) z_E>HeQU7}&xN3il;2L}M?(%xn-t3Y}Kafa$n2-P@TOB{$BOTR$%GN*5EBTdD18)d{ z*T0I?MC5n_D!xW6TAwNFu_lsJ1)(j7Y0RkLe_Xhgl#d@#u3q|zDi0UCeGT*ArPVaj zqS-D4dbH(oYmzDZSc`58vGrs&vF6YeC-4-(X~%s~T6sDL|6mYFf2GV}`9(ZDp`ObE zduH0_KcEMT;OAnMzfl7;Gwv5QMqyuFb~N?J+h5ZW3on!8h6V~kaL~(MiyOe&!5Cm} zC({BiTYltQ4HXcb@TfLyX5HEiJSxa7vgVg`=AxUM8!U6SL^Ps?}$cE&C`-m+5ZssYbW~6n~MUa&25i@PyaF= z02c~*8!q(x1_||U#ZSFS_bo%?`HLgB*WmQe3|h~ydeVGH?U4cuOqJZYGU}0Ur|ofK zfAW~FQ7f>m!B)`k;Sxi6lF;Tap+KvFuJE;_`pu5#Pz0p=9E2lP1sK_f9v8=gDtx4Z z8ZV8{qAm0d_nSL_H^)<{pNQ}l&-VzqC|;{l8%Z*O=l03rwcXtmh^L)xR%g+gR0RO= zLXj@QJWay1bTejT{X0QqVJqkuI@)D|uCuRMmymvG434*)56*t%q17+#O(YRG!ok&h zq8d=%teapTU4Whc{PFF)UncXkIl0InpbDF>SgR*oD5~rdEM5J0H89A@^D}pLhr0YE z;oGGJ|00d9Kd-=-{tg!?>{f@L zcY#1oe!qr}Rzx>oHW{?kQO6YB_Fo96{J%g?m991fWsld&RL#~snG2bj+s>T4odbQ7 zhKw;h#Q{P$9IZK2K%m&18OU zjt0_7!u@Kfo1BG zi(B_8){=B>sFy?F{L)DMiH=*}?n-^0_ASL2BzP?azRHi7!^9!I3NPf9O7<~*m3%w! z2p{nP{{mA|`&{{M)G4tZ2UiNg;N#2F&Tk z0wfY}Epsg9C_|5z@0^e4GcX&<)d+TjRIYs%PwlM(^m-m4S$~U+<&gyJD$Bq$5SGwk zDca`@f>>>>^&DydYRhS)vb7hhr5YEw4%RZ9TqC(D#IE31TRUOu>&^iSAc#3P(&4T; z#G<_|CI>1NB;LFJMfOb#_PLHZt}noVZS6S_ckQ`;*X5`GD@fUcx^Q$bZycp9IO1?M zj84jj!^GlFBtSf5|Fo2I4}jeV-vRs&kYI71txkszKqUBqL=A$s_~ZZiqR%(h!NPx2 zgd^RGuBiDt3N9IAA4g5;Fz}dg0y-;~RpJBu0R)0k>z*9mjq(tDB?{7MM}Sj%`Jl})jV92+wk!&Nc86j z7gQ2#UJwW);PCxN66^g=)%+U_T5^H?s4@#EaA_4$8rhTG2Hh7((Qxl`Hb+|sG*VJH z_lys(60}|STxU%grnjDPglFhG;JaZwPKq70oz1$M-_!0m^#^J-yn_7LQEqbF^N@05 z>6rfrxGplcyPl=D2O$?AjOOK-AZ)A03$b=3GCVz36}N^Oo_&{GdfICocFIdmj>7or zjiy(VF0h|~%$=2Ap0Djr5fSI$GZ9sJvV`fV$kPuHO^}B!-5*_z!2i5#6gu`%RClfv z%1vN1eQ7N)sqbFP1x984&@Q^72rhS!jz&*be^+4AADZ*h}!~eM`$m}<8I5F{ZGsq?)q@3-6nP}$TPYPnV+DFTregEp?sO(ecS{P zOhhCb&bX96IKYFSlCqfDxwJ5C7vi*NA@UAtJBiX(*Rwj}$a1*a%48Fh;!~_P=r%#6 z7Kf)2=d@lxkAIRZw^O`lOu#nHwAKxCg=QxB$ljUNoHtwPtLFD1nF!f(r(CRs>QkWa$$J{tfZ zX=?Y0_bCgd&&nU5@B`4$A+QUWzMkB|@Ad&ZTw88adrHj(ymyxg)Bt*)LB0o$iq7Ld zC4LKRA*w693$%A%`nLkQpTPmZ0NmvpfI=PMCGSHY0C@fc1+W98004kOXFPz=4Dn{? zRRME$<{T6os@BrNie{p@)ry!PMgnecb}DtPt(Uib`; z2B>{Ue`EdzjPs9w%Y5j+-n{8=1H8Lw0TkaN&o7_$*Ddw_usy}!Pu~L1$%mNtdB*~~ zJ?(zG0RJZjfXZi{3&GIeegLHmRsK8xrLTw2lCPWF?}(j6K*$r#MbCSD3qvj8QQo}3 zJiztq5Wx0#z#2RHmw@ZnMnA+aim#ieJRtoEfo4EG0QsBc+50T~-T#B|v}g5ug+TaiQP`GX4dF zcZpV;$gYbv<-!nR;GE4L1vGw0vHDnE#aDccan`%Pr(|j`>zX_EzmcoQKrZ82+1AOg zL_dQukY^G%KNuir|G*79JjU}?!k8kTXMehPzck&G88z<*PL4B6T4Siw!(zu&se=ig z?^_Xj)mMkm(LlmVRD;SW{L12o4;^I}YTQOsg?!&#gF{4kB)nECi6R6oSQF2s^g)O= z#qr3$<)%xq+z*KtVZ0kPHAwMON=<-UQgOAXHbXh+OJ#S|_9IF(yqeW{yzS<&|5hKy z`E7TS8)Am7!$V@dMpkMDbGvE&f6nK>O_i{3ta`qarH#@)%HsKH?&{*t=7Km~alEq} z!SOG?DOT@a7B8%G*Oq^qCEEVSW)3ww)aS(*X-PJabeH@Xj_-O`>-S|_Z&%M6mDZnW zB?eE&DJc)J8;g(3Kp!C(J|_@7H$nN4v#gNkKK@tywF=XluCIPi}2oN81aNa-oIYMIMFv(G5re>lnzINiuD4i}k6jL~c z{(r>szlE434d(d}>RsGhPhhfo{OqVz^TDCvc+Y&Qe@h`7~- z(SiNXe4%!|Kh~ldfGdX{`znFMS1N3d+m^7|#pk%@8bbpxf`VNqG#e{??sY({^}i@Q2fLsq_qWtb4;}u$==~oMiH8MJx9PMu*l5Tx>>np9 zD>r)!7VWxvfas6-2j;UrXuq~zsZA2?_0A6$h5`QoUmEt`A^cYY%fCDQ@?bP4)GW<; z+O8l$`lwvvqe4rU?XATFGvW8ELS2secVfJ-&eW?TcFYY!3=f>{WJ$C1BW747zjk^ql)y(0IxCu9saTL@ znn-0YrP7(=N@ZnARsFIo48xIadpPZgFF50& z40B*pwWs#Xh%b~2gAHBCyPE$*LNQ(o=PwceN)g|)f-ZvTDK}jbsrS49jL{?{$3OH( z<3YwziE!^-mx#XXw%c_7=YE(m&F#x_*g;-DCoj?sZCBaMzZxPC>^c}C1V6uZ^5d@i zzTZaNMQatWnmzI=4T1{yFLAp)loW!8rrt}a;U}>~xV1L_l@r4>A8(U8+p%^}u>6d* z2$w9^+v+m^8YA_D8@rEg;|_~WD%V_lin33w1!XrD?h1m`(F)&Tx4J=mVB}`390i3i!y++IXksm^^k2cdzbb} zFmg;t^Wske3et*$My|W?b@3iT^q*~YL>4iP<<_g-+W5q|IKul6#L?@DJjJpofTMy} z19Ssl9YiVhlShuh0KzIPb0^Bmc~Q16H%(Z%Jek9Y9S$2aP)$GewGB&ZjH?T@o|1hScH+-CFPL!% z{!{4#-Lg*2Jg(7zzFfG%-Dwy&WVSi8l8(WyE9ULF0M@Oe54`{9+^5ky9z3I~4RVr> zyr~c3H8576vOg^O|98^=mqbj`;JS;h#>&P+0!>8phJ#KNoV(^@CKjJ*Cizk$i#!YS z_|_9!OCm z3}}V4`yUj?_Om+&SNbSJLpHlDj)K`k?jV?jONzgR;~?<&sG(wShUPsh!}cg1i|Pmz zK--EAhtX&1TxRr?qf)smqaXMSEaph20huv=)2fd-dTzZvoklN$aLL=6oo}}W7nvh?<1jeSA%0LO6K47MvWzS(%M_xd7jvfrCbRN~ z9yO%VrzD&;8_Vs?^Sgv=*PdX`2V#qq>Tp%@3%*?(343a~*iMhZ{y&S6{HX)}`0vsM z>Jc3LS&CV|Fr|GCn=&tIM@M1E`)C1{cyXmq}RQUXdo3RlT>-299pS zf|$IluNj$tYk!+BlVpKOPe2#7pP=NwmxhU;+esNp_xE-RWg$OGpJdX_iK@yhN1^3+ z7!(p}X=S7LS1zD=@5&LM?6V+ejxq+)##(GAbXmwzsn9zH;jU1h8`?@?fzFy|fD#ScP%1HWF~iY~?UgtG3Rynl|& z4Ef@J#V2WVb{v!qT=)t_oCD$fGbdcBFjkxY7OXwTGm&2sMev!{J4EXJd2%j5ma!sb z)LgKpG2xx*+A64J`Nj@h41K!MinpF?p91eFQ9N14XE#Oq`r*f6*Tv88DsPtEy zr#us@D0x-wl-Tdr?k*>()YLp*y+ty#xh%LxQBK2Ok*1-h5*3D|ZW_XD+eld~t1v1O zIu+_22?vd`lN4t!y=UkXS#cyL2zi&3zTe5##%X++vyG{oP>g^974<~@otbAldSuqj z$H`ZLLz|mYF9prc*mx0Ee(<`dd9ET>E6%xEqkLiz!KeX-`=s6mL^Qn^3V*ZWd}_-t zR+sqfh|XhO#$iMU2d%yDQj|E=c9ELq_fpvsSwRFgIG#eX)JT@qZWkH{kJQhC*o;W+ z@Xuc-JdZeb>_ErqE_ZBQ3|}{9ALy$|7@!WXpzy_gn1?f!yZ-kLlJpBsN>?u$at_In zm)!mDa{L_4ydK=@cy#Y*0BZts^R{-%nBq#DIsf!!6KKgQ?YIMh z-?$=WC9aQBT&;nKct0g}Sx}b?lcb+_wjQv+JD1u|&1dkqhG0H++M zr;GoX&W{8H{mlSJ!GD|6%`UHhH#mLwltjD!>mtW$lfTBiSfxZYk1&2*`hhs3H*tTfi}!!;Z4rywA>XQHtCaaJ+iBHxGDJNfg7CCD9ShoY7NC4(Qy4v^qaTv>vJWL?4dG)Q>noN zu*wBcw0{vUqc$!1p)9G0W5iiC4#%f=X+@e%wilSuWF&}^d28%bH4y0X(N+l&M zl$BZe6_lo#&#U;-o>4-3ZFJ%|)xg%-SwR%*im>dtP3~39Sf+G0Ll=!l2MmTtwNYl* zhbu7wJhbV7KS?AmWqjq8^mL#;`f*K_balOVj%Fp;rKMqjMDE5g6k(c8aYB=Y+fa#g zZTt|!)g5Z=rQHstGTIR8j>e9cGi9>{<{e=F1^s-!CVM}%Z4+ZT&D8_hE`kJ9Um5D! z*9rTm#`Q@`HT#N>a+&=S48|4VE9ul_45F;6OhF2!&Iy{kHdiTXbCalSCu5TmDV6GL zC5XQt(@H8H!uL)B!gttlP8yvr^B2n`q0aN(%}ib%C}1k?H2p9o)*564XhJ0UI2}@7 zr>ETs*grvzPlg$ss;lOn_09>I+x~CcSJQ3CkSFDTof{qrO7MDLRH>$l+L<`#^+4K# zcX{eK&SS&lYI(T@9t#nT@%~uzj1sy1L_?nKWLU2A?Pohc6z)X^YxsX#g5Gb z#=NKaOtagrHL|B>H4alzJMUC23SpvWA>Xo5%S$>y^pA_ft6^+;i0i|i%|EiOHb%>zj9sUnsH1?- ztJ^BJ5dW42ATD=c{6=z_*%|4!K9l+RL+&P%VJ`lYHMoEL<$u+vI_h$Ig8%ZN|7RJ^ z2n6&EC@Rigv8Hmjww;QwkLPINUI^8M7}R_p`U;?-`pxnuzjhYrpC5R78Pa-3Is*?8 z3DZxnXxXOj+)FiClq4{+QuIPh@R&%Gf^S2cAhWu1UA|C~-Z%7p;Ig(hk!P&)+hjEE zF)zY#SA{wgZ3qe{dLY>vW!x+8ciVNb6*;-Jcu>o%VIo9PfwmQ%Ro&dl2ex4$6W4A# zq=&($w{^m;?p7PST+LChF+C1?$E}!P2?~!-U2cm>pWAvO;@VN2Mr$k$CKL?WK^acG@GZMQkvMsqDu(`bShln;x0f8wAsL?hY!(@gsBIN6>x>C3m3 z%e)ByWCwrpRzPWcK2q`&0aSj4CYk<7m>4tIit%{0pU&(VRy^-Lu1-+{>F~qo84hhF zK`rtV7knXLe^I0_;G&i*1NDtK30L2v?#*f-pwoEkBBM6a_GfW7*Cp*3Lx9`3_sviw z^%vuCq~7%?2eM}g86y`fYgonDOw8h~;!j+N8Lx8#okRoF*P$U_jmV%h;Kl)OVbkV9 zrlf;n$1c}Asgb$dopTpc8N}VI@y1Sq#M0MQ0u8j+jQC!*4Q}F9xZDpShtMI zO=!MPc~`*CZv6obPwg!`3XM@Ub@tp1MC9&(SdgO7_=N!a!rA%x_{#HVyQu!fPw>M6 zHOi?la}M=9LGH=P^)8zXoQ!XnhSiJ-@=i$msl~LdI|noBfB-xRan1qI{F*v;pOx;c zd89xHkjX@w&6$WHI|wIZu$iWP1{c4fER>0oq*p}7Y#!h^+@t>fQs{vGLmE$2&=(Oe zPw*V6mnxOKtNKnPbjdoFJ{|BE&*LQOF8<#Ct0W z_cyF|M_z5uY-Bu?Gtdin?o>XRgXtiU) z$zv_)5es9lvW+}+MaDm~55T)zF1GnwQB-J`gqt-IBft_=laTSXQFo=29o7|Z;*D|? z>H@&4YfIv4;Ou<24mc_;T}Dnq>z|62*vpwCbwE$exgD2sdqr4aglZbneIT)VWp62A zf&iosMtt9RJ)qKLTWygr1fyRMYv%g%VnXR=g9e23G2O1S(ej};PZ&ZC-Sa;@ z;fev6mr2C5%z%l&Nrw$fLgx!V*KfGtm~Fj2$fnbi+nDj1JN9{RMhgJ2Yyl`(fVfL3 z@498f^ObAl5RF>JR*Sh7_Pa7ftK%IU9Ii$SI0JKaQ?^_~J}=^i?5BYcx52G{+QmwG zesM5{w3V`fh9b`k~v7Z=4TopdcQ@_ftCkA@q}V z`SnbE_jo!V@C8}#QVTZftdqN_2?IREs%CfXJgP~ zahSlGvO*TC!;WA6?EJ(@`Nr?~0`*8>&n|i)R1I|IwMbPP6#PR`IU67Kl@ zNp?gD3JcCbE|T#n&<^YZWq!@^2*e-A=f=dXupSJrYGt~#96LzSIA)DMQ4eJC4~hbj zz6KTN84`c-8Ln%WuRm(}UIIvOdKdY5+AsOtB6{eUYT-t~dAOr&e}2g7`$;%o_FUVr zqj2qNm(qAjN_Bk`pf)3L@x21O1dk->aR%-18ZFYspd` zF3*2;1B6U>CM+0 zq>Qw_Bw~)XRd2|i6-s$q_3%!e2}v`UA?@d7T!53RrJ%R7Rw)~Gks##bK*o8OrZ{GN z!**tCov8A2r}BE6|ILD!^4d8N-*PB(qBB#ua||(ia2IYn4X;&Gw{H%(U@MN_&p}e( z1D;g@k6pf1C;<;|Tcn-%zincZ4=q{7Zl zRD~|%4*Vxg&%Zd~Q#>TFA&FsAAH4(H%GTg>=eUV4G-j&Ex&N?qFH#m97fK9ltdp{T z4aO*=sG7Rb^ZL`DW3a|__CU63Q8lT>xqDRnh_oZ;S(t+gfAgjuxlQu=#5W;!T*95_ zGQ>`qr)|fC)G2Xm`U7Lu&f}a;a*|C2Im#Z%9I8+V^j*BQ8Rr53d0QhM1W!JUKg3;g zK{tbma**ixcy?5)GvQeVX0kn?j%G{7CrB3e^Xy4ArfIl z<@S|?zw6?#%3E?peT~EeQZS-n_Y?*LZI_8G$z`zZRXOE~{XDx0!}+!43o|2rQ)dz3 z*gc0#z3n1WLN|WYTf2HGQG%V$*s$XaJ}Uh@JrCHST!$O*#@I(9xqB714Cz+Fb<7-# zIVS|_cm*M@{~LSfCZs%CB`F1>oTn8qaxQxP3ITJbr8tZSp7KWfroB~@xxE0j+gAvuu?2fo1 zE{MBXW`DV3H)GoODjV6Q*SDuGhmM&5NQoVOgU7CA(L(Sayk!V1wMdcGp5XH6?HZQ5 z)_@_YJ`B*)m>|~w7QSJK4^WxzzP0DnKja-W;Vos}WLz}n^x25in?qf16=)!C5-YHE zBogSQM!0ts#xiG4YJ0y;Kl=mji3#u^GJ~DadfHNP_tedK4ETb~*X0xK(Gb)UkNwtr zF~f(Q9gdhYhJnR(q5>-(u$kmCm!@Gj#d`!6V0%dv-mIbPQ2#ywQ$f#SOftY_EMyw0>P~yC$TRL)#oCuOMV@R<8D&JIRA?Bh)lR20PwWdpwixjn zwn5BEgzupue>#!Ar?wO*QfS91aVFRsj(8MTK75CjujzoxVbXA9J@qcfP?|-LAI)LL zRIkL{l0SP)xeabbGEz4#-J6WPQ{6a$ma)r(x}V&&`W;det*vccF;`C_uc+u}Tk~=j zTtXOVw-*#@iC8aXl1Z61Mjp4Nn{*XC0687!g+)!X$C_{f$$V8Pu5WaS zWQ(3ESL?r1!D!ERv$i&qUDFV%S$*qlj+nHUzW)10aDsa>9_2|+xuwE7wF5ht-y#?c zuiF3K5OqU2o&Q(EL8mRJL$q^Cb*;%+>w0aJXSLL;q4UCr1k^CI3Y^c+O=X&aXq}rI z=9NGZqJlLFojCIwguoL7Wcpwxs$p{M=Fhk$RR2C5D!8Y0{*&m(C=q(_LTF?!Le#KKuS6U=;KjFk%!)TC z0yHjLvk$zqa+>Q+m+;EfAw|y`#xCE~)Qs!Rgf7;G7o*{Mit(OHpnt(bVoW{gU~wEv8a`#9q#b@kF&uePA;Xl|bmPYRJ_ zI9E34#5|u%pt-g+$9jiJTAq#U`Z>lvsNGNqyC36n#v?G1I|KRHc|Ke#eH1O2`BAlG zi+=^pL)MJFjqE(0J9xl9$pRUNI#u_R6};vlrrbNScp^(?Ubb3*E7w7YMm{f zx1NRW{BW#Rr3eIly#T`rq!+n;a->E+N9z079}ol8Jq!86kTHQK%q2u1YfPv-Q;W-_ z)kv$jKlq_S)5hr(g{-6Ys(QY)0>$jg9|_%Z=C%ZCn|BJ$Vz$6bOZE-btVZh+v=Pdc zi-|3kW3Q}9z5EI*MvKnY)lOabZE0+l5wK@F1uO5`7$A%7*jpX)aqOCI3o$v;#ETpO z;9}g0;g_`s+TJ`$dnsWjepa>LcN`EJiPOsZWUUuv53L>vC z5j=WG;C0RwB=+^B`-A)NhBw6yQfnJ@?YtKg%4KUD0;NY*a@4nIF|$4hp96&xF<zr-3zjw&od4Lvv-g3kii;F=xKd(v|FHIoC{o#(a{;1`QoTBVNV~Cw?&R#yo zW;xyQqr2PWdxBa25Z*z}}MQBXb?hww@ z?431>#ikR?H2FeLm?87CF|%5GI4N=-p_9Fpz-{grnj`MDo66T$6OFg!b~S0IZ8VzOsUXO{_nnajX?Ta6_CY;Dy z5Evc{La8pExydP17jWn|9ac$TUS&tU(-z*{2C%0cCMBD-42;+(ikQ8 zwhx1>3-+g^H5xCU&mkmfk%j5^s8r|lQ7t3{tm;e8DNKO;rlj5%@Po2q`shi9?n^a^ zd`l>!Q*DWLIvieV*PxN5q)%wDzI3S!DlpY8*zYC@vEr%4FM2m)|n$3Jpyc> zD&S)FFv(9J7+AFl%V_zAeRDB~gOk*em&~y;AO9DHGT+;L)%6>GAfi$n|EEwt%1gDB z!LzX*|nrws%-v361SEL^+;auP?z zfY6VooH=beWxM)bs;%p2yRN;=*$+7l0onQ~xr7XA6Qj1YhwV8HmkPYMg>t&n#Y9+1 zBG;#A!Sy7DNNKZ1`v8tgDpJh7zM9j?+v@6@PEo3W$fB*_E$tI1j}u`{4$-!(5oCXm ztCKOW>f%`wCBFz_Uq6f%B>DIKW%l+3P;&A0*n6*Y*{kag;aK=;^6i`8~Ya9Krzp63!hbfu0v6CzM`BYaDxDI{$^-^$5?g;Pyz1>JGwQ- zRA-Uay@^@|`>*VE-TjHFNTFduP)mutnrBOon-TbQJcxv!<;}(EMQ>zIcwUv?S905Q%^~pz!wVf$g93;|>~@(aqhX^(t-_{$4$4qU42mtXj^M@Bv!+iG#n=>|^{T zaIc#)CYgS4H9GLsTqgMwLZz+;C>ut6_^MfNIh%8Ljdz)-`x1L;KWxUgE56PS?|Cuj z{$MJ?WA2$~7hn5&T&O;Af23OF#!}k#TgP^%fLg=vG0RW7ZMZ&zMwtPGzN43)vDm`%) z9Z6_Alhy?HULKkBC7O{glsZ?0D2#=*Y;!zr!*E_eGE0GoS*EQ1izm+~ZbT=~yZ{H> zjIp9|QA;)m;mB_?7c6r4*#w-z6<oIYc zQus_A7)V?2xK!{QFQYTEn0i5b#jbF!6w4wnbZ9u#^SE{Zp8BN5YUBL*Ggi$FhA745 zlopeSM;442o=+6IR>Q`Wbk}QjxRIucA3z)UNcEm^=Wc^PLVYpXgv&cl*vwz&OHh#; zrkzSR(5aqZIhrCn+?s)_+mWV3@k61&xHV=ly#Z8@En+_!LsSWjc?~fCeqV5vK zPngc7SNQEjJ6_^w5A~5D3x(hM<^2GlO#;%yM_|}?wDJv(HHDe?#Z3Y`Gkgm+5v)js zF3tz(I)2q=fK)~e`5I5o-me zCW;#UoDWrdfB+(qn}-*9Mw{mY2qoBVWgV(X*|1uI-PN(I!lQRaSoET0VRc^=DFNz@ z26xoeb)Uc)s>b0C={5t3`Oaw3d$QC;D-bYT*GbnTXb7^DDG^(^?l-Ka;H-FLxHSB+ zAibKvN-uI8MkB%InM!agHAqxaPH!j!orB4X&DpUWX2qG%sYvLZ`zH35VESsRm z$Ppt#)RvqGssYVw%^o}(FTC~+8>0#pnOVH`8Bu#T&1wcXlru8@-G~JElgIzfWi_FGbKv ziBS||c;0V7tU)^$WBvE?tZ-?{n!a@1c*3tsLlDb+SaO1Rpiyf*4C#$x;Jqsu`%XHL z<6|1^v4ym-^D+~Och0t|QExc>a@Vi6%iRr>2*`dw!sI%f2Cm9V8E z-EhB@!!-_7^(r1RlyHKTtNA8@JfLEHMm6x12RWI}$Z`tpyAU~?VM;W7$tt-gaBP;u zM=o_-wPk5l{C<>&tI>EyZ-H%}al+2gu!`CCGC_b6Wsa5=8f zY)WomcfX&)4IS}*X@e0Sqhxk!!r%JmzA!szWp!;n^GFo#9QcB(39~_KejB?biYdGg zO%e)lyCC~bEoNO;Ivq^rJE@uxA&w^aAhpYx}*BEIh zt+03e%pcwZY0jssnrd_Qj|f1)25s52xcJ}%ibcm*(oNCst8oqaWAijZf%}Ey({6Y6 zL^HgYN0qSAVMCof)~UO80S}IyTP9jd-?8$OAp<+>fv=h$qDD+`KV6S#YIWe#LEskz z`aUKz1=%HHiWhUcf!H05x~%!F3RRXv9qU&3d(|8NHKJBs_5|w3s!mL3Yr_uv*{x7< zs>ket!lzq6K?+m1NibKtxXHPBqoc9xST;W*Z;SB&1Dn9tsiz{&Q{}MRm+3&m3*h*8 zk0k0!rASFHo#$F4nKQ?mM@Wn4K?sTIBfz5#An+FSVrWx@{N*I{3na$xQij~*5*<}< zyH>J)l7x^h?KT%cn4m`jIfNwN4epnE)`^%mgf09Ago~_vm!VsBhb=Exo^!Dt7&JtQ za%ytlJFEGEI%#@5MN@{8qMg@#A8r}LiH-qrI}?yL@A|8!3Ea-@0BXfJmzR$Lcg_1V z)UuBxo@CjXHV!RY@Kc3{PDJ?HX-k(*GKWrT)z8mKL_+TNbTg8z2H&UF)BIBK>u$ch zx~i)=Ftbo?a-dXh*MIvXvn+T~%%=(wC=52PuL|^v_B1wN+2vYD+4@srUXS1%3%d8W zrcX?fvHTI4B@0eK?}do*-9onX^9_#R2(^c26US{EI&lJj5SzY{@d;V<&JwF1YEK#% zSp3z`Y^;XhF!LLg`mgYDX|s5EXi1>U>?xBS){^9~bnJkk}M8O-K9< zK?DeRHVJQuB4-o+EEy4Pj*mc~rom;-Hsf(IO@b-yTu52spw4FbKQ20@*4_u=s`t4C z8NU)u$MElbrFKIv844Z^#Y;%CRTTsVK^ zZO27J2jk(?vxOsW>7jiJ9@$-QC137m`Q@2yv9D?Vj9n*|2wXXwi3dpNYvrk9;|+Bz zOAefzd>GzIqIFeApY7e2p{%x2X?6ld7M@6_Y%?;X$ryix22HL!`kl$`+okvWr|gI@ z3m~p@BIr*i#BkG`87W6-u&Q)#7a5e?Itd}>9aybsoXFA9Z(ok_9}8d}ilG7+60rh$q9mAKdA^>jJ!~!hO1}cBBA;4K zKl(jNtENcH%+SZ5v5TLB*uOEGISFz?3o9e8Xud;Mw1@^Bv2U+W& zV2|!-xoR)D+P634CqnXy^?jy1pCC|FCB!lf2nVePU<8+v!yLaV1XyBIx=p5vgaGov zqh1EOE;BkbZ9FZ)-E+QaB+DtL+rT(VTm3%LWGJ@3m}BKFaLQmXbRK`01j6yrx1=@w5C`CX~!<}==%N$9qrK_NSMsx7LL%i~Ic!8Vi$fx^7>G6;tl6P71da+LLk zWWd7kd=k|v(jXdS&Q+FEET!^nck^C$umyO-L<0dXI6*as~nV)R# zItJ!>ArU;K@`#!<@zr_6Jh3|7YCUsJHq;v7O087ePeOZo`le+ydW7gs>d~w#B_#Jl zjj%BBq9IuU<2}6!sJ2T}#2oT$gF3|m;>&`w zm!j_6@yDx=i?RbLaJT1Zk2*DKy~5edd1DR&XB(F5(hIwZ`s(2kl^&8Ab&DFB^P8Q& z*-^?UGc!ZLB-$+PO#a?-V(ok$!$PP|#04UP1^F`l3*FR_61r7mTYu!NGtEEKJIEUG zkh-mprQ_n}*~6Qr??nVuUH!GpYQ=w_Oj6P`o6MPwf@{rP%HG*1hG~}Cw~o&Qub9si zhErORg#ML2UnjBmXv9b(7IkIGkwmnLl~3z=H87RME&WaBju zCm7KV2^@8v*6S74{m&is$gjH5DUJn3PR%^ZEHlAhXwH!JCH^sCv+fbJ^d#{-8 z*Pt_HXF}A`AWJal?T7o3-BBFr>!MJYv->i>flRetfVKlPyD({e8flU5=UZsf9dEJa zdRD!0G=hK_N>L#zpCYDNj9EB#x5AQitcSZ;^s)6mUsI>aeFV`5%eBB84`yLfOr&&<3;eJRk>q9}#SRrUzM#$#x2YRev)Gd{4P5 zq(UWBdYs$dB;96<%~dwXE(R|hQp8?SN&D>gKmsSw7nSTdTh0C{`w)5qlNnr; zWyBH6L_o+JIlxQ-hL9PDHYEs1DV(#g@yRx2H=B3H{X`dBKzYalo?a`}{e3 z1EOASe-d*a3^-~dgkq|GV0_WZ*+OUeVq)a`D$3E`fek>9{Kk3P(bF#GqpVluVi{I( zJZK{wvItz`F?nn#=+NZFx@HfdV#Ilx%#9`6A>EK=IM&AnU>&)gw2Zj;IBHaZ52p_uyP464JG(l^WpCOeK*&ZveYHDyjXUfFRW#NH<;Jw1+*4bQ1N;DQFt)1%&Bf z^F->*;)I7|@~LX{$4fBpw%FM8XFq`cJNwS1Lljc&mzbs(ru5N%xZ#g45;Jd(;JuUv z(xf~SD2#rN4SBmU^&LPJevZo>yq0WGeR~}Q!W&VDMt5DNwUvjnZ;DVdGsj`9>$h|6 zcgO2(ScDN{aV}dC*@;mvH#g2r=F`-@&%=SRS3~*tk9Q}BndK&V5GwpqF@hETlmf>0CUeQuif1E?W+DJSkjjyH zIk%q9(^8(4>6Wras2g8iW?ziknasQE4PHzl``#tGi0f5pfd==XY@ko=#HM}g*#}*M zT^}U9$Gn%!jACEB~ zkpepudcu4l<%r#+O(9EODpLvoNmERub{L_T9+M@iu7qd^1duq|;OS8vM1!jvL$C2V zv!2>s_xy44Npk@S)R7brc)W=20yuok7;fdh8ZjzH(XthGr{Kj`yg^iFH%Jy^w>HiW{4ZmNiD-{ z`%@Rgx=o3kVQ9@atk5_F4Xz%GDIWbc7aF(KY($M=+$NH~#H>m=@ovyD#)6xfphtXO zq$QMa-8JtgkGulOkCcpI2Qf}i6oME=t9`0FQO6uo+iP=m;t%=A^-ifNgO0fItNxVc zU|nd92v&OUSS+}uzV=Ncs&|1|Oxt0U})Trhx6JU(`#wNW{#++(TC-dJNyu2d3(j_dl)q=lY(FE@a!-bV@eOHA zY-ISsU|v5vjkRLuKm4#dpJz=iCqNKP{)gtdz#g$dmg~h%)I+r2ll)4!+vCF+TLjfj z=7e|iFxl_XIAM6TS_AP9Z)^ZzhTquLs?*UQV=m$P0mnh*0@HYAVYt(rd@%3wgzk8Q zo&;yA^Au4xrSquF1yAX86ZRKp0ys9sUD}li-RMfFor_XxR?oF&E$q!IS7i!sBIxMy z(y9)Swr15p)dx)jKH!_tDq-%c}pjJVn3<1~2&SV(94Fdfv50-ySRI-+aZqJ&k z!RF^r%ap<#?IY|N9YK^@~;k*S@os*&Uwd@;V(v=ej*^s0+hg$?;mm|u?Oz!e z2T>s_kR|y$F_DGtOhQJN$AC*XdQW!=X%sz@QiCEnPzNvjWj2AY9@J4aoZw+3Vdtd3 zGijmb;j!;c<_mph`6b641zaoPiKq0PdB_#L+r8;C1x;2-iRfP=^C);!SI}3rzw7 zMwJa7k$@JE3RW4Yj78+{91H6-#Ob%-ec~%`qC^WyA*;0^cw`tS-%Ht@3wMa@1~*C( zL5~eG`yNnoroK`$;x>z$c!pHTE)3k7c8*HfU{?A=c>YGR7@3yZ`d(1y`x#;7J^|4k z?En7Hi!A=bz)3vF%ZifVz+?_GFAa`00Pr!5!+lqp^n@jNOHCGSAB#U#nX>J;-I3;< zSs*ncXmt_VtsuhTv+W9uEj}=#{h>yha&!P!N>Jk{I~scqeN#?lMScmt>6^?Mgld{K zWT-xZ`Dx|XRFh(s>3Zy!8&mx-L_&8H9J%dyFY9jsJd~oU*3st0NxB*AIs_NNFrQOH0-g<>D>VP|sRg2ng z;DOi6;F&+G98BW=^UZjR2)RB{k_h$T7ub%&L=_!dD|Ic*NfZae9JlwbI~Cun|Bkw7rG@%D{($p2%VCJ zri#ZAm|6m|oth}j9J-Kt+@B%6^S}TqB|*v@NS-XaVfP=iXJCQXo(lk_BgIfRs0zwS<;b6dA(cZ#Fi1{BLc?U zHW`V=>lzPZ*EYg>Sg(MCda{#ds2y5}0gNKw_dk0Lyi)4xbBX0mwd@l%;8~F1zJ|6s zE2Pm{Y^$_1XQ_();FLhphF;0TR<*fT4Q+G!m_2ABUecEO_H<9*Q2cfm%eOvRst~X! zMD&5Qa=}oeAl=fWbfX0te!#ftLM4OiQ*g{rCVt$b3MO&6;_-OhX(_r(;|A-u6TdIS zQZ}I;8_UcCaNz7afYxF(*C%1VOD+Sizu7wT|B%@0L3kXnZ>5)V8tWv=jZ9YOZJv7W zAgKCA%turQY=5mJ>52&(U3#kG332!v+J{X~Q!;_8U)<{5Y4>od2_ZggXzlrnvW)w! z0kpi>KV)tD!8?Zrs*;g|zIZAUx4*|$L2XQp2a6p5Q@sgl=a+I1j*R%yoi=TkKqzzV znk&0vJ4Zv7V6l^blo1*%A|?@J21usiVS5M65}Ui_5rG>Cuydjx@SldH4AH$QFyWft z1@jZ+)Vg7k~5iYO{x3l5>Ts?on-Gc^Kzg2sylCQtSr^Z)(DodNZ2vr23lkX zcRekSmz0ALAx0MkYEr4|+NKDZjln+m+jzlinL19EoaQJ^x=xN;92Pg}nP9KjU4oks zA*%o89Owcq4mbH4ipAeU;7@#*PumxFO#-BPVk?p3e)l31$-{ew$-a|61KEv9KWA1e z*6iZkKfEpj8?CR191n>0H2knPZst~9oTG=Gr91ldb~H(p_U@UrX!Y5DZe_zHJ??Yr z2qs|KmMBvBeh=N1-^I10iGvFpZICsko06orc+(KY<&zHnQdJ9r`YI2};_v{OW+tsj zDIcq19zoZ^7*fXx3%riUNyup~;lnlK+1Rxr-%e)X`?bt=X%55BPWgnL()^SCXUEDk z$}g;4@q?py#Z{P8Xe^f1!evEaoNz2Nxw4Fbjoi|YsYj!pC$^u|8|%9^$hY0_euQoP zHm$C65N_2!lrNLomA-N7{I~?z9~?US{LhL}NQ9+|Tzwye`|06~Q#4R_e}%5980#-_ zn3eW*iS*TVDCz<7^_Z2kiG3wGi|RuS@l6J}S1TS#j>MCZf@}ZKx6TTuY}|hs5hy9# zgw=V=Yy*8sNo>r8Q1Yru9tdyOwPzHs-(|Ne0`){K64YI-aJe<9&^F=j32lhKVPcHk z2$Ju&C-_I*+=KH%d7z^g;HZ@CYel05T(=I+JAtkfoCwfE!roFoD1?tys zLC%?+iJ&i7Q!Y|WaOif?x&|@lM@lw{8}Q6a^1^qm-#5{|aR8E5KejAU{%JeBc3cdO zAw!0jYn|DP1uUqKU3G8V#c_OWhy-s~^;1051lYDJ#Yn27wInk%VggDSJlwl1K&rWn z@|n#s)+I}wW=;dCJBMZGsR<9P;qU(}n2H!b0%6EDKM4X!@H-Bal#$#a($&b;_+}#g( zCyM_SMK!w0Y){GU_?}r_w2egyJ{HQKTdY9%2RN-MLLxEFZ!F|N8DNi1_3mpdOob`P zsN2pHi>lX3XA>4)+0q&q&Tzm)Uk*w-oQI$0`JPh48~YaB5KfkNQ_k+h7B3Wm^!`o? zOPkMKiwfK8O-Wqmxw^zwYqOhK3h`qn@>)5P>&w_U@MfdnevbDq6S2`|yn0OFrCuTI z@$$7wdhc4(XX}FX9uj)FDSjzbzCj8qC^}39u&<$16o^K7BER=Kia5< zASGc>!9fbUkR8&@E|KUhmK<;a?UkVxtW2Em!sUV{=TmSB&0=IWZbl71pz04yK8t|s zOkH>1iA{PBl|XkWsuvc5bW6<}GV%wt^=^0=)KwOtN*@_=?=htTsd!{zK6sopajMz* z(FXS4wC#WqZ>0KV64tA1h_s7aZMc#I06}pX0E_otn`zE--1YYFboeWWRlq&(ArlqV z&3tykQplyR+o|x7#S_JDXGCbj*yfl@Yr_%zNVJG!BndBjcmc+=D=eYRJAsQpl~7J* z5;Zku_Vi_Cmd_Q$WFC@KV0subYvT7`CaW?do7Ic`%EndicH)ox+4h(C7%`Z)h`mZr zM4+;cw*c4+6~WSq{ZZ>&8M@E2YvSp47Xgs6B9sy}p%0vLgQVH)5|g$M znW-MNFk)R%#+DGbf)gRgc1TJZC;e6c7zfq}7RZKc_zQ0gwWDjA3c&RtFe`FkgvsAG zndKgx!1v&5A)tE~Jyq+-&bByca;6QP4#q@l%TWf-TM80HthaXQR6%*yRX@*?s&gQP zjCabZ2&zazjNm)%P0}_#Ez4tbZx)#A4RgdP^npHxAvRBP*OKr-$RN`P8b^G1iFKNf z^>*}xT9>@GUtZ!^vDux&W%i*z%x-r4Sd*XoHJKG>$=t_Rmn+ykr?zYcwwuu>6 zb_U@iq5Sz5(gG`)APcjR@2xtd57>P&zeit{*~}S_%XdblO;ooiDA#ILk#@J4L(4Ex zvyU71bBFX@`Dev;pYR_7Z{rWCx=jLl$3a#aHL`2A6C&m3zUM<{y!K8Es0Z;H#aoJkkE|c;tmx5RS+4ueKz!+5El)d_R8YJ>r8!=go z(P27=R6u^5)kc-zpX3s-1F;0}{goy-SGRpTG?hE)_ugJZ@RqOm8}=R~IRV$}L}>;H z6MeF5lP-kM;q7al^(amb-|&mOuo*yIm+u|BN9!jU4 z?>lqSEP`0YYI}^VUNq?J_K;_xk1oMfOK}-d!8dJJf$Le5A!T5IQM&tY%XgGs)cA0oxexazY+2ppR{=g>XOsfk+!H{1o_bsIj3{H190MXv7J zf7O+2Kxf_DlF$6o?Qts^gS(S(CY8FwIlw6XKpgZ!Kzm%5P^{&c6G8H`f+sbp?RU6B zLtaw!n*V3{qyqob7~2Zc-Yp%3NIAG^xqCa+l#WI0K1Wh}0o}|6CgL?uDry9ewRb5- zU#e22H6pl+<`m3s3d{Yf6~gIKi0e}Fr}?r6e_E_ahrh|6)TDsJys;vNd~)b^d*1vJ z%j-)qv7DW5_PdmgmR&DQL<}+H*=x7-Y*#3Jbp3R&OweM#<;hud@!@%yw9rdw*%H*K_IrZfQ0s7 zS=!iW1H0RUcq#Z(A!79_6%}{>yIN;rW$C(A{Gt@9_{hEl-z;i`>%%a^%z+{=TJP9z z(fu?%E>qX*x*dG)NSn;QO&zm=4@?|0QeGP z|0aKP&WcqEJEQ>3-K(cL{LcJnNM+mp-oV0g9@(5tNPwq*K?hOrF z*8a%)^dPgL3UUcZx(K`y`j*L(r3nLdx3&2m;PVMgS+oXaE8DclFiBFlqZvdja3=>R zrY7vJgLddjQ^Z)?FF{5#$77yRMl+{B(O%&vS)Uk(8L3h$lt@dJAc4m1gV!1^GQSXr zleN$2pkRv_+d0#CdnI=R1hOS>CenK%p;oEKPlGj3iI4f>By%Wn|4l1a?lCaccfb~8 zA=P$8A1)ohPwECo*#xZ3Z-S6o&CJzzmxdd`X(n!c>1%Uq{$V}O+?2S@N?fN{%4+8tlMU1- zDNIU4Z#>+t>g_OHR3Q@*0APaF5MXQyETy|XYHTojKvh*?H>cKdz|1$H?tploiQ@Ox zof>~!?V!^j4QCwqXq5xyfaFHHpb2n9W1_ZI%?N66%fo=tijnZgL{ZTntolOqg!9|#sj^((d-eod`}4dK`~xmq1Z(L9prpg!B8PSp zl(>2CmZ^;`?L0aj1|JMkIYXKgA76pOxVh3yr(Po?egdsmA3~&aj#B&e`O>syF=wt2 z9c3f-s5TSKTIFs$M1$uFI-Kxid z-}5FmC+Vjytyj9N*|gsh*%`CsW5e-;S%QF$vMXyp~^%LbJ!#ivXUMgM` z%3~%t=u96K^{JuS^o)ALW>^BVzCm4%(q=xU0btI9@)3IT@MeVs`?{S~1eWj}pR<(y zY1E&I*9&yu;RVPHdIimJ>z@80_vVw>?4546Bg+0^XpXba%rA5M!{w*7`g(r4pq*t0Iwj;gXrt=2o3G10p<9nPbv% zq5-o+!+6$*TrS)7{<(4N-Kb!xXd3e`5^$O1 zt2gWej%4jz%{HG|L1RN%`OLrfKOv^{CDs9E-`Ns_xkyE^RZ3GAUX$P@9 zl2)3m+74JhEBRF^Tprd`?7>|=^C|nSki^QSe?inPuw0lMyvJ9?FIf=vvQrP1-G#*0 zHUZws-pktNTK*s?XE`Vw!=Y=u56;}V60NI&B5fLo1Qd-TNqL?djhVecSfPX}!T!!N zb0eZnG>2sm#aq%=7rR;v-cTyoC1OY?Juw`6Ck#yy65a6*=qn*xU z&Xl>7&DJ(gR1`Yf6;_eX%-`N?Z&eO>m!Fsfm!;mW`$L4x%?y1DZ#T|WmD>AXMejOhC z=G9Y6BiTj+|3rbG>se*NkI{5bKiPaYI&dJzIkF(QJ_0sxCL~A7{EsE2nfvxdk8T0a z(U&a6ZkHf_Mj^3CaDVpXZp!joJLOo^B8AXVzl3J>Cb%?VA-J?+?R#t)OJRG!Qq?~- zLuOD+pqc~8>AOo6)pCj^-Yg4Rel{geSYa#ddrN7=vUAO-3GGTHlrApaBbk`GUk-?g{BHrh7TT6VvO9Wt#e|1+7X^wS(-A|qizR49mrdD*X1;wCJvcX#3%j@Jr#PkdzM)Km%JEQFvT@wj5Vm? z{^8pvK<#K`AHVQHn6P<&wWt5_4I96&_H)jaY~+$VCxZ?bVis-Baf=4{iBfe%4S~P} z4j^;hmFYnl#nKfCQ1I9)PH3E7S`1)YPdM!wK5C%-_e9D+tfApswqiVq`Mc1-H0ZOC z#2?vB|#KbLmuc%0B`qNsg9Q6HLZGS{=ZygyJKOgtGFlx(oV*~rGn`~ zO=}@eMG?nLa|ZZ1|7*VJFBM15_A5t)PXGV_000000000000L#i_f7ks;p=C-?CW-+ z)pr=bd_Ub{`NGxpOyCZS#EY(fff$qst-hR|GUasDlve9GcnsU9=1PMgMX_-6< zA6wou59_qum?HH&6Pt?4@5=N>x8|Z%<@0$Qhzg2hJ>uN8y&7Qz^#-(R`#alL&E>tu z+3RQ`=J3(8p(JdIWNIR~X_NyqUqKW6GpeW>JKx+gls`)qb#T?&P<5cFao&3*_sF0@ z)uE8TNqlWv&s5zdc*tZ$cZv(?W7frqR=f-5gJ)(Vv)g+Bwl7~3omFl0?~SWa;Oj4& z!`pv0RxR?{)^ZzizByes)tSGyFTFjzG^JHwzb85$2MWERn^OQW)T!9_9SmYz2p#?{ zy;L|A@IfIZ(z&HV&+lJ2kRCx$83^ zJ;&&sW~0FVUF39`lz$J*QInOp%O4-kDpjOe8koEO{HhF927G+IhzlG zF{+4ZjuF~}TkeKi#{nNlYe3AAI1Kn>9J4dBlCfg{wVc zqH1S1rV$9pbIY1-y?U2ugmC3DWa+KARH8$ZgZQm&TkGtGlftRWC`<4wzTA&CBviWs2PVbCUTWY3R#CTdX9Ft5SgI_EL%g^UhP01yC_h8c@K zk_MoPCxODY^+R=PKZCzOZU9Xl*=F_|NY#|WEoyXm1vQBFFDkY2pgI^eBhyH1TgQdc z`wyuyy3`jJ8+HMhlzgG=8kG@%0spL&Y2MhVjR;H%TTv4$ER&P|6P@gZM`$BDm-6g3 zyAtesNg*ahgm1+KDGY+AEmV`R?RzHTb0LB#q+0dYeEH7Y`+xKYia~g7p7sYsw*cwS zat-eMcU_FQp>WK90?nYLam%sZe#UyWf&T$`E2$^^8^w8`*q_@y|0Viq4%9Z>bAAaP z{+L-!Qx@|~&XIrthv2<-_2>XF&Acy+v~w>C%63H2ICXE>`;ebJ-E;6+7X#dkakyAH zPMANCkoPwC&pQ@M_s5u3e4H&aUNA)2W&c&nmdwgLnRTUoA3{2>z(2!uBr%2!-128T z)+j3!Z~cY>=nPGiX-D+5}2>;14S(Jo34mo7k z$AEhVM_Ay7$n-v`=}q!*I2W%lvby48@W-jyCN3*>&>TS5p)s{ZdPpeKFe+U+YgfJ8 znXxjI!H%>}-i^WaZMS*fOga$Nak;gJ?+G(!l6OauuUmg5(*gmwo%Q`QTG8uFKmyMoe7gj$ymWvxh+RTvNLkqnVQK^46=1>AN#S0&0`cHRVsn zbHriUg@*Zn9=V40%E_ubwpmXS$f5in7*|NIY2e*)<4oCH(iO^Q9@B*9GIn?KvyWe` zEk)o;s^&N$O{Lb%?H%Z0(kO9Qwq+esrYYSbn6^=rvdm_|aX#qELUb}|OVxM)GDKw` zbpPC1hwSc3_xUi~dIXD-NhW7zsPx@T&8fVZ>WZ^CbB;;FOR<%KG^+tj{!0Cedkipk z(R=4|e@N(Zy*a6kgUoWsVDQv;w50|CZ-+V~YsMO=2b1I`aDh?V=){Rww9khlRaCBN zFJj)cm|1DC!5Dj;{0sNB8WU{97IW_uzPsKLNm|>4%JT<6vgn5sfK5WchX}Car)*oo zqPBNG)l2XSwMxo@2w6k*vLKjZNi0Y#GyyAJcl>0?XI-4Fiq`3uP77wojQf+T)_tum zT}#`7C~OYL1S29vu5gbU<9@!|%^wuC>gVP@C}rLiLlfS+Q2ZWJh9${GGqM_>YDd4B zB9+8`!07A&qz`|`s6?z!?-aA;rxgk8J+FvGjjYK5gId-7%3|maTLR2VnJ?A~k$$jV zMOs1LW&J^}&#}`Okh9FAwMLpB449HMzWl85usZ2V&?=JtvZ4|LA+{bF^EC(e&|>zl zTVWZ-O57cM#eWsXCL2HXdYV`9~?RG7qY@;7cI?zO6fVD0yZh2w`9~@3}-!6|8 zMVBG!0{qLY&H;LW4DMDKacr{@+JFAMJ8^X{6xUWg{ z0h(;vT;HRcZIIbj%=);(rx1@wI0ncxMMzD&Jky8eSHSBI?$g|#n*(a=Fy#)Pw^gJ2 zh@MOLDbc3~+p~Q=5~od2(Z=I(4Z5qqxnbn(NC9d9Fn2U-z7Vma&h}N(3W#3=#@r|% zB`(~uF(=WP%9f3Fb_sM)5JBx=!^U@GQJ%8o>0?XRRuC_MsJTY*tb`tAlR<@-#w79b zYy?5XEcp+$-rz+~AE%mMfXL!+B+F!^FKEQmyG6=<9e#NiA`B3pK7$eI!N_18ub80# zs8AhdsfwP3Xl8*~yThw2#&^*%V3->GxvGwLNc@lx$LLGW7TkzQSPq5b&qh}2c&qr5 zNn6w4<-d-O?+9Vshxe-K|EQ!!LJy{Z6am-M`*fJ#%T24*M{2`BirP7G@Wwl#5T?$- zg8)RRuZvPo@liH|m0-b2o2cfhqBKhZ zs10-pZDw5D14ppAlyyCawK}Mjz+BhBTZla(l?jLYQcUu@iZb75at{gPSvq{#a9+l! zCXO5Rqz1bOBCQ6MLeCo4CA}laGTWSz0B!Eyk9f8QylY~Vu0eyc2=R1^m?{BRs*J-Q z=-BpdaGLuNYU2JVZs2~eNak}q*iqJww_Tc5CQ`WQ%`05ghVA~hTXX*okp5$e^)nCP zSNDsMmrDs58-;P9&Fxc=7zUZEeZT5NWd8c0T@_MXjAGF@ZlWkFWnAy)EbBm+n|JFx z{w5^I82}s+O{aRKy|%^8VZTeV%jQORj%?iQx0#)tYU0|Hpvo2b+s-{=0*F^PvBj`egC@aI zWG6AeDhrNNK{gGNAW$ByH(wNaa{772Q6(qR79vuhO3I~qx&LKmuAe1n1p@V!FRh}K z1wFb4x&9I$-IfaO?Pzg0TBhN*v8AR1GnaJdsd(URjQ81FRb>v9UmoD~bEWs9nOcUd zuC;*cJvdh#WS}H1&%4;i=BDA@37y@|P0Nzhw`I4t*3pqDVo5*m!ZdbRJ!e&7u^<85 zigc!t(f3wBFJ-935%cpwo-LdG-nFf>AL;Q{i!qDKw%0qMxp`P3A6Jy3LXZ?On(WK| z=@K^2d~$P1(jBY^S_AcV_mB@CRm^zc`q9jOo>v+mK8c?br6X9qEL)BnwhHdBsrM12 zrY!B8bzC%1bGe=1Ua^UB%ae5pA^l|A|I-oXUInh0q%d<_Y+u+D2~j+elF2A8ZoL(p zaLu|ZCTHcmBZ7H6`TeKA*XgP{;vUp@1{_T;2J&%&r~4C&(=i^J4az(*^X79AsZmvl z0EwTx&mvR})z!zzW0yvhuyu~eXx~%MKN3$hoHpllfD5&{$LcwYZQ8S$G1Zc{Yx#<1 z`gM%1P&}u`5-`K|%ta`pv_>I0U<5m{KVJz7IEng24V|$qABN)wz(P*8AUC&{ zfcaA}w@_K@edh$$MkaQTZI|RFury5^@Fkos*xQP6`H&0?LW1h&-cp_SqZVuedww6W zecW=sA|MI0DBDw*mm~$wvBTQy@Tni}${qZdJ$lJLJABlAmR6u#>8vIXImnhzl_TTL zABkQJstdnhJH$9UrIVG%L;0HVC(>zZRAFwk_Te=hM=P?=B5ru2T1f^QS>b`VaIDgX zrpO0*`FAC{rSs_ndqah1QnF|^$?xxcHF7vabYfpdHTz9&Y_zNcWJL2iKp#-PLkT@l zBjodp!fw;o>2Ab^8;LxkDWL2NXJt&s_XeOdO7oJHh2K>JcWYiB5>@?xQZE#o!T&c< zK5k<^8hm7LZ>z6#NkVcz<(VhrHTgJjspALW*`A8(6sb+u0gfN0w z%NB#0z5K(6bO0=TJnGeyN~N^zl$jmxEPQl`s2F2Sm3OOU^GBHa9=*SIzfoc*}=%SNZ7Nh!qZeQ(A)e zY393mKKWKR3h&YaCAf=>5+5i(o004MPzOl z;<+MS<=>`_E4Yx2&$mW*gb$PXPnrY%K>$;~N~Tf}c^g-MCK?Q+SA*G(MCr-j)VxJR z6I6r;eJj-wf_6a8G{2G$tmD~i&q&Gu)!BLx6GM+6OLe%*X>_UQbQ~8wVb*-}LJt78 ziujo-+={S*Doy{GA?cY8A*D9BG7xHR?$fM%Lc*V50QPQhS;Ji2Tj#>%W9Hb&>)Y?BnIy2z}Hi@WAZKpvzWJ=ro$5 zWq0oewef9ZMFVp`pZGta*n+*ixJFZt-}ShF?;gjj1}KN%NVxtFXUS)gZhoxe&( z@bhTZny#_s0P;e!D+7}akr!JYq|wgr03&D9#d#8Pz-1H+fiHaOE`;;L^kK89eJ$H{ z{jds;$Og+#d>M?|0i|X!r~=L4f5tR<52X@83=zEJ7;hR}7Z^d;T2_2ytCwXQ9&441 z|IfD~cRB<(Rtsd=50h*^>_yaP;uOv$YyzkAEq2UVgJNd`*-|McCXuD+8|Ro)Z5IOG zj!w;IA`Ge!VEjeo>+=Ip3Gew!c-Aff05vgLC(Q|t#Q^0XiP7bEq4L{ttX+=Kz__6Y zbwv7*4+H*1qm&i(-H&1Np{%yNz^x231P8mzCTIEkX3JC#O<&QuFwoU?QQEZ>2k<{8 z_#sor%ww`yc0Hgn*=Xk1q5)?YeImZeS&Eqc)6S<&oM~$xsK)|%2$sFCxv?}KLbz}g z^}i&x9M&XAPNnIB1zbZX0!CAX&sUT3YsmwQd-I-*hI|C^WEtn=K|}hbn*;5pG9ryM zPn4dTlqD-MiS@M_&k$tI`FX;yOtJMGBSxk! zC4$Mf@&qtvD!}5F?#h6A5;ueEC%&IuhnH}L4OrI*-cwft^WLEps5B|=Vz zflEG_gX8JpVz zal00IvY)Yt7q`PSFx+19AHfS5Dv?6tDn?{H|Kzq*up4=PEh~Nj0~fIbUI>p5br}I< z3kQm?Uq&uOF!h4lxFn3ciAnZ&(zytD@j{P1&2R@AV5iJdutWa`q7S5#FXq5Vq*U4G zJ`{+Pr|$b;feXlnQU{}K9Zo?i&JYenj?4NTQe03^Q2Y4D!xG$RQ2b;z1gNO`VKQ#V zhkF`x9Sl;#auRU0 zqh@WGxCZH;my>W7+fGMHqZr3uk2wzUc(N{JwtHa51^&KxkZybeKhEpD)feDv)>;^a zeQ(Nnf;IlU=W~bH!CL1e5B}^t1~&Z;`XdCLo+E8~-t?-;NSDJb->dGn!T~^?tL*U1 zNixl>l(?ALAFO-JNlfCHQ56@wBm?ZV$+b|nYwrtovj_yOnb57~OtzYZ2v`FT_&eOy z6pW~=MsLZ3NL2Ox5sk?qN61z=u~DDV+rIKeSD1=Mz>lf-=lXL&4r^#Elr9p|llZ+l zJmwhHCq|fbbq0P;IdcDUMa|hN!4JVZgUfwu)8#jS#MyH2&m4{wxb5)gNtKc%LKCS) z5-W*`e&+Kjp2#7|C?zOe1AU*qtI2Wp|67{TF{Hoe%511ftZdU(B{V(S#u^YK?XgHTHDNJ$^3kVt6j;y;7z*KzRSN=cGZ(M0cS9hTI4*wgfCn>rRDJF zr->;uaJX3w!{d8?LXipS%%07)DEYKko04>kv>h%#!PSV8flz8+*GI-#HxeWs<-6!r zlyK5&e|^;yP(%j8kY01H9_J!z*sJUNT23nV4|Gp84;NNW)*7Nem-A zHe4>oeZBXRg!H6-%guPr=Ilm|R~{^=oh{UmI*CO_n7fZUZZ;`kU`u52I_FD6e_#mM z>L2bb#uAj;-LLtr7fDSV4$*Y_Y~1x=o)nr;bFopx6+|217qDd{bd-uo@@i1M1`{tj zyQ;{7P6ET`yFC_?7aX@Of2n>(?#m|>F@+T4+fW0Fj2gIGt_fA$-!$O&A6MRskw+|s zn|E-wg$@oRMXwLxb6i_}jd0% zSHA?y;ltv8UIA@fp#N5-C#xYcncKKN@$&m*Ktg+2V(^20AKAqL)j#iCn$k<$Xl;q!o z9~3z*45aUEA{cZaXo7VfWFC>zb~37L zECetxISGZ5*l+qed$gyXhu$J2N#lV3^VZ5d%za>B0sXVH9ux&3*`60?J7s+Y!hA+F z6OfH4PY=8qD{)0#<}4sDoQsVA(F8Bs=Crr{t5FWG%17uG%$p^D&gncD;>DJFRwTq? z3*qRR-{hGxBR6#UpCo8w@o!k$TXu0!w6b^^QjdQb`$ndC+5X_0VRi?f9XI@;7l*&@ z?BC!)6_W8IXFVMnq#GUF{8CWFisOW`C8EOZII}>J=M0i`BAL`%OieJsw<^XHpsAz; zDTH>Hy*Zhv7Q(~tmjIh0>f~ZwYu1YX#IDhy$9@)YX$XS4VjrP}itoxU`mb#0Yj*5# zuY?}r7~dODIP=hxisG^w0b4Q=2TIVHHX_t9l&LF051{58Oz(#C29Wi@NJ0VS{JlYM znBzJ4A<(Fz^09^WY#MaB)`md9XExT}c*%OIjczzA1|%=erHJH|W-|9~`bHdykZYdu zgBjt{)s$s>d>Uj*^e5#+D~~Gi;%h0%vOP5Hq2B&Lt78xG<>vi?ku7N?vj#D~Hzu#)%HTemv>-%>6ZQ^4VPj zLAF!FO1d=)WnO|WTXsQ9YwJ zcOVrE@4S*{duSXNfod=G3IEe5ofLi5VzG9w)f3s!k}Sf#HgU|#tKqA|9Kkedn0S=x zj9}D%^IGPO;%J*9i8tvmzo(or1qIu_-j5%6-e`!H#pojyKd9zn$(@y$Mn{-7&@pF$ z>B`7&__f%X&Ex6CFX>I^nj7PAU!=q%URgr{o2Xwr<2eKb>;aj^@yvG=eo}CA7Gh=t zN-i4miHL7xXEydQjwU^7@g84E?H$cK_#%8;1HxLE7pwo^ITf2v`=xYWjNtKzZ zO7Ee)Z!i?DY;+&PgUmwF=oD1Fvd2AfwY&Vr!`BrHKhzzf~DxOw#OwB?tlL zBj!ynicHr6ift%7Zj1GK>pZ`0Di?OYjO0wmvz6JXr`3Fh zogDDLl5HJG9^&e-JZn5VwUgHFg4y{FtCXG#lht!`J-ifkNxAa?LOJl`=3wM-BAK2D zKZXB{M+mv3InRlA_0QgZV{a^&iH4G{duWG+sCS3*Wg@G$(75%sac{c5NGr8;JXYTs zQ`nTt6xAkZrnZLW2*%u&beW9430Q*erQ@bp^)Onk!|HQSmrM80TS(#;8nAg=sX za_F#BN_?iO_^_=W-whlt&9jQ(H}b*e~VQhdl8-Y+JX zF^m;Ew(S~!`SVksQa~n`&ke{f6u`j#9l^iYr-({8dfm%nFjvd-^E42)z&I2xL-+!4 zA#w)P9uvd;joSJMfy&7Tx6O$mm$cygBm8Isko|MskymZ6)%v)@sVHvBZ#*tR#4ah; z@7mZXvrF@QL#w#f^EH%+KRSdl^yvHxKeF@mt;nyOJ10~upK7>HA|k=&qK3mEdts}d zUExZaAE(WY-z=sr)M#Q`Ld-XzLbdTlHPld`M(nc>=N@J(&!sBr^}b&@f7AM2-G6X= zV?rVW4ka?Tn;iSN?&#mIBZpXQ3iJ@fP>*ly1?Y97c1*=O%i{!utvzoPT89r_L8p)4 zCL z-0uU>S}OP;I{dv9Ef?JQ^@k<#e1T6O*XKYo-o$WOFy$W@e4pTcknqWL#WUhWIj)2h zlZVY(>^AqOD{OpBQ_aU0dM~MgcinEhfz2h7*75>v7wJ9|+?hHy;*|?Tm6#2go@K}@tWmpd( zDjC%L1M&G7#P66Kw=ql5^M6xwk%(I%5b1t@lqC!efFEmFBkTe5Le5hKZ8rw$1H0P_ zFj!mZ^3CZHT#j_(ailSx3DPDdci~o6O2Y4NXwssjxF6CE$Y64#vFD{_&mA(IOWB;F z_(DG-U-$WSeaGR2CV7!6BFE`{N(&ALyc=)MXfm++tfUjf(H{lC=&%LlQ31svG8&{5 zh~~PIVc^5)|6@hzO&A1!*$QIdwB3JL9&<{tcd$3g7)zr61(WyNJUHMPnZOfnGl<_I z+wRq9<9Q|tE`;$}1pU$LHxDqR_24|^$t|Uz=3ju!112I!JMg%d!IjakU6o&PB$=LC zqLtJoPmjB1wXu)r4v3ljNS<_0ojlF&hmF<(tY3;>u_^D$BH>Rfm{ATW5ZRWXp=Ea` zI!)f*!N4}AYR zA5}QAA3_zYw#~T`TQ9i%0&wOK4?hwO4y;?^HdEs{&N3oDJThef002wV_CooCH;V$3 z^8{+34Idq*J`E15^R6l(8ccmc>!O~18d;QLL66y@TiJi!;rs9hKZ+&1z4@WBYzqX! zc*gt`8NZ~D_EayqE0w5~Rr7N!s}4{ZC;srvG90$%?fxGsqbP|s-o(pRWfAw0I&iYM z4J5^$X|`m2A7_eilRa|v5E_tw5+P*oI3iBvwS^@0_a*(8Yw}8deN-^aSQn%)f|33S ztmaq^aHR%|uRjrX5`O`Gy(%`hY;#}X$gGYW{2lfC5h%7bek9C8dEctcf(PfrStb;I zkF6w*Vr?a-q)&W#Fw>v0Zy zqSunvdUlBG9LpY)<({$cfAln>*|NN)Ay5@>km_PBYKqv|9O#_tWA&p9ZE(Y|{4@26 zo6R<>8jZ!51U#&w9g?44mjG%$F}L(8B0xzey%6(-UN$jYkfW@XK~K~~B}24ldZmB? zWUk7+e99fJSM&#}8UT_bqToLlDaXs~g!eeRvGxM67z;Byen$&Z`fMZ%HzZnAd~Vyt zW*0~%U1VbMotb?>T(+Rfg_hP2XH;r#^nIU7@~?Nv?*d@SY!eKFEQ`v?!$Rl}q19Jn zTmqOF57_QkC0Nod7Gz5OfLny`%%cT>JVlSEen$AXyZcKtz~RQ1iBG9US$XZjqWJNJ z+GJj1;fo$zZc`0BQkyR0aj1THQ{35QrpR*X`*NIXmkcU9D}Xa6v~6+ZM4e17dCZNN YFEx`t^y);7PY}Kk000000000007+8eHvj+t literal 0 HcmV?d00001 diff --git a/muagent/base_configs/prompts/simple_prompts.py b/muagent/base_configs/prompts/simple_prompts.py index d804f21..373ae10 100644 --- a/muagent/base_configs/prompts/simple_prompts.py +++ b/muagent/base_configs/prompts/simple_prompts.py @@ -214,14 +214,7 @@ # 目标 # 我希望你根据输入文本，提供一个输入文本中流程、操作的结构化json表示。可以参考以下步骤思考，但是不要输出每个步骤中间结果，只输出最后的流程图json： 1. 确定流程图节点: 根据输入文本内容，确定流程图的各个节点。节点可以用如下结构表示： -{ - "nodes": { - "节点序号": { - "type": "节点类型", - "content": "节点内容" - }, - } -} +{"nodes":{"节点序号":{"type":"节点类型","content":"节点内容"},}} 其中 nodes 用来存放抽取的节点，每个 node 的 key 通过从0开始对递增序列表示，value 是一个字典，包含 type 和 content 两个属性， type 对应下面定义的四种节点类型，content 为抽取的节点内容。 节点类型定义如下: Schedule: @@ -237,17 +230,10 @@ Analysis节点只能连接在Phenomenon节点之后。 2. 连接流程图节点: 根据输入文本内容，确定流程图的各个节点的连接关系。节点之间的连接关系可以用如下结构表示： -{ - "edges": [ - { - "start": "起始节点序号", - "end": "终止节点序号" - } - ] -} +{"edges":[{"start":"起始节点序号","end":"终止节点序号"}]} edges 用来存放节点间的连接顺序，它是一个列表，每个元素是一个字典，包含 start 和 end 两个属性， start 为起始 node 的 节点序号, end 为结束 node 的 节点序号。 -3. 生成表示流程图的完整json: 将上面[确定流程图节点]和[连接流程图节点]步骤中的结果放到一个json，检查生成的流程图是否符合给定输入文本的内容，优化流程图的结构，合并相邻同类型节点，返回最终的json。 +3. 生成表示流程图的完整json: 将上面[确定流程图节点]和[连接流程图节点]步骤中的结果放到一个json，检查生成的流程图是否符合给定输入文本的内容，优化流程图的结构，合并相邻同类型节点，返回最终的json。注意返回的json内容要紧凑，不要包含额外的空格和换行符。 ############# # 风格 # 流程图节点数尽可能少，保持流程图结构简洁，相邻同类型节点可以合并。流程图节点中的节点内容content要准确、详细、充分。 @@ -260,20 +246,7 @@ ############# # 响应 # 返回json结构定义如下: -{ - "nodes": { - "节点序号": { - "type": "节点类型", - "content": "节点内容" - } - }, - "edges": [ - { - "start": "起始节点序号", - "end": "终止节点序号" - } - ] -} +{"nodes":{"节点序号":{"type":"节点类型","content":"节点内容"}},"edges":[{"start":"起始节点序号","end":"终止节点序号"}]} ############# # 例子 # 以下是几个例子： @@ -283,56 +256,7 @@ 1. 通过观察sofagw网关监控发现，BOLT失败数突增 2. 且失败曲线与退保成功率曲线相关性较高，判定是网络问题。 -输出:{ - "nodes": { - "0": { - "type": "Schedule", - "content": "排查网络问题" - }, - "1": { - "type": "Task", - "content": "查询sofagw网关监控BOLT失败数" - }, - "2": { - "type": "Task", - "content": "查询sofagw网关监控退保成功率" - }, - "3": { - "type": "Task", - "content": "判断两条时序相关性" - }, - "4": { - "type": "Phenomenon", - "content": "相关性较高" - }, - "5": { - "type": "Analysis", - "content": "网络问题" - } - }, - "edges": [ - { - "start": "0", - "end": "1" - }, - { - "start": "1", - "end": "2" - }, - { - "start": "2", - "end": "3" - }, - { - "start": "3", - "end": "4" - }, - { - "start": "4", - "end": "5" - } - ] -} +输出:{"nodes":{"0":{"type":"Schedule","content":"排查网络问题"},"1":{"type":"Task","content":"查询sofagw网关监控BOLT失败数"},"2":{"type":"Task","content":"查询sofagw网关监控退保成功率"},"3":{"type":"Task","content":"判断两条时序相关性"},"4":{"type":"Phenomenon","content":"相关性较高"},"5":{"type":"Analysis","content":"网络问题"}},"edges":[{"start":"0","end":"1"},{"start":"1","end":"2"},{"start":"2","end":"3"},{"start":"3","end":"4"},{"start":"4","end":"5"}]} # 例子2 # 输入文本:二、使用模版创建选品集 @@ -346,40 +270,7 @@ STEP3：按需调整指标模版内的值，完成选品集创建 -输出:{ - "nodes": { - "0": { - "type": "Schedule", - "content": "使用模版创建选品集" - }, - "1": { - "type": "Task", - "content": "创建选品集\n\n注：因为只能选择同类型模版，必须先选择数据类型，才能选择模版创建" - }, - "2": { - "type": "Task", - "content": "按需选择模版后，点击确认\n\n - 我的收藏：个人选择收藏的模版 \n\n - 我的创建：个人创建的模版 \n\n - 模版广场：公开的模版，可以通过名称/创建人搜索到需要的模版并选择使用" - }, - "3": { - "type": "Task", - "content": "按需调整指标模版内的值，完成选品集创建" - } - }, - "edges": [ - { - "start": "0", - "end": "1" - }, - { - "start": "1", - "end": "2" - }, - { - "start": "2", - "end": "3" - } - ] -} +输出:{"nodes":{"0":{"type":"Schedule","content":"使用模版创建选品集"},"1":{"type":"Task","content":"创建选品集\n\n注：因为只能选择同类型模版，必须先选择数据类型，才能选择模版创建"},"2":{"type":"Task","content":"按需选择模版后，点击确认\n\n - 我的收藏：个人选择收藏的模版 \n\n - 我的创建：个人创建的模版 \n\n - 模版广场：公开的模版，可以通过名称/创建人搜索到需要的模版并选择使用"},"3":{"type":"Task","content":"按需调整指标模版内的值，完成选品集创建"}},"edges":[{"start":"0","end":"1"},{"start":"1","end":"2"},{"start":"2","end":"3"}]} # 例子3 # 输入文本:Step1 @@ -402,53 +293,51 @@ - 申请二级场景数据权限，由对应二级场景管理员审批。若二级场景管理员为@小花@小映，按一级场景走申请流程。 - 二级管理员为下图蓝色框②所在位置查看 -输出:{ - "nodes": { - "0": { - "type": "Schedule", - "content": "场景权限申请" - }, - "1": { - "type": "Task", - "content": "点击右侧的左右切换箭头，找到自己所在的站点或业务模块" - }, - "2": { - "type": "Task", - "content": "查询对应一级场景，若没有所需一级场景则联系 [@小明][@小红]添加：具体操作如下：\n 发送邮件给小明和小红，抄送小白，邮件内容包括项目背景，场景名称，场景描述，数据类型和业务管理员" - }, - "3": { - "type": "Task", - "content": "查询对应二级场景，若没有所需二级场景则联系一级场景管理员添加，支持通过搜索二级场景名称和ID快速查询二级场景" - }, - "4": { - "type": "Task", - "content": "申请二级场景数据权限，由对应二级场景管理员审批。若二级场景管理员为@小花@小映，按一级场景走申请流程" - } - }, - "edges": [ - { - "start": "0", - "end": "1" - }, - { - "start": "1", - "end": "2" - }, - { - "start": "2", - "end": "3" - }, - { - "start": "3", - "end": "4" - } - ] -} +输出:{"nodes":{"0":{"type":"Schedule","content":"场景权限申请"},"1":{"type":"Task","content":"点击右侧的左右切换箭头，找到自己所在的站点或业务模块"},"2":{"type":"Task","content":"查询对应一级场景，若没有所需一级场景则联系 [@小明][@小红]添加：具体操作如下：\n 发送邮件给小明和小红，抄送小白，邮件内容包括项目背景，场景名称，场景描述，数据类型和业务管理员"},"3":{"type": "Task","content":"查询对应二级场景，若没有所需二级场景则联系一级场景管理员添加，支持通过搜索二级场景名称和ID快速查询二级场景"},"4":{"type":"Task","content": "申请二级场景数据权限，由对应二级场景管理员审批。若二级场景管理员为@小花@小映，按一级场景走申请流程"}},"edges":[{"start":"0","end":"1"},{"start":"1","end":"2"},{"start":"2","end":"3"},{"start":"3","end":"4"}]} + +# 例子4 # +输入文本:场景：组织一次公司活动 +1.确定活动主题： +确定活动的主要目的（如团建、庆祝活动等）。 + +2.选择活动类型： + +-选项A：户外活动 +选择具体的户外活动（如远足、烧烤、运动会）。 +确定地点和时间。 +安排交通工具和安全措施。 +联系供应商（如餐饮、设备租赁）。 +发出邀请通知。 + +-选项B：室内活动 +选择具体的室内活动（如会议、晚会、游戏）。 +确定场地和时间。 +准备相关的设备（如投影仪、音响）。 +安排餐饮和娱乐节目。 +发出邀请通知。 + +3.预算审核： +计算活动预估费用。 +提交预算给管理层审核。 + +4.活动宣传： +制作宣传材料（如海报、邮件通知）。 +在公司内部推广活动信息。 + +5.活动实施： +根据选择的活动类型，执行相关安排。 +进行现场协调（无论是户外还是室内）。 + +6.活动反馈： +收集参与者的反馈意见。 +总结活动的成功之处和改进建议。 + +输出:{"nodes":{"0":{"type":"Schedule","content":"组织一次公司活动"},"1":{"type":"Task","content":"确定活动主题：确定活动的主要目的（如团建、庆祝活动等）"},"2":{"type":"Task","content":"选择活动类型"},"3":{"type":"Phenomenon","content":"户外活动"},"4":{"type":"Phenomenon","content":"室内活动"},"5":{"type":"Task","content":"选择具体的户外活动（如远足、烧烤、运动会），确定地点和时间，安排交通工具和安全措施，联系供应商（如餐饮、设备租赁），发出邀请通知"},"6":{"type":"Task","content":"选择具体的室内活动（如会议、晚会、游戏），确定场地和时间，准备相关的设备（如投影仪、音响），安排餐饮和娱乐节目，发出邀请通知"},"7":{"type":"Task","content":"预算审核：计算活动预估费用，提交预算给管理层审核"},"8":{"type":"Task","content":"活动宣传：制作宣传材料（如海报、邮件通知），在公司内部推广活动信息"},"9":{"type":"Task","content":"活动实施：根据选择的活动类型，执行相关安排，进行现场协调（无论是户外还是室内）"},"10":{"type":"Task","content":"活动反馈：收集参与者的反馈意见，总结活动的成功之处和改进建议"}},"edges":[{"start":"0","end":"1"},{"start":"1","end":"2"},{"start":"2","end":"3"},{"start":"2","end":"4"},{"start":"3","end":"5"},{"start":"4","end":"6"},{"start":"5","end":"7"},{"start":"6","end":"7"},{"start":"7","end":"8"},{"start":"8","end":"9"},{"start":"9","end":"10"}]} ############# # 开始抽取 # 请根据上述说明和例子来对以下的输入文本抽取结构化流程json: -输入文本:{text} +输入文本: {text} 输出:''' diff --git a/muagent/connector/schema/message.py b/muagent/connector/schema/message.py index e2fa0ce..3868b43 100644 --- a/muagent/connector/schema/message.py +++ b/muagent/connector/schema/message.py @@ -89,10 +89,10 @@ def check_message_index(cls, values): message_index = values.get("message_index") chat_index = values.get("chat_index") if message_index is None or message_index == "": - values["message_index"] = str(uuid.uuid4()) + values["message_index"] = str(uuid.uuid4()).replace("-", "_") if chat_index is None or chat_index == "": - values["chat_index"] = str(uuid.uuid4()) + values["chat_index"] = str(uuid.uuid4()).replace("-", "_") return values diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index ce42900..c26ecf1 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -559,7 +559,7 @@ def update_nodes(self, nodes: List[GNode], teamid: str): if len(tbase_missing_nodeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") for nodeid in tbase_missing_nodeids: - r = self.tb.search(f"@node_id: {nodeid}", index_name=self.node_indexname) + r = self.tb.search(f"@node_id: {nodeid.replace('-', '_')}", index_name=self.node_indexname) teamids_by_nodeid.update({data['node_id']: data["node_str"] for data in r.docs}) tb_result = [] @@ -859,7 +859,7 @@ def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str= sls_nodes, sls_edges = [], [] for node_idx, node_info in node_edge_dict['nodes'].items(): node_type = node_info['type'].lower() - node_id = str(uuid.uuid4()) + node_id = str(uuid.uuid4()).replace("-", "_") node_info['node_id_new'] = node_id ekg_slsdata = EKGGraphSlsSchema( From 3c8569c7e38e65aa3857a253cf8ef5254c748981 Mon Sep 17 00:00:00 2001 From: lightislost Date: Thu, 5 Sep 2024 13:57:24 +0800 Subject: [PATCH 034/128] [update reamd][update license version] --- README.md | 8 ++++---- README_zh.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4f86f98..f58d6a6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

ZH doc EN doc - License + License Open Issues @@ -38,7 +38,7 @@ This framework has been validated in multiple complex DevOps scenarios within An ![](docs/resources/ekg-arch-en.webp) -## 🚀 Quick Start +## 🚀 QuickStart For complete documentation, see: [CodeFuse-muAgent](https://codefuse.ai/docs/api-docs/MuAgent/overview/multi-agent) For more [demos](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_examples) @@ -75,7 +75,7 @@ you can see [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_ -## Features +## 🧭 Features - EKG Builder：Through the design of virtual teams, scene intentions, and semantic nodes, you can experience the differences between online and local documentation, or annotated versus unannotated code handover. For a vast amount of existing documents (text, diagrams, etc.), we support intelligent parsing, which is available for one-click import. - EKG Assets：Through comprehensive KG Schema design—including Intention Nodes, Workflow Nodes, Tool Nodes, and Character Nodes—we can meet various SOP Automation requirements. The inclusion of Tool Nodes in the KG enhances the accuracy of tool selection and parameter filling. Additionally, the incorporation of Characters (whether human or agents) in the KG allows for human-involved process advancement, making it flexible for use in multiplayer text-based games. - EKG Reasoning：Compared to purely model-based or entirely fix-flow Reasoning, our framework allows LLM to operate under human guidance-flexibility, control, and enabling exploration in unknown scenarios. Additionally, successful exploration experiences can be summarized and documented into KG, minimizing detours for similar issues. @@ -83,7 +83,7 @@ you can see [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_ - Memory：Unified message pooling design supports categorized message delivery and subscription based on the needs of different scenarios, like multi-agent. Additionally, through message retrievel, rerank and distillation, it facilitates long-context handling, improving the overall question-answer quality. - ActionSpace：Adhering to Swagger protocol, we provide tool registration, tool categorization, and permission management, facilitating LLM Function Calling. We offer a secure and trustworthy code execution environment, and ensuring precise code generation to meet the demands of various scenarios, including visual plot, numerical calculations, and table editing. -## Contribution +## 🤗 Contribution We are deeply grateful for your interest in the Codefuse project and warmly welcome any suggestions, opinions (including criticism), comments, and contributions. Feel free to raise your suggestions, opinions, and comments directly through GitHub Issues. There are numerous ways to participate in and contribute to the Codefuse project: code implementation, writing tests, process tool improvements, documentation enhancements, etc. diff --git a/README_zh.md b/README_zh.md index 4c6f60d..c8d7313 100644 --- a/README_zh.md +++ b/README_zh.md @@ -7,7 +7,7 @@

ZH doc EN doc - License + License Open Issues @@ -24,7 +24,7 @@ - [🤝 介绍](#-介绍) - [🚀 快速使用](#-快速使用) - [🧭 关键技术](#-关键技术) -- [🤗 贡献](#-贡献) +- [🤗 贡献指南](#-贡献指南) - [🗂 其他](#-其他) - [📱 联系我们](#-联系我们) @@ -77,7 +77,7 @@ pip install codefuse-muagent - 记忆管理：统一消息池设计，支持各类场景所需分门别类消息投递、订阅，隔离且互通，便于多Agent场景消息管理使用；同时面向超长上下文，支持消息检索、排序、蒸馏，提升整体问答质量 - 操作空间：遵循Swagger协议，提供工具注册、权限管理、统一分类，方便LLM在工具调用中接入使用；提供安全可信代码执行环境，同时确保代码精准生成，满足可视绘图、数值计算、图表编辑等各类场景诉求 -## 贡献指南 +## 🤗 贡献指南 非常感谢您对 Codefuse 项目感兴趣，我们非常欢迎您对 Codefuse 项目的各种建议、意见（包括批评）、评论和贡献。 您对 Codefuse 的各种建议、意见、评论可以直接通过 GitHub 的 Issues 提出。 From 583667c55669e68d2be4273537bc3bcc100ee794 Mon Sep 17 00:00:00 2001 From: lightislost Date: Thu, 5 Sep 2024 14:36:49 +0800 Subject: [PATCH 035/128] [update readme][update release content] --- README.md | 14 +++++++------- README_zh.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f58d6a6..cc711b5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ ## 🔔 News - [2024.04.01] codefuse-muAgent is now open source, featuring functionalities such as knowledge base, code library, tool usage, code interpreter, and more -- [2024.09.05] we release muAgent v2.0 about EKG (An Innovative Agent Framework Driven By KG Engine). +- [2024.09.05] we release muAgent v2.0 - EKG: An Innovative Agent Framework Driven By KG Engine. @@ -76,12 +76,12 @@ you can see [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_ ## 🧭 Features -- EKG Builder：Through the design of virtual teams, scene intentions, and semantic nodes, you can experience the differences between online and local documentation, or annotated versus unannotated code handover. For a vast amount of existing documents (text, diagrams, etc.), we support intelligent parsing, which is available for one-click import. -- EKG Assets：Through comprehensive KG Schema design—including Intention Nodes, Workflow Nodes, Tool Nodes, and Character Nodes—we can meet various SOP Automation requirements. The inclusion of Tool Nodes in the KG enhances the accuracy of tool selection and parameter filling. Additionally, the incorporation of Characters (whether human or agents) in the KG allows for human-involved process advancement, making it flexible for use in multiplayer text-based games. -- EKG Reasoning：Compared to purely model-based or entirely fix-flow Reasoning, our framework allows LLM to operate under human guidance-flexibility, control, and enabling exploration in unknown scenarios. Additionally, successful exploration experiences can be summarized and documented into KG, minimizing detours for similar issues. -- Diagnose：After KG editing, visual interface allows for quick debugging, and successful Execution path configurations will be automatically documented, which reduces model interactions, accelerates inference, and minimizes LLM Token costs. Additionally, during online execution, we provide comprehensive end-to-end visual monitoring. -- Memory：Unified message pooling design supports categorized message delivery and subscription based on the needs of different scenarios, like multi-agent. Additionally, through message retrievel, rerank and distillation, it facilitates long-context handling, improving the overall question-answer quality. -- ActionSpace：Adhering to Swagger protocol, we provide tool registration, tool categorization, and permission management, facilitating LLM Function Calling. We offer a secure and trustworthy code execution environment, and ensuring precise code generation to meet the demands of various scenarios, including visual plot, numerical calculations, and table editing. +- **EKG Builder**：Through the design of virtual teams, scene intentions, and semantic nodes, you can experience the differences between online and local documentation, or annotated versus unannotated code handover. For a vast amount of existing documents (text, diagrams, etc.), we support intelligent parsing, which is available for one-click import. +- **EKG Assets**：Through comprehensive KG Schema design—including Intention Nodes, Workflow Nodes, Tool Nodes, and Character Nodes—we can meet various SOP Automation requirements. The inclusion of Tool Nodes in the KG enhances the accuracy of tool selection and parameter filling. Additionally, the incorporation of Characters (whether human or agents) in the KG allows for human-involved process advancement, making it flexible for use in multiplayer text-based games. +- **EKG Reasoning**：Compared to purely model-based or entirely fix-flow Reasoning, our framework allows LLM to operate under human guidance-flexibility, control, and enabling exploration in unknown scenarios. Additionally, successful exploration experiences can be summarized and documented into KG, minimizing detours for similar issues. +- **Diagnose**：After KG editing, visual interface allows for quick debugging, and successful Execution path configurations will be automatically documented, which reduces model interactions, accelerates inference, and minimizes LLM Token costs. Additionally, during online execution, we provide comprehensive end-to-end visual monitoring. +- **Memory**：Unified message pooling design supports categorized message delivery and subscription based on the needs of different scenarios, like multi-agent. Additionally, through message retrievel, rerank and distillation, it facilitates long-context handling, improving the overall question-answer quality. +- **ActionSpace**：Adhering to Swagger protocol, we provide tool registration, tool categorization, and permission management, facilitating LLM Function Calling. We offer a secure and trustworthy code execution environment, and ensuring precise code generation to meet the demands of various scenarios, including visual plot, numerical calculations, and table editing. ## 🤗 Contribution We are deeply grateful for your interest in the Codefuse project and warmly welcome any suggestions, opinions (including criticism), comments, and contributions. diff --git a/README_zh.md b/README_zh.md index c8d7313..1ccecd1 100644 --- a/README_zh.md +++ b/README_zh.md @@ -18,7 +18,7 @@ ## 🔔 更新 - [2024.04.01] CodeFuse-muAgent 开源，支持知识库、代码库、工具使用、代码解释器等功能 -- [2024.09.05] muAgent v2.0 全新版本, 实现了由知识图谱引擎驱动的创新Agent框架 +- [2024.09.05] 发布 muAgent v2.0 - EKG：一款由知识图谱引擎驱动的创新代理框架 ## 📜 目录 - [🤝 介绍](#-介绍) @@ -70,12 +70,12 @@ pip install codefuse-muagent ## 🧭 关键技术 -- 图谱构建：通过虚拟团队构建、场景意图划分，让你体验在线文档VS本地文档的差别；同时，文本语义输入的节点使用方式，让你感受有注释代码VS无注释代码的差别，充分体现在线协同的优势；面向海量存量文档（通用文本、流程画板等），支持文本智能解析、一键导入 -- 图谱资产：通过场景意图、事件流程、统一工具、组织人物四部分的统一图谱设计，满足各类SOP场景所需知识承载；工具在图谱的纳入进一步提升工具选择、参数填充的准确性，人物/智能体在图谱的纳入，让人可加入流程的推进，可灵活应用于多人文本游戏 -- 图谱推理：相比其他Agent框架纯模型推理、纯人工编排的推理模式，让大模型在人的经验/设计指导下做事，灵活、可控，同时面向未知局面，可自由探索，同时将成功探索经验总结、图谱沉淀，面向相似问题，少走弯路；整体流程唤起支持平台对接（规则配置）、语言触发，满足各类诉求 -- 调试运行：图谱编辑完成后，可视调试，快速发现流程错误、修改优化，同时面向调试成功路径，关联配置自动沉淀，减少模型交互、模型开销，加速推理流程；此外，在线运行中，我们提供全链路可视化监控 -- 记忆管理：统一消息池设计，支持各类场景所需分门别类消息投递、订阅，隔离且互通，便于多Agent场景消息管理使用；同时面向超长上下文，支持消息检索、排序、蒸馏，提升整体问答质量 -- 操作空间：遵循Swagger协议，提供工具注册、权限管理、统一分类，方便LLM在工具调用中接入使用；提供安全可信代码执行环境，同时确保代码精准生成，满足可视绘图、数值计算、图表编辑等各类场景诉求 +- **图谱构建**：通过虚拟团队构建、场景意图划分，让你体验在线文档VS本地文档的差别；同时，文本语义输入的节点使用方式，让你感受有注释代码VS无注释代码的差别，充分体现在线协同的优势；面向海量存量文档（通用文本、流程画板等），支持文本智能解析、一键导入 +- **图谱资产**：通过场景意图、事件流程、统一工具、组织人物四部分的统一图谱设计，满足各类SOP场景所需知识承载；工具在图谱的纳入进一步提升工具选择、参数填充的准确性，人物/智能体在图谱的纳入，让人可加入流程的推进，可灵活应用于多人文本游戏 +- **图谱推理**：相比其他Agent框架纯模型推理、纯人工编排的推理模式，让大模型在人的经验/设计指导下做事，灵活、可控，同时面向未知局面，可自由探索，同时将成功探索经验总结、图谱沉淀，面向相似问题，少走弯路；整体流程唤起支持平台对接（规则配置）、语言触发，满足各类诉求 +- **调试运行**：图谱编辑完成后，可视调试，快速发现流程错误、修改优化，同时面向调试成功路径，关联配置自动沉淀，减少模型交互、模型开销，加速推理流程；此外，在线运行中，我们提供全链路可视化监控 +- **记忆管理**：统一消息池设计，支持各类场景所需分门别类消息投递、订阅，隔离且互通，便于多Agent场景消息管理使用；同时面向超长上下文，支持消息检索、排序、蒸馏，提升整体问答质量 +- **操作空间**：遵循Swagger协议，提供工具注册、权限管理、统一分类，方便LLM在工具调用中接入使用；提供安全可信代码执行环境，同时确保代码精准生成，满足可视绘图、数值计算、图表编辑等各类场景诉求 ## 🤗 贡献指南 非常感谢您对 Codefuse 项目感兴趣，我们非常欢迎您对 Codefuse 项目的各种建议、意见（包括批评）、评论和贡献。 From 78018b6340b083a432801df67995c87ce8fefe49 Mon Sep 17 00:00:00 2001 From: lightislost <31849436+lightislost@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:45:33 +0800 Subject: [PATCH 036/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc711b5..4c90263 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 中文  ｜  English 

-#

CodeFuse-muAgent: An Innovative Agent Framework Driven By KG Engine

+#

muAgent: An Innovative Agent Framework Driven by KG Engine

ZH doc From 3901a717e881c167b673a9af5c2959cfe67cea19 Mon Sep 17 00:00:00 2001 From: lightislost <31849436+lightislost@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:46:06 +0800 Subject: [PATCH 037/128] Update README_zh.md --- README_zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_zh.md b/README_zh.md index 1ccecd1..2d04f70 100644 --- a/README_zh.md +++ b/README_zh.md @@ -2,7 +2,7 @@ 中文  ｜  English 

-#

CodeFuse-muAgent: An Innovative Agent Framework Driven By KG Engine

+#

muAgent: An Innovative Agent Framework Driven by KG Engine

ZH doc From 6b2166587936d3cbbe2ce66b62c8ae91356fd8ac Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Thu, 5 Sep 2024 19:47:47 +0800 Subject: [PATCH 038/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c90263..2d9145e 100644 --- a/README.md +++ b/README.md @@ -94,5 +94,5 @@ We welcome any contribution and will add you to the list of contributors. See [C ## 🗂 Miscellaneous ### 📱 Contact Us

- 图片 + 图片
From 7e316838737255a325802a7ff7a3469737b90a34 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Thu, 5 Sep 2024 20:03:07 +0800 Subject: [PATCH 039/128] Update README.md --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2d9145e..e055971 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ ## 🔔 News -- [2024.04.01] codefuse-muAgent is now open source, featuring functionalities such as knowledge base, code library, tool usage, code interpreter, and more -- [2024.09.05] we release muAgent v2.0 - EKG: An Innovative Agent Framework Driven By KG Engine. +- [2024.04.01] muAgent is now open source, focusing on multi-agent orchestration and collaborating with technologies such as FunctionCall, RAG, and CodeInterpreter. +- [2024.09.05] we release muAgent v2.0: An Innovative Agent Framework Driven By KG Engine. @@ -32,11 +32,13 @@ ## 🤝 Introduction - +

A brand new Agent Framework driven by LLM and EKG(Eventic Knowledge Graph, Industry Knowledge Carrier)，collaboratively utilizing MultiAgent, FunctionCall, CodeInterpreter, etc. Through canvas-based drag-and-drop and simple text writing, the large language model can assists you in executing various complex SOP under human guidance. It is compatbile with existing frameworks on the market and can achieve four core differentiating technical functions: Complex Reasoning, Online Collaboration, Human Interaction, Knowledge On-demand. This framework has been validated in multiple complex DevOps scenarios within Ant Group. At the sametime, come and experience the Undercover game we quickly built! - -![](docs/resources/ekg-arch-en.webp) +

+
+ muAgent Architecture +
## 🚀 QuickStart For complete documentation, see: [CodeFuse-muAgent](https://codefuse.ai/docs/api-docs/MuAgent/overview/multi-agent) From 48e9fae4d80fa7138d6417b15eec41b33cc16092 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Thu, 5 Sep 2024 20:06:07 +0800 Subject: [PATCH 040/128] Update README_zh.md --- README_zh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README_zh.md b/README_zh.md index 2d04f70..c1cd872 100644 --- a/README_zh.md +++ b/README_zh.md @@ -17,8 +17,8 @@ ## 🔔 更新 -- [2024.04.01] CodeFuse-muAgent 开源，支持知识库、代码库、工具使用、代码解释器等功能 -- [2024.09.05] 发布 muAgent v2.0 - EKG：一款由知识图谱引擎驱动的创新代理框架 +- [2024.04.01] muAgent正式开源，聚焦多agent编排，协同FunctionCall、RAG、CodeInterpreter等技术 +- [2024.09.05] 发布 muAgent v2.0 - EKG：一款由知识图谱引擎驱动的创新Agent框架 ## 📜 目录 - [🤝 介绍](#-介绍) From aa6a1fff8a2e93eee562a17e3a3903024d0236c9 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:59:43 +0800 Subject: [PATCH 041/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e055971..f0b340a 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ A brand new Agent Framework driven by LLM and EKG(Eventic Knowledge Graph, Indu This framework has been validated in multiple complex DevOps scenarios within Ant Group. At the sametime, come and experience the Undercover game we quickly built!

- muAgent Architecture + muAgent Architecture
## 🚀 QuickStart From 747d15b37e09bae327d712d989a85ef53bc0c215 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:37:48 +0800 Subject: [PATCH 042/128] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f0b340a..f4d577e 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,11 @@ you can see [docs](https://codefuse.ai/docs/api-docs/MuAgent/connector/customed_ - **ActionSpace**：Adhering to Swagger protocol, we provide tool registration, tool categorization, and permission management, facilitating LLM Function Calling. We offer a secure and trustworthy code execution environment, and ensuring precise code generation to meet the demands of various scenarios, including visual plot, numerical calculations, and table editing. ## 🤗 Contribution -We are deeply grateful for your interest in the Codefuse project and warmly welcome any suggestions, opinions (including criticism), comments, and contributions. +Thank you for your interest in the muAgent project! We genuinely appreciate your feedback and invite you to share your suggestions, insights (including constructive criticism), and contributions. -Feel free to raise your suggestions, opinions, and comments directly through GitHub Issues. There are numerous ways to participate in and contribute to the Codefuse project: code implementation, writing tests, process tool improvements, documentation enhancements, etc. +To facilitate this process, we encourage you to submit your feedback directly through GitHub Issues. There are numerous ways to engage with and contribute to the muAgent project, including code implementation, test development, documentation enhancements, and more. -We welcome any contribution and will add you to the list of contributors. See [Contribution Guide...](https://codefuse-ai.github.io/contribution/contribution) +We welcome all forms of contributions and look forward to recognizing your efforts by adding you to our list of contributors. For more details, please refer to our [Contribution Guide](https://codefuse-ai.github.io/contribution/contribution). ## 🗂 Miscellaneous From a4ee39d83d0b9c609aa87731189d7520c03fa4e5 Mon Sep 17 00:00:00 2001 From: wyp311395 Date: Thu, 12 Sep 2024 16:10:01 +0800 Subject: [PATCH 043/128] [features][add utils] --- MANIFEST.in | 1 + docker-compose.yaml | 4 +- .../ekg_construct/ekg_construct_base.py | 4 +- muagent/utils/common_utils.py | 121 +++++++++++++++++- setup.py | 4 +- 5 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..809ffae --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +exclude tests/* \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 66ad9a0..746631e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -129,7 +129,7 @@ services: - 11434:11434 volumes: # - //d/models/ollama:/root/.ollama # windows path - - /Users/yunjiu/ant/models:/root/.ollama # linux/mac path + - /Users/wangyunpeng/Downloads/models:/root/.ollama # linux/mac path networks: - ekg-net restart: on-failure @@ -170,4 +170,4 @@ services: networks: ekg-net: # driver: bridge - external: true + # external: true diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index c26ecf1..ca05073 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -608,7 +608,7 @@ def update_edges(self, edges: List[GEdge], teamid: str): gb_result = [] for edge in edges: # if node.id not in update_tbase_nodeids: continue - SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) + SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) # todo bug，数据不一致问题 DSTID = edge.attributes.pop("DSTID", None) or double_hashing(edge.end_id) resp = self.gb.update_edge( SRCID, DSTID, @@ -699,7 +699,7 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N # tmp iead to filter by teamid nodes = [node for node in nodes if str(teamid) in str(node.attributes)] # select the node which can connect the rootid - nodes = [node for node in nodes if len(self.search_rootpath_by_nodeid(node.id, node.type, f"ekg_team:{teamid}").paths)>0] + nodes = [node for node in nodes if len(self.search_rootpath_by_nodeid(node.id, node.type, f"ekg_team_{teamid}").paths)>0] return nodes def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: diff --git a/muagent/utils/common_utils.py b/muagent/utils/common_utils.py index 0dea3d4..977420b 100644 --- a/muagent/utils/common_utils.py +++ b/muagent/utils/common_utils.py @@ -1,13 +1,17 @@ -import textwrap, time, copy, random, hashlib, json, os +import time, hashlib, json, os from datetime import datetime, timedelta from functools import wraps from loguru import logger from typing import * from pathlib import Path from io import BytesIO -from fastapi import Body, File, Form, Body, Query, UploadFile +from fastapi import UploadFile from tempfile import SpooledTemporaryFile import json +import signal +import contextlib +import sys +import socket DATE_FORMAT = "%Y-%m-%d %H:%M:%S" @@ -124,4 +128,115 @@ def string_to_long_sha256(s: str) -> int: def double_hashing(s: str, modulus: int = 10e12) -> int: hash1 = string_to_long_sha256(s) hash2 = string_to_long_sha256(s[::-1]) # 用字符串的反序进行第二次hash - return int((hash1 + hash2) % modulus) \ No newline at end of file + return int((hash1 + hash2) % modulus) + + +@contextlib.contextmanager +def timer(seconds: Optional[Union[int, float]] = None) -> Generator: + """ + A context manager that limits the execution time of a code block to a + given number of seconds. + The implementation of this contextmanager are borrowed from + https://github.com/openai/human-eval/blob/master/human_eval/execution.py + + Note: + This function only works in Unix and MainThread, + since `signal.setitimer` is only available in Unix. + + """ + if ( + seconds is None + or sys.platform == "win32" + or threading.currentThread().name # pylint: disable=W4902 + != "MainThread" + ): + yield + return + + def signal_handler(*args: Any, **kwargs: Any) -> None: + raise TimeoutError("timed out") + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + + try: + # Enter the context and execute the code block. + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +class ImportErrorReporter: + """Used as a placeholder for missing packages. + When called, an ImportError will be raised, prompting the user to install + the specified extras requirement. + The implementation of this ImportErrorReporter are borrowed from + https://github.com/modelscope/agentscope/src/agentscope/utils/common.py + """ + + def __init__(self, error: ImportError, extras_require: str = None) -> None: + """Init the ImportErrorReporter. + + Args: + error (`ImportError`): the original ImportError. + extras_require (`str`): the extras requirement. + """ + self.error = error + self.extras_require = extras_require + + def __call__(self, *args: Any, **kwds: Any) -> Any: + return self._raise_import_error() + + def __getattr__(self, name: str) -> Any: + return self._raise_import_error() + + def __getitem__(self, __key: Any) -> Any: + return self._raise_import_error() + + def _raise_import_error(self) -> Any: + """Raise the ImportError""" + err_msg = f"ImportError occorred: [{self.error.msg}]." + if self.extras_require is not None: + err_msg += ( + f" Please install [{self.extras_require}] version" + " of agentscope." + ) + raise ImportError(err_msg) + + +def _find_available_port() -> int: + """ + Get an unoccupied socket port number. + The implementation of this _find_available_port are borrowed from + https://github.com/modelscope/agentscope/src/agentscope/utils/common.py + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def _check_port(port: Optional[int] = None) -> int: + """Check if the port is available. + The implementation of this _check_port are borrowed from + https://github.com/modelscope/agentscope/src/agentscope/utils/common.py + + Args: + port (`int`): + the port number being checked. + + Returns: + `int`: the port number that passed the check. If the port is found + to be occupied, an available port number will be automatically + returned. + """ + if port is None: + new_port = _find_available_port() + return new_port + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + if s.connect_ex(("localhost", port)) == 0: + raise RuntimeError("Port is occupied.") + except Exception: + new_port = _find_available_port() + return new_port + return port \ No newline at end of file diff --git a/setup.py b/setup.py index 4dc32d1..1553d9c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="codefuse-muagent", - version="0.0.5", + version="0.1.0", author="shanshi", author_email="wyp311395@antgroup.com", description="A multi-agent framework that facilitates the rapid construction of collaborative teams of agents.", @@ -35,6 +35,8 @@ "notebook", "docker", "sseclient", + "Levenshtein", + "urllib3==1.26.6", # "chromadb==0.4.17", "javalang==0.13.0", From 6a8fa07bdf54c978b95448dc22258d2e755412bb Mon Sep 17 00:00:00 2001 From: wyp311395 Date: Fri, 13 Sep 2024 10:38:22 +0800 Subject: [PATCH 044/128] [bugfix][graph db search circle bug] --- .gitignore | 3 ++- docker-compose.yaml | 6 +++--- .../graph_db_handler/geabase_handler.py | 17 +++++++++++++---- .../vector_db_handler/tbase_handler.py | 2 +- .../service/ekg_construct/ekg_construct_base.py | 4 ++-- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 63afe5c..fe898c4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ dist .ipynb_checkpoints zdatafront* *antgroup* -*ipynb \ No newline at end of file +*ipynb +*log \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 746631e..efe7c0e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '3.4' +version: 'Beta' services: metad0: # image: docker.io/vesoft/nebula-metad:v3.8.0 @@ -129,7 +129,7 @@ services: - 11434:11434 volumes: # - //d/models/ollama:/root/.ollama # windows path - - /Users/wangyunpeng/Downloads/models:/root/.ollama # linux/mac path + # - /Users/wangyunpeng/Downloads/models:/root/.ollama # linux/mac path networks: - ekg-net restart: on-failure @@ -170,4 +170,4 @@ services: networks: ekg-net: # driver: bridge - # external: true + external: true \ No newline at end of file diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index cfcd2f0..9832b55 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -301,17 +301,20 @@ def deduplicate_paths(self, result, block_attributes: List[dict] = {}, select_at # deduplicate the paths path_strs = ["&&".join(_p) for _p in p] new_p = [] + add_path_strs = set() for path_str, _p in zip(path_strs, p): + if path_str in add_path_strs: continue if not any(path_str in other for other in path_strs if path_str != other): new_p.append(_p) + add_path_strs.add(path_str) # 根据保留路径进行合并 nodeid2type = {i["id"]: i["type"] for i in n0+n1} unique_node_ids = [j for i in new_p for j in i] if reverse: - last_node_ids = list(set([i[0] for i in new_p if len(i)>=hop])) + last_node_ids = list(set([i[0] for i in new_p if len(i)>=hop and i[0] not in i[1:]])) else: - last_node_ids = list(set([i[-1] for i in new_p if len(i)>=hop])) + last_node_ids = list(set([i[-1] for i in new_p if len(i)>=hop and i[-1] not in i[:-1]])) last_node_types = [nodeid2type[i] for i in last_node_ids] new_n0 = deduplicate_dict([i for i in n0 if i["id"] in unique_node_ids]) @@ -393,12 +396,17 @@ def decode_result(self, geabase_result, gql: str) -> Dict: def decode_path(self, col_data, k) -> List: steps = col_data.get("pathVal", {}).get("steps", []) connections = {} - for step in steps: + head = None + path = [] + for idx, step in enumerate(steps): props = step["props"] start = props["original_src_id1__"].get("strVal", "") or props["original_src_id1__"].get("intVal", -1) end = props["original_dst_id2__"].get("strVal", "") or props["original_dst_id2__"].get("intVal", -1) connections[start] = end + head = start if idx==0 else head + path = [start] if idx==0 else path + # 找到头部（1） for k in connections: if k not in connections.values(): @@ -409,7 +417,8 @@ def decode_path(self, col_data, k) -> List: while head in connections: head = connections[head] path.append(head) - + if head == path[0]: + break return path def decode_vertex(self, col_data, k) -> Dict: diff --git a/muagent/db_handler/vector_db_handler/tbase_handler.py b/muagent/db_handler/vector_db_handler/tbase_handler.py index 1f46ce2..71517b2 100644 --- a/muagent/db_handler/vector_db_handler/tbase_handler.py +++ b/muagent/db_handler/vector_db_handler/tbase_handler.py @@ -63,7 +63,7 @@ def insert_data_hash( self, data_list: Union[list[dict], dict], key: str = "message_index", - expire_time: int = 86400, + expire_time: int = None, need_etime: bool = True ): ''' diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index ca05073..3937bce 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -630,8 +630,8 @@ def get_graph_by_nodeid( ) -> Graph: if hop<2: raise Exception(f"hop must be smaller than 2, now hop is {hop}") - if hop >= 14: - raise Exception(f"hop can't be larger than 14, now hop is {hop}") + if hop >= 20: + raise Exception(f"hop can't be larger than 20, now hop is {hop}") # filter the node which dont match teamid result = self.gb.get_hop_infos( {'id': nodeid}, node_type=node_type, From d1d0dd47d380bf8b0f1d39183093b2b72de6e3f3 Mon Sep 17 00:00:00 2001 From: wyp311395 Date: Mon, 23 Sep 2024 11:07:27 +0800 Subject: [PATCH 045/128] [features-service][add atomic crud] --- docker-compose.yaml | 4 +- examples/muagent_examples/codechat_example.py | 12 +- .../prompts/intention_template_prompt.py | 8 +- muagent/connector/agents/base_agent.py | 2 +- muagent/connector/agents/executor_agent.py | 2 +- muagent/connector/memory_manager.py | 4 +- muagent/connector/phase/base_phase.py | 3 +- .../graph_db_handler/geabase_handler.py | 25 +- .../graph_db_handler/nebula_handler.py | 6 + muagent/schemas/ekg/ekg_graph.py | 22 +- .../ekg_construct/ekg_construct_base.py | 394 ++++++++--- muagent/service/ekg_inference/__init__.py | 5 + .../ekg_inference/intention_match_rule.py | 15 - .../service/ekg_inference/intention_router.py | 447 ++++++++----- tests/service/ekg_construct_test_3.py | 621 ++++++++++++++++++ tests/service/intention_router_test.py | 187 ++++++ 16 files changed, 1474 insertions(+), 283 deletions(-) create mode 100644 muagent/service/ekg_inference/__init__.py create mode 100644 tests/service/ekg_construct_test_3.py create mode 100644 tests/service/intention_router_test.py diff --git a/docker-compose.yaml b/docker-compose.yaml index efe7c0e..395de46 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -127,9 +127,9 @@ services: TZ: "${TZ}" ports: - 11434:11434 - volumes: + # volumes: # - //d/models/ollama:/root/.ollama # windows path - # - /Users/wangyunpeng/Downloads/models:/root/.ollama # linux/mac path + # - /Users/xxxx/Desktop/ollama:/root/.ollama # linux/mac path networks: - ekg-net restart: on-failure diff --git a/examples/muagent_examples/codechat_example.py b/examples/muagent_examples/codechat_example.py index 9fefc95..a256ae7 100644 --- a/examples/muagent_examples/codechat_example.py +++ b/examples/muagent_examples/codechat_example.py @@ -31,11 +31,11 @@ embeddings = None logger.error(f"{e}") -# # test local code -# src_dir = os.path.join( -# os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# ) -# sys.path.append(src_dir) +# test local code +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) from muagent.base_configs.env_config import CB_ROOT_PATH from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.connector.phase import BasePhase @@ -58,7 +58,7 @@ # delete codebase codebase_name = 'client_local' -code_path = "D://chromeDownloads/devopschat-bot/client_v2/client" +code_path = "{add_your_code_path}" # initialize codebase use_nh = True do_interpret = True diff --git a/muagent/base_configs/prompts/intention_template_prompt.py b/muagent/base_configs/prompts/intention_template_prompt.py index b177253..f6836b1 100644 --- a/muagent/base_configs/prompts/intention_template_prompt.py +++ b/muagent/base_configs/prompts/intention_template_prompt.py @@ -97,15 +97,17 @@ def get_intention_prompt( INTENTIONS_CONSULT_WHICH = [ - ('整体计划查询', '用户询问关于某个解决方案的完整流程或步骤，包含但不限于“整个流程”、“步骤”、“流程图”等词汇或概念。'), - ('下一步任务查询', '用户询问在某个解决方案的特定步骤中应如何操作或处理，通常会提及“下一步”、“具体操作”、“如何做”等，且明确指向解决方案中的某个特定环节。'), + ('整体计划查询', '用户想要获取某个问题的答案，或某个解决方案的完整流程（步骤）。'), + ('下一步任务查询', '用户询问某个问题或方案的特定步骤，通常会提及“下一步”、“具体操作”等'), ('闲聊', '用户询问的内容与当前的技术问题或解决方案无关，更多是出于兴趣或社交性质的交流。') ] CONSULT_WHICH_PROMPT = get_intention_prompt( '作为运维领域的客服，您的职责是根据用户询问的内容，精准判断其背后的意图，以便提供最恰当的服务和支持。', INTENTIONS_CONSULT_WHICH, { + '如何组织一次活动？': '整体计划查询', '系统升级的整个流程是怎样的？': '整体计划查询', + '为什么我没有收到红包？请告诉我方案': '整体计划查询', '听说你们采用了新工具，能讲讲它的特点吗？': '闲聊' } ) @@ -119,7 +121,7 @@ def get_intention_prompt( INTENTIONS_WHETHER_EXEC, { '为什么我的优惠券使用失败？': '执行', - '我想知道如何才能更好地优化我的服务器性能，你们有什么建议吗？': '询问' + '为什么我的优惠券使用失败？请告诉我方案': '询问' } ) diff --git a/muagent/connector/agents/base_agent.py b/muagent/connector/agents/base_agent.py index 148307c..b323360 100644 --- a/muagent/connector/agents/base_agent.py +++ b/muagent/connector/agents/base_agent.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Union,Generator import importlib import re, os import copy diff --git a/muagent/connector/agents/executor_agent.py b/muagent/connector/agents/executor_agent.py index 1e39756..b9b081a 100644 --- a/muagent/connector/agents/executor_agent.py +++ b/muagent/connector/agents/executor_agent.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Union, Generator import copy import uuid from loguru import logger diff --git a/muagent/connector/memory_manager.py b/muagent/connector/memory_manager.py index 22018de..1b95138 100644 --- a/muagent/connector/memory_manager.py +++ b/muagent/connector/memory_manager.py @@ -1,6 +1,6 @@ from abc import abstractmethod, ABC from typing import List, Dict -import os, sys, copy, json, uuid +import os, sys, copy, json, uuid, random from jieba.analyse import extract_tags from collections import Counter from loguru import logger @@ -238,7 +238,7 @@ def __init__( # self.unique_name = unique_name # self.memory_type = memory_type self.db_config = db_config - self.vb_config = vb_config + self.vb_config = vb_config or VBConfig(vb_type="LocalFaissHandler") self.gb_config = gb_config self.tb_config = tb_config self.do_init = do_init diff --git a/muagent/connector/phase/base_phase.py b/muagent/connector/phase/base_phase.py index d1ef766..d4713f2 100644 --- a/muagent/connector/phase/base_phase.py +++ b/muagent/connector/phase/base_phase.py @@ -92,7 +92,8 @@ def __init__( ) else: self.memory_manager: BaseMemoryManager = LocalMemoryManager( - unique_name=phase_name, do_init=True, kb_root_path = kb_root_path, embed_config=embed_config, llm_config=llm_config + # unique_name=phase_name, + do_init=True, kb_root_path = kb_root_path, embed_config=embed_config, llm_config=llm_config ) self.conv_summary_agent = BaseAgent( role=role_configs["conv_summary"].role, diff --git a/muagent/db_handler/graph_db_handler/geabase_handler.py b/muagent/db_handler/graph_db_handler/geabase_handler.py index 9832b55..0f5d92d 100644 --- a/muagent/db_handler/graph_db_handler/geabase_handler.py +++ b/muagent/db_handler/graph_db_handler/geabase_handler.py @@ -25,11 +25,24 @@ def __init__( self.project = gb_config.extra_kwargs.get("project") self.city = gb_config.extra_kwargs.get("city") self.lib_path = gb_config.extra_kwargs.get("lib_path") - - GeaBaseEnv.init(self.lib_path) - self.geabase_client = GeaBaseClient( - self.metaserver_address, self.project,self.city - ) + self.graph_name = gb_config.extra_kwargs.get("graph_name") + + try: + GeaBaseEnv.init(self.lib_path) + except Exception as e: + logger.error(f"{e}") + + if self.graph_name: + self.geabase_client = GeaBaseClient( + self.metaserver_address, self.project,self.city, + graph_name=self.graph_name + ) + self.hop_max = 6 + else: + self.geabase_client = GeaBaseClient( + self.metaserver_address, self.project,self.city, + ) + self.hop_max = 8 # option 指定 self.option = GeaBaseEnv.QueryRequestOption.newBuilder().gqlType(GeaBaseEnv.QueryProtocol.GQLType.GQL_ISO).build() @@ -227,7 +240,7 @@ def get_hop_infos(self, attributes: dict, node_type: str = None, hop: int = 2, b ''' hop >= 2， 表面需要至少两跳 ''' - hop_max = 8 + hop_max = self.hop_max # where_str = ' and '.join([f"n0.{k}='{v}'" for k, v in attributes.items()]) if reverse: diff --git a/muagent/db_handler/graph_db_handler/nebula_handler.py b/muagent/db_handler/graph_db_handler/nebula_handler.py index 0731da6..fb31ca0 100644 --- a/muagent/db_handler/graph_db_handler/nebula_handler.py +++ b/muagent/db_handler/graph_db_handler/nebula_handler.py @@ -71,6 +71,12 @@ def execute_cypher(self, cypher: str, space_name: str = '', format_res: str = 'a resp = resp.dict_for_vis() return resp + def add_hosts(self, hostname, port): + with self.connection_pool.session_context(self.username, self.password) as session: + cypher = f'ADD HOSTS "{hostname}":{port}' + resp = session.execute(cypher) + return resp + def close_connection(self): self.connection_pool.close() diff --git a/muagent/schemas/ekg/ekg_graph.py b/muagent/schemas/ekg/ekg_graph.py index 17b831f..9226a38 100644 --- a/muagent/schemas/ekg/ekg_graph.py +++ b/muagent/schemas/ekg/ekg_graph.py @@ -165,9 +165,9 @@ class EKGNodeTbaseSchema(BaseModel): # node_str = 'graph_id={graph_id}'/teamids, use for searching by graph_id/teamids node_str: str name_keyword: str - desc_keyword: str + description_keyword: str name_vector: List - desc_vector: List + description_vector: List class EKGEdgeTbaseSchema(BaseModel): @@ -204,6 +204,24 @@ class EKGSlsData(BaseModel): } +############################### +##### tbase & gbase status ##### +############################### + +class TbaseExecStatus(BaseModel): + errorMessage: Optional[str] = None + statusCode: Optional[int] = None + # + total: Optional[int] = None + docs: Optional[List[dict]] = None + +class GbaseExecStatus(BaseModel): + errorMessage: Optional[str] = None + statusCode: Optional[int] = None + + + + ##################### ##### yuque dsl ##### ##################### diff --git a/muagent/service/ekg_construct/ekg_construct_base.py b/muagent/service/ekg_construct/ekg_construct_base.py index 3937bce..9a52392 100644 --- a/muagent/service/ekg_construct/ekg_construct_base.py +++ b/muagent/service/ekg_construct/ekg_construct_base.py @@ -1,7 +1,8 @@ from loguru import logger import re import json -from typing import List, Dict +from typing import List, Dict, Optional, Tuple, Literal + import numpy as np import random import uuid @@ -55,7 +56,7 @@ def __init__( gb_config: GBConfig = None, tb_config: TBConfig = None, sls_config: SLSConfig = None, - intention_router: IntentionRouter = None, + intention_router: Optional[IntentionRouter] = None, do_init: bool = False, kb_root_path: str = KB_ROOT_PATH, ): @@ -74,7 +75,7 @@ def __init__( # get llm model self.model = getChatModelFromConfig(self.llm_config) - self.intention_router = IntentionRouter(self.model, embed_config=self.embed_config) + self.intention_router = intention_router or IntentionRouter(self.model, embed_config=self.embed_config) # init db handler self.init_handler() @@ -106,15 +107,17 @@ def init_tb(self, do_init: bool=None): "DIM": DIM, "DISTANCE_METRIC": "COSINE" }), - VectorField("desc_vector", + VectorField("description_vector", 'FLAT', { "TYPE": "FLOAT32", "DIM": DIM, "DISTANCE_METRIC": "COSINE" }), + TextField("ekg_type",), + TextField("graph_id",), TagField(name='name_keyword', separator='|'), - TagField(name='desc_keyword', separator='|') + TagField(name='description_keyword', separator='|') ] EDGE_SCHEMA = [ @@ -123,6 +126,7 @@ def init_tb(self, do_init: bool=None): TextField("edge_source", ), TextField("edge_target", ), TextField("edge_str", ), + TextField("ekg_type",), ] @@ -130,16 +134,20 @@ def init_tb(self, do_init: bool=None): tb_dict = {"TbaseHandler": TbaseHandler} tb_class = tb_dict.get(self.tb_config.tb_type, TbaseHandler) self.tb: TbaseHandler = tb_class( - tb_config=self.tb_config, index_name=self.tb_config.index_name, - definition_value=self.tb_config.extra_kwargs.get("definition_value", "muagent_ekg") + tb_config=self.tb_config, + index_name=self.tb_config.index_name, + definition_value=self.tb_config.extra_kwargs.get( + "definition_value", "muagent_ekg") ) # # create index if not self.tb.is_index_exists(self.node_indexname): - res = self.tb.create_index(index_name=self.node_indexname, schema=NODE_SCHEMA) + res = self.tb.create_index( + index_name=self.node_indexname, schema=NODE_SCHEMA) logger.info(f"tb init: {res}") if not self.tb.is_index_exists(self.edge_indexname): - res = self.tb.create_index(index_name=self.edge_indexname, schema=EDGE_SCHEMA) + res = self.tb.create_index( + index_name=self.edge_indexname, schema=EDGE_SCHEMA) logger.info(f"tb init: {res}") else: self.tb = None @@ -152,6 +160,9 @@ def init_gb(self, do_init: bool=None): initialize_space = True # True or False if initialize_space and self.gb_config.gb_type=="NebulaHandler": + self.gb.add_hosts('storaged0', 9779) + print('增加NebulaGraph Storage主机中，等待20秒') + time.sleep(20) # 初始化space # self.gb.drop_space('client') self.gb.create_space('client') @@ -161,7 +172,6 @@ def init_gb(self, do_init: bool=None): print('Node Tags和Edge Types初始化中，等待20秒......') time.sleep(20) - else: self.gb = None @@ -190,7 +200,7 @@ def init_sls(self, do_init: bool=None): else: self.sls = None - def _get_local_graph(self, nodes: List[GNode], edges: List[GEdge], rootid): + def _get_local_graph(self, nodes: List[GNode], edges: List[GEdge], rootid) -> Tuple[List[str], Graph]: # search and delete unconnect nodes and edges connections = {} for edge in edges: @@ -211,28 +221,34 @@ def _dfs(node, current_path: List): visited.add(node) current_path.append(node) rootid_can_arrive_nodeids.append(node) - # 假设终止条件是没有更多的邻居 + # stop condition, there is no more neightbos if not connections.get(node, []): - # 到达终止节点，保存当前路径的副本 + # when arrive the endpoiond, save the copy of current path paths.append(list(current_path)) else: for neighbor in connections.get(node, []): _dfs(neighbor, current_path) - # 回溯：移除最后一个节点 + # recursive：remove the last node current_path.pop() - # 初始化 DFS + # init DFS _dfs(rootid, []) logger.info(f"graph paths, {paths}") - logger.info(f"rootid can not arrive nodeids, {[n for n in nodes if n.id not in rootid_can_arrive_nodeids]}") + logger.info(f"rootid can not arrive nodeids, " + f"{[n for n in nodes if n.id not in rootid_can_arrive_nodeids]}") graph = Graph( nodes=[n for n in nodes if n.id in rootid_can_arrive_nodeids], - edges=[e for e in edges if e.start_id in rootid_can_arrive_nodeids and e.end_id in rootid_can_arrive_nodeids], + edges=[ + e for e in edges + if e.start_id in rootid_can_arrive_nodeids and + e.end_id in rootid_can_arrive_nodeids + ], paths=paths ) return rootid_can_arrive_nodeids, graph + def create_gb_tags_and_edgetypes(self): #print('create_gb_tags_and_edgetypes') @@ -311,8 +327,15 @@ def update_graph( # get add nodes & edges and filter those nodes/edges cant be arrived from rootid add_nodes = [node for node in new_nodes if node.id not in origin_nodeids] add_nodes = [n for n in add_nodes if n.id in rootid_can_arrive_nodeids] - add_edges = [edge for edge in new_edges if f"{edge.start_id}__{edge.end_id}" not in origin_edgeids] - add_edges = [edge for edge in add_edges if (edge.start_id in rootid_can_arrive_nodeids) and (edge.end_id in rootid_can_arrive_nodeids)] + add_edges = [ + edge for edge in new_edges + if f"{edge.start_id}__{edge.end_id}" not in origin_edgeids + ] + add_edges = [ + edge for edge in add_edges + if (edge.start_id in rootid_can_arrive_nodeids) and + (edge.end_id in rootid_can_arrive_nodeids) + ] # get delete nodes & edges delete_nodes = [node for node in origin_nodes if node.id not in nodeids] @@ -409,60 +432,95 @@ def update_graph( } - def add_nodes(self, nodes: List[GNode], teamid: str): + def add_nodes(self, nodes: List[GNode], teamid: str, ekg_type: str="ekgnode") -> Dict: + ''' + add new nodes into tbase and graph base + :param nodes: new nodes + :param teamid: teamid + ''' nodes = self._update_new_attr_for_nodes(nodes, teamid, do_check=True) tbase_nodes = [] for node in nodes: + # get the node's teamids + r = self.tb.search( + f"@node_id: *{node.id}*", index_name=self.node_indexname + ) + + teamids = [ + i.strip() + for i in r.docs[0]["node_str"].replace("graph_id=", "").split(",") + if i.strip() + ] if r.docs else [] + teamids = list(set(teamids+[teamid])) + tbase_nodes.append({ **{ "ID": node.attributes.get("ID", 0) or double_hashing(node.id), - "node_id": f"{node.id}", + "node_id": node.id, "node_type": node.type, - "node_str": f"graph_id={teamid}", + "node_str": ', '.join(teamids), + "graph_id": ', '.join(teamids), + "ekg_type": ekg_type, }, **self._update_tbase_attr_for_nodes(node.attributes) - }) + }) tb_result, gb_result = [], [] try: gb_result = [self.gb.add_node(node) for node in nodes] - tb_result.append(self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False)) + tb_result.append( + self.tb.insert_data_hash(tbase_nodes, key='node_id', need_etime=False) + ) except Exception as e: logger.error(e) + + # todo return nodes' infomation return {"gb_result": gb_result, "tb_result": tb_result} - def add_edges(self, edges: List[GEdge], teamid: str): + def add_edges(self, edges: List[GEdge], teamid: str, ekg_type: str="ekgedge"): edges = self._update_new_attr_for_edges(edges) tbase_edges = [{ - # 'edge_id': f"ekg_edge:{teamid}{edge.start_id}:{edge.end_id}", 'edge_id': f"{edge.start_id}__{edge.end_id}", 'edge_type': edge.type, 'edge_source': edge.start_id, 'edge_target': edge.end_id, - 'edge_str': f'graph_id={teamid}' + 'edge_str': f'graph_id={teamid}', + "ekg_type": ekg_type, } for edge in edges ] tb_result, gb_result = [], [] try: + # bug: there is gap between zhizhu and geabase gb_result = [self.gb.add_edge(edge) for edge in edges] - tb_result.append(self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False)) + tb_result.append( + self.tb.insert_data_hash(tbase_edges, key="edge_id", need_etime=False) + ) except Exception as e: logger.error(e) + + # todo return nodes' infomation return {"gb_result": gb_result, "tb_result": tb_result} def delete_nodes(self, nodes: List[GNode], teamid: str=''): # delete tbase nodes - r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) + r = self.tb.search( + f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes) + ) tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 delete_nodeids = [node.id for node in nodes] - tbase_missing_nodeids = [nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids] + tbase_missing_nodeids = [ + nodeid for nodeid in delete_nodeids + if nodeid not in tbase_nodeids + ] if len(tbase_missing_nodeids) > 0: - logger.error(f"there must something wrong! ID not match, such as {tbase_missing_nodeids}") + logger.error( + f"there must something wrong! ID not match, such as {tbase_missing_nodeids}" + ) # node_neighbor_lens = [ # len([ @@ -492,12 +550,6 @@ def delete_nodes(self, nodes: List[GNode], teamid: str=''): # resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") # tb_result.append(resp) - # # directly delete extra_delete_edges in tbase - # tb_result = [] - # for edge in extra_delete_edges: - # resp = self.tb.delete(f"{edge.start_id}__{edge.end_id}") - # tb_result.append(resp) - # delete the nodeids in tbase # tb_result = [] # for node, node_len in zip(nodes, node_neighbor_lens): @@ -548,11 +600,15 @@ def delete_edges(self, edges: List[GEdge], teamid: str): return {"gb_result": gb_result, "tb_result": tb_result} def update_nodes(self, nodes: List[GNode], teamid: str): - # delete tbase nodes + ''' + update nodes with new attributes and teamid + :param nodes: + :param teamid: + ''' r = self.tb.search(f"@node_str: *{teamid}*", index_name=self.node_indexname, limit=len(nodes)) teamids_by_nodeid = {data['node_id']: data["node_str"] for data in r.docs} - tbase_nodeids = [data['node_id'] for data in r.docs] # 附带了definition信息 + tbase_nodeids = [data['node_id'] for data in r.docs] update_nodeids = [node.id for node in nodes] tbase_missing_nodeids = [nodeid for nodeid in update_nodeids if nodeid not in tbase_nodeids] @@ -562,17 +618,27 @@ def update_nodes(self, nodes: List[GNode], teamid: str): r = self.tb.search(f"@node_id: {nodeid.replace('-', '_')}", index_name=self.node_indexname) teamids_by_nodeid.update({data['node_id']: data["node_str"] for data in r.docs}) - tb_result = [] + tbase_datas = [] for node in nodes: - # tbase_data = {} tbase_data["node_id"] = node.id + + if node.id not in teamids_by_nodeid: + raise ValueError(f"this id {node.id} not in graph, please check your input") + if teamid not in teamids_by_nodeid[node.id]: - teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()]+[teamid])) - tbase_data["teamids"] = ", ".join(teamids) # teamids_by_nodeid[node.id] + f", {teamid}" - # tbase_data["teamids"] = teamids_by_nodeid[node.id] + f", {teamid}" + teamids = [ + i.strip() for i in teamids_by_nodeid[node.id].replace("graph_id=", "").split(",") + if i.strip() + ] + teamids = list(set(teamids+[teamid])) + tbase_data["node_str"] = ', '.join(teamids) + # tbase_data["teamids"] = f"graph_id={', '.join(teamids)}", tbase_data.update(self._update_tbase_attr_for_nodes(node.attributes)) - # + tbase_datas.append(tbase_data) + + tb_result = [] + for tbase_data in tbase_datas: resp = self.tb.insert_data_hash(tbase_data, key="node_id", need_etime=False) tb_result.append(resp) @@ -580,7 +646,6 @@ def update_nodes(self, nodes: List[GNode], teamid: str): nodes = self._update_new_attr_for_nodes(nodes, teamid, teamids_by_nodeid, do_check=False) gb_result = [] for node in nodes: - # if node.id not in update_tbase_nodeids: continue ID = node.attributes.pop("ID", None) or double_hashing(node.id) resp = self.gb.update_node( {}, node.attributes, node_type=node.type, @@ -591,24 +656,19 @@ def update_nodes(self, nodes: List[GNode], teamid: str): def update_edges(self, edges: List[GEdge], teamid: str): r = self.tb.search(f"@edge_str: *{teamid}*", index_name=self.node_indexname, limit=len(edges)) - # teamids_by_edgeid = {data['edge_id']: data["edge_str"] for data in r.docs} - tbase_edgeids = [data['edge_id'] for data in r.docs] delete_edgeids = [f"{edge.start_id}__{edge.end_id}" for edge in edges] tbase_missing_edgeids = [edgeid for edgeid in delete_edgeids if edgeid not in tbase_edgeids] if len(tbase_missing_edgeids) > 0: logger.error(f"there must something wrong! ID not match, such as {tbase_missing_edgeids}") - # for edge in edges: - # r = self.tb.search(f"@edge_id: {edge.start_id}__{edge.end_id}", index_name=self.node_indexname) - # teamids_by_edgeid.update({data['edge_id']: data["node_str"] for data in r.docs}) # update the nodeids in geabase edges = self._update_new_attr_for_edges(edges, do_check=False, do_update=True) gb_result = [] for edge in edges: - # if node.id not in update_tbase_nodeids: continue - SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) # todo bug，数据不一致问题 + # todo bug, there is gap between zhizhu and graph base + SRCID = edge.attributes.pop("SRCID", None) or double_hashing(edge.start_id) DSTID = edge.attributes.pop("DSTID", None) or double_hashing(edge.end_id) resp = self.gb.update_edge( SRCID, DSTID, @@ -617,9 +677,81 @@ def update_edges(self, edges: List[GEdge], teamid: str): gb_result.append(resp) return {"gb_result": gb_result, "tb_result": []} - def get_node_by_id(self, nodeid: str, node_type:str = None) -> GNode: - node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) - return self._normalized_nodes_type(nodes=[node])[0] + def delete_nodes_v2(self, nodes: List[GNode], teamid: str=''): + ''' + delete tbase nodes + :param nodes: + :param teamid: + ''' + r = self.tb.search( + f"@node_str: *{teamid}*", + index_name=self.node_indexname, + limit=len(nodes) + ) + + tbase_nodeids = [data['node_id'] for data in r.docs] + delete_nodeids = [node.id for node in nodes] + tbase_missing_nodeids = [ + nodeid for nodeid in delete_nodeids if nodeid not in tbase_nodeids + ] + + if len(tbase_missing_nodeids) > 0: + logger.error(f"there must something wrong! " + f"ID not match, such as {tbase_missing_nodeids}") + + node_upstream_nodes = { + node.id: [ + n.id # reverse neighbor nodes which are not in delete nodes + for n in self.gb.get_neighbor_nodes( + {"id": node.id}, node.type, reverse=True) + if n.id not in delete_nodeids] + for node in nodes + } + node_downstream_nodes = { + node.id: [ + n.id # reverse neighbor nodes which are not in delete nodes + for n in self.gb.get_neighbor_nodes( + {"id": node.id}, node.type, reverse=False) + if n.id not in delete_nodeids] + for node in nodes + } + + # delete the nodeids in tbase + tb_result = [] + for node in nodes: + if len(node_upstream_nodes.get(node.id, []))>0: continue + resp = self.tb.delete(node.id) + tb_result.append(resp) + + # delete the nodeids in geabase + gb_result = [] + for node in nodes: + if len(node_upstream_nodes.get(node.id, []))>0 or \ + len(node_downstream_nodes.get(node.id, []))>0: + continue + gb_result.append(self.gb.delete_node( + {"id": node.id}, node.type, ID=node.attributes.get("ID") or double_hashing(node.id) + )) + return {"gb_result": gb_result, "tb_result": tb_result} + + def get_node_by_id( + self, nodeid: str, node_type:str = None, service_type: Literal["gbase", "tbase"]="gbase", + ) -> GNode: + if service_type=="gbase": + node = self.gb.get_current_node({'id': nodeid}, node_type=node_type) + node = self._normalized_nodes_type(nodes=[node])[0] + else: + node = GNode(id=nodeid, type="", attributes={}) + # tbase search + r = self.tb.search(f"@node_id: *{nodeid}*", index_name=self.node_indexname) + teamids = [ + i.strip() + for i in r.docs[0]["node_str"].replace("graph_id=", "").split(",") + if i.strip() + ] if r.docs else [] + + node.attributes["teamids"] = teamids + return node def get_graph_by_nodeid( self, @@ -630,8 +762,8 @@ def get_graph_by_nodeid( ) -> Graph: if hop<2: raise Exception(f"hop must be smaller than 2, now hop is {hop}") - if hop >= 20: - raise Exception(f"hop can't be larger than 20, now hop is {hop}") + if hop >= 30: + raise Exception(f"hop can't be larger than 30, now hop is {hop}") # filter the node which dont match teamid result = self.gb.get_hop_infos( {'id': nodeid}, node_type=node_type, @@ -642,14 +774,17 @@ def get_graph_by_nodeid( result.nodes.append(current_node) if block_attributes: - leaf_nodeids = [node.id for node in result.nodes if node.type=="opsgptkg_schedule"] + leaf_nodeids = [ + node.id for node in result.nodes if node.type=="opsgptkg_schedule" + ] else: leaf_nodeids = [path[-1] for path in result.paths if len(path)==hop+1] nodes = self._normalized_nodes_type(result.nodes) for node in nodes: if node.id in leaf_nodeids: - neighbor_nodes = self.gb.get_neighbor_nodes({"id": node.id}, node_type=node.type) + neighbor_nodes = self.gb.get_neighbor_nodes( + {"id": node.id}, node_type=node.type) node.attributes["cnode_nums"] = len(neighbor_nodes) edges = self._normalized_edges_type(result.edges) @@ -657,7 +792,9 @@ def get_graph_by_nodeid( result.edges = edges return result - def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = None, top_k=5) -> List[GNode]: + def search_nodes_by_text( + self, text: str, node_type: str = None, teamid: str = None, top_k=5 + ) -> List[GNode]: if text is None: return [] @@ -668,11 +805,15 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N query_embedding = np.array(vector_dict[text]).astype(dtype=np.float32).tobytes() nodeid_with_dist = [] - for key in ["name_vector", "desc_vector"]: - base_query = f'(@node_str: *{teamid}*)=>[KNN {top_k} @{key} $vector AS distance]' + for key in ["name_vector", "description_vector"]: + + base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' if teamid is None \ + else f'(@node_str: *{teamid}*)=>[KNN {top_k} @{key} $vector AS distance]' # base_query = f'(*)=>[KNN {top_k} @{key} $vector AS distance]' query_params = {"vector": query_embedding} - r = self.tb.vector_search(base_query, index_name=self.node_indexname, query_params=query_params) + r = self.tb.vector_search( + base_query, index_name=self.node_indexname, query_params=query_params + ) for i in r.docs: data_dict = i.__dict__ @@ -687,7 +828,7 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N # search keyword by jieba spliting text keywords = extract_tags(text) keyword = "|".join(keywords) - for key in ["name_keyword", "desc_keyword"]: + for key in ["name_keyword", "description_keyword"]: query = f"(@node_str: *{teamid}*)(@{key}:{{{keyword}}})" r = self.tb.search(query, index_name=self.node_indexname, limit=30) for i in r.docs: @@ -699,11 +840,20 @@ def search_nodes_by_text(self, text: str, node_type: str = None, teamid: str = N # tmp iead to filter by teamid nodes = [node for node in nodes if str(teamid) in str(node.attributes)] # select the node which can connect the rootid - nodes = [node for node in nodes if len(self.search_rootpath_by_nodeid(node.id, node.type, f"ekg_team_{teamid}").paths)>0] + nodes = [ + node for node in nodes + if len(self.search_rootpath_by_nodeid( + node.id, node.type, f"ekg_team_{teamid}" + ).paths) > 0 + ] return nodes - def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> Graph: - result = self.gb.get_hop_infos({"id": nodeid}, node_type=node_type, hop=15, reverse=True) + def search_rootpath_by_nodeid( + self, nodeid: str, node_type: str, rootid: str + ) -> Graph: + result = self.gb.get_hop_infos( + {"id": nodeid}, node_type=node_type, hop=15, reverse=True + ) # paths must be ordered from start to end paths = result.paths @@ -718,7 +868,10 @@ def search_rootpath_by_nodeid(self, nodeid: str, node_type: str, rootid: str) -> nodeid_set = set([nodeid for path in paths for nodeid in path]) new_nodes = [node for node in result.nodes if node.id in nodeid_set] - new_edges = [edge for edge in result.edges if edge.start_id in nodeid_set and edge.end_id in nodeid_set] + new_edges = [ + edge for edge in result.edges + if edge.start_id in nodeid_set and edge.end_id in nodeid_set + ] new_nodes = self._normalized_nodes_type(new_nodes) new_edges = self._normalized_edges_type(new_edges) @@ -758,7 +911,9 @@ def create_ekg( def dsl2graph(self, ): pass - def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], teamid: str) -> dict: + def text2graph( + self, text: str, intents: List[str], all_intent_list: List[str], teamid: str + ) -> dict: # generate graph by llm result = self.get_graph_by_text(text, ) # convert llm contet to database schema @@ -767,20 +922,26 @@ def text2graph(self, text: str, intents: List[str], all_intent_list: List[str], dsl_graph = self.transform2dsl(sls_graph, intents, all_intent_list, teamid=teamid) return {"tbase_graph": tbase_graph, "sls_graph": sls_graph, "dsl_graph": dsl_graph} - def write2kg(self, ekg_sls_data: EKGSlsData, teamid: str, graphid: str="", do_save: bool=False) -> Graph: + def write2kg( + self, ekg_sls_data: EKGSlsData, teamid: str, graphid: str="", do_save: bool=False + ) -> Graph: ''' :param graphid: str, use for record the new path ''' # everytimes, it will add new nodes and edges - gbase_nodes: List[EKGNodeSchema] = [TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes] + gbase_nodes: List[EKGNodeSchema] = [ + TYPE2SCHEMA.get(node.type,)(**node.dict()) for node in ekg_sls_data.nodes + ] gbase_nodes: List[GNode] = [ GNode( id=node.id, type=node.type, attributes=node.attributes() if graphid else {**node.attributes(), **{"graphid": f"{graphid}"}} ) for node in gbase_nodes] - gbase_edges: List[EKGEdgeSchema] = [TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges] + gbase_edges: List[EKGEdgeSchema] = [ + TYPE2SCHEMA.get("edge",)(**edge.dict()) for edge in ekg_sls_data.edges + ] gbase_edges = [ GEdge(start_id=edge.original_src_id1__, end_id=edge.original_dst_id2__, type="opsgptkg_"+edge.type.split("_")[2] + "_route_" + "opsgptkg_"+edge.type.split("_")[3], @@ -808,14 +969,23 @@ def returndsl(self, graph_datas_by_path: dict, intents: List[str], ) -> dict: 'dsl': graph_datas["dsl_graph"], 'sls': graph_datas["sls_graph"], } - merge_dsl_nodes.extend([node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets]) + merge_dsl_nodes.extend([ + node for node in graph_datas["dsl_graph"].nodes if node.id not in id_sets + ]) id_sets.update([i.id for i in graph_datas["dsl_graph"].nodes]) - merge_dsl_edges.extend([edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets]) + merge_dsl_edges.extend([ + edge for edge in graph_datas["dsl_graph"].edges if edge.id not in id_sets + ]) id_sets.update([i.id for i in graph_datas["dsl_graph"].edges]) - merge_gbase_nodes.extend([node for node in graph_datas["graph"].nodes if node.id not in gid_sets]) + merge_gbase_nodes.extend([ + node for node in graph_datas["graph"].nodes if node.id not in gid_sets + ]) gid_sets.update([i.id for i in graph_datas["graph"].nodes]) - merge_gbase_edges.extend([edge for edge in graph_datas["graph"].edges if f"{edge.start_id}__{edge.end_id}" not in gid_sets]) + merge_gbase_edges.extend([ + edge for edge in graph_datas["graph"].edges + if f"{edge.start_id}__{edge.end_id}" not in gid_sets + ]) gid_sets.update([f"{i.start_id}__{i.end_id}" for i in graph_datas["graph"].edges]) res["dsl"] = {"nodes": merge_dsl_nodes, "edges": merge_dsl_edges} @@ -854,7 +1024,9 @@ def get_graph_by_text(self, text: str) -> EKGSlsData: return node_edge_dict - def transform2sls(self, node_edge_dict: dict, pnode_ids: List[str], teamid: str='') -> EKGSlsData: + def transform2sls( + self, node_edge_dict: dict, pnode_ids: List[str], teamid: str='' + ) -> EKGSlsData: # type类型处理也要注意下 sls_nodes, sls_edges = [], [] for node_idx, node_info in node_edge_dict['nodes'].items(): @@ -920,16 +1092,16 @@ def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData name = node.name description = node.description name_vector = self._get_embedding(name) - desc_vector = self._get_embedding(description) + description_vector = self._get_embedding(description) tbase_nodes.append( EKGNodeTbaseSchema( node_id=node.id, node_type=node.type, - node_str=f'graph_id={teamid}', + node_str=teamid, name_keyword=" | ".join(extract_tags(name, topK=None)), - desc_keyword=" | ".join(extract_tags(description, topK=None)), + description_keyword=" | ".join(extract_tags(description, topK=None)), name_vector= name_vector[name], - desc_vector= desc_vector[description], + description_vector= description_vector[description], ) ) for edge in ekg_sls_data.edges: @@ -945,7 +1117,13 @@ def transform2tbase(self, ekg_sls_data: EKGSlsData, teamid: str) -> EKGTbaseData ) return EKGTbaseData(nodes=tbase_nodes, edges=tbase_edges) - def transform2dsl(self, ekg_sls_data: EKGSlsData, pnode_ids: List[str], all_intents: List[str], teamid: str) -> YuqueDslDatas: + def transform2dsl( + self, + ekg_sls_data: EKGSlsData, + pnode_ids: List[str], + all_intents: List[str], + teamid: str + ) -> YuqueDslDatas: '''define your personal dsl format and code''' def get_md5(s): import hashlib @@ -967,7 +1145,11 @@ def get_md5(s): for node in ekg_sls_data.nodes: # 需要注意下 dsl的id md编码 nodes.append( - YuqueDslNodeData(id=f"ekg_node:{node.type}:{node.id}", type=type_dict.get(node.type.split("opsgptkg_")[-1]), label=node.description) + YuqueDslNodeData( + id=f"ekg_node:{node.type}:{node.id}", + type=type_dict.get(node.type.split("opsgptkg_")[-1]), + label=node.description + ) ) # 添加意图节点 @@ -1068,20 +1250,26 @@ def _update_tbase_attr_for_nodes(self, attrs): if k in attrs: text = attrs.get(k, "") text_vector = self._get_embedding(text) - tbase_attrs[f"{k}_vector"] = np.array(text_vector[text]).astype(dtype=np.float32).tobytes() - tbase_attrs[f"{k}_keyword"] = " | ".join(extract_tags(text, topK=None)) + tbase_attrs[f"{k}_vector"] = np.array(text_vector[text]).\ + astype(dtype=np.float32).tobytes() + tbase_attrs[f"{k}_keyword"] = " | ".join( + extract_tags(text, topK=None) + ) return tbase_attrs - def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by_nodeid={}, do_check=False): + def _update_new_attr_for_nodes( + self, nodes: List[GNode], teamid: str, teamids_by_nodeid={}, do_check=False + ): '''update new attributes for nodes''' nodetype2fields_dict = {} for node in nodes: node_type = node.type if node.id in teamids_by_nodeid: - teamids = list(set([i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()]+ [teamid])) + teamids = list(set( + [i.strip() for i in teamids_by_nodeid[node.id].split(",") if i.strip()]+ [teamid] + )) node.attributes["teamids"] = ", ".join(teamids) - # node.attributes["teamids"] = teamids_by_nodeid.get(node.id, "").split("=")[1] + f", {teamid}" else: node.attributes["teamids"] = f"{teamid}" @@ -1096,6 +1284,7 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by fields = list(getClassFields(schema)) nodetype2fields_dict[node_type] = fields + fields = [field for field in fields if field not in ["__slots__"]] missing_fields = [ field for field in fields @@ -1103,7 +1292,10 @@ def _update_new_attr_for_nodes(self, nodes: List[GNode], teamid: str, teamids_by and field not in node.attributes ] if len(missing_fields)>0 and do_check: - raise Exception(f"node is wrong, type is {node_type}, missing_fields is {missing_fields}, fields is {fields}, data is {node.attributes}") + raise Exception( + f"node is wrong, type is {node_type}, missing_fields is {missing_fields}, " + f"fields is {fields}, data is {node.attributes}" + ) # update extra infomations to extra extra_fields = [k for k in node.attributes.keys() if k not in fields] @@ -1137,14 +1329,19 @@ def _update_new_attr_for_edges(self, edges: List[GEdge], do_check=True, do_updat fields = list(getClassFields(schema)) edgetype2fields_dict[edge_type] = fields + fields = [field for field in fields if field not in ["__slots__"]] + check_fields = ["type", "dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"] missing_fields = [ field for field in fields - if field not in ["type", "dst_id", "src_id", "DSTID", "SRCID", "timestamp", "ID", "id", "extra"] + if field not in check_fields and field not in edge.attributes ] if len(missing_fields)>0 and do_check: - raise Exception(f"edge is wrong, type is {edge_type}, missing_fields is {missing_fields}, fields is {fields}, data is {edge.attributes}") + raise Exception( + f"edge is wrong, type is {edge_type}, missing_fields is {missing_fields}, " + f"fields is {fields}, data is {edge.attributes}" + ) # update extra infomations to extra extra_fields = [k for k in edge.attributes.keys() if k not in fields+["@timestamp"]] @@ -1166,7 +1363,10 @@ def _normalized_nodes_type(self, nodes: List[GNode]) -> List[GNode]: for node in nodes: node_type = node.type node_data_dict = {**{"id": node.id, "type": node_type}, **node.attributes} - node_data_dict = {k: 'False' if k in ["enable", "summaryswitch"] and v=="" else v for k,v in node_data_dict.items()} + node_data_dict = { + k: 'False' if k in ["enable", "summaryswitch"] and v=="" else v + for k,v in node_data_dict.items() + } node_data: EKGNodeSchema = TYPE2SCHEMA[node_type](**node_data_dict) valid_node = GNode(id=node.id, type=node_type, attributes=node_data.attributes()) valid_nodes.append(valid_node) @@ -1176,7 +1376,13 @@ def _normalized_edges_type(self, edges: List[GEdge]) -> GEdge: valid_edges = [] for edge in edges: edge_data: EKGEdgeSchema = TYPE2SCHEMA["edge"]( - **{**{"original_src_id1__": edge.start_id, "original_dst_id2__": edge.end_id, "type": edge.type}, **edge.attributes} + **{ + **{ + "original_src_id1__": edge.start_id, + "original_dst_id2__": edge.end_id, + "type": edge.type}, + **edge.attributes + } ) valid_edge = GEdge( start_id=edge_data.original_src_id1__, end_id=edge_data.original_dst_id2__, diff --git a/muagent/service/ekg_inference/__init__.py b/muagent/service/ekg_inference/__init__.py new file mode 100644 index 0000000..5595792 --- /dev/null +++ b/muagent/service/ekg_inference/__init__.py @@ -0,0 +1,5 @@ +from .intention_router import IntentionRouter +from .intention_match_rule import MatchRule + + +__all__ = ['IntentionRouter', 'MatchRule'] diff --git a/muagent/service/ekg_inference/intention_match_rule.py b/muagent/service/ekg_inference/intention_match_rule.py index 17293ec..f94b6ab 100644 --- a/muagent/service/ekg_inference/intention_match_rule.py +++ b/muagent/service/ekg_inference/intention_match_rule.py @@ -24,18 +24,3 @@ def edit_distance(cls, node: GNode, pattern=None, **kwargs): @classmethod def edit_distance_integer(cls, node: GNode, **kwargs): return cls.edit_distance(node, pattern='\d+', **kwargs) - - -class RuleDict(dict): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def save(self): - """save the rules""" - raise NotImplementedError - - def load(self, **kwargs): - """load the rules""" - raise NotImplementedError - -rule_dict = RuleDict() diff --git a/muagent/service/ekg_inference/intention_router.py b/muagent/service/ekg_inference/intention_router.py index c6c2aab..81698b4 100644 --- a/muagent/service/ekg_inference/intention_router.py +++ b/muagent/service/ekg_inference/intention_router.py @@ -1,16 +1,20 @@ import re import numpy as np +import pandas as pd import muagent.base_configs.prompts.intention_template_prompt as itp +from collections import defaultdict, deque from loguru import logger +from jieba.analyse import extract_tags from dataclasses import dataclass, field, asdict from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Callable, Union, Optional, Any from muagent.db_handler.graph_db_handler.base_gb_handler import GBHandler from muagent.db_handler.vector_db_handler.tbase_handler import TbaseHandler -from muagent.schemas.ekg.ekg_graph import NodeTypesEnum, TYPE2SCHEMA -from muagent.schemas.common import GNode +from muagent.schemas.ekg.ekg_graph import NodeTypesEnum +from muagent.schemas.common import GNode, GEdge from muagent.llm_models.get_embedding import get_embedding -from .intention_match_rule import rule_dict, MatchRule +from muagent.utils.common_utils import double_hashing +from .intention_match_rule import MatchRule @dataclass @@ -31,166 +35,142 @@ class RuleRetInfo: class IntentionRouter: Rule_type = Optional[str] - def __init__(self, agent=None, gb_handler: GBHandler=None, tb_handler: TbaseHandler=None, embed_config=None): + + def __init__( + self, agent=None, gb_handler: GBHandler = None, tb_handler: TbaseHandler = None, + embed_config=None + ): self.agent = agent self.gb_handler = gb_handler self.tb_handler = tb_handler self.embed_config = embed_config self._node_type = NodeTypesEnum.INTENT.value - self._max_num_tb_retrieval = 10 + self._max_num_tb_retrieval = 5 self._filter_max_depth = 5 + self._dis_threshold = 16 - def add_rule(self, node_id2rule: dict[str, str], gb_handler: Optional[GBHandler] = None): - gb_handler = gb_handler if gb_handler is not None else self.gb_handler - ori_len = len(rule_dict) - fail_nodes, fail_rules = [], [] - for node_id, rule in node_id2rule.items(): - try: - gb_handler.get_current_node({'id': node_id}, node_type=self._node_type) - except IndexError: - fail_nodes.append(node_id) - continue - - rule_func = _get_rule_by_str(rule) - if rule_func is None: - fail_rules.append(node_id) - continue - - rule_dict[node_id] = rule - - if len(rule_dict) > ori_len: - rule_dict.save() - - error_msg = '' - if fail_nodes: - error_msg += ', '.join([ - f'Node(id={node_id}, node_type={self._node_type})' - for node_id in fail_nodes - ]) - error_msg += ' do not exist! ' - if fail_rules: - error_msg += 'Rule of ' - error_msg += ', '.join([ - f'Node(id={node_id}, node_type={self._node_type})' - for node_id in fail_rules - ]) - error_msg += ' is not valid!' - return error_msg - - def get_intention_by_info2id(self, gb_handler: GBHandler, rule: Union[str, Callable]=':', **kwargs) -> str: + def get_intention_by_info2id( + self, gb_handler: GBHandler, rule: Union[str, Callable] = ':', **kwargs + ) -> str: gb_handler = gb_handler if gb_handler is not None else self.gb_handler - node_id = rule.join(kwargs.values()) if isinstance(rule, str) else rule(**kwargs) - try: - gb_handler.get_current_node({'id': node_id}, node_type=self._node_type) - except IndexError: + node_id = rule.join(kwargs.values()) if isinstance( + rule, str) else rule(**kwargs) + if not self._node_exist(node_id): logger.error(f'Node(id={node_id}, node_type={self._node_type}) does not exist!') return None return node_id def _get_intention_by_node_info_match( - self, gb_handler: GBHandler, root_node_id: str, - rule: Rule_type = None, **kwargs + self, gb_handler: GBHandler, root_node_id: str, rule: Rule_type = None, **kwargs ) -> str: def _func(node: GNode, rule: Callable): return rule(node, **kwargs), node.id error_msg, rule_str = '', rule + rule = self._get_rule_by_str(rule) if rule is None: - if root_node_id in rule_dict: - rule = rule_dict(root_node_id) - else: - rule = MatchRule.edit_distance - rule_str = 'edit_distance' - if isinstance(rule, str): - rule = _get_rule_by_str(rule) - if isinstance(rule, str): - try: - rule = getattr(MatchRule, rule) - except AttributeError: - rule = None - if rule is None: - error_msg = f'Rule {rule_str} is not valid!' - return None, error_msg + error_msg = f'Rule {rule_str} is not valid!' + return None, error_msg intention_nodes = gb_handler.get_neighbor_nodes({'id': root_node_id}, self._node_type) - intention_nodes = [node for node in intention_nodes if node.type == self._node_type] + intention_nodes = [ + node for node in intention_nodes if node.type == self._node_type + ] if len(intention_nodes) == 0: return root_node_id, error_msg elif len(intention_nodes) == 1: return intention_nodes[0].id, error_msg - elif len(intention_nodes) < 20: + elif len(intention_nodes) < self._dis_threshold: scores = [_func(node, rule) for node in intention_nodes] else: params = [(node, rule) for node in intention_nodes] scores = _execute_func_distributed(_func, params) - return max(scores)[-1], error_msg + max_score = max(scores)[0] + select_nodes = [x[-1] for x in scores if x[0] == max_score] + select_node = self._equal_score_route_info_match(select_nodes, gb_handler) + return select_node, error_msg def get_intention_by_node_info_match( - self, root_node_id: str, filter_attribute: Optional[dict]=None, gb_handler: Optional[GBHandler] = None, - rule: Union[Rule_type, list[Rule_type]]=None, **kwargs + self, root_node_id: str, filter_attribute: Optional[dict] = None, + gb_handler: Optional[GBHandler] = None, + rule: Union[Rule_type, list[Rule_type]] = None, **kwargs ) -> dict[str, Any]: gb_handler = gb_handler if gb_handler is not None else self.gb_handler - root_node_id = self._filter_from_root_node(gb_handler, root_node_id, filter_attribute) + root_node_id = self._filter_from_root_node( + gb_handler, root_node_id, filter_attribute + ) is_leaf = False if len(kwargs) == 0: error_msg = 'No information in query to be matched.' - return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + return asdict( + RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) args_list = _parse_kwargs(**kwargs) if not isinstance(rule, (list, tuple)): rule = [rule] * len(args_list) if len(rule) != len(args_list): error_msg = 'Length of rule should be equal to the length of Arguments.' - return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + return asdict( + RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) if not root_node_id: error_msg = f'No node matches attribute {filter_attribute}.' - return asdict(RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) + return asdict( + RuleRetInfo(node_id=root_node_id, error_msg=error_msg)) for cur_kw_arg, cur_rule in zip(args_list, rule): next_node_id, error_msg = self._get_intention_by_node_info_match( - gb_handler, root_node_id, cur_rule, **cur_kw_arg - ) + gb_handler, root_node_id, cur_rule, **cur_kw_arg) if next_node_id is None or next_node_id == root_node_id: is_leaf = True if next_node_id else False - return asdict(RuleRetInfo(node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg)) + return asdict(RuleRetInfo( + node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg + )) root_node_id = next_node_id next_node_id = root_node_id while next_node_id: root_node_id = next_node_id - intention_nodes = gb_handler.get_neighbor_nodes({'id': next_node_id}, self._node_type) + intention_nodes = gb_handler.get_neighbor_nodes( + {'id': next_node_id}, self._node_type) intention_nodes = [ node for node in intention_nodes if node.type == self._node_type ] if len(intention_nodes) == 1: next_node_id = intention_nodes[0].id - else: - next_node_id = None - if len(intention_nodes) == 0: - is_leaf = True - + continue + next_node_id = None + if len(intention_nodes) == 0: + is_leaf = True + if not is_leaf: error_msg = 'Not enough to arrive the leaf node.' - return asdict(RuleRetInfo(node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg)) + return asdict( + RuleRetInfo(node_id=root_node_id, is_leaf=is_leaf, error_msg=error_msg) + ) def get_intention_by_node_info_nlp( - self, root_node_id: str, query: str, start_from_root: bool = False, - gb_handler: Optional[GBHandler] = None, tb_handler: Optional[TbaseHandler] = None, agent=None, + self, + root_node_id: str, + query: str, + start_from_root: bool = False, + gb_handler: Optional[GBHandler] = None, + tb_handler: Optional[TbaseHandler] = None, + agent=None, ) -> dict[str, Any]: gb_handler = gb_handler if gb_handler is not None else self.gb_handler tb_handler = tb_handler if tb_handler is not None else self.tb_handler agent = agent if agent is not None else self.agent if start_from_root: - return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + return self._get_intention_by_nlp_from_root( + gb_handler, agent, root_node_id, query) nodes_tb = self._tb_match(tb_handler, query, self._node_type) - filter_nodes_tb = self._filter_ancestors(gb_handler, set(nodes_tb), root_node_id) - + filter_nodes_tb = self._filter_ancestors(gb_handler, nodes_tb, root_node_id) filter_nodes_tb = { k: v for k, v in filter_nodes_tb.items() if self.is_node_valid(k, gb_handler) @@ -199,35 +179,48 @@ def get_intention_by_node_info_nlp( if len(filter_nodes_tb) == 0: error_msg = 'No intention matched after tb_handler retrieval.' ans = self._get_agent_ans_no_ekg(agent, query) - return asdict(NLPRetInfo(root_node_id, answer=ans, error_msg=error_msg)) + return asdict( + NLPRetInfo(root_node_id, answer=ans, error_msg=error_msg)) elif len(filter_nodes_tb) > 1: error_msg = 'More than one intention matched after tb_handler retrieval.' desc_list = [] for k, v in filter_nodes_tb.items(): - node_desc = gb_handler.get_current_node({'id': k}, node_type=self._node_type) + node_desc = gb_handler.get_current_node( + {'id': k}, node_type=self._node_type) node_desc = node_desc.attributes.get('description', '') - desc_list.append({'description': node_desc, 'path': ' -> '.join(v)}) - return asdict(NLPRetInfo(root_node_id, nodes_to_choose=desc_list, error_msg=error_msg)) + desc_list.append({ + 'description': node_desc, + 'path': ' -> '.join(v) + }) + return asdict( + NLPRetInfo(root_node_id, nodes_to_choose=desc_list, error_msg=error_msg) + ) root_node_id = list(filter_nodes_tb.keys())[0] return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) def _get_intention_by_nlp_from_root( - self, gb_handler: GBHandler, agent, root_node_id: str, query: str, + self, + gb_handler: GBHandler, + agent, + root_node_id: str, + query: str, ) -> dict[str, Any]: - canditates = gb_handler.get_neighbor_nodes({'id': root_node_id}, self._node_type) + canditates = gb_handler.get_neighbor_nodes({'id': root_node_id}, + self._node_type) canditates = [n for n in canditates if n.type == self._node_type] if len(canditates) == 0: return asdict(NLPRetInfo(root_node_id, True)) elif len(canditates) == 1: root_node_id = canditates[0].id - return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + return self._get_intention_by_nlp_from_root( + gb_handler, agent, root_node_id, query) desc_list = [x.attributes.get('description', '') for x in canditates] desc_list.append('与上述意图都不匹配，属于其他类型的询问意图。') query_intention = itp.get_intention_prompt( - '作为运维领域的客服，您需要根据用户询问判断其主要意图，以确定接下来的运维流程。', desc_list - ).format(query=query) + '作为智能助手，您需要根据用户询问判断其主要意图，以确定接下来的行动。', + desc_list).format(query=query) ans = agent.predict(query_intention).strip() ans = re.search('\d+', ans) @@ -235,11 +228,13 @@ def _get_intention_by_nlp_from_root( ans = int(ans.group(0)) - 1 if ans < len(desc_list) - 1: root_node_id = canditates[ans].id - return self._get_intention_by_nlp_from_root(gb_handler, agent, root_node_id, query) + return self._get_intention_by_nlp_from_root( + gb_handler, agent, root_node_id, query) error_msg = f'No intention matched at Node(id={root_node_id}).' ans = self._get_agent_ans_no_ekg(agent, query) - return asdict(NLPRetInfo(root_node_id, answer=ans, error_msg=error_msg)) + return asdict(NLPRetInfo(root_node_id, answer=ans, + error_msg=error_msg)) def get_intention_whether_execute(self, query: str, agent=None) -> bool: agent = agent if agent else self.agent @@ -250,7 +245,6 @@ def get_intention_whether_execute(self, query: str, agent=None) -> bool: ans = int(ans.group(0)) - 1 if ans < len(itp.INTENTIONS_WHETHER_EXEC): return itp.INTENTIONS_WHETHER_EXEC[ans][0] == '执行' - return False def get_intention_consult_which(self, query: str, agent=None) -> str: @@ -271,9 +265,15 @@ def _filter_from_root_node( if attribute is None or len(attribute) == 0: return root_node_id canditates = gb_handler.get_hop_infos( - {'id': root_node_id}, self._node_type, hop=self._filter_max_depth, + { + 'id': root_node_id + }, + self._node_type, + hop=self._filter_max_depth, ).nodes - canditates = [node for node in canditates if node.type == self._node_type] + canditates = [ + node for node in canditates if node.type == self._node_type + ] for node in canditates: count = len(attribute) @@ -285,29 +285,47 @@ def _filter_from_root_node( return None - def _tb_match(self, tb_handler: TbaseHandler, query: str, node_type: str) -> list: - base_query = f'(*)=>[KNN {self._max_num_tb_retrieval} @desc_vector $query AS distance]' - - query_vector = get_embedding( - self.embed_config.embed_engine, [query], - self.embed_config.embed_model_path, self.embed_config.model_device, - self.embed_config - )[query] - - query_params = {'query': np.array(query_vector).astype(dtype=np.float32).tobytes()} + def _tb_match(self, tb_handler: TbaseHandler, query: str, node_type: str, teamid=None) -> set: + def _vector_search(query_vector: bytes, key: str): + prefix = f'(@node_str: *{teamid}*)' if teamid else '' + prefix += '(@node_type: {})'.format(node_type) + base_query = f'{prefix}=>[KNN {self._max_num_tb_retrieval} @{key} $vector AS distance]' + query_params = {"vector": query_vector} + r = tb_handler.vector_search(base_query, query_params=query_params) + ret = [x.node_id for x in r.docs] + return ret + + def _keyword_search(query: str, key: str): + prefix = f'(@node_str: *{teamid}*)' if teamid else '' + query = prefix + f'(@node_type: {node_type})(@{key}:{{{keyword}}})' + r = tb_handler.search(query, limit=self._max_num_tb_retrieval) + ret = [x.node_id for x in r.docs] + return ret + + tb_results = [] + if self.embed_config: + query_vector = self._get_embedding(query) + ret = _execute_func_distributed( + _vector_search, + [(query_vector, key) for key in ('desc_vector', 'name_vector')] + ) + for temp_ret in ret: + tb_results.extend(temp_ret) + + keyword = '|'.join(extract_tags(query)) + ret = _execute_func_distributed( + _keyword_search, + [(keyword, key) for key in ('desc_keyword', 'name_keyword')] + ) + for temp_ret in ret: + tb_results.extend(temp_ret) - canditates = tb_handler.vector_search( - base_query, limit=self._max_num_tb_retrieval, query_params=query_params - ).docs - canditates = [ - node.node_id for node in canditates - if node.node_type == node_type - ] - return canditates + return set(tb_results) def is_node_valid(self, node_id: str, gb_handler: Optional[GBHandler] = None) -> bool: gb_handler = gb_handler if gb_handler is not None else self.gb_handler - canditates = gb_handler.get_neighbor_nodes({'id': node_id}, self._node_type) + canditates = gb_handler.get_neighbor_nodes({'id': node_id}, + self._node_type) if len(canditates) == 0: return False canditates = [n.id for n in canditates if n.type == self._node_type] @@ -320,12 +338,12 @@ def _get_agent_ans_no_ekg(self, agent, query: str) -> str: ans = agent.predict(query).strip() ans += f'\n\n以上内容由语言模型生成，仅供参考。' return ans - - def _filter_ancestors_hop( - self, gb_handler: GBHandler, nodes: set, root_node: str - ) -> dict[str, list[str]]: + + def _filter_ancestors_hop(self, gb_handler: GBHandler, nodes: set, root_node: str) -> dict[str, list[str]]: gb_ret = gb_handler.get_hop_infos( - {'id': root_node}, self._node_type, hop=self._filter_max_depth, + {'id': root_node}, + self._node_type, + hop=self._filter_max_depth, ) paths, nodes = gb_ret.paths, gb_ret.nodes if len(paths) == 0: @@ -349,10 +367,8 @@ def _filter_ancestors_hop( ret_dict[k] = [id2name[x] for x in v] return ret_dict - - def _filter_ancestors( - self, gb_handler: GBHandler, nodes: set, root_node: str - ) -> dict[str, list[str]]: + + def _filter_ancestors(self, gb_handler: GBHandler, nodes: set, root_node: str) -> dict[str, list[str]]: split = '<->' def _dfs(s: str, ancestor: str, path: str, out: dict, visited: set): @@ -376,7 +392,7 @@ def _dfs(s: str, ancestor: str, path: str, out: dict, visited: set): _dfs(child, temp_ancestor, child_path, out, visited) if s in nodes: visited.add(s) - + if len(nodes) == 0: return dict() filter_nodes = dict() @@ -386,12 +402,159 @@ def _dfs(s: str, ancestor: str, path: str, out: dict, visited: set): filter_nodes[k] = v.split(split) return filter_nodes + def _node_exist(self, node_id: str, gb_handler): + try: + gb_handler.get_current_node({'id': node_id}, node_type=self._node_type) + except IndexError: + return False + return True + + def _get_embedding(self, text: str): + text_vector = get_embedding( + self.embed_config.embed_engine, [text], self.embed_config.embed_model_path, + self.embed_config.model_device, self.embed_config + )[text] + return np.array(text_vector).astype(dtype=np.float32).tobytes() + + def _get_rule_by_str(self, rule: str) -> Optional[Callable]: + if rule is None: + return None + has_func = re.search('def ([a-zA-Z0-9_]+)\(.*\):', rule) + if not has_func: + return getattr(MatchRule, rule.strip(), None) + + func_name = has_func.group(1) + try: + exec(rule) + except Exception as e: + logger.info(f'Rule {rule} cannot be executed!') + logger.error(e) + return None + + return locals().get(func_name, None) + + def _equal_score_route_info_match(self, node_ids: Union[list, tuple], gb_handler): + if len(node_ids) == 1: + return node_ids[0] + for node_id in node_ids: + if self.is_node_valid(node_id, gb_handler): + return node_id + return node_ids[-1] + + def intention_df2graph( + self, data_df: pd.DataFrame, teamid: str, root_node_id: Optional[str] = None, + gb_handler: Optional[GBHandler] = None, + tb_handler: Optional[TbaseHandler] = None, + embed_config=None + ): + def _check_columns(): + df_columns = set(data_df.columns) + tar_columns = ('id', 'description', 'name', 'child_ids') + tar_col_absent = set(tar_columns) - df_columns + return tar_col_absent + + def _exist_cycle(in_degree: dict, out_graph: dict): + in_degree = in_degree.copy() + stack = deque( + [node for node in out_graph if node not in in_degree]) + while stack: + node = stack.popleft() + if not out_graph[node]: + continue + for i in out_graph[node]: + in_degree[i] -= 1 + if in_degree[i] == 0: + stack.append(i) + in_degree.pop(i) + return bool(in_degree) + + def _cat_id(*node_ids): + node_ids = [x for x in node_ids if x] + return '_'.join(node_ids) + + def _check_fail_num_gb(gb_results: list): + ret = sum([ + int(x["status"]["errorMessage"] not in ["GDB_SUCCEED"]) + for x in gb_results + ]) + return ret + + gb_handler = gb_handler if gb_handler is not None else self.gb_handler + tb_handler = tb_handler if tb_handler is not None else self.tb_handler + embed_config = embed_config if embed_config is not None else self.embed_config + + absent_cols = _check_columns() + if absent_cols: + return 'Columns {} do not exist!'.format(', '.join(absent_cols)) + + in_degrees, out_graphs = defaultdict(int), dict() + for _, row in data_df.iterrows(): + for child_id in row['child_ids']: + in_degrees[child_id] += 1 + out_graphs[row['id']] = row['child_ids'] + if _exist_cycle(in_degrees, out_graphs): + return 'Cycle exists!' + + edge_type = f'{self._node_type}_extend_{self._node_type}' + add_nodes, update_nodes, add_edges, del_edges = [], [], [], [] + for _, row in data_df.iterrows(): + node_id = _cat_id(root_node_id, row['id']) + node = GNode( + id=node_id, type=self._node_type, + attributes={'description': row['description'], 'name': row['name']} + ) + if self._node_exist(node_id, gb_handler): + update_nodes.append(node) + child_nodes = gb_handler.get_neighbor_nodes({'id': node_id}, self._node_type) + child_nodes = [node for node in child_nodes if node.node_type == self._node_type] + del_edges.extend([ + (double_hashing(node_id), double_hashing(child_node), edge_type) + for child_node in child_nodes + ]) + else: + add_nodes.append(node) + for child_id in row['child_ids']: + child_id = _cat_id(root_node_id, child_id) + add_edges.append(GEdge( + start_id=node_id, end_id=child_id, + type=edge_type, + attributes=dict() + )) + if not in_degrees[row['id']] and root_node_id: + add_edges.append(GEdge( + start_id=root_node_id, end_id=node_id, + type=edge_type, + attributes=dict() + )) + + from muagent.service.ekg_construct import EKGConstructService + ekg_construct = EKGConstructService(embed_config=embed_config) + ekg_construct.gb = gb_handler + ekg_construct.tb = tb_handler + + ret_add_nodes = ekg_construct.add_nodes(add_nodes, teamid=teamid)['gb_result'] + ret_update_nodes = ekg_construct.update_nodes(update_nodes, teamid=teamid)['gb_result'] + ret_del_edges = _execute_func_distributed(gb_handler.delete_edge, del_edges) + add_edges = ekg_construct._update_new_attr_for_edges(add_edges) + ret_add_edges = _execute_func_distributed(gb_handler.add_edge, add_edges) + + status, error_msg = True, '' + for gb_ret, prefix in zip( + (ret_add_nodes + ret_update_nodes, ret_add_edges, ret_del_edges), + ('Add Nodes', 'Add Edges', 'Delete Edges') + ): + fail_num = _check_fail_num_gb(gb_ret) + status &= fail_num == 0 + error_msg += '{}\tSuccess {}\tFail {}\n'.format(prefix, len(gb_ret) - fail_num, fail_num) + + return status, error_msg.strip() + def _parse_kwargs(**kwargs) -> list: keys = list(kwargs.keys()) if not isinstance(kwargs[keys[0]], (list, tuple)): return [kwargs] - + ret_list = [] max_len = max([len(v) for v in kwargs.values()]) for i in range(max_len): @@ -400,7 +563,7 @@ def _parse_kwargs(**kwargs) -> list: if i < len(v): cur_kwargs[k] = v[i] ret_list.append(cur_kwargs) - + return ret_list @@ -413,24 +576,8 @@ def _execute_func_distributed(func: Callable, params: list[Union[tuple, dict]]): else: task = exec.submit(func, **param) tasks.append(task) - + results = [] for task in as_completed(tasks): results.append(task.result()) return results - - -def _get_rule_by_str(rule: str) -> Union[str, Callable]: - has_func = re.search('def ([a-zA-Z0-9_]+)\(.*\):', rule) - if not has_func: - return getattr(MatchRule, rule.strip()) - - func_name = has_func.group(1) - try: - exec(rule) - except Exception as e: - logger.info(f'Rule {rule} cannot be executed!') - logger.error(e) - return None - - return locals().get(func_name, None) diff --git a/tests/service/ekg_construct_test_3.py b/tests/service/ekg_construct_test_3.py new file mode 100644 index 0000000..ec1c2c7 --- /dev/null +++ b/tests/service/ekg_construct_test_3.py @@ -0,0 +1,621 @@ +###################################################### +###################### 校验原子能力##################### +###################################################### + + +import time +import sys, os +from loguru import logger + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +sys.path.append(src_dir) + +import os, sys +from loguru import logger + +sys.path.append("/ossfs/workspace/muagent") +from muagent.schemas.common import GNode, GEdge +from muagent.schemas.db import GBConfig, TBConfig +from muagent.service.ekg_construct import EKGConstructService +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig + + + + + +# 初始化 GeaBaseHandler 实例 +gb_config = GBConfig( + gb_type="GeaBaseHandler", + extra_kwargs={ + 'metaserver_address': os.environ['metaserver_address'], + 'project': os.environ['project'], + 'city': os.environ['city'], + 'lib_path': os.environ['lib_path'], + } +) + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="teamida_test", + host=os.environ['host'], + port=os.environ['port'], + username=os.environ['username'], + password=os.environ['password'], + extra_kwargs={ + 'host': os.environ['host'], + 'port': os.environ['port'], + 'username': os.environ['username'] , + 'password': os.environ['password'], + 'definition_value': os.environ['definition_value'] + } +) + +# llm config +llm_config = LLMConfig( + model_name=model_name, model_engine=model_engine, api_key=api_key, api_base_url=api_base_url, temperature=0.3, +) + + +# emebdding config +# embed_config = EmbedConfig( +# embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path +# ) + +# embed_config = EmbedConfig( +# embed_model="default", +# langchain_embeddings=embeddings +# ) +embed_config = None + +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) + + +def generate_node(id, type): + extra_attr = {"tmp": "hello"} + if type == "opsgptkg_schedule": + extra_attr["enable"] = False + + if type == "opsgptkg_task": + extra_attr["accesscriteria"] = "hello" + extra_attr["executetype"] = "hello" + + if type == "opsgptkg_analysis": + extra_attr["accesscriteria"] = "hello" + extra_attr["summaryswitch"] = False + extra_attr['dsltemplate'] = "hello" + + return GNode(**{ + "id": id, + "type": type, + "attributes": {**{ + "path": id, + "name": id, + "description": id, + }, **extra_attr} + }) + +def generate_edge(node1, node2): + type_connect = "extend" if node1.type == "opsgptkg_intent" and node2.type == "opsgptkg_intent" else "route" + return GEdge(**{ + "start_id": node1.id, + "end_id": node2.id, + "type": f"{node1.type}_{type_connect}_{node2.type}", + "attributes": { + "lat": "hello", + "attr": "hello" + } + }) + +nodetypes = [ + 'opsgptkg_intent', 'opsgptkg_schedule', 'opsgptkg_task', + 'opsgptkg_phenomenon', 'opsgptkg_analysis' +] + +nodes_dict = {} +for nodetype in nodetypes: + for i in range(8): + nodes_dict[f"teamida_{nodetype}_{i}"] = generate_node(f"teamida_{nodetype}_{i}", nodetype) + +for nodetype in nodetypes: + for i in range(8): + nodes_dict[f"teamidb_{nodetype}_{i}"] = generate_node(f"teamidb_{nodetype}_{i}", nodetype) + +for nodetype in nodetypes: + for i in range(8): + nodes_dict[f"teamidc_{nodetype}_{i}"] = generate_node(f"teamidc_{nodetype}_{i}", nodetype) + + + +edge_ids = [ + ["teamida_opsgptkg_intent_0", "teamida_opsgptkg_intent_1"], + ["teamida_opsgptkg_intent_1", "teamida_opsgptkg_intent_2"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_0"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_1"], + ["teamida_opsgptkg_schedule_1", "teamida_opsgptkg_analysis_3"], + ["teamida_opsgptkg_schedule_0", "teamida_opsgptkg_task_0"], + ["teamida_opsgptkg_task_0", "teamida_opsgptkg_task_1"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_analysis_0"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_phenomenon_0"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_phenomenon_1"], + ["teamida_opsgptkg_phenomenon_0", "teamida_opsgptkg_task_2"], + ["teamida_opsgptkg_phenomenon_1", "teamida_opsgptkg_task_3"], + ["teamida_opsgptkg_task_2", "teamida_opsgptkg_analysis_1"], + ["teamida_opsgptkg_task_3", "teamida_opsgptkg_analysis_2"], +] + + +nodeid_set = set() +origin_edges = [] +origin_nodes = [] +for src_id, dst_id in edge_ids: + origin_edges.append(generate_edge(nodes_dict[src_id], nodes_dict[dst_id])) + if src_id not in nodeid_set: + nodeid_set.add(src_id) + origin_nodes.append(nodes_dict[src_id]) + if dst_id not in nodeid_set: + nodeid_set.add(dst_id) + origin_nodes.append(nodes_dict[dst_id]) + + +flags = [] +### 测试原子能力 +teamid = "teamida" + +# 测试边的添加功能 +## 1、先加节点 +for node in origin_nodes: + result = ekg_construct_service.add_nodes([node], teamid) + +error_cnt = 0 +for node in origin_nodes: + try: + result = ekg_construct_service.get_node_by_id(node.id, node.type) + except: + error_cnt+=1 +flags.append(error_cnt==0) +print("节点添加功能正常" if error_cnt==0 else "节点添加功能异常") + +## 2、添加边 +for edge in origin_edges: + result = ekg_construct_service.add_edges([edge], teamid) + +neighbors_by_nodeid = {} +for edge in origin_edges: + neighbors_by_nodeid.setdefault(edge.start_id, []).append(edge.end_id) + +error_cnt = 0 +for nodeid, neighbors in neighbors_by_nodeid.items(): + + result = ekg_construct_service.gb.get_neighbor_nodes( + {"id": nodeid}, nodes_dict[nodeid].type) + if set(neighbors)!=set([n.id for n in result]): + error_cnt += 1 +flags.append(error_cnt==0) +print("边添加功能正常" if error_cnt==0 else "边添加功能异常") + + +## 3、修改节点 +for node in origin_nodes: + node.attributes["name"] = "muagenttest" + result = ekg_construct_service.update_nodes([node], teamid) + +error_cnt = 0 +for node in origin_nodes: + try: + result = ekg_construct_service.get_node_by_id(node.id, node.type) + error_cnt += result.attributes["name"] != "muagenttest" + except: + error_cnt+=1 +flags.append(error_cnt==0) +print("节点修改功能正常" if error_cnt==0 else "节点修改功能异常") + +## 4、路径查询 +connections = {} +for edge in origin_edges: + connections.setdefault(edge.start_id, []).append(edge.end_id) + +visited = set() +rootid_can_arrive_nodeids = [] +paths = [] +def _dfs(node, current_path): + if node not in visited: + visited.add(node) + current_path.append(node) + rootid_can_arrive_nodeids.append(node) + # stop condition, there is no more neightbos + if not connections.get(node, []): + # when arrive the endpoiond, save the copy of current path + paths.append(list(current_path)) + else: + for neighbor in connections.get(node, []): + _dfs(neighbor, current_path) + + # recursive：remove the last node + current_path.pop() + +# init DFS +_dfs("teamida_opsgptkg_intent_0", []) +graph = ekg_construct_service.get_graph_by_nodeid("teamida_opsgptkg_intent_0", "opsgptkg_intent", 29) +set([', '.join(i) for i in graph.paths])==set([', '.join(i) for i in paths]) +flags.append(set([', '.join(i) for i in graph.paths])==set([', '.join(i) for i in paths])) +print("路径查询功能正常" if set([', '.join(i) for i in graph.paths])==set([', '.join(i) for i in paths]) else "路径查询功能异常") + + + + +## 5、测试边的删除功能 +for edge in origin_edges: + result = ekg_construct_service.delete_edges([edge], teamid) + +neighbors_by_nodeid = {} +for edge in origin_edges: + neighbors_by_nodeid.setdefault(edge.start_id, []).append(edge.end_id) + +error_cnt = 0 +for nodeid, neighbors in neighbors_by_nodeid.items(): + + result = ekg_construct_service.gb.get_neighbor_nodes( + {"id": nodeid}, nodes_dict[nodeid].type) + if set(neighbors)==set([n.id for n in result]): + error_cnt += 1 +flags.append(error_cnt==0) +print("边删除功能正常" if error_cnt==0 else "边删除功能异常") + +## 6、测试节点的删除功能 +for node in origin_nodes: + result = ekg_construct_service.delete_nodes([node], teamid) + +error_cnt = 0 +for node in origin_nodes: + try: + result = ekg_construct_service.get_node_by_id(node.id, node.type) + except: + error_cnt += 1 +flags.append(error_cnt==len(origin_nodes)) +print("节点删除功能正常" if error_cnt==len(origin_nodes) else "节点删除功能异常") + + + +if sum(flags) != len(flags): + sys.exit(f"存在功能异常, {flags}") + + +############################## +###### 测试交叉团队的增删改 ##### +############################## + +edge_ids_by_teamid = { + "teamida": [ + ["teamida_opsgptkg_intent_0", "teamida_opsgptkg_intent_1"], + ["teamida_opsgptkg_intent_1", "teamida_opsgptkg_intent_2"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_0"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_1"], + ["teamida_opsgptkg_schedule_1", "teamida_opsgptkg_analysis_3"], + ["teamida_opsgptkg_schedule_0", "teamida_opsgptkg_task_0"], + ["teamida_opsgptkg_task_0", "teamida_opsgptkg_task_1"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_analysis_0"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_phenomenon_0"], + ["teamida_opsgptkg_task_1", "teamida_opsgptkg_phenomenon_1"], + ["teamida_opsgptkg_phenomenon_0", "teamida_opsgptkg_task_2"], + ["teamida_opsgptkg_phenomenon_1", "teamida_opsgptkg_task_3"], + ["teamida_opsgptkg_task_2", "teamida_opsgptkg_analysis_1"], + ["teamida_opsgptkg_task_3", "teamida_opsgptkg_analysis_2"], + ], + "teamidb": [ + ["teamidb_opsgptkg_intent_0", "teamidb_opsgptkg_intent_1"], + ["teamidb_opsgptkg_intent_1", "teamidb_opsgptkg_intent_2"], + ["teamidb_opsgptkg_intent_2", "teamidb_opsgptkg_schedule_0"], + ["teamidb_opsgptkg_intent_2", "teamidb_opsgptkg_schedule_1"], + ["teamidb_opsgptkg_schedule_1", "teamidb_opsgptkg_analysis_3"], + ["teamidb_opsgptkg_schedule_0", "teamidb_opsgptkg_task_0"], + ["teamidb_opsgptkg_task_0", "teamidb_opsgptkg_task_1"], + ["teamidb_opsgptkg_task_1", "teamidb_opsgptkg_analysis_0"], + ["teamidb_opsgptkg_task_1", "teamidb_opsgptkg_phenomenon_0"], + ["teamidb_opsgptkg_task_1", "teamidb_opsgptkg_phenomenon_1"], + ["teamidb_opsgptkg_phenomenon_0", "teamidb_opsgptkg_task_2"], + ["teamidb_opsgptkg_phenomenon_1", "teamidb_opsgptkg_task_3"], + ["teamidb_opsgptkg_task_2", "teamidb_opsgptkg_analysis_1"], + ["teamidb_opsgptkg_task_3", "teamidb_opsgptkg_analysis_2"], + ], + "teamidc": [ + ["teamidc_opsgptkg_intent_0", "teamidc_opsgptkg_intent_1"], + ["teamidc_opsgptkg_intent_1", "teamidc_opsgptkg_intent_2"], + ["teamidc_opsgptkg_intent_2", "teamidc_opsgptkg_schedule_0"], + ["teamidc_opsgptkg_intent_2", "teamidc_opsgptkg_schedule_1"], + ["teamidc_opsgptkg_schedule_1", "teamidc_opsgptkg_analysis_3"], + ["teamidc_opsgptkg_schedule_0", "teamidc_opsgptkg_task_0"], + ["teamidc_opsgptkg_task_0", "teamidc_opsgptkg_task_1"], + ["teamidc_opsgptkg_task_1", "teamidc_opsgptkg_analysis_0"], + ["teamidc_opsgptkg_task_1", "teamidc_opsgptkg_phenomenon_0"], + ["teamidc_opsgptkg_task_1", "teamidc_opsgptkg_phenomenon_1"], + ["teamidc_opsgptkg_phenomenon_0", "teamidc_opsgptkg_task_2"], + ["teamidc_opsgptkg_phenomenon_1", "teamidc_opsgptkg_task_3"], + ["teamidc_opsgptkg_task_2", "teamidc_opsgptkg_analysis_1"], + ["teamidc_opsgptkg_task_3", "teamidc_opsgptkg_analysis_2"], + ], + "mixer": [ + ["teamida_opsgptkg_intent_2", "teamidb_opsgptkg_schedule_0"], + ["teamida_opsgptkg_intent_2", "teamidc_opsgptkg_schedule_0"], + ["teamidb_opsgptkg_intent_2", "teamidc_opsgptkg_schedule_0"], + ["teamidb_opsgptkg_schedule_1", "teamidc_opsgptkg_task_0"], + ["teamidb_opsgptkg_schedule_0", "teamidc_opsgptkg_task_1"], + ], +} + +neighbors_by_nodeid = {} +for teamid, origin_edges in edge_ids_by_teamid.items(): + for edge in origin_edges: + neighbors_by_nodeid.setdefault(edge.start_id, []).append(edge.end_id) + + +flags = [] +# 测试边的添加功能 +## 1、先加节点 +for node in origin_nodes: + teamid = node.id.split("_")[0] + result = ekg_construct_service.add_nodes([node], teamid) + +error_cnt = 0 +for node in origin_nodes: + try: + result = ekg_construct_service.get_node_by_id(node.id, node.type) + except: + error_cnt+=1 +flags.append(error_cnt==0) +print("节点添加功能正常" if error_cnt==0 else "节点添加功能异常") + +## 2、添加边 +for edge in origin_edges: + teamid = edge.start_id.split("_")[0] + result = ekg_construct_service.add_edges([edge], teamid) + +neighbors_by_nodeid = {} +for edge in origin_edges: + neighbors_by_nodeid.setdefault(edge.start_id, []).append(edge.end_id) + +error_cnt = 0 +for nodeid, neighbors in neighbors_by_nodeid.items(): + + result = ekg_construct_service.gb.get_neighbor_nodes( + {"id": nodeid}, nodes_dict[nodeid].type) + if set(neighbors)!=set([n.id for n in result]): + error_cnt += 1 +flags.append(error_cnt==0) +print("边添加功能正常" if error_cnt==0 else "边添加功能异常") + + + + +delete_edges_by_teamid = { + "teamidc": ["teamidb_opsgptkg_schedule_0", "teamidc_opsgptkg_task_1"], +} +# delete edge: teamidb_opsgptkg_schedule_0, teamidc_opsgptkg_task_1 +delete_edgeid = ["teamidb_opsgptkg_schedule_0", "teamidc_opsgptkg_task_1"] +nodeid = "teamidb_opsgptkg_schedule_0" +for edge in origin_edges: + do_delete = False + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + do_delete = True + break + +result = ekg_construct_service.delete_edges([edge], teamid="teamida") +# check +result = ekg_construct_service.gb.get_neighbor_nodes( + {"id": nodeid}, nodes_dict[nodeid].type) +print("跨团队删除边功能正常" if set(neighbors_by_nodeid[nodeid])!=set([n.id for n in result]) else "跨团队删除边功能正常") +# add back to test other functions +ekg_construct_service.add_edges([edge], teamid="teamida") + + +delete_nodes_by_teamid = { + "teamida": ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_analysis_3"], + "teamidb": ["teamidb_opsgptkg_schedule_1"], + "teamidc": ["teamidc_opsgptkg_intent_1", "teamidc_opsgptkg_schedule_0"], +} + +# delete teamida_opsgptkg_intent_2 +delete_edgeids = [ + ["teamida_opsgptkg_intent_1", "teamida_opsgptkg_intent_2"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_0"], + ["teamida_opsgptkg_intent_2", "teamida_opsgptkg_schedule_1"] +] +delete_nodeid = "teamida_opsgptkg_intent_2" +delete_edges = [] +for edge in origin_edges: + do_delete = False + for delete_edgeid in delete_edgeids: + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + delete_edges.append(edge) + break + +result = ekg_construct_service.delete_edges(delete_edges, teamid="teamida") +result = ekg_construct_service.delete_nodes_v2([nodes_dict[delete_nodeid]], teamid="teamida") +try: + result = ekg_construct_service.get_node_by_id(delete_nodeid, nodes_dict[delete_nodeid].type) + print("删除节点逻辑正常") +except Exception as e: + print(e) + print("删除节点逻辑异常") + +result = ekg_construct_service.add_nodes([nodes_dict[delete_nodeid]], teamid="teamida") +result = ekg_construct_service.add_edges(delete_edges, teamid="teamida") + + + + + +# delete teamida_opsgptkg_analysis_3 +delete_edgeids = [ + ["teamida_opsgptkg_schedule_1", "teamida_opsgptkg_analysis_3"], +] +delete_nodeid = "teamida_opsgptkg_analysis_3" +delete_edges = [] +for edge in origin_edges: + do_delete = False + for delete_edgeid in delete_edgeids: + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + delete_edges.append(edge) + break + +result = ekg_construct_service.delete_edges(delete_edges, teamid="teamida") +result = ekg_construct_service.delete_nodes_v2([nodes_dict[delete_nodeid]], teamid="teamida") +try: + result = ekg_construct_service.get_node_by_id(delete_nodeid, nodes_dict[delete_nodeid].type) + print("删除节点逻辑异常") +except Exception as e: + print(e) + print("删除节点逻辑正常") + +result = ekg_construct_service.add_nodes([nodes_dict[delete_nodeid]], teamid="teamida") +result = ekg_construct_service.add_edges(delete_edges, teamid="teamida") + + +# delete node: teamidb_opsgptkg_schedule_1 +delete_edgeids = [ + ["teamidb_opsgptkg_schedule_1", "teamidb_opsgptkg_analysis_3"], + ["teamidb_opsgptkg_intent_2", "teamidb_opsgptkg_schedule_1"], +] +delete_nodeid = "teamidb_opsgptkg_schedule_1" +teamid = "teamidb" +delete_edges = [] +for edge in origin_edges: + do_delete = False + for delete_edgeid in delete_edgeids: + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + delete_edges.append(edge) + break + +result = ekg_construct_service.delete_edges(delete_edges, teamid=teamid) +result = ekg_construct_service.delete_nodes_v2([nodes_dict[delete_nodeid]], teamid=teamid) +try: + result = ekg_construct_service.get_node_by_id(delete_nodeid, nodes_dict[delete_nodeid].type) + print("删除节点逻辑正常") +except Exception as e: + print(e) + print("删除节点逻辑异常") + +result = ekg_construct_service.add_nodes([nodes_dict[delete_nodeid]], teamid=teamid) +result = ekg_construct_service.add_edges(delete_edges, teamid=teamid) + + +# delete node: teamidc_opsgptkg_intent_1 +delete_edgeids = [ + ["teamidc_opsgptkg_intent_0", "teamidc_opsgptkg_intent_1"], + ["teamidc_opsgptkg_intent_1", "teamidc_opsgptkg_intent_2"], +] +delete_nodeid = "teamidc_opsgptkg_intent_1" +teamid = "teamidc" +delete_edges = [] +for edge in origin_edges: + do_delete = False + for delete_edgeid in delete_edgeids: + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + delete_edges.append(edge) + break + +result = ekg_construct_service.delete_edges(delete_edges, teamid=teamid) +result = ekg_construct_service.delete_nodes_v2([nodes_dict[delete_nodeid]], teamid=teamid) +try: + result = ekg_construct_service.get_node_by_id(delete_nodeid, nodes_dict[delete_nodeid].type) + print("删除节点逻辑异常") +except Exception as e: + print(e) + print("删除节点逻辑正常") + +result = ekg_construct_service.add_nodes([nodes_dict[delete_nodeid]], teamid=teamid) +result = ekg_construct_service.add_edges(delete_edges, teamid=teamid) + + + +# delete node: teamidc_opsgptkg_schedule_0 +delete_edgeids = [ + ["teamidc_opsgptkg_schedule_0", "teamidc_opsgptkg_task_0"], + ["teamidc_opsgptkg_intent_2", "teamidc_opsgptkg_schedule_0"], +] +delete_nodeid = "teamidc_opsgptkg_schedule_0" +teamid = "teamidc" +delete_edges = [] +for edge in origin_edges: + do_delete = False + for delete_edgeid in delete_edgeids: + if delete_edgeid[0]==edge.start_id and delete_edgeid[1] == edge.end_id: + delete_edges.append(edge) + break + +result = ekg_construct_service.delete_edges(delete_edges, teamid=teamid) +result = ekg_construct_service.delete_nodes_v2([nodes_dict[delete_nodeid]], teamid=teamid) +try: + result = ekg_construct_service.get_node_by_id(delete_nodeid, nodes_dict[delete_nodeid].type) + print("删除节点逻辑正常") +except Exception as e: + print(e) + print("删除节点逻辑异常") + +result = ekg_construct_service.add_nodes([nodes_dict[delete_nodeid]], teamid=teamid) +result = ekg_construct_service.add_edges(delete_edges, teamid=teamid) + + +# 保证数据全部被删除 +## 5、测试边的删除功能 +for edge in origin_edges: + teamid = edge.start_id.split("_")[0] + result = ekg_construct_service.delete_edges([edge], teamid) + +neighbors_by_nodeid = {} +for edge in origin_edges: + neighbors_by_nodeid.setdefault(edge.start_id, []).append(edge.end_id) + +error_cnt = 0 +for nodeid, neighbors in neighbors_by_nodeid.items(): + + result = ekg_construct_service.gb.get_neighbor_nodes( + {"id": nodeid}, nodes_dict[nodeid].type) + if set(neighbors)==set([n.id for n in result]): + error_cnt += 1 +flags.append(error_cnt==0) +print("边删除功能正常" if error_cnt==0 else "边删除功能异常") + +## 6、测试节点的删除功能 +for node in origin_nodes: + teamid = node.id.split("_")[0] + result = ekg_construct_service.delete_nodes([node], teamid) + +error_cnt = 0 +for node in origin_nodes: + try: + result = ekg_construct_service.get_node_by_id(node.id, node.type) + except: + error_cnt += 1 +flags.append(error_cnt==len(origin_nodes)) +print("节点删除功能正常" if error_cnt==len(origin_nodes) else "节点删除功能异常") \ No newline at end of file diff --git a/tests/service/intention_router_test.py b/tests/service/intention_router_test.py new file mode 100644 index 0000000..bde6a5f --- /dev/null +++ b/tests/service/intention_router_test.py @@ -0,0 +1,187 @@ +import os +import sys +from loguru import logger + +src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +if src_dir not in sys.path: + sys.path.append(src_dir) + +from muagent.service.ekg_inference import IntentionRouter +from muagent.schemas.db import GBConfig, TBConfig +from muagent.llm_models.llm_config import EmbedConfig, LLMConfig +from muagent.service.ekg_construct import EKGConstructService +from muagent.schemas.ekg.ekg_graph import NodeTypesEnum +from muagent.schemas.common import GNode, GEdge +from langchain.llms.base import LLM + + +try: + src_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + sys.path.append(src_dir) + import test_config + api_key = os.environ["OPENAI_API_KEY"] + api_base_url= os.environ["API_BASE_URL"] + model_name = os.environ["model_name"] + model_engine = os.environ["model_engine"] + embed_model = os.environ["embed_model"] + embed_model_path = os.environ["embed_model_path"] +except Exception as e: + # set your config + api_key = "" + api_base_url= "" + model_name = "" + model_engine = os.environ["model_engine"] + model_engine = "" + embed_model = "" + embed_model_path = "" + logger.error(f"{e}") + + +# 初始化 TbaseHandler 实例 +tb_config = TBConfig( + tb_type="TbaseHandler", + index_name="muagent_test", + host=os.environ['host'], + port=os.environ['port'], + username=os.environ['username'], + password=os.environ['password'], + extra_kwargs={ + 'host': os.environ['host'], + 'port': os.environ['port'], + 'username': os.environ['username'] , + 'password': os.environ['password'], + 'definition_value': os.environ['definition_value'] + } +) + +# 初始化 NebulaHandler 实例 +gb_config = GBConfig( + gb_type="NebulaHandler", + extra_kwargs={ + 'host': os.environ['nb_host'], + 'port': os.environ['nb_port'], + 'username': os.environ['nb_username'] , + 'password': os.environ['nb_password'], + "space": os.environ['nb_space'], + 'definition_value': os.environ['nb_definition_value'] + + } +) + +# llm config +llm_config = LLMConfig( + model_name=model_name, + model_engine=model_engine, + api_key=api_key, + api_base_url=api_base_url, + temperature=0.3, +) + +# embed config +embed_config = None + +ekg_construct_service = EKGConstructService( + embed_config=embed_config, + llm_config=llm_config, + tb_config=tb_config, + gb_config=gb_config, +) + +intention_router = IntentionRouter( + ekg_construct_service.model, + ekg_construct_service.gb, + ekg_construct_service.tb, + embed_config +) + + +# 生成节点和意图树 +def generate_intention_node(id: str, attrs: dict): + assert 'description' in attrs + return GNode(**{ + 'id': id, + 'type': NodeTypesEnum.INTENT.value, + 'attributes': attrs + }) + + +def generate_edge(node1: GNode, node2: GNode): + type_connect = "extend" if node1.type == "opsgptkg_intent" and node2.type == "opsgptkg_intent" else "route" + return GEdge(**{ + "start_id": node1.id, + "end_id": node2.id, + "type": f"{node1.type}_{type_connect}_{node2.type}", + "attributes": { + "lat": "hello", + "attr": "hello" + } + }) + +def generate_intention_flow(descs_dict: dict): + def _generate_intention_flow(desc_dict, out_node: dict, out_edge: list, start_idx=0): + for k, v in desc_dict.items(): + out_node[k] = generate_intention_node( + id=f'intention_test_{start_idx}', + attrs={'description': k, 'name': ''} + ) + start_idx += 1 + if v is not None: + start_idx = _generate_intention_flow(v, out_node, out_edge, start_idx) + for k_1 in v: + out_edge.append(generate_edge(out_node[k], out_node[k_1])) + return start_idx + + nodes, edges = dict(), [] + _generate_intention_flow(descs_dict, nodes, edges, 0) + return nodes, edges + + +descriptions_flow = { + '节假日安排': { + '当前时间在上午9:00~12:00之间': {'心情不错': None, '心情很一般': None, '心情很糟糕': None}, + '当前时间在下午14:00~18:00之间': {'很有活力': None, '无精打采': None}, + '当前时间在晚上8点~11点之间': {'今天是周六': None, '今天不是周六': None}, + '现在已经晚上23点过了': {'该睡觉了': None} + } +} +query = '现在是周六晚上9点' +intention_nodes, intention_edges = generate_intention_flow(descriptions_flow) +print('Nodes...') +for v in intention_nodes.values(): + print(v) +print('Edges') +for edge in intention_edges: + print(edge) + +# 添加节点和边 +ekg_construct_service.add_nodes(list(intention_nodes.values()), teamid='intention_test') +ekg_construct_service.add_edges(intention_edges, teamid='intention_test') + +# nlp 路由 +out = intention_router.get_intention_by_node_info_nlp( + root_node_id=intention_nodes[next(iter(descriptions_flow))].id, + query=query, + start_from_root=True +) +print(out) + + +# 路由匹配 +# rule = """import re +# def func(node: GNode, query: str): +# nums = re.findall('[1-9]+', getattr(node, 'description', '')) +# if not nums: +# return -float('inf') +# query_time = re.findall('[1-9]+', query)[0] +# return int(query_time > nums[0]) +# """ +# out = intention_router.get_intention_by_node_info_match( +# root_node_id=intention_nodes[next(iter(descriptions_flow))].id, +# rule=rule, +# query=query +# ) +# print(out) From 834dbe809ceec2031b7719599f85062cca06a407 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Mon, 23 Sep 2024 14:04:21 +0800 Subject: [PATCH 046/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4d577e..d08b014 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ cd CodeFuse-muAgent docker-compose up -d ``` -The current image version includes only the basic EKG service. We expect to provide a front-end interface and back-end interaction services in September. +The current image version includes only the basic EKG service. We plan to launch the front-end interface and back-end interaction services in Octorber. To Be Continued! From d141a2e0ea50c52a21f5d69ecff638e4638ac694 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Mon, 23 Sep 2024 14:05:28 +0800 Subject: [PATCH 047/128] Update README_zh.md --- README_zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_zh.md b/README_zh.md index c1cd872..e9715cc 100644 --- a/README_zh.md +++ b/README_zh.md @@ -55,7 +55,7 @@ cd CodeFuse-muAgent docker-compose up -d ``` -当前镜像版本仅包含了EKG基础服务。我们将会在9月底提供前端交互和后端交互的镜像服务。 +当前镜像版本仅包含了EKG基础服务。我们将会在10月份提供前端交互和后端交互的镜像服务。 敬请期待! From 097fcbdf6820c2eab230e2f3617cbd434fab758f Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:26:19 +0800 Subject: [PATCH 048/128] Update README_zh.md --- README_zh.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README_zh.md b/README_zh.md index e9715cc..5989ea6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -30,10 +30,12 @@ ## 🤝 介绍 +

全新体验的 Agent 框架，将KG从知识获取来源直接升级为Agent编排引擎！基于 LLM+ EKG（Eventic Knowledge Graph 行业知识承载）驱动，协同 MultiAgent、FunctionCall、CodeInterpreter等技术，通过画布式拖拽、轻文字编写，让大模型在人的经验指导下帮助你实现各类复杂 SOP 流程。兼容现有市面各类 Agent 框架，同时可实现复杂推理、在线协同、人工交互、知识即用四大核心差异技术功能。这套框架目前在蚂蚁集团内多个复杂DevOps场景落地验证，同时来体验下我们快速搭建的谁是卧底游戏吧。 - - -![](docs/resources/ekg-arch-zh.webp) +

+
+ muAgent Architecture +
## 🚀 快速使用 @@ -88,6 +90,6 @@ pip install codefuse-muagent ## 🗂 其他 ### 📱 联系我们
- 图片 + 图片
From 6f7e17b1f884017268087d9f6cf83fee9bd99a01 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:47:27 +0800 Subject: [PATCH 049/128] Update README_zh.md --- README_zh.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README_zh.md b/README_zh.md index 5989ea6..c4a05dc 100644 --- a/README_zh.md +++ b/README_zh.md @@ -18,7 +18,7 @@ ## 🔔 更新 - [2024.04.01] muAgent正式开源，聚焦多agent编排，协同FunctionCall、RAG、CodeInterpreter等技术 -- [2024.09.05] 发布 muAgent v2.0 - EKG：一款由知识图谱引擎驱动的创新Agent框架 +- [2024.09.05] 发布 muAgent v2.0 - EKG：一款由知识图谱引擎驱动的创新Agent产品框架 ## 📜 目录 - [🤝 介绍](#-介绍) @@ -31,7 +31,7 @@ ## 🤝 介绍

-全新体验的 Agent 框架，将KG从知识获取来源直接升级为Agent编排引擎！基于 LLM+ EKG（Eventic Knowledge Graph 行业知识承载）驱动，协同 MultiAgent、FunctionCall、CodeInterpreter等技术，通过画布式拖拽、轻文字编写，让大模型在人的经验指导下帮助你实现各类复杂 SOP 流程。兼容现有市面各类 Agent 框架，同时可实现复杂推理、在线协同、人工交互、知识即用四大核心差异技术功能。这套框架目前在蚂蚁集团内多个复杂DevOps场景落地验证，同时来体验下我们快速搭建的谁是卧底游戏吧。 +全新体验的Agent框架，将KG从知识获取来源直接升级为Agent编排引擎！基于LLM+ EKG（Eventic Knowledge Graph 行业知识承载）驱动，协同MultiAgent、FunctionCall、CodeInterpreter等技术，通过画布式拖拽、轻文字编写，让大模型在人的经验指导下帮助你实现各类复杂 SOP 流程。兼容现有市面各类Agent框架，同时可实现复杂推理、在线协同、人工交互、知识即用四大核心差异技术功能。这套框架目前在蚂蚁集团内多个复杂DevOps场景落地验证，同时来体验下我们快速搭建的谁是卧底游戏吧。

muAgent Architecture @@ -80,12 +80,11 @@ pip install codefuse-muagent - **操作空间**：遵循Swagger协议，提供工具注册、权限管理、统一分类，方便LLM在工具调用中接入使用；提供安全可信代码执行环境，同时确保代码精准生成，满足可视绘图、数值计算、图表编辑等各类场景诉求 ## 🤗 贡献指南 -非常感谢您对 Codefuse 项目感兴趣，我们非常欢迎您对 Codefuse 项目的各种建议、意见（包括批评）、评论和贡献。 +感谢您对muAgent项目的关注！我们欢迎您的任何建议、意见（包括批评）和贡献。 -您对 Codefuse 的各种建议、意见、评论可以直接通过 GitHub 的 Issues 提出。 - -参与 Codefuse 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表。详见[Contribution Guide...](https://codefuse-ai.github.io/zh-CN/contribution/issue) +为了更好促进项目的发展，我们鼓励您通过GitHub的Issues提交您对项目的各种建议、意见和评论。 +参与Codefuse项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表。详见[Contribution Guide...](https://codefuse-ai.github.io/zh-CN/contribution/issue) ## 🗂 其他 ### 📱 联系我们 From e8fc9e8cc0ea2f084c0df158442d417e3698b053 Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:39:56 +0800 Subject: [PATCH 050/128] Update README_zh.md --- README_zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_zh.md b/README_zh.md index c4a05dc..5585eb9 100644 --- a/README_zh.md +++ b/README_zh.md @@ -84,7 +84,7 @@ pip install codefuse-muagent 为了更好促进项目的发展，我们鼓励您通过GitHub的Issues提交您对项目的各种建议、意见和评论。 -参与Codefuse项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表。详见[Contribution Guide...](https://codefuse-ai.github.io/zh-CN/contribution/issue) +参与Codefuse项目并为其作出贡献的方法有很多：代码实现、测试编写、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表，详见[Contribution Guide](https://codefuse-ai.github.io/contribution/contribution)。 ## 🗂 其他 ### 📱 联系我们 From 64a52fa81d3150fbcd31efeca54391a48b9f6d99 Mon Sep 17 00:00:00 2001 From: lightislost <31849436+lightislost@users.noreply.github.com> Date: Thu, 10 Oct 2024 19:04:26 +0800 Subject: [PATCH 051/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d08b014..a06348b 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ git clone https://github.com/codefuse-ai/CodeFuse-muAgent.git cd CodeFuse-muAgent # step3. start all container services, it might cost some time -docker-compose up -d +docker network create ekg-net && docker-compose up -d ``` The current image version includes only the basic EKG service. We plan to launch the front-end interface and back-end interaction services in Octorber. From f7f84355c2104457c96537831a3049f571fc999a Mon Sep 17 00:00:00 2001 From: cresting1222 <131508362+cresting1222@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:48:35 +0800 Subject: [PATCH 052/128] Update README_zh.md --- README_zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_zh.md b/README_zh.md index 5585eb9..031000b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -72,7 +72,7 @@ pip install codefuse-muagent ## 🧭 关键技术 -- **图谱构建**：通过虚拟团队构建、场景意图划分，让你体验在线文档VS本地文档的差别；同时，文本语义输入的节点使用方式，让你感受有注释代码VS无注释代码的差别，充分体现在线协同的优势；面向海量存量文档（通用文本、流程画板等），支持文本智能解析、一键导入 +- **图谱构建**：通过虚拟团队构建、场景意图划分，让你体验在线文档VS本地文档的差别；同时，文本语义输入的节点使用方式，让你感受有注释代码VS无注释代码的差别，充分体现在线协同的优势；面向海量存量文档（通用文本、流程画板等），支持文本智能解析、一键导入，以及经验拆分泛化 - **图谱资产**：通过场景意图、事件流程、统一工具、组织人物四部分的统一图谱设计，满足各类SOP场景所需知识承载；工具在图谱的纳入进一步提升工具选择、参数填充的准确性，人物/智能体在图谱的纳入，让人可加入流程的推进，可灵活应用于多人文本游戏 - **图谱推理**：相比其他Agent框架纯模型推理、纯人工编排的推理模式，让大模型在人的经验/设计指导下做事，灵活、可控，同时面向未知局面，可自由探索，同时将成功探索经验总结、图谱沉淀，面向相似问题，少走弯路；整体流程唤起支持平台对接（规则配置）、语言触发，满足各类诉求 - **调试运行**：图谱编辑完成后，可视调试，快速发现流程错误、修改优化，同时面向调试成功路径，关联配置自动沉淀，减少模型交互、模型开销，加速推理流程；此外，在线运行中，我们提供全链路可视化监控 From 2733dbddbb703e8464effbac69bd9e908adba3bb Mon Sep 17 00:00:00 2001 From: wyp311395 Date: Wed, 16 Oct 2024 12:24:56 +0800 Subject: [PATCH 053/128] [bugfix]del base_agent params --- muagent/connector/agents/base_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/muagent/connector/agents/base_agent.py b/muagent/connector/agents/base_agent.py index b323360..402550f 100644 --- a/muagent/connector/agents/base_agent.py +++ b/muagent/connector/agents/base_agent.py @@ -162,7 +162,7 @@ def init_memory_manager(self, memory_manager): ) else: memory_manager = LocalMemoryManager( - unique_name=self.role.role_name, + # unique_name=self.role.role_name, do_init=True, kb_root_path = self.kb_root_path, embed_config=self.embed_config, From 5acb05d15e276f899921a612d1e4d5c8e5e00395 Mon Sep 17 00:00:00 2001 From: wyp311395 Date: Sun, 27 Oct 2024 11:36:28 +0800 Subject: [PATCH 054/128] feat: add frontend and backend bugfix: del error message bugifx: fix nebula config bugifx: replace oss url by local picture --- .gitignore | 21 +- Dockerfile | 1 - Dockerfile_frontend | 18 + docker-compose.yaml | 38 +- examples/ekg_examples/ekg.yaml | 44 - examples/ekg_examples/start.py | 190 +- examples/ekg_examples/who_is_spy_game.py | 267 ++ examples/test_config.py.example | 81 +- frontend/.eslintrc.js | 3 + frontend/.prettierignore | 3 + frontend/.prettierrc | 8 + frontend/.stylelintrc.js | 3 + frontend/.umirc.ts | 46 + frontend/README.md | 14 + frontend/package.json | 41 + frontend/src/app.ts | 22 + frontend/src/assets/.gitkeep | 0 frontend/src/components/.gitkeep | 0 frontend/src/constants/index.tsx | 1 + .../EKG/Common/Chat/AgentPrologue/index.tsx | 36 + .../EKG/Common/Chat/AgentPrologue/style.ts | 35 + .../src/pages/EKG/Common/Chat/ChatContent.tsx | 134 + .../src/pages/EKG/Common/Chat/ChatFooter.tsx | 289 ++ .../Common/Chat/TemplateCode/index copy.tsx | 53 + .../EKG/Common/Chat/TemplateCode/index.tsx | 72 + .../EKG/Common/Chat/TemplateCode/style.ts | 45 + frontend/src/pages/EKG/Common/Chat/index.tsx | 15 + frontend/src/pages/EKG/Common/Chat/sseport.ts | 204 + frontend/src/pages/EKG/Common/Chat/style.ts | 95 + .../src/pages/EKG/Common/ChatDetail/index.tsx | 399 ++ .../src/pages/EKG/Common/ChatDetail/style.ts | 42 + frontend/src/pages/EKG/Common/index.tsx | 425 ++ frontend/src/pages/EKG/Common/style.ts | 101 + .../pages/EKG/Flow/components/AddNodeList.tsx | 54 + .../components/ExecutionConditions/index.tsx | 155 + .../components/ExecutionConditions/style.ts | 38 + .../pages/EKG/Flow/components/NodeHeader.tsx | 304 ++ .../src/pages/EKG/Flow/components/style.ts | 176 + frontend/src/pages/EKG/Flow/index.tsx | 515 +++ .../src/pages/EKG/Flow/nodes/BranchNode.tsx | 101 + frontend/src/pages/EKG/Flow/nodes/EndNode.tsx | 138 + .../src/pages/EKG/Flow/nodes/StartNode.tsx | 39 + .../src/pages/EKG/Flow/nodes/TaskNode.tsx | 107 + frontend/src/pages/EKG/Flow/nodes/style.ts | 30 + frontend/src/pages/EKG/Flow/store/index.tsx | 22 + frontend/src/pages/EKG/Flow/style.ts | 9 + .../pages/EKG/Flow/utils/ChildNodeTemplate.ts | 41 + frontend/src/pages/EKG/Flow/utils/index.tsx | 135 + frontend/src/pages/EKG/Flow/utils/random.tsx | 8 + .../src/pages/EKG/Flow/utils/typeFlow.tsx | 4 + frontend/src/pages/EKG/NodeTemplate.ts | 33 + .../pages/EKG/components/EKGHeader/index.tsx | 122 + .../pages/EKG/components/EKGHeader/style.ts | 42 + .../components/EditKnowledgeModal/index.tsx | 243 + .../components/EditKnowledgeModal/style.ts | 37 + .../NodeDetailModal/SceneModalInfo.tsx | 228 + .../EKG/components/NodeDetailModal/index.tsx | 163 + .../EKG/components/NodeDetailModal/style.ts | 109 + .../pages/EKG/components/SearchNode/index.tsx | 58 + frontend/src/pages/EKG/flow.d.ts | 23 + frontend/src/pages/EKG/index.tsx | 649 +++ .../EKG/nodes/OperationPlanNode/index.tsx | 251 ++ .../EKG/nodes/OperationPlanNode/style.ts | 108 + .../src/pages/EKG/nodes/SceneNode/index.tsx | 334 ++ .../src/pages/EKG/nodes/SceneNode/style.ts | 156 + .../src/pages/EKG/nodes/StartNode/index.tsx | 159 + .../src/pages/EKG/nodes/StartNode/style.ts | 50 + frontend/src/pages/EKG/service/index.ts | 12 + frontend/src/pages/EKG/store/index.tsx | 78 + frontend/src/pages/EKG/style.ts | 65 + frontend/src/pages/EKG/utils/format.tsx | 223 + frontend/src/pages/EKG/utils/index.ts | 319 ++ frontend/src/pages/EKG/utils/nodeAction.ts | 48 + frontend/src/pages/EKG/utils/userStore.ts | 84 + frontend/src/services/.gitkeep | 0 .../opsconvobus/EkgGraphController.ts | 38 + .../services/afs2demo/opsconvobus/index.ts | 6 + .../src/services/nexa/EkgProdController.ts | 244 + .../nexa/PortalAgentOperationController.ts | 556 +++ .../nexa/PortalConversationController.ts | 241 + .../nexa/PortalKnowledgeController.ts | 217 + frontend/src/services/nexa/typings.d.ts | 3990 +++++++++++++++++ .../services/opsgpt/ConversationController.ts | 379 ++ frontend/src/services/opsgpt/typings.d.ts | 3821 ++++++++++++++++ frontend/src/services/user.ts | 11 + frontend/src/utils/.gitkeep | 0 frontend/tsconfig.json | 3 + frontend/typings.d.ts | 1 + .../prompts/intention_template_prompt.py | 44 +- muagent/connector/agents/base_agent.py | 2 +- muagent/connector/memory_manager.py | 12 + .../graph_db_handler/base_gb_handler.py | 20 +- .../graph_db_handler/geabase_handler.py | 57 +- .../graph_db_handler/nebula_handler.py | 417 +- .../vector_db_handler/tbase_handler.py | 1 - muagent/httpapis/ekg_construct/api.py | 309 +- muagent/llm_models/openai_model.py | 19 +- muagent/memory/__init__.py | 5 + muagent/memory/hierarchical_memory_manager.py | 275 ++ muagent/schemas/apis/ekg_api_schema.py | 76 +- muagent/schemas/common/__init__.py | 2 +- .../common/auto_extract_graph_schema.py | 32 +- muagent/schemas/ekg/ekg_graph.py | 22 +- .../ekg_construct/ekg_construct_base.py | 261 +- .../service/ekg_inference/intention_router.py | 496 +- .../geabase_handler/geabase_handlerplus.py | 491 ++ .../src/graph_search/call_old_fuction.py | 51 + .../src/graph_search/geabase_search_plus.py | 1372 ++++++ .../src/graph_search/graph_search_main.py | 1029 +++++ .../graphstructure/graphstrcturesearchfun.py | 391 ++ .../intention_recognition_tool.py | 152 + .../src/memory_handler/ekg_memory_handler.py | 681 +++ .../src/question_answer/qa_function.py | 281 ++ .../ekg_reasoning/src/utils/call_llm.py | 159 + .../service/ekg_reasoning/src/utils/crypt.py | 16 + .../service/ekg_reasoning/src/utils/logger.py | 16 + .../ekg_reasoning/src/utils/normalize.py | 38 + requirements.txt | 5 +- runtime/.gitignore | 41 + runtime/Dockerfile | 13 + runtime/Dockerfile.no-package | 16 + runtime/bootstrap/pom.xml | 92 + .../bootstrap/BootstrapApplication.java | 25 + .../config/application-default.properties | 17 + .../resources/config/application.properties | 4 + .../src/main/resources/log4j2-spring.xml | 81 + .../main/resources/static/avatar/lijing.png | Bin 0 -> 178719 bytes .../src/main/resources/static/avatar/nex.png | Bin 0 -> 1109522 bytes .../main/resources/static/avatar/referee.png | Bin 0 -> 148286 bytes .../main/resources/static/avatar/wangpeng.png | Bin 0 -> 159756 bytes .../main/resources/static/avatar/zhangwei.png | Bin 0 -> 53136 bytes .../src/main/resources/static/index.html | 10 + .../bootstrap/src/main/resources/tools/1.json | 49 + .../resources/tools/dispatch_position.json | 49 + .../src/main/resources/tools/ekg-query.json | 49 + .../src/main/resources/tools/qwen110.json | 128 + .../resources/tools/system.select_tool.json | 49 + .../tools/undercover.dispatch_keyword.json | 49 + .../tools/undercover.dispatch_position.json | 49 + .../resources/tools/undercover.judge.json | 49 + .../resources/tools/undercover.lijing.json | 49 + .../undercover.show_key_information.json | 49 + .../resources/tools/undercover.summary.json | 49 + .../resources/tools/undercover.wangpeng.json | 49 + .../resources/tools/undercover.zhangwei.json | 49 + runtime/model/pom.xml | 38 + .../com/alipay/muagent/model/agent/Agent.java | 119 + .../muagent/model/chat/ChatContent.java | 12 + .../muagent/model/chat/ChatRequest.java | 76 + .../muagent/model/chat/ChatResponse.java | 53 + .../model/chat/SessionCreateRequest.java | 25 + .../model/chat/content/JsonContent.java | 19 + .../chat/content/RoleResponseContent.java | 61 + .../model/chat/content/TextContent.java | 23 + .../model/connector/http/HttpParameters.java | 44 + .../muagent/model/ekg/BaseEkgResponse.java | 19 + .../muagent/model/ekg/EkgAlgorithmResult.java | 47 + .../muagent/model/ekg/EkgFeaturesRequest.java | 17 + .../com/alipay/muagent/model/ekg/EkgNode.java | 40 + .../alipay/muagent/model/ekg/EkgNodeType.java | 28 + .../muagent/model/ekg/EkgQueryRequest.java | 49 + .../muagent/model/ekg/EkgQuestionContent.java | 29 + .../model/ekg/EkgQuestionDescription.java | 32 + .../alipay/muagent/model/ekg/EkgRequest.java | 21 + .../alipay/muagent/model/ekg/EkgResponse.java | 17 + .../model/ekg/EkgResponseResultMap.java | 17 + .../model/ekg/EkgSceneSessionRequest.java | 25 + .../muagent/model/ekg/EkgToolResponse.java | 23 + .../muagent/model/ekg/ExeNodeResponse.java | 21 + .../model/ekg/configuration/Config.java | 21 + .../muagent/model/ekg/storage/GraphEdge.java | 35 + .../muagent/model/ekg/storage/GraphGraph.java | 27 + .../muagent/model/ekg/storage/GraphNode.java | 32 + .../model/ekg/storage/GraphUpdateRequest.java | 36 + .../model/enums/chat/ChatExtendedKeyEnum.java | 17 + .../enums/chat/ChatTypeBelongingTypeEnum.java | 17 + .../model/enums/chat/ChatTypeEnum.java | 36 + .../model/enums/ekg/ToolPlanTypeEnum.java | 24 + .../model/enums/refactor/ResultCodeEnum.java | 82 + .../scheduler/TaskSchedulerTypeEnum.java | 14 + .../model/enums/tool/ToolProtocolEnum.java | 17 + .../muagent/model/exception/BizException.java | 68 + .../model/scheduler/SubmitTaskRequest.java | 37 + .../model/scheduler/TaskExeResponse.java | 19 + .../muagent/model/shell/ShellRequest.java | 21 + .../muagent/model/shell/ShellResponse.java | 21 + .../muagent/model/tool/TaskExeContext.java | 35 + .../muagent/model/tool/ToolExeRequest.java | 12 + .../muagent/model/tool/ToolExeResponse.java | 19 + .../model/tool/ToolInvokeResponse.java | 17 + .../model/tool/callback/CallBackConfig.java | 15 + .../muagent/model/tool/meta/ApiIvkSchema.java | 48 + .../model/tool/meta/ApiIvkSchemaInfo.java | 30 + .../model/tool/meta/ApiIvkSchemaServer.java | 21 + .../model/tool/meta/ManifestSchema.java | 57 + .../muagent/model/tool/meta/Protocol.java | 47 + .../model/tool/meta/ProtocolParameter.java | 25 + .../model/tool/meta/ProtocolSchema.java | 22 + .../muagent/model/tool/meta/Protocols.java | 26 + .../alipay/muagent/model/tool/meta/Tool.java | 95 + .../muagent/model/tool/meta/ToolDef.java | 32 + .../muagent/model/tool/meta/ToolDefParam.java | 48 + runtime/pom.xml | 86 + runtime/service/pom.xml | 48 + .../muagent/service/agent/AgentService.java | 18 + .../service/agent/impl/AgentServiceImpl.java | 31 + .../muagent/service/chat/ChatService.java | 33 + .../chat/configuration/EkgConfiguration.java | 33 + .../service/chat/impl/EkgChatServiceImpl.java | 257 ++ .../chat/impl/MockChatServiceImpl.java | 70 + .../muagent/service/connector/Connector.java | 20 + .../service/connector/ConnectorManager.java | 36 + .../connector/impl/GroovyConnector.java | 61 + .../service/connector/impl/HttpConnector.java | 127 + .../service/ekgmanager/EkgGraphManager.java | 85 + .../ekgmanager/EkgGraphStorageClient.java | 69 + .../ekgmanager/impl/EkgGraphManagerImpl.java | 147 + .../impl/EkgGraphStorageClientImpl.java | 222 + .../muagent/service/sheduler/Scheduler.java | 75 + .../service/sheduler/SchedulerManager.java | 36 + .../service/sheduler/impl/BaseScheduler.java | 115 + .../muagent/service/shell/ShellService.java | 17 + .../service/shell/impl/ShellServiceImpl.java | 104 + .../service/thread/ThreadPoolConfig.java | 31 + .../service/tool/loader/ToolLoader.java | 24 + .../configuration/LoaderConfiguration.java | 23 + .../tool/loader/impl/LocalToolLoader.java | 71 + runtime/util/pom.xml | 53 + .../com/alipay/muagent/util/GsonUtils.java | 237 + .../com/alipay/muagent/util/HttpUtil.java | 200 + .../com/alipay/muagent/util/LoggerUtil.java | 95 + .../java/com/alipay/muagent/util/MapUtil.java | 132 + .../com/alipay/muagent/util/StringUtils.java | 61 + runtime/web/pom.xml | 45 + .../alipay/muagent/web/AgentController.java | 35 + .../muagent/web/ConversationController.java | 108 + .../muagent/web/EkgStorageController.java | 139 + .../alipay/muagent/web/TaskController.java | 39 + .../alipay/muagent/web/ToolController.java | 55 + .../muagent/web/base/BaseController.java | 56 + .../muagent/web/base/ControllerProcessor.java | 16 + .../com/alipay/muagent/web/model/Result.java | 156 + .../web/src/main/resources/spring/spring.xml | 11 + tests/db_handler/geabase_hanlder_test.py | 12 +- tests/db_handler/nebulahandler_test.py | 439 +- tests/httpapis/api_func_test.py | 598 +++ tests/httpapis/fastapi_connet_test_ekg.py | 378 ++ tests/httpapis/fastapi_connet_test_llm.py | 62 + tests/httpapis/fastapi_connet_test_qa.py | 139 + .../service/EKG_test_construct_data_zq_02.py | 352 ++ .../service/EKG_test_construct_data_zq_03.py | 266 ++ tests/service/ekg_construct_test_2_nebula.py | 554 ++- tests/service/test_main_sswd.py | 309 ++ tests/service/test_main_sswd_long.py | 638 +++ tests/test_config.py.example | 82 +- 255 files changed, 33610 insertions(+), 894 deletions(-) create mode 100644 Dockerfile_frontend delete mode 100644 examples/ekg_examples/ekg.yaml create mode 100644 examples/ekg_examples/who_is_spy_game.py create mode 100644 frontend/.eslintrc.js create mode 100644 frontend/.prettierignore create mode 100644 frontend/.prettierrc create mode 100644 frontend/.stylelintrc.js create mode 100644 frontend/.umirc.ts create mode 100644 frontend/README.md create mode 100644 frontend/package.json create mode 100644 frontend/src/app.ts create mode 100644 frontend/src/assets/.gitkeep create mode 100644 frontend/src/components/.gitkeep create mode 100644 frontend/src/constants/index.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/AgentPrologue/index.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/AgentPrologue/style.ts create mode 100644 frontend/src/pages/EKG/Common/Chat/ChatContent.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/ChatFooter.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/TemplateCode/index copy.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/TemplateCode/index.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/TemplateCode/style.ts create mode 100644 frontend/src/pages/EKG/Common/Chat/index.tsx create mode 100644 frontend/src/pages/EKG/Common/Chat/sseport.ts create mode 100644 frontend/src/pages/EKG/Common/Chat/style.ts create mode 100644 frontend/src/pages/EKG/Common/ChatDetail/index.tsx create mode 100644 frontend/src/pages/EKG/Common/ChatDetail/style.ts create mode 100644 frontend/src/pages/EKG/Common/index.tsx create mode 100644 frontend/src/pages/EKG/Common/style.ts create mode 100644 frontend/src/pages/EKG/Flow/components/AddNodeList.tsx create mode 100644 frontend/src/pages/EKG/Flow/components/ExecutionConditions/index.tsx create mode 100644 frontend/src/pages/EKG/Flow/components/ExecutionConditions/style.ts create mode 100644 frontend/src/pages/EKG/Flow/components/NodeHeader.tsx create mode 100644 frontend/src/pages/EKG/Flow/components/style.ts create mode 100644 frontend/src/pages/EKG/Flow/index.tsx create mode 100644 frontend/src/pages/EKG/Flow/nodes/BranchNode.tsx create mode 100644 frontend/src/pages/EKG/Flow/nodes/EndNode.tsx create mode 100644 frontend/src/pages/EKG/Flow/nodes/StartNode.tsx create mode 100644 frontend/src/pages/EKG/Flow/nodes/TaskNode.tsx create mode 100644 frontend/src/pages/EKG/Flow/nodes/style.ts create mode 100644 frontend/src/pages/EKG/Flow/store/index.tsx create mode 100644 frontend/src/pages/EKG/Flow/style.ts create mode 100644 frontend/src/pages/EKG/Flow/utils/ChildNodeTemplate.ts create mode 100644 frontend/src/pages/EKG/Flow/utils/index.tsx create mode 100644 frontend/src/pages/EKG/Flow/utils/random.tsx create mode 100644 frontend/src/pages/EKG/Flow/utils/typeFlow.tsx create mode 100644 frontend/src/pages/EKG/NodeTemplate.ts create mode 100644 frontend/src/pages/EKG/components/EKGHeader/index.tsx create mode 100644 frontend/src/pages/EKG/components/EKGHeader/style.ts create mode 100644 frontend/src/pages/EKG/components/EditKnowledgeModal/index.tsx create mode 100644 frontend/src/pages/EKG/components/EditKnowledgeModal/style.ts create mode 100644 frontend/src/pages/EKG/components/NodeDetailModal/SceneModalInfo.tsx create mode 100644 frontend/src/pages/EKG/components/NodeDetailModal/index.tsx create mode 100644 frontend/src/pages/EKG/components/NodeDetailModal/style.ts create mode 100644 frontend/src/pages/EKG/components/SearchNode/index.tsx create mode 100644 frontend/src/pages/EKG/flow.d.ts create mode 100644 frontend/src/pages/EKG/index.tsx create mode 100644 frontend/src/pages/EKG/nodes/OperationPlanNode/index.tsx create mode 100644 frontend/src/pages/EKG/nodes/OperationPlanNode/style.ts create mode 100644 frontend/src/pages/EKG/nodes/SceneNode/index.tsx create mode 100644 frontend/src/pages/EKG/nodes/SceneNode/style.ts create mode 100644 frontend/src/pages/EKG/nodes/StartNode/index.tsx create mode 100644 frontend/src/pages/EKG/nodes/StartNode/style.ts create mode 100644 frontend/src/pages/EKG/service/index.ts create mode 100644 frontend/src/pages/EKG/store/index.tsx create mode 100644 frontend/src/pages/EKG/style.ts create mode 100644 frontend/src/pages/EKG/utils/format.tsx create mode 100644 frontend/src/pages/EKG/utils/index.ts create mode 100644 frontend/src/pages/EKG/utils/nodeAction.ts create mode 100644 frontend/src/pages/EKG/utils/userStore.ts create mode 100644 frontend/src/services/.gitkeep create mode 100644 frontend/src/services/afs2demo/opsconvobus/EkgGraphController.ts create mode 100644 frontend/src/services/afs2demo/opsconvobus/index.ts create mode 100644 frontend/src/services/nexa/EkgProdController.ts create mode 100644 frontend/src/services/nexa/PortalAgentOperationController.ts create mode 100644 frontend/src/services/nexa/PortalConversationController.ts create mode 100644 frontend/src/services/nexa/PortalKnowledgeController.ts create mode 100644 frontend/src/services/nexa/typings.d.ts create mode 100644 frontend/src/services/opsgpt/ConversationController.ts create mode 100644 frontend/src/services/opsgpt/typings.d.ts create mode 100644 frontend/src/services/user.ts create mode 100644 frontend/src/utils/.gitkeep create mode 100644 frontend/tsconfig.json create mode 100644 frontend/typings.d.ts create mode 100644 muagent/memory/__init__.py create mode 100644 muagent/memory/hierarchical_memory_manager.py create mode 100644 muagent/service/ekg_reasoning/src/geabase_handler/geabase_handlerplus.py create mode 100644 muagent/service/ekg_reasoning/src/graph_search/call_old_fuction.py create mode 100644 muagent/service/ekg_reasoning/src/graph_search/geabase_search_plus.py create mode 100644 muagent/service/ekg_reasoning/src/graph_search/graph_search_main.py create mode 100644 muagent/service/ekg_reasoning/src/graphstructure/graphstrcturesearchfun.py create mode 100644 muagent/service/ekg_reasoning/src/intention_recognition/intention_recognition_tool.py create mode 100644 muagent/service/ekg_reasoning/src/memory_handler/ekg_memory_handler.py create mode 100644 muagent/service/ekg_reasoning/src/question_answer/qa_function.py create mode 100644 muagent/service/ekg_reasoning/src/utils/call_llm.py create mode 100644 muagent/service/ekg_reasoning/src/utils/crypt.py create mode 100644 muagent/service/ekg_reasoning/src/utils/logger.py create mode 100644 muagent/service/ekg_reasoning/src/utils/normalize.py create mode 100644 runtime/.gitignore create mode 100644 runtime/Dockerfile create mode 100644 runtime/Dockerfile.no-package create mode 100644 runtime/bootstrap/pom.xml create mode 100644 runtime/bootstrap/src/main/java/com/alipay/muagent/bootstrap/BootstrapApplication.java create mode 100644 runtime/bootstrap/src/main/resources/config/application-default.properties create mode 100644 runtime/bootstrap/src/main/resources/config/application.properties create mode 100644 runtime/bootstrap/src/main/resources/log4j2-spring.xml create mode 100644 runtime/bootstrap/src/main/resources/static/avatar/lijing.png create mode 100644 runtime/bootstrap/src/main/resources/static/avatar/nex.png create mode 100644 runtime/bootstrap/src/main/resources/static/avatar/referee.png create mode 100644 runtime/bootstrap/src/main/resources/static/avatar/wangpeng.png create mode 100644 runtime/bootstrap/src/main/resources/static/avatar/zhangwei.png create mode 100644 runtime/bootstrap/src/main/resources/static/index.html create mode 100644 runtime/bootstrap/src/main/resources/tools/1.json create mode 100644 runtime/bootstrap/src/main/resources/tools/dispatch_position.json create mode 100644 runtime/bootstrap/src/main/resources/tools/ekg-query.json create mode 100644 runtime/bootstrap/src/main/resources/tools/qwen110.json create mode 100644 runtime/bootstrap/src/main/resources/tools/system.select_tool.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.dispatch_keyword.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.dispatch_position.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.judge.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.lijing.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.show_key_information.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.summary.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.wangpeng.json create mode 100644 runtime/bootstrap/src/main/resources/tools/undercover.zhangwei.json create mode 100644 runtime/model/pom.xml create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/agent/Agent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/ChatContent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/ChatRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/ChatResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/SessionCreateRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/content/JsonContent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/content/RoleResponseContent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/chat/content/TextContent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/connector/http/HttpParameters.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/BaseEkgResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgAlgorithmResult.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgFeaturesRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgNode.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgNodeType.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgQueryRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgQuestionContent.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgQuestionDescription.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgResponseResultMap.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgSceneSessionRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/EkgToolResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/ExeNodeResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/configuration/Config.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/storage/GraphEdge.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/storage/GraphGraph.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/storage/GraphNode.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/ekg/storage/GraphUpdateRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/chat/ChatExtendedKeyEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/chat/ChatTypeBelongingTypeEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/chat/ChatTypeEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/ekg/ToolPlanTypeEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/refactor/ResultCodeEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/scheduler/TaskSchedulerTypeEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/enums/tool/ToolProtocolEnum.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/exception/BizException.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/scheduler/SubmitTaskRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/scheduler/TaskExeResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/shell/ShellRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/shell/ShellResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/TaskExeContext.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/ToolExeRequest.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/ToolExeResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/ToolInvokeResponse.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/callback/CallBackConfig.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ApiIvkSchema.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ApiIvkSchemaInfo.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ApiIvkSchemaServer.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ManifestSchema.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/Protocol.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ProtocolParameter.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ProtocolSchema.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/Protocols.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/Tool.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ToolDef.java create mode 100644 runtime/model/src/main/java/com/alipay/muagent/model/tool/meta/ToolDefParam.java create mode 100644 runtime/pom.xml create mode 100644 runtime/service/pom.xml create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/agent/AgentService.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/agent/impl/AgentServiceImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/chat/ChatService.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/chat/configuration/EkgConfiguration.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/chat/impl/EkgChatServiceImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/chat/impl/MockChatServiceImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/connector/Connector.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/connector/ConnectorManager.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/connector/impl/GroovyConnector.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/connector/impl/HttpConnector.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/ekgmanager/EkgGraphManager.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/ekgmanager/EkgGraphStorageClient.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/ekgmanager/impl/EkgGraphManagerImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/ekgmanager/impl/EkgGraphStorageClientImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/sheduler/Scheduler.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/sheduler/SchedulerManager.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/sheduler/impl/BaseScheduler.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/shell/ShellService.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/shell/impl/ShellServiceImpl.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/thread/ThreadPoolConfig.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/tool/loader/ToolLoader.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/tool/loader/configuration/LoaderConfiguration.java create mode 100644 runtime/service/src/main/java/com/alipay/muagent/service/tool/loader/impl/LocalToolLoader.java create mode 100644 runtime/util/pom.xml create mode 100644 runtime/util/src/main/java/com/alipay/muagent/util/GsonUtils.java create mode 100644 runtime/util/src/main/java/com/alipay/muagent/util/HttpUtil.java create mode 100644 runtime/util/src/main/java/com/alipay/muagent/util/LoggerUtil.java create mode 100644 runtime/util/src/main/java/com/alipay/muagent/util/MapUtil.java create mode 100644 runtime/util/src/main/java/com/alipay/muagent/util/StringUtils.java create mode 100644 runtime/web/pom.xml create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/AgentController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/ConversationController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/EkgStorageController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/TaskController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/ToolController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/base/BaseController.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/base/ControllerProcessor.java create mode 100644 runtime/web/src/main/java/com/alipay/muagent/web/model/Result.java create mode 100644 runtime/web/src/main/resources/spring/spring.xml create mode 100644 tests/httpapis/api_func_test.py create mode 100644 tests/httpapis/fastapi_connet_test_ekg.py create mode 100644 tests/httpapis/fastapi_connet_test_llm.py create mode 100644 tests/httpapis/fastapi_connet_test_qa.py create mode 100644 tests/service/EKG_test_construct_data_zq_02.py create mode 100644 tests/service/EKG_test_construct_data_zq_03.py create mode 100644 tests/service/test_main_sswd.py create mode 100644 tests/service/test_main_sswd_long.py diff --git a/.gitignore b/.gitignore index fe898c4..0d35ad7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,23 @@ dist zdatafront* *antgroup* *ipynb -*log \ No newline at end of file +*log + + +# frontend +frontend/node_modules +frontend/.env.local +frontend/.umirc.local.ts +frontend/config/config.local.ts +frontend/src/.umi +frontend/src/.umi-production +frontend/src/.umi-test +frontend/.umi +frontend/.umi-production +frontend/.umi-test +frontend/dist +frontend/.mfsu +frontend/.swc +frontend/pnpm-lock.yaml + +*.jar \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fdf707a..dc4c484 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY ./requirements.txt /home/user/docker_requirements.txt # RUN wget https://oss-cdn.nebula-graph.com.cn/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb # RUN dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb - RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple RUN pip install -r /home/user/docker_requirements.txt --retries 5 --timeout 120 diff --git a/Dockerfile_frontend b/Dockerfile_frontend new file mode 100644 index 0000000..acb7a23 --- /dev/null +++ b/Dockerfile_frontend @@ -0,0 +1,18 @@ +From node:20.18.0-bookworm + +WORKDIR /home/user + +# +COPY frontend /home/user/frontend +RUN ls /home/user && ls /home/user/frontend + +# +RUN npm config set registry https://registry.npmmirror.com +RUN npm install -g pnpm +RUN pnpm config set registry https://registry.npmmirror.com +# +RUN cd /home/user/frontend && npm i +# 新增删除 +RUN cd /home/user/frontend && rm -rf !\(node_modules\) + +CMD ["bash"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 395de46..805f6fc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -143,18 +143,32 @@ services: # count: all # 或者您想要的数量，例如 1 # capabilities: [gpu] + runtime: + build: + context: ./runtime + dockerfile: Dockerfile.no-package + image: runtime:0.1.0 + container_name: runtime + environment: + USER: + root + ports: + # - 5050:3737 + - 8080:8080 + networks: + - ekg-net + ekgservice: build: context: . dockerfile: Dockerfile container_name: ekgservice image: muagent:0.1.0 - # image: muagent:0.2.0 environment: USER: root TZ: "${TZ}" + OLLAMA_HOST: http://ollama:11434 ports: - # - 5050:3737 - 3737:3737 volumes: - ./examples:/home/user/muagent/examples @@ -164,10 +178,26 @@ services: networks: - ekg-net command: ["python", "/home/user/muagent/examples/ekg_examples/start.py"] # 指定要执行的脚本 - # command: ["python", "/home/user/muagent/tests/httpapis/fastapi_test.py"] # 指定要执行的脚本 + ekgfrontend: + build: + context: . + dockerfile: Dockerfile_frontend + container_name: ekgfrontend + image: ekgfrontend:0.1.0 + environment: + USER: root + TZ: "${TZ}" + ports: + - 8000:8000 + volumes: + - ./examples:/home/user/muagent/examples + - ./frontend:/home/user/muagent/frontend + restart: on-failure + networks: + - ekg-net + command: ["sh", "-c", "cp -rf /home/user/muagent/frontend /home/user && cd /home/user/frontend && npm run start"] networks: ekg-net: - # driver: bridge external: true \ No newline at end of file diff --git a/examples/ekg_examples/ekg.yaml b/examples/ekg_examples/ekg.yaml deleted file mode 100644 index e7df07c..0000000 --- a/examples/ekg_examples/ekg.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# geabase config -geabase_config: - metaserver_address: 'deafault' - project: 'deafault' - city: 'deafault' - lib_path: 'deafault' - - -# nebula config -nebula_config: - host: 'graphd' - port: '9669' - username: 'root' - password: 'nebula' - space_name: 'client' - -# tbase config -tbase_config: - # host: 'localhost' - host: 'redis-stack' - port: '6379' - username: 'default' - password: '' - definition_value: 'opsgptkg' - - -# model -llm: - model_type: 'openai' - model_name: 'default' - stop: '' - temperature: 0.3 - top_p: 0.95 - top_k: 50 - url: '' - token: '' - - -# embedding -embedding: - embedding_type: 'openai' - model_name: 'default' - url: '' - token: '' diff --git a/examples/ekg_examples/start.py b/examples/ekg_examples/start.py index de9a287..74c56f2 100644 --- a/examples/ekg_examples/start.py +++ b/examples/ekg_examples/start.py @@ -5,38 +5,74 @@ from typing import List from loguru import logger from tqdm import tqdm +import ollama +import shutil + from concurrent.futures import ThreadPoolExecutor from langchain.llms.base import LLM from langchain.embeddings.base import Embeddings -src_dir = os.path.join( +grandparent_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) -sys.path.append(src_dir) +sys.path.append(grandparent_dir) +parent_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +sys.path.append(parent_dir) +logger.info(f'parent_dir is {parent_dir}' ) try: - import os, sys - src_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - sys.path.append(src_dir) import test_config except Exception as e: # set your config logger.error(f"{e}") + logger.error(f"please set your test_config") + + if not os.path.exists(os.path.join(parent_dir, "test_config.py")): + shutil.copy( + os.path.join(parent_dir, "test_config.py.example"), + os.path.join(parent_dir, "test_config.py") + ) + import test_config from muagent.schemas.db import * +from muagent.db_handler import * from muagent.llm_models.llm_config import EmbedConfig, LLMConfig from muagent.service.ekg_construct.ekg_construct_base import EKGConstructService +from muagent.service.ekg_inference import IntentionRouter +from muagent.connector.memory_manager import TbaseMemoryManager +from muagent.llm_models import getChatModelFromConfig from pydantic import BaseModel + + + +cur_dir = os.path.dirname(__file__) +# print(cur_dir) + +# # 要打开的YAML文件路径 +# file_path = 'ekg.yaml' + +# if not os.path.exists(os.path.join(cur_dir, file_path)): +# shutil.copy( +# os.path.join(cur_dir, "ekg.yaml.example"), +# os.path.join(cur_dir, "ekg.yaml") +# ) +# with open(os.path.join(cur_dir, file_path), 'r') as file: +# # 加载YAML文件内容 +# config_data = yaml.safe_load(file) + + + + # llm config class CustomLLM(LLM, BaseModel): - url: str = "http://localhost:11434/api/generate" - model_name: str = "qwen2:1b" - model_type: str = "ollama" - api_key: str = "" + url: str = os.environ["API_BASE_URL"]# "http://ollama:11434/api/generate" + model_name: str=os.environ["model_name"] #str = "qwen2.5:0.5b" + model_type: str = os.environ["model_engine"]#"ollama" + api_key: str = os.environ["OPENAI_API_KEY"]#"" stop: str = None temperature: float = 0.3 top_k: int = 50 @@ -68,12 +104,16 @@ def _call(self, prompt: str, stop = stop or self.stop if self.model_type == "ollama": - data = { - "model": self.model_name, - "prompt": prompt - } - r = requests.post(self.url, json=data, ) - return r.json() + stream = ollama.chat( + model=self.model_name, + messages=[{'role': 'user', 'content': prompt}], + stream=True, + ) + answer = "" + for chunk in stream: + answer += chunk['message']['content'] + + return answer elif self.model_type == "openai": from muagent.llm_models.openai_model import getChatModelFromConfig llm_config = LLMConfig( @@ -86,11 +126,11 @@ def _call(self, prompt: str, ) model = getChatModelFromConfig(llm_config) return model.predict(prompt, stop=self.stop) - elif self.model_type == "lingyiwangwu": + elif self.model_type in ["lingyiwanwu", "kimi", "moonshot", "qwen"]: from muagent.llm_models.openai_model import getChatModelFromConfig llm_config = LLMConfig( model_name=self.model_name, - model_engine="lingyiwangwu", + model_engine=self.model_type, api_key=self.api_key, api_base_url=self.url, temperature=self.temperature, @@ -106,10 +146,10 @@ def _call(self, prompt: str, class CustomEmbeddings(Embeddings): # ollama embeddings - url = "http://localhost:11434/api/embeddings" + url = "http://ollama:11434/api/embeddings" # embedding_type = "ollama" - model_name = "" + model_name = "qwen2.5:0.5b" api_key = "" def params(self): @@ -128,12 +168,12 @@ def _get_sentence_emb(self, sentence: str) -> dict: 调用句子向量提取服务 """ if self.embedding_type == "ollama": - data = { - "model": self.model_name, - "prompt": sentence - } - r = requests.post(self.url, json=data, ) - return r.json() + ollama_embeddings = ollama.embed( + model=self.model_name, + input=sentence + ) + return ollama_embeddings["embeddings"][0] + elif self.embedding_type == "openai": from muagent.llm_models.get_embedding import get_embedding os.environ["OPENAI_API_KEY"] = self.api_key @@ -181,20 +221,6 @@ def embed_query(self, text: str) -> List[float]: return embedding - -cur_dir = os.path.dirname(__file__) -print(cur_dir) - -# 要打开的YAML文件路径 -file_path = 'ekg.yaml' - -# 使用 'with' 语句确保文件正确关闭 -with open(os.path.join(cur_dir, file_path), 'r') as file: - # 加载YAML文件内容 - config_data = yaml.safe_load(file) - - - # gb_config = GBConfig( # gb_type="GeaBaseHandler", # extra_kwargs={ @@ -205,21 +231,15 @@ def embed_query(self, text: str) -> List[float]: # } # ) - -# gb_config = GBConfig( -# gb_type="NebulaHandler", -# extra_kwargs={} -# ) - # 初始化 NebulaHandler 实例 gb_config = GBConfig( gb_type="NebulaHandler", extra_kwargs={ - 'host': config_data["nebula_config"]['host'], - 'port': config_data["nebula_config"]['port'], - 'username': config_data["nebula_config"]['username'] , - 'password': config_data["nebula_config"]['password'], - "space": config_data["nebula_config"]['space_name'], + 'host': os.environ["nb_host"], # config_data["nebula_config"]['host'], + 'port': os.environ["nb_port"], # config_data["nebula_config"]['port'], + 'username': os.environ["nb_username"], # config_data["nebula_config"]['username'] , + 'password': os.environ["nb_password"], # config_data["nebula_config"]['password'], + "space": os.environ["nb_space"], # config_data["nebula_config"]['space_name'], } ) @@ -227,16 +247,16 @@ def embed_query(self, text: str) -> List[float]: tb_config = TBConfig( tb_type="TbaseHandler", index_name="muagent_test", - host=config_data["tbase_config"]["host"], - port=config_data["tbase_config"]['port'], - username=config_data["tbase_config"]['username'], - password=config_data["tbase_config"]['password'], + host=os.environ["tb_host"], # config_data["tbase_config"]["host"], + port=os.environ["tb_port"], # config_data["tbase_config"]['port'], + username=os.environ["tb_username"], # config_data["tbase_config"]['username'], + password=os.environ["tb_password"], # config_data["tbase_config"]['password'], extra_kwargs={ - 'host': config_data["tbase_config"]['host'], - 'port': config_data["tbase_config"]['port'], - 'username': config_data["tbase_config"]['username'] , - 'password': config_data["tbase_config"]['password'], - 'definition_value': config_data["tbase_config"]['definition_value'] + 'host': os.environ["tb_host"], # config_data["tbase_config"]['host'], + 'port': os.environ["tb_port"], # config_data["tbase_config"]['port'], + 'username': os.environ["tb_username"], # config_data["tbase_config"]['username'] , + 'password': os.environ["tb_password"], # config_data["tbase_config"]['password'], + 'definition_value': os.environ["tb_definition_value"], # config_data["tbase_config"]['definition_value'] } ) @@ -261,5 +281,51 @@ def embed_query(self, text: str) -> List[float]: gb_config=gb_config, ) + + + + + +# 指定index_name +index_name = os.environ["tb_index_name"] +th = TbaseHandler(tb_config, index_name, definition_value=os.environ['tb_definition_value']) +# 5、memory 接口配置 +# create tbase memory manager +memory_manager = TbaseMemoryManager( + unique_name="EKG", + embed_config=embed_config, + llm_config=llm_config, + tbase_handler=th, + use_vector=False + ) + + +intention_router = IntentionRouter( + ekg_construct_service.model, + ekg_construct_service.gb, + ekg_construct_service.tb, + embed_config +) + + +memory_manager = memory_manager +#geabase_handler = GeaBaseHandler(gb_config) +geabase_handler = ekg_construct_service.gb +intention_router = intention_router + + +from who_is_spy_game import load_whoisspy_datas, test_whoisspy_datas +load_whoisspy_datas(ekg_construct_service) +test_whoisspy_datas(ekg_construct_service) + + from muagent.httpapis.ekg_construct import create_api -create_api(llm, embeddings, ekg_construct_service) \ No newline at end of file +create_api( + llm, + llm_config, + embeddings, + ekg_construct_service, + memory_manager, + geabase_handler, + intention_router +) \ No newline at end of file diff --git a/examples/ekg_examples/who_is_spy_game.py b/examples/ekg_examples/who_is_spy_game.py new file mode 100644 index 0000000..878ba84 --- /dev/null +++ b/examples/ekg_examples/who_is_spy_game.py @@ -0,0 +1,267 @@ +from loguru import logger + + +from muagent.schemas.common import GNode, GEdge + + +import math +import hashlib + +def normalize(lis): + s = sum([i * i for i in lis]) + if s == 0: + raise ValueError('Sum of lis is 0') + + s_sqrt = math.sqrt(s) + res = [i / s_sqrt for i in lis] + return res +def md5_hash(text): + m = hashlib.md5() + m.update(text.encode('utf-8')) + return m.hexdigest() +def hash_id(nodeId, sessionId='', otherstr = None): + test_res = '' + test_all = nodeId + sessionId + test_res = test_res + (md5_hash(test_all)) + if otherstr == None: + return test_res + else: + test_res = test_res + otherstr + return test_res + + + +new_nodes_1 = [] +new_edges_1 = [] + + +new_nodes_1 = \ +[GNode(id='haPvrjEkz4LARZyR7OAuPmVMHMIQPMew', type='opsgptkg_intent', attributes={'ID': 79745654784, 'extra': '{}', 'teamids': '8400001', 'gdb_timestamp': '1729496822', 'description': '需要公司多人参与的事务，以及相关的问题', 'name': '公司事务'}), + GNode(id='dicVRAk5rT3y9LxcmBCN2jDi1TjHc5rm', type='opsgptkg_intent', attributes={'ID': 6028807454720, 'description': '与个人有关的事务(如个人贷款），或遇到的个人问题，不涉及公司事务', 'name': '个人事务', 'extra': '{}', 'teamids': '8400001', 'gdb_timestamp': '1729496798'}), + GNode(id='ClKvwjBRZUJC7ttSZaiT0dh7lhSujNWi', type='opsgptkg_intent', attributes={'ID': 7905077215232, 'gdb_timestamp': '1729496904', 'description': '公司活动', 'name': '公司活动', 'extra': '{}', 'teamids': '8400001'}), + GNode(id='NyBXAHQckQx1xL5lnSgBGlotbZkkQ9C7', type='opsgptkg_intent', attributes={'ID': 5876942970880, 'gdb_timestamp': '1729496969', 'description': '金融（如借款、存款、贷款等）', 'name': '金融', 'extra': '{}', 'teamids': '8400001'}), + GNode(id='6sa4zJCnVKJxKMtOtypapjZk4sdo93QU', type='opsgptkg_intent', attributes={'ID': 8505664413696, 'extra': '{}', 'teamids': '8400001', 'gdb_timestamp': '1729496951', 'description': '医疗(包括预约、挂号、看病、诊断等)', 'name': '医疗'}), + GNode(id='a8d85669_141a_4f54_ab8c_209c08d27c35', type='opsgptkg_schedule', attributes={'ID': 8284119801856, 'extra': '{"graphid": "", "cnode_nums": 1}', 'teamids': '8400001', 'gdb_timestamp': '1729497515', 'description': '组织一次公司活动', 'name': '组织一次公司活动', 'enable': 'False'}), + GNode(id='2b8df337_f29e_4d49_865f_84088c3a94e7', type='opsgptkg_schedule', attributes={'ID': 8971031896064, 'teamids': '8400001', 'gdb_timestamp': '1729497515', 'description': '在线申请贷款', 'name': '在线申请贷款', 'enable': 'False', 'extra': '{"graphid": "", "cnode_nums": 1}'}), + GNode(id='b9fe38f1_33f6_468b_a1dd_43efdfd8e2d1', type='opsgptkg_schedule', attributes={'ID': 2223780347904, 'extra': '{"graphid": "", "cnode_nums": 1}', 'teamids': '8400001', 'gdb_timestamp': '1729497515', 'description': '预约医生', 'name': '预约医生', 'enable': 'False'}), + GNode(id='98234102_4e4a_4997_9b1e_3cda6382b1c7', type='opsgptkg_task', attributes={'ID': 711254376448, 'description': '确定活动主题：确定活动的主要目的（如团建、庆祝活动等）', 'name': '确定活动主题：确定活动的主要目的（如团建、庆祝活动等）', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': ''}), + GNode(id='59030678_760d_4a10_8d61_0d4e4cc5fbcb', type='opsgptkg_task', attributes={'ID': 3166553161728, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '访问贷款平台：输入网址并访问贷款申请网站', 'name': '访问贷款平台：输入网址并访问贷款申请网站', 'accesscriteria': ''}), + GNode(id='5afab73b_8f03_422f_856e_386f183bdd71', type='opsgptkg_task', attributes={'ID': 6192381952, 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541526', 'executetype': '', 'description': '选择医院/医生：访问医院官网或APP，查找相关科室和医生', 'name': '选择医院/医生：访问医院官网或APP，查找相关科室和医生'}), + GNode(id='95ec00ef_cc9c_4947_a21c_88eeb9a71af5', type='opsgptkg_task', attributes={'ID': 1876388151296, 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': '', 'description': '选择活动类型', 'name': '选择活动类型', 'accesscriteria': ''}), + GNode(id='5504af87_416e_4ee5_bfce_86b969a63433', type='opsgptkg_task', attributes={'ID': 6316483026944, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '注册/登录：如果你已经注册,输入用户名和密码进行登录。如果你还没有注册,点击“注册”按钮，填写个人信息，创建账户', 'name': '注册/登录：如果你已经注册,输入用户名和密码进行登录。如果你还没有注册,点击“注册”按钮，填写个人信息，创建账户', 'accesscriteria': ''}), + GNode(id='3ff8f54a_fa65_4368_86ce_d65058035dd0', type='opsgptkg_task', attributes={'ID': 7833205129216, 'teamids': '8400001', 'gdb_timestamp': '1725541526', 'executetype': '', 'description': '查看可预约时间：点击医生姓名，查看可预约时段', 'name': '查看可预约时间：点击医生姓名，查看可预约时段', 'accesscriteria': '', 'extra': '{"graphid": ""}'}), + GNode(id='d5e760b4_ae82_410d_a73d_4c0c98926ae5', type='opsgptkg_phenomenon', attributes={'ID': 4450592235520, 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'description': '室内活动', 'name': '室内活动'}), + GNode(id='2a37b90a_fd96_4548_989c_7c1e8fa9d881', type='opsgptkg_phenomenon', attributes={'ID': 5557429084160, 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'description': '户外活动', 'name': '户外活动'}), + GNode(id='88d4cf2b_7cf5_4e40_b54e_59268f119f63', type='opsgptkg_task', attributes={'ID': 5471766437888, 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '选择贷款类型：浏览可用的贷款类型（如个人贷款、汽车贷款、房屋贷款），选择适合自己的贷款类型', 'name': '选择贷款类型：浏览可用的贷款类型（如个人贷款、汽车贷款、房屋贷款），选择适合自己的贷款类型'}), + GNode(id='39021995_6e63_4907_9d67_26ba50d0cd44', type='opsgptkg_task', attributes={'ID': 24387616768, 'description': '填写个人信息：输入姓名、联系方式等，选择预约时间', 'name': '填写个人信息：输入姓名、联系方式等，选择预约时间', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541526', 'executetype': ''}), + GNode(id='59fe9c1d_0731_403e_936a_2e2bbba4b3ee', type='opsgptkg_task', attributes={'ID': 7311708086272, 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': '', 'description': '选择具体的室内活动（如会议、晚会、游戏），确定场地和时间，准备相关的设备（如投影仪、音响），安排餐饮和娱乐节目，发出邀请通知', 'name': '选择具体的室内活动（如会议、晚会、游戏），确定场地和时间，准备相关的设备（如投影仪、音响），安排餐饮和娱乐节目，发出邀请通知'}), + GNode(id='60163dc6_87af_4972_b350_6b9275975c83', type='opsgptkg_task', attributes={'ID': 7678175674368, 'description': '选择具体的户外活动（如远足、烧烤、运动会），确定地点和时间，安排交通工具和安全措施，联系供应商（如餐饮、设备租赁），发出邀请通知', 'name': '选择具体的户外活动（如远足、烧烤、运动会），确定地点和时间，安排交通工具和安全措施，联系供应商（如餐饮、设备租赁），发出邀请通知', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': ''}), + GNode(id='910f3634_b999_4cf3_94c9_346a67b0d5ed', type='opsgptkg_task', attributes={'ID': 9145981739008, 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '填写申请表：提供个人信息（如姓名、年龄、收入等），提供贷款金额和贷款目的', 'name': '填写申请表：提供个人信息（如姓名、年龄、收入等），提供贷款金额和贷款目的', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001'}), + GNode(id='1330ad69_dfc3_4538_864e_6867a3fd8dd4', type='opsgptkg_task', attributes={'ID': 4367816081408, 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541526', 'executetype': '', 'description': '确认预约：检查预约信息，点击“确认预约”按钮', 'name': '确认预约：检查预约信息，点击“确认预约”按钮'}), + GNode(id='fcbc3e04_ad8c_4aad_9f75_191f8037ced8', type='opsgptkg_task', attributes={'ID': 868313473024, 'description': '预算审核：计算活动预估费用，提交预算给管理层审核', 'name': '预算审核：计算活动预估费用，提交预算给管理层审核', 'accesscriteria': '{"type":"OR"}', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': ''}), + GNode(id='2c7a0d7b_a490_41b9_a6f8_e71b5212e0be', type='opsgptkg_task', attributes={'ID': 5168810909696, 'description': '提交资料：上传所需文件（如身份证、收入证明等）', 'name': '提交资料：上传所需文件（如身份证、收入证明等）', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': ''}), + GNode(id='3cd46fb7_e11c_4181_8670_2f080a453142', type='opsgptkg_phenomenon', attributes={'ID': 7081063784448, 'teamids': '8400001', 'gdb_timestamp': '1725541526', 'description': '接收通知：收到预约确认短信或邮件', 'name': '接收通知：收到预约确认短信或邮件', 'extra': '{"graphid": ""}'}), + GNode(id='0f4610cd_cf6a_475b_8ac0_80166569a292', type='opsgptkg_task', attributes={'ID': 7170049368064, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '审核资料：系统开始审核申请', 'name': '审核资料：系统开始审核申请', 'accesscriteria': ''}), + GNode(id='b9f81925_b43a_459d_9902_1bc4b024f5a1', type='opsgptkg_phenomenon', attributes={'ID': 2310638313472, 'description': '审核通过', 'name': '审核通过', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385'}), + GNode(id='191687cd_1b76_4e77_9f2a_e67936dd372e', type='opsgptkg_phenomenon', attributes={'ID': 3083664605184, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'description': '审核失败', 'name': '审核失败'}), + GNode(id='18c33ec1_08ef_4df8_b938_7244852d19c8', type='opsgptkg_task', attributes={'ID': 1978980810752, 'description': '用户收到“申请通过”的通知，前往下一步选择贷款期限和还款方式', 'name': '用户收到“申请通过”的通知，前往下一步选择贷款期限和还款方式', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': ''}), + GNode(id='b73c2551_0890_40fb_b0ca_04912bc21b65', type='opsgptkg_task', attributes={'ID': 8995127967744, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '提供反馈，建议修改后重新申请', 'name': '提供反馈，建议修改后重新申请', 'accesscriteria': ''}), + GNode(id='e95adaa2_d177_435b_bac7_a8b6047ecc3d', type='opsgptkg_task', attributes={'ID': 8207832350720, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '确认贷款条件：查看贷款条款和条件', 'name': '确认贷款条件：查看贷款条款和条件', 'accesscriteria': ''}), + GNode(id='0c561d68_ee31_49d2_82c1_1dac81e731ff', type='opsgptkg_phenomenon', attributes={'ID': 2242499641344, 'gdb_timestamp': '1725541385', 'description': '拒绝条款', 'name': '拒绝条款', 'extra': '{"graphid": ""}', 'teamids': '8400001'}), + GNode(id='81f579ac_851d_4b85_8608_d2732a2612ff', type='opsgptkg_phenomenon', attributes={'ID': 6852580294656, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'description': '接受条款', 'name': '接受条款'}), + GNode(id='1f0b64aa_5d45_4cf5_bcdd_084b8c125889', type='opsgptkg_task', attributes={'ID': 6605313998848, 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '选择“拒绝”并退出申请流程', 'name': '选择“拒绝”并退出申请流程', 'accesscriteria': '', 'extra': '{"graphid": ""}'}), + GNode(id='5fd5901a_8adc_4b76_aea2_dcf18884ea0e', type='opsgptkg_task', attributes={'ID': 8029006143488, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '点击“接受”并继续', 'name': '点击“接受”并继续', 'accesscriteria': ''}), + GNode(id='8c999c60_baa7_4e74_903b_f10f148dd12f', type='opsgptkg_task', attributes={'ID': 3902763704320, 'extra': '{"graphid": ""}', 'teamids': '8400001', 'gdb_timestamp': '1725541385', 'executetype': '', 'description': '签署合同：在线签署贷款合同', 'name': '签署合同：在线签署贷款合同', 'accesscriteria': ''}), + GNode(id='e1004c60_5c0c_4f32_b765_a57cc4d39dcc', type='opsgptkg_analysis', attributes={'ID': 9098881261568, 'accesscriteria': '', 'summaryswitch': 'False', 'extra': '{"graphid": ""}', 'teamids': '8400001', 'dsltemplate': '', 'gdb_timestamp': '1725541526', 'description': '根据提示前往医院就诊', 'name': '根据提示前往医院就诊'}), + GNode(id='c50ff5e3_aa01_4a6c_96d7_d8645303846d', type='opsgptkg_task', attributes={'ID': 1676448784384, 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': '', 'description': '活动宣传：制作宣传材料（如海报、邮件通知），在公司内部推广活动信息', 'name': '活动宣传：制作宣传材料（如海报、邮件通知），在公司内部推广活动信息'}), + GNode(id='4f540a57_f73d_451e_aafb_43f1335a18a7', type='opsgptkg_task', attributes={'ID': 4546036121600, 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': '', 'description': '活动实施：根据选择的活动类型，执行相关安排，进行现场协调（无论是户外还是室内）', 'name': '活动实施：根据选择的活动类型，执行相关安排，进行现场协调（无论是户外还是室内）', 'accesscriteria': '', 'extra': '{"graphid": ""}'}), + GNode(id='c9952fa7_7f82_4737_8cfd_bdbb2dabb20e', type='opsgptkg_task', attributes={'ID': 8324758257664, 'description': '活动反馈：收集参与者的反馈意见，总结活动的成功之处和改进建议', 'name': '活动反馈：收集参与者的反馈意见，总结活动的成功之处和改进建议', 'accesscriteria': '', 'extra': '{"graphid": ""}', 'teamids': '8400001, graph_id=8400001', 'gdb_timestamp': '1725541672', 'executetype': ''}), + GNode(id='ekg_team_default', type='opsgptkg_intent', attributes={'ID': 9015207174144, 'teamids': '8400001', 'gdb_timestamp': '1729497515', 'description': '团队起始节点', 'name': '开始', 'extra': '{"isTeamRoot": true}'})] + + +new_edges_1 = \ +[GEdge(start_id='ekg_team_default', end_id='haPvrjEkz4LARZyR7OAuPmVMHMIQPMew', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 9015207174144, 'DSTID': 79745654784, 'gdb_timestamp': '1729496799', 'extra': '{"sourceHandle": "0", "targetHandle": "2"}'}), + GEdge(start_id='ekg_team_default', end_id='dicVRAk5rT3y9LxcmBCN2jDi1TjHc5rm', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 9015207174144, 'DSTID': 6028807454720, 'extra': '{"sourceHandle": "0", "targetHandle": "2"}', 'gdb_timestamp': '1729496767'}), + GEdge(start_id='haPvrjEkz4LARZyR7OAuPmVMHMIQPMew', end_id='ClKvwjBRZUJC7ttSZaiT0dh7lhSujNWi', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 79745654784, 'DSTID': 7905077215232, 'gdb_timestamp': '1729496887', 'extra': '{"sourceHandle": "0", "targetHandle": "2"}'}), + GEdge(start_id='dicVRAk5rT3y9LxcmBCN2jDi1TjHc5rm', end_id='NyBXAHQckQx1xL5lnSgBGlotbZkkQ9C7', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 6028807454720, 'DSTID': 5876942970880, 'extra': '{"sourceHandle": "0", "targetHandle": "2"}', 'gdb_timestamp': '1729496951'}), + GEdge(start_id='dicVRAk5rT3y9LxcmBCN2jDi1TjHc5rm', end_id='6sa4zJCnVKJxKMtOtypapjZk4sdo93QU', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 6028807454720, 'DSTID': 8505664413696, 'extra': '{"sourceHandle": "0", "targetHandle": "2"}', 'gdb_timestamp': '1729496931'}), + GEdge(start_id='ClKvwjBRZUJC7ttSZaiT0dh7lhSujNWi', end_id='a8d85669_141a_4f54_ab8c_209c08d27c35', type='opsgptkg_intent_route_opsgptkg_schedule', attributes={'SRCID': 7905077215232, 'DSTID': 8284119801856, 'extra': '{"sourceHandle": "0", "targetHandle": "2"}', 'gdb_timestamp': '1729496918'}), + GEdge(start_id='NyBXAHQckQx1xL5lnSgBGlotbZkkQ9C7', end_id='2b8df337_f29e_4d49_865f_84088c3a94e7', type='opsgptkg_intent_route_opsgptkg_schedule', attributes={'SRCID': 5876942970880, 'DSTID': 8971031896064, 'gdb_timestamp': '1729496975', 'extra': '{"sourceHandle": "0", "targetHandle": "2"}'}), + GEdge(start_id='6sa4zJCnVKJxKMtOtypapjZk4sdo93QU', end_id='b9fe38f1_33f6_468b_a1dd_43efdfd8e2d1', type='opsgptkg_intent_route_opsgptkg_schedule', attributes={'SRCID': 8505664413696, 'DSTID': 2223780347904, 'gdb_timestamp': '1729496969', 'extra': '{"sourceHandle": "0", "targetHandle": "2"}'}), + GEdge(start_id='a8d85669_141a_4f54_ab8c_209c08d27c35', end_id='98234102_4e4a_4997_9b1e_3cda6382b1c7', type='opsgptkg_schedule_route_opsgptkg_task', attributes={'SRCID': 8284119801856, 'DSTID': 711254376448, 'extra': '{}', 'gdb_timestamp': '1725541635'}), + GEdge(start_id='2b8df337_f29e_4d49_865f_84088c3a94e7', end_id='59030678_760d_4a10_8d61_0d4e4cc5fbcb', type='opsgptkg_schedule_route_opsgptkg_task', attributes={'SRCID': 8971031896064, 'DSTID': 3166553161728, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='b9fe38f1_33f6_468b_a1dd_43efdfd8e2d1', end_id='5afab73b_8f03_422f_856e_386f183bdd71', type='opsgptkg_schedule_route_opsgptkg_task', attributes={'SRCID': 2223780347904, 'DSTID': 6192381952, 'gdb_timestamp': '1725541526', 'extra': '{}'}), + GEdge(start_id='98234102_4e4a_4997_9b1e_3cda6382b1c7', end_id='95ec00ef_cc9c_4947_a21c_88eeb9a71af5', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 711254376448, 'DSTID': 1876388151296, 'extra': '{}', 'gdb_timestamp': '1725541635'}), + GEdge(start_id='59030678_760d_4a10_8d61_0d4e4cc5fbcb', end_id='5504af87_416e_4ee5_bfce_86b969a63433', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 3166553161728, 'DSTID': 6316483026944, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='5afab73b_8f03_422f_856e_386f183bdd71', end_id='3ff8f54a_fa65_4368_86ce_d65058035dd0', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 6192381952, 'DSTID': 7833205129216, 'gdb_timestamp': '1725541526', 'extra': '{}'}), + GEdge(start_id='95ec00ef_cc9c_4947_a21c_88eeb9a71af5', end_id='d5e760b4_ae82_410d_a73d_4c0c98926ae5', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 1876388151296, 'DSTID': 4450592235520, 'gdb_timestamp': '1725541635', 'extra': '{}'}), + GEdge(start_id='95ec00ef_cc9c_4947_a21c_88eeb9a71af5', end_id='2a37b90a_fd96_4548_989c_7c1e8fa9d881', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 1876388151296, 'DSTID': 5557429084160, 'gdb_timestamp': '1725541635', 'extra': '{}'}), + GEdge(start_id='5504af87_416e_4ee5_bfce_86b969a63433', end_id='88d4cf2b_7cf5_4e40_b54e_59268f119f63', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 6316483026944, 'DSTID': 5471766437888, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='3ff8f54a_fa65_4368_86ce_d65058035dd0', end_id='39021995_6e63_4907_9d67_26ba50d0cd44', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 7833205129216, 'DSTID': 24387616768, 'gdb_timestamp': '1725541526', 'extra': '{}'}), + GEdge(start_id='d5e760b4_ae82_410d_a73d_4c0c98926ae5', end_id='59fe9c1d_0731_403e_936a_2e2bbba4b3ee', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 4450592235520, 'DSTID': 7311708086272, 'extra': '{}', 'gdb_timestamp': '1725541635'}), + GEdge(start_id='2a37b90a_fd96_4548_989c_7c1e8fa9d881', end_id='60163dc6_87af_4972_b350_6b9275975c83', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 5557429084160, 'DSTID': 7678175674368, 'gdb_timestamp': '1725541635', 'extra': '{}'}), + GEdge(start_id='88d4cf2b_7cf5_4e40_b54e_59268f119f63', end_id='910f3634_b999_4cf3_94c9_346a67b0d5ed', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 5471766437888, 'DSTID': 9145981739008, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='39021995_6e63_4907_9d67_26ba50d0cd44', end_id='1330ad69_dfc3_4538_864e_6867a3fd8dd4', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 24387616768, 'DSTID': 4367816081408, 'gdb_timestamp': '1725541526', 'extra': '{}'}), + GEdge(start_id='59fe9c1d_0731_403e_936a_2e2bbba4b3ee', end_id='fcbc3e04_ad8c_4aad_9f75_191f8037ced8', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 7311708086272, 'DSTID': 868313473024, 'gdb_timestamp': '1725541635', 'extra': '{}'}), + GEdge(start_id='60163dc6_87af_4972_b350_6b9275975c83', end_id='fcbc3e04_ad8c_4aad_9f75_191f8037ced8', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 7678175674368, 'DSTID': 868313473024, 'gdb_timestamp': '1725541635', 'extra': '{}'}), + GEdge(start_id='910f3634_b999_4cf3_94c9_346a67b0d5ed', end_id='2c7a0d7b_a490_41b9_a6f8_e71b5212e0be', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 9145981739008, 'DSTID': 5168810909696, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='1330ad69_dfc3_4538_864e_6867a3fd8dd4', end_id='3cd46fb7_e11c_4181_8670_2f080a453142', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 4367816081408, 'DSTID': 7081063784448, 'extra': '{}', 'gdb_timestamp': '1725541526'}), + GEdge(start_id='2c7a0d7b_a490_41b9_a6f8_e71b5212e0be', end_id='0f4610cd_cf6a_475b_8ac0_80166569a292', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 5168810909696, 'DSTID': 7170049368064, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='0f4610cd_cf6a_475b_8ac0_80166569a292', end_id='b9f81925_b43a_459d_9902_1bc4b024f5a1', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 7170049368064, 'DSTID': 2310638313472, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='0f4610cd_cf6a_475b_8ac0_80166569a292', end_id='191687cd_1b76_4e77_9f2a_e67936dd372e', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 7170049368064, 'DSTID': 3083664605184, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='b9f81925_b43a_459d_9902_1bc4b024f5a1', end_id='18c33ec1_08ef_4df8_b938_7244852d19c8', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 2310638313472, 'DSTID': 1978980810752, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='191687cd_1b76_4e77_9f2a_e67936dd372e', end_id='b73c2551_0890_40fb_b0ca_04912bc21b65', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 3083664605184, 'DSTID': 8995127967744, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='18c33ec1_08ef_4df8_b938_7244852d19c8', end_id='e95adaa2_d177_435b_bac7_a8b6047ecc3d', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 1978980810752, 'DSTID': 8207832350720, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='e95adaa2_d177_435b_bac7_a8b6047ecc3d', end_id='0c561d68_ee31_49d2_82c1_1dac81e731ff', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 8207832350720, 'DSTID': 2242499641344, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='e95adaa2_d177_435b_bac7_a8b6047ecc3d', end_id='81f579ac_851d_4b85_8608_d2732a2612ff', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': 8207832350720, 'DSTID': 6852580294656, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='0c561d68_ee31_49d2_82c1_1dac81e731ff', end_id='1f0b64aa_5d45_4cf5_bcdd_084b8c125889', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 2242499641344, 'DSTID': 6605313998848, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='81f579ac_851d_4b85_8608_d2732a2612ff', end_id='5fd5901a_8adc_4b76_aea2_dcf18884ea0e', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 6852580294656, 'DSTID': 8029006143488, 'gdb_timestamp': '1725541385', 'extra': '{}'}), + GEdge(start_id='5fd5901a_8adc_4b76_aea2_dcf18884ea0e', end_id='8c999c60_baa7_4e74_903b_f10f148dd12f', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 8029006143488, 'DSTID': 3902763704320, 'extra': '{}', 'gdb_timestamp': '1725541385'}), + GEdge(start_id='3cd46fb7_e11c_4181_8670_2f080a453142', end_id='e1004c60_5c0c_4f32_b765_a57cc4d39dcc', type='opsgptkg_phenomenon_route_opsgptkg_analysis', attributes={'SRCID': 7081063784448, 'DSTID': 9098881261568, 'extra': '{}', 'gdb_timestamp': '1725541526'}), + GEdge(start_id='fcbc3e04_ad8c_4aad_9f75_191f8037ced8', end_id='c50ff5e3_aa01_4a6c_96d7_d8645303846d', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 868313473024, 'DSTID': 1676448784384, 'extra': '{}', 'gdb_timestamp': '1725541635'}), + GEdge(start_id='c50ff5e3_aa01_4a6c_96d7_d8645303846d', end_id='4f540a57_f73d_451e_aafb_43f1335a18a7', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 1676448784384, 'DSTID': 4546036121600, 'extra': '{}', 'gdb_timestamp': '1725541635'}), + GEdge(start_id='4f540a57_f73d_451e_aafb_43f1335a18a7', end_id='c9952fa7_7f82_4737_8cfd_bdbb2dabb20e', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 4546036121600, 'DSTID': 8324758257664, 'gdb_timestamp': '1725541635', 'extra': '{}'})] + + +new_nodes_2 = \ +[GNode(id='剧本杀/谁是卧底', type='opsgptkg_intent', attributes={'ID': -5201231166222141228, 'teamids': '', 'gdb_timestamp': '1725088421109', 'description': '谁是卧底', 'name': '谁是卧底', 'extra': ''}), + GNode(id='剧本杀/狼人杀', type='opsgptkg_intent', attributes={'ID': 5476827419397129797, 'description': '狼人杀', 'name': '狼人杀', 'extra': '', 'teamids': '', 'gdb_timestamp': '1724815561170'}), + GNode(id='剧本杀/谁是卧底/智能交互', type='opsgptkg_schedule', attributes={'ID': 603563742932974030, 'extra': '', 'teamids': '', 'gdb_timestamp': '1725088469126', 'description': '智能交互', 'name': '智能交互', 'enable': ''}), + GNode(id='剧本杀/狼人杀/智能交互', type='opsgptkg_schedule', attributes={'ID': -5931163481230280444, 'extra': '', 'teamids': '', 'gdb_timestamp': '1724815624907', 'description': '智能交互', 'name': '智能交互', 'enable': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/分配座位', type='opsgptkg_task', attributes={'ID': 2011080219630105469, 'extra': '{"dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728912109030', 'executetype': '', 'description': '分配座位', 'name': '分配座位', 'accesscriteria': ''}), + GNode(id='剧本杀/狼人杀/智能交互/位置选择', type='opsgptkg_task', attributes={'ID': 2541178858602010284, 'description': '位置选择', 'name': '位置选择', 'accesscriteria': '', 'extra': '{"memory_tag": "all"}', 'teamids': '', 'gdb_timestamp': '1724849735167', 'executetype': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/角色分配和单词分配', type='opsgptkg_task', attributes={'ID': -1817533533893637377, 'accesscriteria': '', 'extra': '{"memory_tag": "None","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728912123682', 'executetype': '', 'description': '角色分配和单词分配', 'name': '角色分配和单词分配'}), + GNode(id='剧本杀/狼人杀/智能交互/角色选择', type='opsgptkg_task', attributes={'ID': -8695604495489305484, 'description': '角色选择', 'name': '角色选择', 'accesscriteria': '', 'extra': '{"memory_tag": "None"}', 'teamids': '', 'gdb_timestamp': '1724849085296', 'executetype': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/通知身份', type='opsgptkg_task', attributes={'ID': 8901447933395410622, 'extra': '{"pattern": "react","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728912141060', 'executetype': '', 'description': '##角色##\n你正在参与“谁是卧底”这个游戏，你的角色是[主持人]。你熟悉“谁是卧底”游戏的完整流程，你需要完成[任务]，保证游戏的顺利进行。\n目前已经完成 1)位置分配; 2)角色分配和单词分配。\n##任务##\n向所有玩家通知信息他们的 座位信息和单词信息。\n发送格式是： 【身份通知】你是{player_name}, 你的位置是{位置号}号， 你分配的单词是{单词}\n##详细步骤##\nstep1.依次向所有玩家通知信息他们的 座位信息和单词信息。发送格式是： 你是{player_name}, 你的位置是{位置号}号， 你分配的单词是{单词}\nstpe2.所有玩家信息都发送后，结束\n\n##注意##\n1. 每条信息只能发送给对应的玩家，其他人无法看到。\n2. 不要告诉玩家的角色信息，即不要高斯他是平民还是卧底角色\n3. 在将每个人的信息通知到后，本阶段任务结束\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为JSON，格式为\n[{"action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}, ...]\n\n关键词含义如下：\n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n#example#\n如果是玩家发言，则用 {"action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{ "memory_tag":["agent_name_a","agent_name_b"],"content": "str"}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["agent_name_a","agent_name_b"], "content": "str",}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请输出jsonstr，不用输出markdown格式\n5. 结合已有的步骤，每次只输出下一个步骤，即一个 {"action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}', 'name': '通知身份', 'accesscriteria': ''}), + GNode(id='剧本杀/狼人杀/智能交互/向玩家通知消息', type='opsgptkg_task', attributes={'ID': -4014299322597660132, 'extra': '{"pattern": "react"}', 'teamids': '', 'gdb_timestamp': '1725092109310', 'executetype': '', 'description': '##角色##\n你正在参与狼人杀这个游戏，你的角色是[主持人]。你熟悉狼人杀游戏的完整流程，你需要完成[任务]，保证狼人杀游戏的顺利进行。\n目前已经完成位置分配和角色分配。\n##任务##\n向所有玩家通知信息他们的座位信息和角色信息。\n发送格式是： 你是{player_name}, 你的位置是{位置号}号，你的身份是{角色名}\n##注意##\n1. 每条信息只能发送给对应的玩家，其他人无法看到。\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为Python可解析的JSON，格式为\n\n[{"action": {player_name, agent_name}, "observation" or "Dungeon_Master": [{content, memory_tag}, ...]}]\n\n关键词含义如下：\n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n##example##\n如果是玩家发言，则用 {"action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{"content": "str", "memory_tag":["agent_name_a","agent_name_b"]}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{"content": "str", memory_tag:["agent_name_a","agent_name_b"]}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请直接输出jsonstr，不用输出markdown格式\n\n##结果##', 'name': '向玩家通知消息', 'accesscriteria': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/关键信息_1', type='opsgptkg_task', attributes={'ID': 3196717310525578616, 'gdb_timestamp': '1728913619628', 'executetype': '', 'description': '关键信息', 'name': '关键信息', 'accesscriteria': '', 'extra': '{"ignorememory":"True","dodisplay":"True"}', 'teamids': ''}), + GNode(id='剧本杀/狼人杀/智能交互/狼人时刻', type='opsgptkg_task', attributes={'ID': 8926130661368382825, 'accesscriteria': 'OR', 'extra': '{"pattern": "react"}', 'teamids': '', 'gdb_timestamp': '1725092131051', 'executetype': '', 'description': '##背景##\n在狼人杀游戏中，主持人通知当前存活的狼人玩家指认一位击杀对象，所有狼人玩家给出击杀目标，主持人确定最终结果。\n\n##任务##\n整个流程分为6个步骤：\n1. 存活狼人通知：主持人向所有的狼人玩家广播，告知他们当前存活的狼人玩家有哪些。\n2. 第一轮讨论：主持人告知所有存活的狼人玩家投票，从当前存活的非狼人玩家中，挑选一个想要击杀的玩家。\n3. 第一轮投票：按照座位顺序，每一位存活的狼人为自己想要击杀的玩家投票。\n4. 第一轮结果反馈：主持人统计所有狼人的票数分布，确定他们是否达成一致。若达成一致，告知所有狼人最终被击杀的玩家的player_name，流程结束；否则，告知他们票数的分布情况，并让所有狼人重新投票指定击杀目标，主持人需要提醒他们，若该轮还不能达成一致，则取票数最大的目标为最终击杀对象。\n5. 第二轮投票：按照座位顺序，每一位存活的狼人为自己想要击杀的玩家投票。\n6. 第二轮结果反馈：主持人统计第二轮投票中所有狼人的票数分布，取票数最大的玩家为最终击杀对象，如果存在至少两个对象的票数最大且相同，取座位号最大的作为最终击杀对象。主持人告知所有狼人玩家最终被击杀的玩家的player_name。\n\n该任务的参与者只有狼人玩家和主持人，信息可见对象是所有狼人玩家。\n\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为Python可解析的JSON，格式为\n\n[{"action": {player_name, agent_name}, "observation" or "Dungeon_Master": [{content, memory_tag}, ...]}]\n\n关键词含义如下：\n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n##example##\n如果是玩家发言，则用 {"action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{"content": "str", "memory_tag":["agent_name_a","agent_name_b"]}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{"content": "str", memory_tag:["agent_name_a","agent_name_b"]}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请直接输出jsonstr，不用输出markdown格式\n\n##结果##', 'name': '狼人时刻'}), + GNode(id='剧本杀/谁是卧底/智能交互/开始新一轮的讨论', type='opsgptkg_task', attributes={'ID': -6077057339616293423, 'accesscriteria': 'OR', 'extra': '{"pattern": "react", "endcheck": "True",\n"memory_tag":"all",\n"dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913634866', 'executetype': '', 'description': '###以上为本局游戏记录###\n\n\n##背景##\n你正在参与“谁是卧底”这个游戏，你的角色是[主持人]。你熟悉“谁是卧底”游戏的完整流程，你需要完成[任务]，保证游戏的顺利进行。\n\n##任务##\n以结构化的语句来模拟进行 谁是卧底的讨论环节。 在这一个环节里，所有主持人先宣布目前存活的玩家，然后每位玩家按照座位顺序发言\n\n\n##详细步骤##\nstep1. 主持人根据本局游戏历史记录，感知最开始所有的玩家 以及 在前面轮数中已经被票选死亡的玩家。注意死亡的玩家不能参与本轮游戏。得到当前存活的玩家个数以及其player_name。 并告知所有玩家当前存活的玩家个数以及其player_name。\nstep2. 主持人确定发言规则并告知所有玩家，发言规则步骤如下: 存活的玩家按照座位顺序由小到大进行发言\n（一个例子：假设总共有5个玩家，如果3号位置处玩家死亡，则发言顺序为：1_>2_>4_>5）\nstep3. 存活的的玩家按照顺序依次发言\nstpe4. 在每一位存活的玩家都发言后，结束\n\n \n \n##注意##\n1.之前的游戏轮数可能已经投票选中了某位/某些玩家，被票选中的玩家会立即死亡，不再视为存活玩家,死亡的玩家不能参与本轮游戏 \n2.你要让所有存活玩家都参与发言，不能遗漏任何存活玩家。在本轮所有玩家只发言一次\n3.该任务的参与者为主持人和所有存活的玩家，信息可见对象为所有玩家。\n4.不仅要模拟主持人的发言，还需要模拟玩家的发言\n5.每一位存活的玩家均发完言后，本阶段结束\n\n\n\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为JSON，格式为\n[ {"thought": str, "action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}] }, ...]\n\n\n\n\n关键词含义如下：\n_ thought (str): 主持人执行行动的一些思考,包括分析玩家的存活状态，对历史对话信息的理解，对当前任务情况的判断等。 \n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空 ;否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为本条信息的可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n##example##\n如果是玩家发言，则用 {"thought": "str", "action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{ "memory_tag":["agent_name_a","agent_name_b"],"content": "str"}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"thought": "str", "action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["agent_name_a","agent_name_b"], "content": "str",}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请输出jsonstr，不用输出markdown格式\n5. 结合已有的步骤，每次只输出下一个步骤，即一个 {"thought": str, "action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}\n6. 如果是人类玩家发言， 一定要选择类似 agent_人类玩家 这样的agent_name', 'name': '开始新一轮的讨论'}), + GNode(id='剧本杀/狼人杀/智能交互/天亮讨论', type='opsgptkg_task', attributes={'ID': 274796810216558717, 'gdb_timestamp': '1725106348469', 'executetype': '', 'description': '##角色##\n你正在参与狼人杀这个游戏，你的角色是[主持人]。你熟悉狼人杀游戏的完整流程，你需要完成[任务]，保证狼人杀游戏的顺利进行。\n##任务##\n你的任务如下: \n1. 告诉玩家昨晚发生的情况: 首先告诉玩家天亮了，然后你需要根据过往信息，告诉所有玩家，昨晚是否有玩家死亡。如果有，则向所有人宣布死亡玩家的名字，你只能宣布死亡玩家是谁如："昨晚xx玩家死了"，不要透露任何其他信息。如果没有，则宣布昨晚是平安夜。\n2. 确定发言规则并告诉所有玩家:\n确定发言规则步骤如下: \n第一步：确定第一个发言玩家，第一个发言的玩家为死者的座位号加1位置处的玩家(注意：最后一个位置+1的位置号为1号座位），如无人死亡，则从1号玩家开始。\n第二步：告诉所有玩家从第一个发言玩家开始发言，除了死亡玩家，每个人都需要按座位号依次讨论，只讨论一轮，所有人发言完毕后结束。注意不能遗忘指挥任何存活玩家发言！\n以下是一个例子：\n```\n总共有5个玩家，如果3号位置处玩家死亡，则第一个发言玩家为4号位置处玩家，因此从他开始发言，发言顺序为：4_>5_>1_>2\n```\n3. 依次指定存活玩家依次发言\n4. 被指定的玩家依次发言\n##注意##\n1. 你必须根据规则确定第一个发言玩家是谁，然后根据第一个发言玩家的座位号，确定所有人的发言顺序并将具体发言顺序并告知所有玩家，不要做任何多余解释\n2. 你要让所有存活玩家都参与发言，不能遗漏任何存活玩家\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为Python可解析的JSON，格式为\n\n[{"action": {player_name, agent_name}, "observation" or "Dungeon_Master": [{content, memory_tag}, ...]}]\n\n关键词含义如下：\n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n##example##\n如果是玩家发言，则用 {"action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{"content": "str", "memory_tag":["agent_name_a","agent_name_b"]}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{"content": "str", memory_tag:["agent_name_a","agent_name_b"]}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请直接输出jsonstr，不用输出markdown格式\n\n##结果（请直接在后面输出，如果后面已经有部分结果，请续写。一定要保持续写后的内容结合前者能构成一个合法的 jsonstr）##', 'name': '天亮讨论', 'accesscriteria': '', 'extra': '{"pattern": "react"}', 'teamids': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/关键信息_2', type='opsgptkg_task', attributes={'ID': -8309123437761850283, 'description': '关键信息', 'name': '关键信息', 'accesscriteria': '', 'extra': '{"ignorememory":"True","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913648645', 'executetype': ''}), + GNode(id='剧本杀/狼人杀/智能交互/票选凶手', type='opsgptkg_task', attributes={'ID': 1492108834523573937, 'accesscriteria': '', 'extra': '{"pattern": "react"}', 'teamids': '', 'gdb_timestamp': '1725106389716', 'executetype': '', 'description': '##角色##\n你正在参与“谁是卧底”这个游戏，你的角色是[主持人]。你熟悉“谁是卧底”游戏的完整流程，你需要完成[任务]，保证游戏的顺利进行。\n\n##任务##\n你的任务如下:\n1. 告诉玩家投票规则，规则步骤如下: \nstep1: 确定讨论阶段第一个发言的玩家A\nstep2: 从A玩家开始，按座位号依次投票，每个玩家只能对一个玩家进行投票，投票这个玩家表示认为该玩家是“卧底”。每个玩家只能投一次票。\nstep3: 将完整投票规则告诉所有玩家\n2. 指挥存活玩家依次投票。\n3. 被指定的玩家进行投票\n4. 主持人统计投票结果，并告知所有玩家，投出的角色是谁。\n\n该任务的参与者为主持人和所有存活的玩家，信息可见对象是所有玩家。\n\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为Python可解析的JSON，格式为\n```\n{"action": {player_name, agent_name}, "observation" or "Dungeon_Master": [{content, memory_tag}, ...]}\n```\n关键词含义如下：\n_ player_name (str): 行动方的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): 行动方的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n\n##example##\n如果是玩家发言，则用 {"action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{"content": "str", "memory_tag":["agent_name_a","agent_name_b"]}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{"content": "str", memory_tag:["agent_name_a","agent_name_b"]}]}\n\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请直接输出jsonstr，不用输出markdown格式\n\n##结果##\n', 'name': '票选凶手'}), + GNode(id='剧本杀/谁是卧底/智能交互/票选卧底_1', type='opsgptkg_task', attributes={'ID': 267468674566989196, 'teamids': '', 'gdb_timestamp': '1728913670477', 'executetype': '', 'description': '##以上为本局游戏历史记录##\n##角色##\n你是一个统计票数大师，你正在参与“谁是卧底”这个游戏，你的角色是[主持人]。你熟悉“谁是卧底”游戏的完整流程，你需要完成[任务]，保证游戏的顺利进行。 现在是投票阶段。\n\n##任务##\n以结构化的语句来模拟进行 谁是卧底的投票环节， 也仅仅只模拟投票环节，投票环节结束后就本阶段就停止了，由后续的阶段继续进行游戏。 在这一个环节里，由主持人先告知大家投票规则，然后组织每位存活玩家按照座位顺序发言投票, 所有人投票后，本阶段结束。 \n##详细步骤##\n你的任务如下:\nstep1. 向所有玩家通知现在进入了票选环节，在这个环节，每个人都一定要投票指定某一个玩家为卧底\nstep2. 主持人确定投票顺序并告知所有玩家。 投票顺序基于如下规则: 1: 存活的玩家按照座位顺序由小到大进行投票（一个例子：假设总共有5个玩家，如果3号位置处玩家死亡，则投票顺序为：1_>2_>4_>5）2: 按座位号依次投票，每个玩家只能对一个玩家进行投票。每个玩家只能投一次票。3：票数最多的玩家会立即死亡\n\nstep3. 存活的的玩家按照顺序进行投票\nstep4. 所有存活玩家发言完毕,主持人宣布投票环节结束\n该任务的参与者为主持人和所有存活的玩家，信息可见对象是所有玩家。\n##注意##\n\n1.之前的游戏轮数可能已经投票选中了某位/某些玩家，被票选中的玩家会立即死亡，不再视为存活玩家 \n2.你要让所有存活玩家都参与投票，不能遗漏任何存活玩家。在本轮每一位玩家只投票一个人\n3.该任务的参与者为主持人和所有存活的玩家，信息可见对象为所有玩家。\n4.不仅要模拟主持人的发言，还需要模拟玩家的发言\n5.不允许玩家自己投自己，如果出现了这种情况，主持人会提醒玩家重新投票。\n\n\n\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为JSON，格式为\n["thought": str, {"action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}, ...]\n关键词含义如下：\n_ thought (str): 主持人执行行动的一些思考,包括分析玩家的存活状态，对历史对话信息的理解，对当前任务情况的判断。 \n_ player_name (str): ***的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): ***的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n##example##\n如果是玩家发言，则用 {"thought": "str", "action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{ "memory_tag":["agent_name_a","agent_name_b"],"content": "str"}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"thought": "str", "action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["agent_name_a","agent_name_b"], "content": "str",}]}\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请输出jsonstr，不用输出markdown格式\n5. 结合已有的步骤，每次只输出下一个步骤，即一个 {"thought": str, "action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}\n6. 如果是人类玩家发言， 一定要选择类似 人类agent 这样的agent_name', 'name': '票选卧底', 'accesscriteria': '', 'extra': '{"pattern": "react", "endcheck": "True", "memory_tag":"all","dodisplay":"True"}'}), + GNode(id='剧本杀/谁是卧底/智能交互/关键信息_4', type='opsgptkg_task', attributes={'ID': -4669093152651945828, 'extra': '{"ignorememory":"True","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913685959', 'executetype': '', 'description': '关键信息_4', 'name': '关键信息_4', 'accesscriteria': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/统计票数', type='opsgptkg_task', attributes={'ID': -6836070348442528830, 'teamids': '', 'gdb_timestamp': '1728913701092', 'executetype': '', 'description': '##以上为本局游戏历史记录##\n##角色##\n你是一个统计票数大师，你非常擅长计数以及统计信息。你正在参与“谁是卧底”这个游戏，你的角色是[主持人]。你熟悉“谁是卧底”游戏的完整流程，你需要完成[任务]，保证游戏的顺利进行。 现在是票数统计阶段\n\n##任务##\n以结构化的语句来模拟进行 谁是卧底的票数统计阶段， 也仅仅只票数统计阶段环节，票数统计阶段结束后就本阶段就停止了，由后续的阶段继续进行游戏。 在这一个环节里，由主持人根据上一轮存活的玩家投票结果统计票数。 \n##详细步骤##\n你的任务如下:\nstep1. 主持人感知上一轮投票环节每位玩家的发言, 统计投票结果，格式为[{"player_name":票数}]. \nstep2 然后，主持人宣布死亡的玩家，以最大票数为本轮被投票的目标，如果票数相同，则取座位号高的角色死亡。并告知所有玩家本轮被投票玩家的player_name。（格式为【重要通知】本轮死亡的玩家为XXX）同时向所有玩家宣布，被投票中的角色会视为立即死亡（即不再视为存活角色）\nstep3. 在宣布死亡玩家后，本阶段流程结束，由后续阶段继续推进游戏\n该任务的参与者为主持人和所有存活的玩家，信息可见对象是所有玩家。\n##注意##\n1.如果有2个或者两个以上的被玩家被投的票数相同，则取座位号高的玩家死亡。并告知大家原因：票数相同，取座位号高的玩家死亡\n2.在统计票数时，首先确认存活玩家的数量，再先仔细回忆，谁被投了。 最后统计每位玩家被投的次数。 由于每位玩家只有一票，所以被投次数的总和等于存活玩家的数量 \n3.通知完死亡玩家是谁后，本阶段才结束，由后续阶段继续推进游戏。输出 {"action": "taskend"}即可\n4.主持人只有当通知本轮死亡的玩家时，才使用【重要通知】的前缀，其他情况下不要使用【重要通知】前缀\n5.只统计上一轮投票环节的情况\n##example##\n{"thought": "在上一轮中, 存活玩家有 小北,李光,赵鹤,张良 四个人。 其中 小北投了李光, 赵鹤投了小北, 张良投了李光, 李光投了张良。总结被投票数为： 李光:2票; 小北:1票,张良:1票. Check一下，一共有四个人投票了，被投的票是2（李光）+1（小北）+1（张良）=4，总结被投票数没有问题。 因此李光的票最多", "action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["all"], "content": "李光:2票; 小北:1票,张良:1票 .因此李光的票最多.【重要通知】本轮死亡玩家是李光",}]}\n\n##example##\n{"thought": "在上一轮中, 存活玩家有 小北,人类玩家,赵鹤,张良 四个人。 其中 小北投了人类玩家, 赵鹤投了小北, 张良投了小北, 人类玩家投了张良。总结被投票数为：小北:2票,人类玩家:1票,张良:0票 .Check一下，一共有四个人投票了，被投的票是2（小北）+1（人类玩家）+张良（0）=3，总结被投票数有问题。 更正总结被投票数为：小北:2票,人类玩家:1票,张良:1票。因此小北的票最多", "action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["all"], "content": "小北:2票,人类玩家:1票,张良:1票 .因此小北的票最多.【重要通知】本轮死亡玩家是小北",}]}\n\n\n##输出##\n请以列表的形式，给出参与者的所有行动。每个行动表示为JSON，格式为\n["thought": str, {"action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}, ...]\n关键词含义如下：\n_ thought (str): 主持人执行行动的一些思考,包括分析玩家的存活状态，对历史对话信息的理解，对当前任务情况的判断。 \n_ player_name (str): ***的 player_name，若行动方为主持人，为空，否则为玩家的 player_name；\n_ agent_name (str): ***的 agent_name，若为主持人，则 agent_name 为 "主持人"，否则为玩家的 agent_name。\n_ content (str): 行动方的具体行为，若为主持人，content 为告知信息；否则，content 为玩家的具体行动。\n_ memory_tag (List[str]): 无论行动方是主持人还是玩家，memory_tag 固定为**所有**信息可见对象的agent_name, 如果信息可见对象为所有玩家，固定为 ["all"]\n##example##\n如果是玩家发言，则用 {"thought": "str", "action": {"agent_name": "agent_name_c", "player_name":"player_name_d"}, "observation": [{ "memory_tag":["agent_name_a","agent_name_b"],"content": "str"}]} 格式表示。content是玩家发出的信息；memory_tag是这条信息可见的对象，需要填写agent名。不要填写 agent_description\n如果agent_name是主持人，则无需输入player_name， 且observation变为 Dungeon_Master。即{"thought": "str", "action": {"agent_name": "主持人", "player_name":""}, "Dungeon_Master": [{ "memory_tag":["agent_name_a","agent_name_b"], "content": "str",}]}\n##注意事项##\n1. 所有玩家的座位、身份、agent_name、存活状态等信息在开头部分已给出。\n2. "observation" or "Dungeon_Master"如何选择？若 agent_name 为"主持人"，则为"Dungeon_Master"，否则为 "observation"。\n3. 输出列表的最后一个元素一定是{"action": "taskend"}。\n4. 整个list是一个jsonstr，请输出jsonstr，不用输出markdown格式\n5. 结合已有的步骤，每次只输出下一个步骤，即一个 {"thought": str, "action": {"player_name":str, "agent_name":str}, "observation" or "Dungeon_Master": [{"memory_tag":str,"content":str}]}\n6. 如果是人类玩家发言， 一定要选择类似 人类agent 这样的agent_name', 'name': '统计票数', 'accesscriteria': '', 'extra': '{"pattern": "react", "endcheck": "True", "memory_tag":"all","model_name":"gpt_4","dodisplay":"True"}'}), + GNode(id='剧本杀/谁是卧底/智能交互/关键信息_3', type='opsgptkg_task', attributes={'ID': -4800215480474522940, 'accesscriteria': '', 'extra': '{"ignorememory":"True","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913715255', 'executetype': '', 'description': '关键信息', 'name': '关键信息'}), + GNode(id='剧本杀/谁是卧底/智能交互/判断游戏是否结束', type='opsgptkg_task', attributes={'ID': -5959590132883379159, 'description': '判断游戏是否结束', 'name': '判断游戏是否结束', 'accesscriteria': '', 'extra': '{"memory_tag": "None","dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913728308', 'executetype': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/事实_1', type='opsgptkg_phenomenon', attributes={'ID': -525629912140732688, 'description': '是', 'name': '是', 'extra': '', 'teamids': '', 'gdb_timestamp': '1725089138724'}), + GNode(id='剧本杀/谁是卧底/智能交互/事实_2', type='opsgptkg_phenomenon', attributes={'ID': 4216433814773851843, 'teamids': '', 'gdb_timestamp': '1725089593085', 'description': '否', 'name': '否', 'extra': ''}), + GNode(id='剧本杀/谁是卧底/智能交互/给出每个人的单词以及最终胜利者', type='opsgptkg_task', attributes={'ID': 8878899410716129093, 'extra': '{"dodisplay":"True"}', 'teamids': '', 'gdb_timestamp': '1728913745186', 'executetype': '', 'description': '给出每个人的单词以及最终胜利者', 'name': '给出每个人的单词以及最终胜利者', 'accesscriteria': ''}), + GNode(id='剧本杀/狼人杀/智能交互/判断游戏是否结束', type='opsgptkg_task', attributes={'ID': -2316854558435035646, 'description': '判断游戏是否结束 ', 'name': '判断游戏是否结束 ', 'accesscriteria': '', 'extra': '{"memory_tag": "None"}', 'teamids': '', 'gdb_timestamp': '1725092210244', 'executetype': ''}), + GNode(id='剧本杀/狼人杀/智能交互/事实_2', type='opsgptkg_phenomenon', attributes={'ID': -6298561983042120406, 'extra': '', 'teamids': '', 'gdb_timestamp': '1724816562165', 'description': '否', 'name': '否'}), + GNode(id='剧本杀/狼人杀/智能交互/事实_1', type='opsgptkg_phenomenon', attributes={'ID': 6987562967613654408, 'gdb_timestamp': '1724816495297', 'description': '是', 'name': '是', 'extra': '', 'teamids': ''}), + GNode(id='剧本杀/l狼人杀/智能交互/宣布游戏胜利者', type='opsgptkg_task', attributes={'ID': -758955621725402723, 'extra': '', 'teamids': '', 'gdb_timestamp': '1725097362872', 'executetype': '', 'description': '判断游戏是否结束', 'name': '判断游戏是否结束', 'accesscriteria': ''}), + GNode(id='剧本杀', type='opsgptkg_intent', attributes={'ID': -3388526698926684245, 'description': '文本游戏相关（如狼人杀等）', 'name': '剧本杀', 'extra': '', 'teamids': '', 'gdb_timestamp': '1724815537102'})] + + + +new_edges_2 = \ +[GEdge(start_id='剧本杀', end_id='剧本杀/谁是卧底', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': -3388526698926684245, 'DSTID': -5201231166222141228, 'gdb_timestamp': '1725088433347', 'extra': ''}), + GEdge(start_id='剧本杀', end_id='剧本杀/狼人杀', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': -3388526698926684245, 'DSTID': 5476827419397129797, 'gdb_timestamp': '1724815572710', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底', end_id='剧本杀/谁是卧底/智能交互', type='opsgptkg_intent_route_opsgptkg_schedule', attributes={'SRCID': -5201231166222141228, 'DSTID': 603563742932974030, 'gdb_timestamp': '1725088478251', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀', end_id='剧本杀/狼人杀/智能交互', type='opsgptkg_intent_route_opsgptkg_schedule', attributes={'SRCID': 5476827419397129797, 'DSTID': -5931163481230280444, 'gdb_timestamp': '1724815633494', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互', end_id='剧本杀/谁是卧底/智能交互/分配座位', type='opsgptkg_schedule_route_opsgptkg_task', attributes={'SRCID': 603563742932974030, 'DSTID': 2011080219630105469, 'gdb_timestamp': '1725088659469', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀/智能交互', end_id='剧本杀/狼人杀/智能交互/位置选择', type='opsgptkg_schedule_route_opsgptkg_task', attributes={'SRCID': -5931163481230280444, 'DSTID': 2541178858602010284, 'gdb_timestamp': '1724815720186', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/分配座位', end_id='剧本杀/谁是卧底/智能交互/角色分配和单词分配', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 2011080219630105469, 'DSTID': -1817533533893637377, 'gdb_timestamp': '1725088761379', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀/智能交互/位置选择', end_id='剧本杀/狼人杀/智能交互/角色选择', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 2541178858602010284, 'DSTID': -8695604495489305484, 'extra': '', 'gdb_timestamp': '1724815828424'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/角色分配和单词分配', end_id='剧本杀/谁是卧底/智能交互/通知身份', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -1817533533893637377, 'DSTID': 8901447933395410622, 'gdb_timestamp': '1725088813780', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀/智能交互/角色选择', end_id='剧本杀/狼人杀/智能交互/向玩家通知消息', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -8695604495489305484, 'DSTID': -4014299322597660132, 'gdb_timestamp': '1724815943792', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/通知身份', end_id='剧本杀/谁是卧底/智能交互/关键信息_1', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 8901447933395410622, 'DSTID': 3196717310525578616, 'extra': '', 'gdb_timestamp': '1725364881808'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/向玩家通知消息', end_id='剧本杀/狼人杀/智能交互/狼人时刻', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -4014299322597660132, 'DSTID': 8926130661368382825, 'gdb_timestamp': '1724815952503', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/关键信息_1', end_id='剧本杀/谁是卧底/智能交互/开始新一轮的讨论', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 3196717310525578616, 'DSTID': -6077057339616293423, 'extra': '', 'gdb_timestamp': '1725364891197'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/狼人时刻', end_id='剧本杀/狼人杀/智能交互/天亮讨论', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 8926130661368382825, 'DSTID': 274796810216558717, 'gdb_timestamp': '1724911515908', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/开始新一轮的讨论', end_id='剧本杀/谁是卧底/智能交互/关键信息_2', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -6077057339616293423, 'DSTID': -8309123437761850283, 'extra': '', 'gdb_timestamp': '1725364966817'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/天亮讨论', end_id='剧本杀/狼人杀/智能交互/票选凶手', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 274796810216558717, 'DSTID': 1492108834523573937, 'extra': '', 'gdb_timestamp': '1724816423574'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/关键信息_2', end_id='剧本杀/谁是卧底/智能交互/票选卧底_1', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -8309123437761850283, 'DSTID': 267468674566989196, 'gdb_timestamp': '1725507894066', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/票选卧底_1', end_id='剧本杀/谁是卧底/智能交互/关键信息_4', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 267468674566989196, 'DSTID': -4669093152651945828, 'gdb_timestamp': '1725507901109', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/关键信息_4', end_id='剧本杀/谁是卧底/智能交互/统计票数', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -4669093152651945828, 'DSTID': -6836070348442528830, 'extra': '', 'gdb_timestamp': '1725507907343'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/统计票数', end_id='剧本杀/谁是卧底/智能交互/关键信息_3', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -6836070348442528830, 'DSTID': -4800215480474522940, 'extra': '', 'gdb_timestamp': '1725507917664'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/关键信息_3', end_id='剧本杀/谁是卧底/智能交互/判断游戏是否结束', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': -4800215480474522940, 'DSTID': -5959590132883379159, 'extra': '', 'gdb_timestamp': '1725365051574'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/判断游戏是否结束', end_id='剧本杀/谁是卧底/智能交互/事实_1', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': -5959590132883379159, 'DSTID': -525629912140732688, 'extra': '', 'gdb_timestamp': '1725089153218'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/判断游戏是否结束', end_id='剧本杀/谁是卧底/智能交互/事实_2', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': -5959590132883379159, 'DSTID': 4216433814773851843, 'extra': '', 'gdb_timestamp': '1725089603500'}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/事实_1', end_id='剧本杀/谁是卧底/智能交互/给出每个人的单词以及最终胜利者', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': -525629912140732688, 'DSTID': 8878899410716129093, 'gdb_timestamp': '1725089654391', 'extra': ''}), + GEdge(start_id='剧本杀/谁是卧底/智能交互/事实_2', end_id='剧本杀/谁是卧底/智能交互/开始新一轮的讨论', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 4216433814773851843, 'DSTID': -6077057339616293423, 'extra': '', 'gdb_timestamp': '1725089612866'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/票选凶手', end_id='剧本杀/狼人杀/智能交互/判断游戏是否结束', type='opsgptkg_task_route_opsgptkg_task', attributes={'SRCID': 1492108834523573937, 'DSTID': -2316854558435035646, 'extra': '', 'gdb_timestamp': '1724816464917'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/判断游戏是否结束', end_id='剧本杀/狼人杀/智能交互/事实_2', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': -2316854558435035646, 'DSTID': -6298561983042120406, 'gdb_timestamp': '1724816570641', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀/智能交互/判断游戏是否结束', end_id='剧本杀/狼人杀/智能交互/事实_1', type='opsgptkg_task_route_opsgptkg_phenomenon', attributes={'SRCID': -2316854558435035646, 'DSTID': 6987562967613654408, 'gdb_timestamp': '1724816506031', 'extra': ''}), + GEdge(start_id='剧本杀/狼人杀/智能交互/事实_2', end_id='剧本杀/狼人杀/智能交互/狼人时刻', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': -6298561983042120406, 'DSTID': 8926130661368382825, 'extra': '', 'gdb_timestamp': '1724816585403'}), + GEdge(start_id='剧本杀/狼人杀/智能交互/事实_1', end_id='剧本杀/l狼人杀/智能交互/宣布游戏胜利者', type='opsgptkg_phenomenon_route_opsgptkg_task', attributes={'SRCID': 6987562967613654408, 'DSTID': -758955621725402723, 'gdb_timestamp': '1724911404270', 'extra': ''})] + +new_edges_3 = [GEdge(start_id='ekg_team_default', end_id='剧本杀', type='opsgptkg_intent_route_opsgptkg_intent', attributes={'SRCID': 9015207174144, 'DSTID': -3388526698926684245, 'gdb_timestamp': '1724816506031', 'extra': ''})] + +new_nodes = new_nodes_1 + new_nodes_2 +new_edges = new_edges_1 + new_edges_2 + new_edges_3 + +teamid = "default" + + +def add_nodes(ekg_service, nodes: list[GNode]): + + logger.info('尝试查插入节点') + for one_node in nodes: + one_node.attributes['description'] = one_node.attributes['description'] + one_node.attributes['gdb_timestamp'] = int(one_node.attributes['gdb_timestamp'] ) + if one_node.id != "ekg_team_default": + one_node.id = hash_id(one_node.id ) + + if one_node.type == 'opsgptkg_analysis': + + one_node.attributes['summaryswitch'] = False + + if one_node.type == 'opsgptkg_schedule': + one_node.attributes['enable'] = True + + ekg_service.add_nodes([one_node], teamid=teamid) + # ekg_service.gb.add_node(one_node) + + +def add_edges(ekg_service, edges): + + logger.info('尝试查插入边') + for one_edge in edges: + one_edge.attributes['gdb_timestamp'] = int(one_edge.attributes['gdb_timestamp'] ) + + if one_edge.start_id != "ekg_team_default": + one_edge.start_id = hash_id(one_edge.start_id ) + + if one_edge.end_id != "ekg_team_default": + one_edge.end_id = hash_id(one_edge.end_id ) + # one_edge.start_id = hash_id(one_edge.start_id ) + # one_edge.end_id = hash_id(one_edge.end_id ) + one_edge.attributes['type'] = 'opsgptkg_intent_extend_opsgptkg_intent' + + if one_edge.type == 'opsgptkg_phenomenon_conclude_opsgptkg_analysis': + one_edge.type = 'opsgptkg_phenomenon_route_opsgptkg_analysis' + # ekg_service.gb.add_edge(one_edge) + ekg_service.add_edges([one_edge], teamid=teamid) + +def test_whoisspy_datas(ekg_service, ): + logger.info('尝试查找一阶近邻') + start_nodetype =' opsgptkg_intent' + start_nodeid = hash_id('剧本杀') + + neighbor_nodes = ekg_service.gb.get_neighbor_nodes(attributes={"id": start_nodeid,}, + node_type=start_nodetype) + + current_nodes = ekg_service.gb.get_current_nodes(attributes={"id": start_nodeid,}, + node_type=start_nodetype) + + logger.info(neighbor_nodes) + logger.info(current_nodes) + + + +def load_whoisspy_datas(ekg_service,): + neighbor_nodes = ekg_service.gb.get_neighbor_nodes(attributes={"id": hash_id("剧本杀"),}, + node_type="opsgptkg_intent") + logger.info(f"load_game_datas: {neighbor_nodes}") + + if len(neighbor_nodes) == 1: + return + add_nodes(ekg_service, new_nodes) + add_edges(ekg_service, new_edges) + + neighbor_nodes = ekg_service.gb.get_neighbor_nodes(attributes={"id": hash_id("剧本杀"),}, + node_type="opsgptkg_intent") + logger.info(f"load_game_datas: {neighbor_nodes}") \ No newline at end of file diff --git a/examples/test_config.py.example b/examples/test_config.py.example index 1b96b3f..c1347ee 100644 --- a/examples/test_config.py.example +++ b/examples/test_config.py.example @@ -1,17 +1,90 @@ import os, openai, base64 from loguru import logger -# +# 兜底大模型配置 OPENAI_API_BASE = "https://api.openai.com/v1" os.environ["API_BASE_URL"] = OPENAI_API_BASE os.environ["OPENAI_API_KEY"] = "sk-xxx" openai.api_key = "sk-xxx" os.environ["model_name"] = "gpt-3.5-turbo" -os.environ["model_engine] = "openai" +os.environ["model_engine"] = "openai" -# +# 涉及react模式时需要配置主持人的模型，起到起承转合的作用 +# 推荐gpt-4 +os.environ["gpt4-API_BASE_URL"] = OPENAI_API_BASE +os.environ["gpt4-OPENAI_API_KEY"] = "" +os.environ["gpt4-model_name"] = "gpt-4" +os.environ["gpt4-model_engine"] = "openai" +os.environ["gpt4-llm_temperature"] = "0.0" + + + +#### NebulaHandler #### +os.environ['nb_host'] = 'graphd' +os.environ['nb_port'] = '9669' +os.environ['nb_username'] = 'root' +os.environ['nb_password'] = 'nebula' +os.environ['nb_space'] = "client" + +#### tbasehandler #### + +## opensource-default### +os.environ['tb_host'] = 'redis-stack' +os.environ['tb_port'] = '6379' +os.environ['tb_username'] = '' +os.environ['tb_password'] = '' +os.environ['tb_definition_value'] = "opsgptkg" + +# tbase-memory +os.environ["tb_type"] = "TbaseHandler" +os.environ["tb_index_name"] = "ekg_migration_new" +os.environ['tb_definition_value'] = 'message_test_new' +os.environ['tb_expire_time'] = '604800' #86400*7 + + + +######################################## +########## 以下参数暂不涉及无需配置 ######## +######################################## os.environ["embed_model"] = "{{embed_model_name}}" os.environ["embed_model_path"] = "{{embed_model_path}}" # -os.environ["DUCKDUCKGO_PROXY"] = os.environ.get("DUCKDUCKGO_PROXY") or "socks5h://127.0.0.1:13659" \ No newline at end of file +os.environ["DUCKDUCKGO_PROXY"] = os.environ.get("DUCKDUCKGO_PROXY") or "socks5h://127.0.0.1:13659" + +os.environ['operation_mode'] = 'open_source' # 'open_source' or 'antcode' +os.environ['config_path'] = "" + + +### call llm ### +os.environ['aes_encrypt_iv'] = "" + +#gpt-4# +os.environ['gpt4_serviceName'] = "" +os.environ['gpt4_visitDomain'] = "" +os.environ['gpt4_visitBiz'] = "" +os.environ['gpt4_visitBizLine'] = "" +os.environ['gpt4_cacheInterval'] = "" +os.environ['gpt4_model'] = "" +os.environ['gpt4_api_key'] = "" + +os.environ['gpt4_url']= '' +os.environ['gpt4_key']= "" + +#modelops# +os.environ['modelops_url']= "" +os.environ['modelops_qwen_7b_chat_gpt_token'] ='' +os.environ['modelops_qwen_7b_chat_gpt_Cookie']= '' + +###意图识别### +os.environ['intention_url'] = '' + + +###old function### +OLD_PARAMS_STRING_EXAMPLE = {} + +os.environ['oldfunction_url'] = '' + + +### qa ### +os.environ['sre_agent_flow_url'] = '' \ No newline at end of file diff --git a/frontend/.eslintrc.js b/frontend/.eslintrc.js new file mode 100644 index 0000000..85ba500 --- /dev/null +++ b/frontend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: require.resolve('@umijs/max/eslint'), +}; diff --git a/frontend/.prettierignore b/frontend/.prettierignore new file mode 100644 index 0000000..5892e28 --- /dev/null +++ b/frontend/.prettierignore @@ -0,0 +1,3 @@ +node_modules +.umi +.umi-production diff --git a/frontend/.prettierrc b/frontend/.prettierrc new file mode 100644 index 0000000..70767cd --- /dev/null +++ b/frontend/.prettierrc @@ -0,0 +1,8 @@ +{ + "printWidth": 80, + "singleQuote": true, + "trailingComma": "all", + "proseWrap": "never", + "overrides": [{ "files": ".prettierrc", "options": { "parser": "json" } }], + "plugins": ["prettier-plugin-organize-imports", "prettier-plugin-packagejson"] +} diff --git a/frontend/.stylelintrc.js b/frontend/.stylelintrc.js new file mode 100644 index 0000000..08bc02c --- /dev/null +++ b/frontend/.stylelintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: require.resolve('@umijs/max/stylelint'), +}; diff --git a/frontend/.umirc.ts b/frontend/.umirc.ts new file mode 100644 index 0000000..7eb50c9 --- /dev/null +++ b/frontend/.umirc.ts @@ -0,0 +1,46 @@ +import { defineConfig } from '@umijs/max'; +const mockSpaceId = 1; + +export default defineConfig({ + styledComponents: {}, + valtio: {}, + access: {}, + model: {}, + initialState: {}, + request: {}, + layout: false, + locale: { + default: 'zh-CN', + antd: true, + }, + proxy: { + '/api': { + // target: 'https://nexa-api-pre.alipay.com', + // target: 'http://30.230.0.179:8080', + target: 'http://runtime:8080', + // target: 'http://30.98.121.212:8083', + changeOrigin: true, + }, + }, + routes: [ + { + path: '/', + redirect: '/space/default/ekg/default', + }, + { + path: `/space/:spaceId/ekg/:ekgId`, + component: './EKG/Common', + routes: [ + { + path: '', + component: './EKG/index', + }, + { + path: 'flow/:flowId', + component: './EKG/Flow/index', + }, + ], + }, + ], + npmClient: 'pnpm', +}); diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..048973d --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,14 @@ +# README +本项目是`@umijs/max` 模板项目，更多功能参考 [Umi Max 简介](https://umijs.org/docs/max/introduce) + +# 安装依赖 +`pnpm i` + +# 启动项目 +`pnpm dev` + +# 打开页面 +前端服务运行在`http://localhost:8000/` + + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..940b634 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,41 @@ +{ + "private": true, + "author": "will.jc ", + "scripts": { + "build": "max build", + "dev": "max dev", + "format": "prettier --cache --write .", + "postinstall": "max setup", + "setup": "max setup", + "start": "npm run dev" + }, + "dependencies": { + "@ant-design/icons": "^5.0.1", + "@ant-design/pro-components": "^2.4.4", + "@microsoft/fetch-event-source": "^2.0.1", + "@umijs/max": "^4.3.20", + "antd": "^5.20.6", + "d3-force": "^3.0.0", + "dagre": "^0.8.5", + "fetch-event-source": "^1.0.0-alpha.2", + "highlight.js": "^10.5.0", + "js-cookie": "^3.0.5", + "lodash": "^4.17.21", + "markdown-it": "^14.1.0", + "nanoid": "^5.0.7", + "react-json-editor-ajrm": "^2.5.14", + "react-markdown": "^9.0.1", + "reactflow": "^11.11.3" + }, + "devDependencies": { + "@types/js-cookie": "^3.0.6", + "@types/lodash": "^4.17.9", + "@types/react": "^18.0.33", + "@types/react-dom": "^18.0.11", + "prettier": "^2.8.7", + "prettier-plugin-organize-imports": "^3.2.2", + "prettier-plugin-packagejson": "^2.4.3", + "typescript": "^5.0.3" + }, + "repository": "git@code.alipay.com:codefuse_opensource/muagent.git" +} diff --git a/frontend/src/app.ts b/frontend/src/app.ts new file mode 100644 index 0000000..012a39a --- /dev/null +++ b/frontend/src/app.ts @@ -0,0 +1,22 @@ +import { createGlobalStyle } from '@umijs/max'; +// 运行时配置 + +// 全局初始化数据配置，用于 Layout 用户信息和权限初始化 +// 更多信息见文档：https://umijs.org/docs/api/runtime-config#getinitialstate +// export async function getInitialState(): Promise<{ name: string }> { +// return { name: '@umijs/max' }; +// } + +export const styledComponents = { + GlobalStyle: createGlobalStyle` +html, +body { + height: 100%; + background-color: #eceff6; + margin: 0; +} +* { + box-sizing: border-box; +} + `, +}; diff --git a/frontend/src/assets/.gitkeep b/frontend/src/assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/frontend/src/components/.gitkeep b/frontend/src/components/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/frontend/src/constants/index.tsx b/frontend/src/constants/index.tsx new file mode 100644 index 0000000..4172c4f --- /dev/null +++ b/frontend/src/constants/index.tsx @@ -0,0 +1 @@ +export const BACKEND_URL = 'http://nexa-114-gz00b.cloudide.dev.alipay.net'; \ No newline at end of file diff --git a/frontend/src/pages/EKG/Common/Chat/AgentPrologue/index.tsx b/frontend/src/pages/EKG/Common/Chat/AgentPrologue/index.tsx new file mode 100644 index 0000000..dabd5b6 --- /dev/null +++ b/frontend/src/pages/EKG/Common/Chat/AgentPrologue/index.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import { Avatar, Tooltip } from 'antd'; +import { AgentPrologueWrapper } from './style'; + +const AgentPrologue = ({ agent }: any) => { + return ( + +
+ +

+ {agent.agentConfig?.name || agent?.agentName} +

+
+ {agent?.agentDesc} +
+
+ + ); +}; + +export default AgentPrologue; diff --git a/frontend/src/pages/EKG/Common/Chat/AgentPrologue/style.ts b/frontend/src/pages/EKG/Common/Chat/AgentPrologue/style.ts new file mode 100644 index 0000000..6518f44 --- /dev/null +++ b/frontend/src/pages/EKG/Common/Chat/AgentPrologue/style.ts @@ -0,0 +1,35 @@ +import { styled } from '@umijs/max'; + +export const AgentPrologueWrapper = styled.div` + width: 100%; + height: 100%; + text-align: center; + display: flex; + justify-content: center; + align-items: center; + .agent_dec { + width: 100%; + display: -webkit-box; + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + -webkit-line-clamp: 2; + font-size: 14px; + padding: 0 12px; + } + .agent_questionSample { + width: 100%; + height: auto; + padding: 14px 24px; + margin: 14px 0; + background: #fff; + border-radius: 16px; + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + -webkit-line-clamp: 1; + display: flex; + justify-content: space-between; + align-items: center; + } +`; diff --git a/frontend/src/pages/EKG/Common/Chat/ChatContent.tsx b/frontend/src/pages/EKG/Common/Chat/ChatContent.tsx new file mode 100644 index 0000000..2778fd4 --- /dev/null +++ b/frontend/src/pages/EKG/Common/Chat/ChatContent.tsx @@ -0,0 +1,134 @@ +import { dataToJson, jsonToData } from '../../utils/format'; +import { getMyEmpId } from '../../utils/userStore'; +import { Avatar, Button, Space } from 'antd'; +import React, { useContext, useState } from 'react'; +import { + CheckCircleFilled, + CloseCircleFilled, + ExclamationCircleFilled, +} from '@ant-design/icons'; +import type { CommonContextType } from '../../Common'; +import { CommonContext } from '../../Common'; +import AgentPrologue from './AgentPrologue'; +import { ContentWrapper } from './style'; +import TemplateCode from './TemplateCode'; + +const ChatContent = () => { + const { msgList, selectedAgent, setHeaderType } = useContext( + CommonContext, + ) as CommonContextType; + + const formatMsgContent = (dataItem: NEX_MAIN_API.EKGChatJSONMsg) => { + const { content, type } = dataItem; + if (type === 'json') { + const { costMs, output, input, status } = + content as NEX_MAIN_API.EKGChatJSONMsgContent; + return { + costMs, + text: output, + input, + status, + }; + } + if (type === 'text') { + return { text: jsonToData(content as unknown as string)?.data?.text }; + } + if (type === 'role_response') { + if (content.response?.type === 'text') { + return { text: content.response.content.text } + } + return { text: content.response.text } + } + return { text: '解析错误' }; + }; + + const renderAvatarSrc = (item: any) => { + if (item?.content?.role === 'user') { + return `https://work.alibaba-inc.com/photo/${getMyEmpId()}.100x100.jpg`; + } + if (item?.type === "role_response") { + return item?.content?.url; + } + return selectedAgent?.avatar; + }; + + const renderAvatarTitle = (item: any) => { + if (item?.type === "role_response") { + return item?.content?.name; + } + }; + + return ( + + {!dataToJson(msgList).flag && } + {msgList?.length > 0 && ( + <> + {msgList?.map((items: NEX_MAIN_API.EKGChatJSONMsg, index: number) => { + console.log('debug_msgList渲染>>>>', msgList); + return ( +
+ + + {renderAvatarTitle(items)} + + + {formatMsgContent(items)?.text === '输出中' ? ( + + ) : ( +
+ +
+ )} + +
+ ); + })} + + )} + + ); +}; + +export default ChatContent; diff --git a/frontend/src/pages/EKG/Common/Chat/ChatFooter.tsx b/frontend/src/pages/EKG/Common/Chat/ChatFooter.tsx new file mode 100644 index 0000000..260b4bf --- /dev/null +++ b/frontend/src/pages/EKG/Common/Chat/ChatFooter.tsx @@ -0,0 +1,289 @@ +import { createSession } from '@/services/nexa/PortalConversationController'; +import { getMyEmpId, getMyLoginName } from '../../utils/userStore'; +import { useParams, useRequest } from '@umijs/max'; +import { + Button, + Form, + Input, + message, + Space, + Spin, +} from 'antd'; +import React, { + createContext, + useContext, + useState, +} from 'react'; +import { CloseOutlined } from '@ant-design/icons'; +import { cloneDeep } from 'lodash'; +import type { CommonContextType } from '../../Common'; +import { CommonContext } from '../../Common'; +import { ssePost } from './sseport'; +import { FooterWrapper } from './style'; + +interface InputContextType { + type: 'ANTEMC' | 'NLP'; + setType: React.Dispatch>; + form: any; +} + +const InputContext = createContext(undefined); + +const ChatFooter = () => { + const [type, setType] = useState<'ANTEMC' | 'NLP'>('NLP'); + const [loading, setLoading] = useState(false); + + const { selectedAgent, setMsgList, msgList } = useContext( + CommonContext, + ) as CommonContextType; + + const [form] = Form.useForm(); + const { TextArea } = Input; + const { spaceId } = useParams(); + + const RenderTextBtn = () => { + return ( + + ); + }; + + const RenderDebugBtn = () => { + return ( +
+ + + 应急场景调试 + + { + form.resetFields(); + setType('NLP'); + }} + /> +
+ ); + }; + + const renderHeader = () => { + if (type === 'NLP') { + return ; + } + if (type === 'ANTEMC') { + return ; + } + }; + + /** + * 发送普通会话 + * @param props_sessionId + * @param inputValueStr + */ + const sendSession = (props_sessionId: string, inputValueStr?: string) => { + setMsgList((preMsg: NEX_MAIN_API.EKGChatMsgList) => { + const newMsg = cloneDeep(preMsg); + return [ + ...newMsg, + { + type: 'json', + msgId: new Date().getTime(), + content: { + output: inputValueStr, + input: inputValueStr, + role: 'user', + messageType: 'json', + conversationId: new Date().getTime(), + status: 'EXECUTING', + }, + traceId: new Date().getTime(), + chatResultTypeCode: 'cover', + streamingDisplay: false, + }, + { + type: 'json', + msgId: new Date().getTime(), + content: { + output: '输出中', + input: inputValueStr, + role: 'assistant', + messageType: 'json', + conversationId: new Date().getTime(), + status: 'EXECUTING', + }, + traceId: new Date().getTime(), + chatResultTypeCode: 'cover', + streamingDisplay: false, + }, + ]; + }); + ssePost( + '/api/portal/conversation/chat', + { + sessionId: props_sessionId, + agentId: selectedAgent?.agentId, + userId: getMyLoginName(), + empId: getMyEmpId(), + type: 'text', + submitType: 'NORMAL', + content: { text: inputValueStr }, + stream: false, + extendContext: { + debugMode: true, + ekgMode: true, + ekgSource: type, + spaceId, + ...(msgList[msgList.length - 1]?.extendContext || {}) + }, + }, + { + setLoading, + props_sessionId, + setMsgList, + }, + ); + form.resetFields(); + }; + + /** + * 创建普通会话 + */ + const createSession_action = useRequest(createSession, { + manual: true, + formatResult: (res: any) => res, + onSuccess: (res: any, params: any) => { + if (res?.success) { + sendSession(res?.data, params[0]?.summary); + } else { + message.error(res?.errorMessage); + } + }, + }); + + const sendMessage = (inputValueStr: string) => { + if (inputValueStr) { + setLoading(true); + if (!msgList || msgList?.length === 0) { + createSession_action.run({ + userId: getMyLoginName(), + empId: getMyEmpId(), + summary: inputValueStr, + debugMode: true, + }); + } else { + const preData = msgList[msgList.length - 1]; + sendSession(preData?.extendContext?.chatUnidId, inputValueStr); + } + } else { + message.warning('请先问一个问题'); + } + }; + + return ( + + +
+
+ {/*
+ {renderHeader()} +
*/} +
+
+ +