diff --git a/README.md b/README.md index 0a89a45d0..706043aa9 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,8 @@ At present, we have introduced several key features to showcase our current capa We offer extensive model support, including dozens of large language models (LLMs) from both open-source and API agents, such as LLaMA/LLaMA2, Baichuan, ChatGLM, Wenxin, Tongyi, Zhipu, and many more. - News + - 🔥🔥🔥 [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) + - 🔥🔥🔥 [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) - 🔥🔥🔥 [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat) - 🔥🔥🔥 [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat) - 🔥🔥🔥 [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta) diff --git a/README.zh.md b/README.zh.md index 162b8654c..4775af49b 100644 --- a/README.zh.md +++ b/README.zh.md @@ -152,6 +152,8 @@ 海量模型支持，包括开源、API代理等几十种大语言模型。如LLaMA/LLaMA2、Baichuan、ChatGLM、文心、通义、智谱等。当前已支持如下模型: - 新增支持模型 + - 🔥🔥🔥 [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) + - 🔥🔥🔥 [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) - 🔥🔥🔥 [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat) - 🔥🔥🔥 [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat) - 🔥🔥🔥 [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta) @@ -164,6 +166,7 @@ - [更多开源模型](https://www.yuque.com/eosphoros/dbgpt-docs/iqaaqwriwhp6zslc#qQktR) - 支持在线代理模型 + - [x] [月之暗面.Moonshot](https://platform.moonshot.cn/docs/) - [x] [零一万物.Yi](https://platform.lingyiwanwu.com/docs) - [x] [OpenAI·ChatGPT](https://api.openai.com/) - [x] [百川·Baichuan](https://platform.baichuan-ai.com/) diff --git a/dbgpt/configs/model_config.py b/dbgpt/configs/model_config.py index ed8d296f8..4da990a42 100644 --- a/dbgpt/configs/model_config.py +++ b/dbgpt/configs/model_config.py @@ -72,6 +72,10 @@ def get_device() -> str: "llama-2-7b": os.path.join(MODEL_PATH, "Llama-2-7b-chat-hf"), "llama-2-13b": os.path.join(MODEL_PATH, "Llama-2-13b-chat-hf"), "llama-2-70b": os.path.join(MODEL_PATH, "Llama-2-70b-chat-hf"), + # https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct + "meta-llama-3-8b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-8B-Instruct"), + # https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct + "meta-llama-3-70b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-70B-Instruct"), "baichuan-13b": os.path.join(MODEL_PATH, "Baichuan-13B-Chat"), # please rename "fireballoon/baichuan-vicuna-chinese-7b" to "baichuan-7b" "baichuan-7b": os.path.join(MODEL_PATH, "baichuan-7b"), diff --git a/dbgpt/datasource/rdbms/conn_clickhouse.py b/dbgpt/datasource/rdbms/conn_clickhouse.py index 5f785e009..13659af46 100644 --- a/dbgpt/datasource/rdbms/conn_clickhouse.py +++ b/dbgpt/datasource/rdbms/conn_clickhouse.py @@ -153,7 +153,7 @@ def get_columns(self, table_name: str) -> List[Dict]: @property def dialect(self) -> str: """Return string representation of dialect to use.""" - pass + return "" def get_fields(self, table_name) -> List[Tuple]: """Get column fields about specified table.""" diff --git a/dbgpt/model/adapter/hf_adapter.py b/dbgpt/model/adapter/hf_adapter.py index 090eb5827..60ed6a043 100644 --- a/dbgpt/model/adapter/hf_adapter.py +++ b/dbgpt/model/adapter/hf_adapter.py @@ -270,9 +270,48 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None): ) +class Llama3Adapter(NewHFChatModelAdapter): + """ + https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct + https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct + """ + + support_4bit: bool = True + support_8bit: bool = True + + def do_match(self, lower_model_name_or_path: Optional[str] = None): + return lower_model_name_or_path and "llama-3" in lower_model_name_or_path + + def get_str_prompt( + self, + params: Dict, + messages: List[ModelMessage], + tokenizer: Any, + prompt_template: str = None, + convert_to_compatible_format: bool = False, + ) -> Optional[str]: + str_prompt = super().get_str_prompt( + params, + messages, + tokenizer, + prompt_template, + convert_to_compatible_format, + ) + terminators = [ + tokenizer.eos_token_id, + tokenizer.convert_tokens_to_ids("<|eot_id|>"), + ] + exist_token_ids = params.get("stop_token_ids", []) + terminators.extend(exist_token_ids) + # TODO(fangyinc): We should modify the params in the future + params["stop_token_ids"] = terminators + return str_prompt + + register_model_adapter(YiAdapter) register_model_adapter(Mixtral8x7BAdapter) register_model_adapter(SOLARAdapter) register_model_adapter(GemmaAdapter) register_model_adapter(StarlingLMAdapter) register_model_adapter(QwenAdapter) +register_model_adapter(Llama3Adapter) diff --git a/dbgpt/model/llm_out/hf_chat_llm.py b/dbgpt/model/llm_out/hf_chat_llm.py index e8fb58054..5c994a05a 100644 --- a/dbgpt/model/llm_out/hf_chat_llm.py +++ b/dbgpt/model/llm_out/hf_chat_llm.py @@ -20,6 +20,8 @@ def huggingface_chat_generate_stream( top_p = float(params.get("top_p", 1.0)) echo = params.get("echo", False) max_new_tokens = int(params.get("max_new_tokens", 2048)) + stop_token_ids = params.get("stop_token_ids", []) + do_sample = params.get("do_sample", None) input_ids = tokenizer(prompt).input_ids # input_ids = input_ids.to(device) @@ -39,13 +41,22 @@ def huggingface_chat_generate_stream( streamer = TextIteratorStreamer( tokenizer, skip_prompt=not echo, skip_special_tokens=True ) - generate_kwargs = { - "input_ids": input_ids, + + base_kwargs = { "max_length": context_len, "temperature": temperature, "streamer": streamer, + "top_p": top_p, } + if stop_token_ids: + base_kwargs["eos_token_id"] = stop_token_ids + if do_sample is not None: + base_kwargs["do_sample"] = do_sample + + logger.info(f"Predict with parameters: {base_kwargs}") + + generate_kwargs = {"input_ids": input_ids, **base_kwargs} thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start() out = "" diff --git a/setup.py b/setup.py index 0cefec7dc..dd027a891 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ "BUILD_FROM_SOURCE_URL_FAST_CHAT", "git+https://github.com/lm-sys/FastChat.git" ) BUILD_VERSION_OPENAI = os.getenv("BUILD_VERSION_OPENAI") +INCLUDE_QUANTIZATION = os.getenv("INCLUDE_QUANTIZATION", "true").lower() == "true" def parse_requirements(file_name: str) -> List[str]: @@ -552,7 +553,9 @@ def quantization_requires(): # TODO(yyhhyy): Add autoawq install method for CUDA version 11.8 quantization_pkgs.extend(["autoawq", _build_autoawq_requires(), "optimum"]) - setup_spec.extras["quantization"] = ["cpm_kernels"] + quantization_pkgs + setup_spec.extras["quantization"] = ( + ["cpm_kernels"] + quantization_pkgs + setup_spec.extras["bitsandbytes"] + ) def all_vector_store_requires(): @@ -659,7 +662,9 @@ def default_requires(): setup_spec.extras["default"] += setup_spec.extras["rag"] setup_spec.extras["default"] += setup_spec.extras["datasource"] setup_spec.extras["default"] += setup_spec.extras["torch"] - setup_spec.extras["default"] += setup_spec.extras["quantization"] + if INCLUDE_QUANTIZATION: + # Add quantization extra to default, default is True + setup_spec.extras["default"] += setup_spec.extras["quantization"] setup_spec.extras["default"] += setup_spec.extras["cache"]