Skip to content

Commit

Permalink
feat(model): Support Llama-3 (eosphoros-ai#1436)
Browse files Browse the repository at this point in the history
  • Loading branch information
fangyinc authored and Hopshine committed Sep 10, 2024
1 parent 2841011 commit 547ccba
Show file tree
Hide file tree
Showing 7 changed files with 69 additions and 5 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ At present, we have introduced several key features to showcase our current capa
We offer extensive model support, including dozens of large language models (LLMs) from both open-source and API agents, such as LLaMA/LLaMA2, Baichuan, ChatGLM, Wenxin, Tongyi, Zhipu, and many more.

- News
- 🔥🔥🔥 [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)
- 🔥🔥🔥 [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
- 🔥🔥🔥 [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)
- 🔥🔥🔥 [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat)
- 🔥🔥🔥 [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta)
Expand Down
3 changes: 3 additions & 0 deletions README.zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@
海量模型支持,包括开源、API代理等几十种大语言模型。如LLaMA/LLaMA2、Baichuan、ChatGLM、文心、通义、智谱等。当前已支持如下模型:

- 新增支持模型
- 🔥🔥🔥 [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)
- 🔥🔥🔥 [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
- 🔥🔥🔥 [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)
- 🔥🔥🔥 [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat)
- 🔥🔥🔥 [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta)
Expand All @@ -164,6 +166,7 @@
- [更多开源模型](https://www.yuque.com/eosphoros/dbgpt-docs/iqaaqwriwhp6zslc#qQktR)

- 支持在线代理模型
- [x] [月之暗面.Moonshot](https://platform.moonshot.cn/docs/)
- [x] [零一万物.Yi](https://platform.lingyiwanwu.com/docs)
- [x] [OpenAI·ChatGPT](https://api.openai.com/)
- [x] [百川·Baichuan](https://platform.baichuan-ai.com/)
Expand Down
4 changes: 4 additions & 0 deletions dbgpt/configs/model_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ def get_device() -> str:
"llama-2-7b": os.path.join(MODEL_PATH, "Llama-2-7b-chat-hf"),
"llama-2-13b": os.path.join(MODEL_PATH, "Llama-2-13b-chat-hf"),
"llama-2-70b": os.path.join(MODEL_PATH, "Llama-2-70b-chat-hf"),
# https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct
"meta-llama-3-8b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-8B-Instruct"),
# https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
"meta-llama-3-70b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-70B-Instruct"),
"baichuan-13b": os.path.join(MODEL_PATH, "Baichuan-13B-Chat"),
# please rename "fireballoon/baichuan-vicuna-chinese-7b" to "baichuan-7b"
"baichuan-7b": os.path.join(MODEL_PATH, "baichuan-7b"),
Expand Down
2 changes: 1 addition & 1 deletion dbgpt/datasource/rdbms/conn_clickhouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def get_columns(self, table_name: str) -> List[Dict]:
@property
def dialect(self) -> str:
"""Return string representation of dialect to use."""
pass
return ""

def get_fields(self, table_name) -> List[Tuple]:
"""Get column fields about specified table."""
Expand Down
39 changes: 39 additions & 0 deletions dbgpt/model/adapter/hf_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,9 +270,48 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
)


class Llama3Adapter(NewHFChatModelAdapter):
"""
https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct
https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
"""

support_4bit: bool = True
support_8bit: bool = True

def do_match(self, lower_model_name_or_path: Optional[str] = None):
return lower_model_name_or_path and "llama-3" in lower_model_name_or_path

def get_str_prompt(
self,
params: Dict,
messages: List[ModelMessage],
tokenizer: Any,
prompt_template: str = None,
convert_to_compatible_format: bool = False,
) -> Optional[str]:
str_prompt = super().get_str_prompt(
params,
messages,
tokenizer,
prompt_template,
convert_to_compatible_format,
)
terminators = [
tokenizer.eos_token_id,
tokenizer.convert_tokens_to_ids("<|eot_id|>"),
]
exist_token_ids = params.get("stop_token_ids", [])
terminators.extend(exist_token_ids)
# TODO(fangyinc): We should modify the params in the future
params["stop_token_ids"] = terminators
return str_prompt


register_model_adapter(YiAdapter)
register_model_adapter(Mixtral8x7BAdapter)
register_model_adapter(SOLARAdapter)
register_model_adapter(GemmaAdapter)
register_model_adapter(StarlingLMAdapter)
register_model_adapter(QwenAdapter)
register_model_adapter(Llama3Adapter)
15 changes: 13 additions & 2 deletions dbgpt/model/llm_out/hf_chat_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ def huggingface_chat_generate_stream(
top_p = float(params.get("top_p", 1.0))
echo = params.get("echo", False)
max_new_tokens = int(params.get("max_new_tokens", 2048))
stop_token_ids = params.get("stop_token_ids", [])
do_sample = params.get("do_sample", None)

input_ids = tokenizer(prompt).input_ids
# input_ids = input_ids.to(device)
Expand All @@ -39,13 +41,22 @@ def huggingface_chat_generate_stream(
streamer = TextIteratorStreamer(
tokenizer, skip_prompt=not echo, skip_special_tokens=True
)
generate_kwargs = {
"input_ids": input_ids,

base_kwargs = {
"max_length": context_len,
"temperature": temperature,
"streamer": streamer,
"top_p": top_p,
}

if stop_token_ids:
base_kwargs["eos_token_id"] = stop_token_ids
if do_sample is not None:
base_kwargs["do_sample"] = do_sample

logger.info(f"Predict with parameters: {base_kwargs}")

generate_kwargs = {"input_ids": input_ids, **base_kwargs}
thread = Thread(target=model.generate, kwargs=generate_kwargs)
thread.start()
out = ""
Expand Down
9 changes: 7 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"BUILD_FROM_SOURCE_URL_FAST_CHAT", "git+https://github.com/lm-sys/FastChat.git"
)
BUILD_VERSION_OPENAI = os.getenv("BUILD_VERSION_OPENAI")
INCLUDE_QUANTIZATION = os.getenv("INCLUDE_QUANTIZATION", "true").lower() == "true"


def parse_requirements(file_name: str) -> List[str]:
Expand Down Expand Up @@ -552,7 +553,9 @@ def quantization_requires():
# TODO(yyhhyy): Add autoawq install method for CUDA version 11.8
quantization_pkgs.extend(["autoawq", _build_autoawq_requires(), "optimum"])

setup_spec.extras["quantization"] = ["cpm_kernels"] + quantization_pkgs
setup_spec.extras["quantization"] = (
["cpm_kernels"] + quantization_pkgs + setup_spec.extras["bitsandbytes"]
)


def all_vector_store_requires():
Expand Down Expand Up @@ -659,7 +662,9 @@ def default_requires():
setup_spec.extras["default"] += setup_spec.extras["rag"]
setup_spec.extras["default"] += setup_spec.extras["datasource"]
setup_spec.extras["default"] += setup_spec.extras["torch"]
setup_spec.extras["default"] += setup_spec.extras["quantization"]
if INCLUDE_QUANTIZATION:
# Add quantization extra to default, default is True
setup_spec.extras["default"] += setup_spec.extras["quantization"]
setup_spec.extras["default"] += setup_spec.extras["cache"]


Expand Down

0 comments on commit 547ccba

Please sign in to comment.