Skip to content

Commit 6a494b5

Browse files
kaiyuxkaiyux
authored
Update main branch (triton-inference-server#11)
* Update * Update submodule
1 parent aa4d2a6 commit 6a494b5

File tree

12 files changed

+1163
-121
lines changed

12 files changed

+1163
-121
lines changed

‎README.md

Lines changed: 316 additions & 113 deletions
Large diffs are not rendered by default.

‎ci/L0_backend_trtllm/generate_engines.sh

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/bash
2+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
BASE_DIR=/opt/tritonserver/tensorrtllm_backend/ci/L0_backend_trtllm
29+
GPT_DIR=/opt/tritonserver/tensorrtllm_backend/tensorrt_llm/examples/gpt
30+
31+
function build_base_model {
32+
cd ${GPT_DIR}
33+
rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2
34+
pushd gpt2 && rm pytorch_model.bin model.safetensors && wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin && popd
35+
python3 hf_gpt_convert.py -i gpt2 -o ./c-model/gpt2 --tensor-parallelism 1 --storage-type float16
36+
cd ${BASE_DIR}
37+
}
38+
39+
function build_tensorrt_engine_inflight_batcher {
40+
cd ${GPT_DIR}
41+
# ./c-model/gpt2/ must already exist (it will if build_base_model
42+
# has already been run)
43+
python3 build.py --model_dir=./c-model/gpt2/1-gpu/ \
44+
--dtype float16 \
45+
--use_inflight_batching \
46+
--use_gpt_attention_plugin float16 \
47+
--paged_kv_cache \
48+
--use_gemm_plugin float16 \
49+
--remove_input_padding \
50+
--use_layernorm_plugin float16 \
51+
--hidden_act gelu \
52+
--output_dir=inflight_single_gpu/
53+
cd ${BASE_DIR}
54+
55+
}
56+
57+
function build_tensorrt_engine_inflight_batcher_multi_gpu {
58+
cd ${GPT_DIR}
59+
python3 hf_gpt_convert.py -p 8 -i gpt2 -o ./c-model/gpt2 --tensor-parallelism 4 --storage-type float16
60+
python3 build.py --model_dir=./c-model/gpt2/4-gpu/ \
61+
--world_size=4 \
62+
--dtype float16 \
63+
--use_inflight_batching \
64+
--use_gpt_attention_plugin float16 \
65+
--paged_kv_cache \
66+
--use_gemm_plugin float16 \
67+
--remove_input_padding \
68+
--use_layernorm_plugin float16 \
69+
--hidden_act gelu \
70+
--parallel_build \
71+
--output_dir=inflight_multi_gpu/
72+
cd ${BASE_DIR}
73+
}
74+
75+
# Install TRT LLM
76+
# FIXME: Update the url
77+
pip install git+https://github.com/NVIDIA/TensorRT-LLM.git@${TENSORRTLLM_BACKEND_REPO_TAG}
78+
mkdir /usr/local/lib/python3.10/dist-packages/tensorrt_llm/libs/
79+
cp /opt/tritonserver/backends/tensorrtllm/* /usr/local/lib/python3.10/dist-packages/tensorrt_llm/libs/
80+
81+
export LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:$LD_LIBRARY_PATH
82+
export TRT_ROOT=/usr/local/tensorrt
83+
84+
# Generate the TRT_LLM model engines
85+
build_base_model
86+
build_tensorrt_engine_inflight_batcher
87+
build_tensorrt_engine_inflight_batcher_multi_gpu
88+
89+
# Move the TRT_LLM model engines to the CI directory
90+
mkdir engines
91+
mv ${GPT_DIR}/inflight_single_gpu engines/
92+
mv ${GPT_DIR}/inflight_multi_gpu engines/
93+
94+
# Move the tokenizer into the CI directory
95+
mkdir tokenizer
96+
mv ${GPT_DIR}/gpt2/* tokenizer/
97+
98+
# Now that the engines are generated, we should remove the
99+
# tensorrt_llm module to ensure the C++ backend tests are
100+
# not using it
101+
rm -rf /usr/local/lib/python3.10/dist-packages/tensorrt_llm

‎ci/L0_backend_trtllm/simple_data.json

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
[
2+
{
3+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
4+
"instruction": "Summarize the following news article:",
5+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
6+
},
7+
{
8+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
9+
"instruction": "Summarize the following news article:",
10+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
11+
},
12+
{
13+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
14+
"instruction": "Summarize the following news article:",
15+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
16+
},
17+
{
18+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
19+
"instruction": "Summarize the following news article:",
20+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
21+
},
22+
{
23+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
24+
"instruction": "Summarize the following news article:",
25+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
26+
},
27+
{
28+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
29+
"instruction": "Summarize the following news article:",
30+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
31+
},
32+
{
33+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
34+
"instruction": "Summarize the following news article:",
35+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
36+
},
37+
{
38+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
39+
"instruction": "Summarize the following news article:",
40+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
41+
},
42+
{
43+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
44+
"instruction": "Summarize the following news article:",
45+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
46+
},
47+
{
48+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
49+
"instruction": "Summarize the following news article:",
50+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
51+
},
52+
{
53+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
54+
"instruction": "Summarize the following news article:",
55+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
56+
},
57+
{
58+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
59+
"instruction": "Summarize the following news article:",
60+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
61+
},
62+
{
63+
"input": "MADRID, Spain -- Lionel Messi scored for the sixth game in a row as Barcelona defeated big-spending Atletico Madrid 3-0 to stay in touch with Primera Liga leaders Real Madrid. Messi (left) is congratulated by Ronaldinho after scoring again in Barcelona's 3-0 win over Atletico Madrid. Barcelona had thumped Atletico 6-0 on their own ground last season and the visitors were out for revenge -- but conceded twice in four minutes. After 15 minutes, Italian goalkeeper Christian Abbiati let a routine Messi cross slip out of his hands and Deco rolled home into the empty net. Four minutes later Messi played a great one-two with Ronaldinho and rifled a shot past Abbiati for his sixth goal of the season and Xavi added a third late on.",
64+
"instruction": "Summarize the following news article:",
65+
"output": "Lionel Messi scores for the sixth game in a row as Barca defeat Atletico 3-0 ."
66+
}
67+
]

0 commit comments

Comments
 (0)