Skip to content

Commit e81766a

Browse files
author
Guang Yang
committed
Add compatible HuggingFace models to benchmark workflow
1 parent c707e4c commit e81766a

File tree

4 files changed

+121
-56
lines changed

4 files changed

+121
-56
lines changed

.ci/scripts/test_hf_model.sh

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
# shellcheck source=/dev/null
11+
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
12+
13+
# Input parameter: Hugging Face model repo (e.g., 'google/gemma-2b')
14+
HF_MODEL_REPO=$1
15+
UPLOAD_DIR=${2:-}
16+
17+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
18+
PYTHON_EXECUTABLE=python
19+
fi
20+
which "${PYTHON_EXECUTABLE}"
21+
22+
# Extract the model name from the HF_MODEL_REPO by splitting on '/' and replacing '_' with '-'
23+
ET_MODEL_NAME=$(echo "$HF_MODEL_REPO" | awk -F'/' '{print $2}' | sed 's/_/-/g')
24+
# Add the suffix "_xnnpack_fp32" to the model name (currently supported delegate and dtype)
25+
OUT_ET_MODEL_NAME="${ET_MODEL_NAME}_xnnpack_fp32"
26+
27+
# Files to be handled
28+
TOKENIZER_FILE="tokenizer.model"
29+
OUT_TOKENIZER_BIN_FILE="tokenizer.bin"
30+
31+
# Download the tokenizer model using Hugging Face hub
32+
DOWNLOADED_TOKENIZER_FILE_PATH=$("${PYTHON_EXECUTABLE}" << EOF
33+
from huggingface_hub import hf_hub_download
34+
35+
# Download the tokenizer file from the Hugging Face Hub
36+
try:
37+
downloaded_path = hf_hub_download(
38+
repo_id='${HF_MODEL_REPO}',
39+
filename='${TOKENIZER_FILE}'
40+
)
41+
print(downloaded_path)
42+
except Exception as e:
43+
print(f"Error: {str(e)}")
44+
EOF
45+
)
46+
47+
# Check if the tokenizer file was successfully downloaded
48+
if [ -f "$DOWNLOADED_TOKENIZER_FILE_PATH" ]; then
49+
echo "${TOKENIZER_FILE} downloaded successfully at: $DOWNLOADED_TOKENIZER_FILE_PATH"
50+
51+
# Convert the tokenizer to binary using the Python module
52+
echo "Convert the tokenizer to binary format"
53+
"${PYTHON_EXECUTABLE}" -m extension.llm.tokenizer.tokenizer -t "$DOWNLOADED_TOKENIZER_FILE_PATH" -o "./${OUT_TOKENIZER_BIN_FILE}"
54+
ls "./${OUT_TOKENIZER_BIN_FILE}"
55+
else
56+
echo "Failed to download ${TOKENIZER_FILE} from ${HF_MODEL_REPO}."
57+
exit 1
58+
fi
59+
60+
# Export the Hugging Face model
61+
echo "Export the Hugging Face model ${HF_MODEL_REPO} to ExecuTorch"
62+
"${PYTHON_EXECUTABLE}" -m extension.export_util.export_hf_model -hfm="$HF_MODEL_REPO" -o "$OUT_ET_MODEL_NAME"
63+
ls -All "./${OUT_ET_MODEL_NAME}.pte"
64+
65+
if [ -n "$UPLOAD_DIR" ]; then
66+
echo "Preparing for uploading generated artifacs"
67+
zip -j model.zip "${OUT_ET_MODEL_NAME}.pte" "${OUT_TOKENIZER_BIN_FILE}"
68+
mkdir -p "${UPLOAD_DIR}"
69+
mv model.zip "${UPLOAD_DIR}"
70+
fi
71+
72+
cmake_install_executorch_libraries() {
73+
echo "Installing libexecutorch.a, libextension_module.so, libportable_ops_lib.a"
74+
rm -rf cmake-out
75+
retry cmake \
76+
-DCMAKE_INSTALL_PREFIX=cmake-out \
77+
-DCMAKE_BUILD_TYPE=Release \
78+
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
79+
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
80+
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
81+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
82+
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
83+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
84+
-DEXECUTORCH_BUILD_XNNPACK=ON \
85+
-DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
86+
-Bcmake-out .
87+
cmake --build cmake-out -j9 --target install --config Release
88+
}
89+
90+
cmake_build_llama_runner() {
91+
echo "Building llama runner"
92+
dir="examples/models/llama2"
93+
retry cmake \
94+
-DCMAKE_INSTALL_PREFIX=cmake-out \
95+
-DCMAKE_BUILD_TYPE=Release \
96+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
97+
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
98+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
99+
-DEXECUTORCH_BUILD_XNNPACK=ON \
100+
-DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
101+
-Bcmake-out/${dir} \
102+
${dir}
103+
cmake --build cmake-out/${dir} -j9 --config Release
104+
}
105+
106+
cmake_install_executorch_libraries
107+
cmake_build_llama_runner
108+
109+
./cmake-out/examples/models/llama2/llama_main --model_path="${OUT_ET_MODEL_NAME}.pte" --tokenizer_path="${OUT_TOKENIZER_BIN_FILE}" --prompt="My name is"

.github/workflows/android-perf.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,11 @@ jobs:
157157
BUILD_MODE="cmake"
158158
DTYPE="fp32"
159159
160-
if [[ ${{ matrix.model }} =~ ^stories* ]]; then
160+
if [[ ${{ matrix.model }} =~ ^[^/]+/[^/]+$ ]]; then
161+
# HuggingFace model. Assume the pattern is always like "<org>/<repo>"
162+
HF_MODEL_REPO=${{ matrix.model }}
163+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_hf_model.sh ${{ matrix.model }} ${ARTIFACTS_DIR_NAME}
164+
elif [[ ${{ matrix.model }} =~ ^stories* ]]; then
161165
# Install requirements for export_llama
162166
PYTHON_EXECUTABLE=python bash examples/models/llama2/install_requirements.sh
163167
# Test llama2

.github/workflows/apple-perf.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,11 @@ jobs:
157157
BUILD_MODE="cmake"
158158
DTYPE="fp32"
159159
160+
if [[ ${{ matrix.model }} =~ ^[^/]+/[^/]+$ ]]; then
161+
# HuggingFace model. Assume the pattern is always like "<org>/<repo>"
162+
HF_MODEL_REPO=${{ matrix.model }}
163+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_hf_model.sh ${{ matrix.model }} ${ARTIFACTS_DIR_NAME}
164+
elif [[ ${{ matrix.model }} =~ ^stories* ]]; then
160165
if [[ ${{ matrix.model }} =~ ^stories* ]]; then
161166
# Install requirements for export_llama
162167
PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \

.github/workflows/trunk.yml

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -373,36 +373,6 @@ jobs:
373373
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
374374
conda activate "${CONDA_ENV}"
375375
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh cmake
376-
377-
echo "Installing libexecutorch.a, libextension_module.so, libportable_ops_lib.a"
378-
rm -rf cmake-out
379-
cmake \
380-
-DCMAKE_INSTALL_PREFIX=cmake-out \
381-
-DCMAKE_BUILD_TYPE=Release \
382-
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
383-
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
384-
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
385-
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
386-
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
387-
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
388-
-DEXECUTORCH_BUILD_XNNPACK=ON \
389-
-DPYTHON_EXECUTABLE=python \
390-
-Bcmake-out .
391-
cmake --build cmake-out -j9 --target install --config Release
392-
393-
echo "Build llama runner"
394-
dir="examples/models/llama2"
395-
cmake \
396-
-DCMAKE_INSTALL_PREFIX=cmake-out \
397-
-DCMAKE_BUILD_TYPE=Release \
398-
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
399-
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
400-
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
401-
-DEXECUTORCH_BUILD_XNNPACK=ON \
402-
-DPYTHON_EXECUTABLE=python \
403-
-Bcmake-out/${dir} \
404-
${dir}
405-
cmake --build cmake-out/${dir} -j9 --config Release
406376
echo "::endgroup::"
407377
408378
echo "::group::Set up HuggingFace Dependencies"
@@ -415,29 +385,6 @@ jobs:
415385
echo "::endgroup::"
416386
417387
echo "::group::Export to ExecuTorch"
418-
TOKENIZER_FILE=tokenizer.model
419-
TOKENIZER_BIN_FILE=tokenizer.bin
420-
ET_MODEL_NAME=et_model
421-
# Fetch the file using a Python one-liner
422-
DOWNLOADED_TOKENIZER_FILE_PATH=$(python -c "
423-
from huggingface_hub import hf_hub_download
424-
# Download the file from the Hugging Face Hub
425-
downloaded_path = hf_hub_download(
426-
repo_id='${{ matrix.hf_model_repo }}',
427-
filename='${TOKENIZER_FILE}'
428-
)
429-
print(downloaded_path)
430-
")
431-
if [ -f "$DOWNLOADED_TOKENIZER_FILE_PATH" ]; then
432-
echo "${TOKENIZER_FILE} downloaded successfully at: $DOWNLOADED_TOKENIZER_FILE_PATH"
433-
python -m extension.llm.tokenizer.tokenizer -t $DOWNLOADED_TOKENIZER_FILE_PATH -o ./${TOKENIZER_BIN_FILE}
434-
ls ./tokenizer.bin
435-
else
436-
echo "Failed to download ${TOKENIZER_FILE} from ${{ matrix.hf_model_repo }}."
437-
exit 1
438-
fi
439-
440-
python -m extension.export_util.export_hf_model -hfm=${{ matrix.hf_model_repo }} -o ${ET_MODEL_NAME}
441-
442-
cmake-out/examples/models/llama2/llama_main --model_path=${ET_MODEL_NAME}.pte --tokenizer_path=${TOKENIZER_BIN_FILE} --prompt="My name is"
388+
# HuggingFace model. Assume the pattern is always like "<org>/<repo>"
389+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_hf_model.sh ${{ matrix.hf_model_repo }}
443390
echo "::endgroup::"

0 commit comments

Comments
 (0)