Skip to content

Commit

Permalink
Merge pull request #208 from leseb/rm-list_models_in_directory_op
Browse files Browse the repository at this point in the history 
feat: remove list_models_in_directory_op
  • Loading branch information
@tumido
tumido authored Nov 22, 2024
2 parents f3d0942 + dd6018a commit fef6ff1
Show file tree
Hide file tree
Showing 6 changed files with 64 additions and 157 deletions.
15 changes: 6 additions & 9 deletions eval/mt_bench/components.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,13 @@

@component(base_image=RHELAI_IMAGE, install_kfp_package=False)
def run_mt_bench_op(
models_path_prefix: str,
merge_system_user_message: bool,
# generate_answers,judgment uses a magic word for its mt_bench evaluator - 'auto'
# with 'auto', number of gpus allocated for serving is calculated based on environment
# https://github.com/instructlab/eval/blob/main/src/instructlab/eval/mt_bench.py#L36
max_workers: str,
models_folder: str,
output_path: str = "/output/mt_bench_data.json",
models_list: List[str] = None,
models_folder: Optional[str] = None,
best_score_file: Optional[str] = None,
) -> NamedTuple("outputs", best_model=str, best_score=float):
import json
Expand Down Expand Up @@ -136,8 +134,7 @@ def shutdown_vllm(process: subprocess.Popen, timeout: int = 20):

print(f"GPU Available: {gpu_available}, {gpu_name}")

if models_list is None and models_folder:
models_list = os.listdir(models_folder)
models_list = os.listdir(models_folder)

judge_api_key = os.getenv("JUDGE_API_KEY", "")
judge_model_name = os.getenv("JUDGE_NAME")
Expand All @@ -162,7 +159,7 @@ def shutdown_vllm(process: subprocess.Popen, timeout: int = 20):
models_list = [model for model in models_list if not model.endswith(".jsonl")]
for model_name in models_list:
print(f"Serving candidate model: {model_name}")
model_path = f"{models_path_prefix}/{model_name}"
model_path = f"{models_folder}/{model_name}"

vllm_process, vllm_server = launch_vllm(model_path, gpu_count)

Expand Down Expand Up @@ -214,12 +211,12 @@ def shutdown_vllm(process: subprocess.Popen, timeout: int = 20):

# Rename the best model directory to "candidate_model" for the next step
# So we know which model to use for the final evaluation
if os.path.exists(os.path.join(models_path_prefix, "candidate_model")):
if os.path.exists(os.path.join(models_folder, "candidate_model")):
print("candidate_model already exists. Skipping renaming")
else:
os.rename(
os.path.join(models_path_prefix, best_model),
os.path.join(models_path_prefix, "candidate_model"),
os.path.join(models_folder, best_model),
os.path.join(models_folder, "candidate_model"),
)

return outputs(best_model=best_model, best_score=best_score)
18 changes: 3 additions & 15 deletions pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ def pipeline_wrapper(mock: List[Literal[MOCKED_STAGES]]):
# Imports for evaluation
from eval.final import run_final_eval_op
from eval.mt_bench import run_mt_bench_op
from utils import list_models_in_directory_op

@dsl.pipeline(
display_name="InstructLab",
Expand Down Expand Up @@ -350,22 +349,10 @@ def pipeline(
mount_path="/output",
)

models_list_2_task = list_models_in_directory_op(
models_folder="/output/phase_2/model/hf_format",
)
models_list_2_task.set_caching_options(False)
models_list_2_task.after(training_phase_2)
mount_pvc(
task=models_list_2_task,
pvc_name=output_pvc_task.output,
mount_path="/output",
)

# MT_Bench Evaluation of models

run_mt_bench_task = run_mt_bench_op(
models_list=models_list_2_task.output,
models_path_prefix="/output/phase_2/model/hf_format",
models_folder="/output/phase_2/model/hf_format",
max_workers=mt_bench_max_workers,
merge_system_user_message=mt_bench_merge_system_user_message,
)
Expand All @@ -379,6 +366,7 @@ def pipeline(
run_mt_bench_task.set_accelerator_type("nvidia.com/gpu")
run_mt_bench_task.set_accelerator_limit(1)
run_mt_bench_task.set_caching_options(False)
run_mt_bench_task.after(training_phase_2)
use_config_map_as_env(
run_mt_bench_task,
JUDGE_CONFIG_MAP,
Expand Down Expand Up @@ -530,7 +518,7 @@ def gen_standalone():
"exec-sdg-op": 'sdg_op(num_instructions_to_generate={num_instructions_to_generate}, pipeline="{sdg_pipeline}", repo_branch="{exec_git_clone_op_repo_branch or ""}", repo_pr={exec_git_clone_op_repo_pr or 0}, taxonomy_path="{TAXONOMY_DATA_PATH}", sdg_path="{DATA_PVC_SDG_PATH}", sdg_sampling_size={sdg_sampling_size})',
"exec-git-clone-op": {},
"exec-huggingface-importer-op": 'huggingface_importer_op(repo_name="{REPO_GRANITE_7B_IMAGE}", model_path="{DATA_PVC_MODEL_PATH}")',
"exec-run-mt-bench-op": 'run_mt_bench_op(best_score_file="{MT_BENCH_SCORES_PATH}",output_path="{MT_BENCH_OUTPUT_PATH}",models_folder="{CANDIDATE_MODEL_PATH_PREFIX}",models_path_prefix="{CANDIDATE_MODEL_PATH_PREFIX}", max_workers="{MAX_WORKERS}", merge_system_user_message={MERGE_SYSTEM_USER_MESSAGE})',
"exec-run-mt-bench-op": 'run_mt_bench_op(best_score_file="{MT_BENCH_SCORES_PATH}",output_path="{MT_BENCH_OUTPUT_PATH}",models_folder="{CANDIDATE_MODEL_PATH_PREFIX}", max_workers="{MAX_WORKERS}", merge_system_user_message={MERGE_SYSTEM_USER_MESSAGE})',
"exec-run-final-eval-op": 'run_final_eval_op(mmlu_branch_output="{MMLU_BRANCH_SCORES_PATH}", mt_bench_branch_output="{MT_BENCH_BRANCH_SCORES_PATH}", candidate_model="{CANDIDATE_MODEL_PATH}", taxonomy_path="{TAXONOMY_PATH}", sdg_path="{DATA_PVC_SDG_PATH}", base_branch="", candidate_branch="", base_model_dir="{DATA_PVC_MODEL_PATH}", max_workers="{MAX_WORKERS}", merge_system_user_message={MERGE_SYSTEM_USER_MESSAGE}, few_shots={FEW_SHOTS}, batch_size="{BATCH_SIZE}")',
}

Expand Down
160 changes: 48 additions & 112 deletions pipeline.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,16 +304,6 @@ components:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
comp-list-models-in-directory-op:
executorLabel: exec-list-models-in-directory-op
inputDefinitions:
parameters:
models_folder:
parameterType: STRING
outputDefinitions:
parameters:
Output:
parameterType: LIST
comp-pvc-to-model-op:
executorLabel: exec-pvc-to-model-op
inputDefinitions:
Expand Down Expand Up @@ -489,12 +479,6 @@ components:
merge_system_user_message:
parameterType: BOOLEAN
models_folder:
isOptional: true
parameterType: STRING
models_list:
isOptional: true
parameterType: LIST
models_path_prefix:
parameterType: STRING
output_path:
defaultValue: /output/mt_bench_data.json
Expand Down Expand Up @@ -707,29 +691,6 @@ deploymentSpec:
- /bin/sh
- -c
image: registry.access.redhat.com/ubi9/toolbox
exec-list-models-in-directory-op:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- list_models_in_directory_op
command:
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef list_models_in_directory_op(models_folder: str) -> List[str]:\n\
\ import os\n\n models = os.listdir(models_folder)\n return models\n\
\n"
image: quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111
exec-pvc-to-model-op:
container:
args:
Expand Down Expand Up @@ -1438,46 +1399,46 @@ deploymentSpec:
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef run_mt_bench_op(\n models_path_prefix: str,\n merge_system_user_message:\
\ bool,\n # generate_answers,judgment uses a magic word for its mt_bench\
\ evaluator - 'auto'\n # with 'auto', number of gpus allocated for serving\
\ is calculated based on environment\n # https://github.com/instructlab/eval/blob/main/src/instructlab/eval/mt_bench.py#L36\n\
\ max_workers: str,\n output_path: str = \"/output/mt_bench_data.json\"\
,\n models_list: List[str] = None,\n models_folder: Optional[str]\
\ = None,\n best_score_file: Optional[str] = None,\n) -> NamedTuple(\"\
outputs\", best_model=str, best_score=float):\n import json\n import\
\ os\n import subprocess\n\n import torch\n from instructlab.eval.mt_bench\
\ import MTBenchEvaluator\n\n if judge_ca_cert := os.getenv(\"JUDGE_CA_CERT_PATH\"\
):\n import httpx\n import openai\n\n # Create a custom\
\ HTTP client\n class CustomHttpClient(httpx.Client):\n \
\ def __init__(self, *args, **kwargs):\n # Use the custom\
\ CA certificate\n kwargs.setdefault(\"verify\", judge_ca_cert)\n\
\ super().__init__(*args, **kwargs)\n\n # Create a\
\ new OpenAI class that uses the custom HTTP client\n class CustomOpenAI(openai.OpenAI):\n\
\ def __init__(self, *args, **kwargs):\n custom_client\
\ = CustomHttpClient()\n super().__init__(http_client=custom_client,\
\ *args, **kwargs)\n\n # Monkey patch the OpenAI class in the openai\
\ module, so that the eval lib can use it\n openai.OpenAI = CustomOpenAI\n\
\n def launch_vllm(\n model_path: str, gpu_count: int, retries:\
\ int = 120, delay: int = 10\n ) -> tuple:\n import subprocess\n\
\ import sys\n import time\n\n import requests\n \
\ from instructlab.model.backends.common import free_tcp_ipv4_port\n\
\n free_port = free_tcp_ipv4_port(\"127.0.0.1\")\n port =\
\ str(free_port)\n vllm_server = f\"http://127.0.0.1:{port}/v1\"\n\
\n command = [\n sys.executable,\n \"-m\",\n\
\ \"vllm.entrypoints.openai.api_server\",\n \"--port\"\
,\n port,\n \"--model\",\n model_path,\n\
\ ]\n if gpu_count > 0:\n command += [\n \
\ \"--tensor-parallel-size\",\n str(gpu_count),\n\
\ ]\n\n process = subprocess.Popen(args=command)\n\n \
\ print(f\"Waiting for vLLM server to start at {vllm_server}...\"\
)\n\n for attempt in range(retries):\n try:\n \
\ response = requests.get(f\"{vllm_server}/models\")\n \
\ if response.status_code == 200:\n print(f\"vLLM\
\ server is up and running at {vllm_server}.\")\n return\
\ process, vllm_server\n except requests.ConnectionError:\n \
\ pass\n\n print(\n f\"Server not\
\ available yet, retrying in {delay} seconds (Attempt {attempt + 1}/{retries})...\"\
\ *\n\ndef run_mt_bench_op(\n merge_system_user_message: bool,\n #\
\ generate_answers,judgment uses a magic word for its mt_bench evaluator\
\ - 'auto'\n # with 'auto', number of gpus allocated for serving is\
\ calculated based on environment\n # https://github.com/instructlab/eval/blob/main/src/instructlab/eval/mt_bench.py#L36\n\
\ max_workers: str,\n models_folder: str,\n output_path: str =\
\ \"/output/mt_bench_data.json\",\n best_score_file: Optional[str] =\
\ None,\n) -> NamedTuple(\"outputs\", best_model=str, best_score=float):\n\
\ import json\n import os\n import subprocess\n\n import torch\n\
\ from instructlab.eval.mt_bench import MTBenchEvaluator\n\n if judge_ca_cert\
\ := os.getenv(\"JUDGE_CA_CERT_PATH\"):\n import httpx\n import\
\ openai\n\n # Create a custom HTTP client\n class CustomHttpClient(httpx.Client):\n\
\ def __init__(self, *args, **kwargs):\n # Use\
\ the custom CA certificate\n kwargs.setdefault(\"verify\"\
, judge_ca_cert)\n super().__init__(*args, **kwargs)\n\n\
\ # Create a new OpenAI class that uses the custom HTTP client\n\
\ class CustomOpenAI(openai.OpenAI):\n def __init__(self,\
\ *args, **kwargs):\n custom_client = CustomHttpClient()\n\
\ super().__init__(http_client=custom_client, *args, **kwargs)\n\
\n # Monkey patch the OpenAI class in the openai module, so that\
\ the eval lib can use it\n openai.OpenAI = CustomOpenAI\n\n def\
\ launch_vllm(\n model_path: str, gpu_count: int, retries: int =\
\ 120, delay: int = 10\n ) -> tuple:\n import subprocess\n \
\ import sys\n import time\n\n import requests\n \
\ from instructlab.model.backends.common import free_tcp_ipv4_port\n\n\
\ free_port = free_tcp_ipv4_port(\"127.0.0.1\")\n port = str(free_port)\n\
\ vllm_server = f\"http://127.0.0.1:{port}/v1\"\n\n command\
\ = [\n sys.executable,\n \"-m\",\n \"\
vllm.entrypoints.openai.api_server\",\n \"--port\",\n \
\ port,\n \"--model\",\n model_path,\n \
\ ]\n if gpu_count > 0:\n command += [\n \
\ \"--tensor-parallel-size\",\n str(gpu_count),\n \
\ ]\n\n process = subprocess.Popen(args=command)\n\n \
\ print(f\"Waiting for vLLM server to start at {vllm_server}...\")\n\n\
\ for attempt in range(retries):\n try:\n \
\ response = requests.get(f\"{vllm_server}/models\")\n \
\ if response.status_code == 200:\n print(f\"vLLM server\
\ is up and running at {vllm_server}.\")\n return process,\
\ vllm_server\n except requests.ConnectionError:\n \
\ pass\n\n print(\n f\"Server not available\
\ yet, retrying in {delay} seconds (Attempt {attempt + 1}/{retries})...\"\
\n )\n time.sleep(delay)\n\n raise RuntimeError(\n\
\ f\"Failed to start vLLM server at {vllm_server} after {retries}\
\ retries.\"\n )\n\n def shutdown_vllm(process: subprocess.Popen,\
Expand All @@ -1500,9 +1461,8 @@ deploymentSpec:
\ torch.cuda.get_device_name(torch.cuda.current_device())\n if\
\ gpu_available\n else \"No GPU available\"\n )\n gpu_count\
\ = torch.cuda.device_count() if gpu_available else 0\n\n print(f\"GPU\
\ Available: {gpu_available}, {gpu_name}\")\n\n if models_list is None\
\ and models_folder:\n models_list = os.listdir(models_folder)\n\n\
\ judge_api_key = os.getenv(\"JUDGE_API_KEY\", \"\")\n judge_model_name\
\ Available: {gpu_available}, {gpu_name}\")\n\n models_list = os.listdir(models_folder)\n\
\n judge_api_key = os.getenv(\"JUDGE_API_KEY\", \"\")\n judge_model_name\
\ = os.getenv(\"JUDGE_NAME\")\n judge_endpoint = os.getenv(\"JUDGE_ENDPOINT\"\
)\n\n scores = {}\n all_mt_bench_data = []\n\n # generate_answers,judgment\
\ uses a magic word for its mt_bench evaluator - 'auto'\n # with 'auto',\
Expand All @@ -1515,7 +1475,7 @@ deploymentSpec:
\ to ignore any jsonl files present in the directory\n models_list =\
\ [model for model in models_list if not model.endswith(\".jsonl\")]\n \
\ for model_name in models_list:\n print(f\"Serving candidate model:\
\ {model_name}\")\n model_path = f\"{models_path_prefix}/{model_name}\"\
\ {model_name}\")\n model_path = f\"{models_folder}/{model_name}\"\
\n\n vllm_process, vllm_server = launch_vllm(model_path, gpu_count)\n\
\n # model ID is the model_path value in vLLM\n evaluator\
\ = MTBenchEvaluator(\n model_name=model_path,\n judge_model_name=judge_model_name,\n\
Expand All @@ -1540,10 +1500,10 @@ deploymentSpec:
utf-8\") as f:\n json.dump({\"best_model\": best_model, \"best_score\"\
: best_score}, f, indent=4)\n\n # Rename the best model directory to\
\ \"candidate_model\" for the next step\n # So we know which model to\
\ use for the final evaluation\n if os.path.exists(os.path.join(models_path_prefix,\
\ use for the final evaluation\n if os.path.exists(os.path.join(models_folder,\
\ \"candidate_model\")):\n print(\"candidate_model already exists.\
\ Skipping renaming\")\n else:\n os.rename(\n os.path.join(models_path_prefix,\
\ best_model),\n os.path.join(models_path_prefix, \"candidate_model\"\
\ Skipping renaming\")\n else:\n os.rename(\n os.path.join(models_folder,\
\ best_model),\n os.path.join(models_folder, \"candidate_model\"\
),\n )\n\n return outputs(best_model=best_model, best_score=best_score)\n\
\n"
env:
Expand Down Expand Up @@ -1805,20 +1765,6 @@ root:
- data-processing-op
taskInfo:
name: knowledge-processed-data-to-artifact-op
list-models-in-directory-op:
cachingOptions: {}
componentRef:
name: comp-list-models-in-directory-op
dependentTasks:
- createpvc-3
- pytorchjob-manifest-op-2
inputs:
parameters:
models_folder:
runtimeValue:
constant: /output/phase_2/model/hf_format
taskInfo:
name: list-models-in-directory-op
pvc-to-model-op:
cachingOptions: {}
componentRef:
Expand Down Expand Up @@ -1987,18 +1933,14 @@ root:
name: comp-run-mt-bench-op
dependentTasks:
- createpvc-3
- list-models-in-directory-op
- pytorchjob-manifest-op-2
inputs:
parameters:
max_workers:
componentInputParameter: mt_bench_max_workers
merge_system_user_message:
componentInputParameter: mt_bench_merge_system_user_message
models_list:
taskOutputParameter:
outputParameterKey: Output
producerTask: list-models-in-directory-op
models_path_prefix:
models_folder:
runtimeValue:
constant: /output/phase_2/model/hf_format
taskInfo:
Expand Down Expand Up @@ -2263,12 +2205,6 @@ platforms:
taskOutputParameter:
outputParameterKey: name
producerTask: createpvc
exec-list-models-in-directory-op:
pvcMount:
- mountPath: /output
taskOutputParameter:
outputParameterKey: name
producerTask: createpvc-3
exec-pvc-to-model-op:
pvcMount:
- mountPath: /output
Expand Down
Loading

0 comments on commit fef6ff1

Please sign in to comment.