Skip to content

[llm-d] Add dry-run capability #132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kpouget merged 4 commits into from
Jul 20, 2026
Merged

[llm-d] Add dry-run capability #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bdba8e9
[kserve] toolbox: deploy_llmisvc: accept a dry-run parameter to only …
kpouget Jul 16, 2026
8e7bfaa
[kserve] toolbox: deploy_llmisvc: update to avoid building a custom d…
kpouget Jul 16, 2026
76de0e3
[llm_d] orchestration: add a kserve dry-run flag
kpouget Jul 16, 2026
9b6ab21
[guidellm] toolbox: run_smoke_request: save to file directly
kpouget Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions projects/guidellm/toolbox/run_smoke_request/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ def capture_smoke_pod_debug_info(args, ctx):

try:
# Capture pod YAML
yaml_result = oc(
oc(
"get",
"pod",
ctx.pod_name,
Expand All @@ -248,20 +248,17 @@ def capture_smoke_pod_debug_info(args, ctx):
"yaml",
check=False,
)
if yaml_result.returncode == 0 and yaml_result.stdout:
write_text(artifacts_dir / f"{ctx.pod_name}.yaml", yaml_result.stdout)

# Capture pod description
desc_result = oc(
oc(
"describe",
"pod",
ctx.pod_name,
"-n",
args.namespace,
check=False,
stdout_dest=artifacts_dir / f"{ctx.pod_name}.description.txt",
Comment on lines +253 to +260

@coderabbitai coderabbitai Bot Jul 18, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
rg -n -A30 'def oc\(' projects/core/dsl/utils/k8s.py
rg -n 'stdout_dest' projects/core/dsl/utils/k8s.py projects/guidellm/toolbox/run_smoke_request/main.py

Repository: openshift-psap/forge

Length of output: 1204

🏁 Script executed:

#!/bin/bash
sed -n '220,290p' projects/guidellm/toolbox/run_smoke_request/main.py
printf '\n---\n'
sed -n '1,120p' projects/core/dsl/utils/k8s.py
printf '\n---\n'
rg -n "except .*TypeError|TypeError|stdout_dest" projects/guidellm/toolbox/run_smoke_request/main.py projects/core/dsl/utils/k8s.py projects/core/dsl/utils -g '!**/__pycache__/**'

Repository: openshift-psap/forge

Length of output: 5114

Pass stdout_dest through oc or call shell.run directly projects/guidellm/toolbox/run_smoke_request/main.py:253-260 passes stdout_dest into oc, but projects/core/dsl/utils/k8s.py:19-48 doesn’t accept or forward that keyword, so this raises TypeError and the debug-info task silently skips writing the pod description artifact.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@projects/guidellm/toolbox/run_smoke_request/main.py` around lines 253 - 260,
Update the oc invocation in the debug-info task to ensure stdout_dest reaches
the underlying command runner: either extend oc to accept and forward
stdout_dest, or call shell.run directly with the existing arguments. Preserve
writing the pod description artifact to artifacts_dir and avoid the current
TypeError.

)
if desc_result.returncode == 0 and desc_result.stdout:
write_text(artifacts_dir / f"{ctx.pod_name}.description.txt", desc_result.stdout)

return f"Captured debug info for pod {ctx.pod_name}"

Expand Down
146 changes: 74 additions & 72 deletions projects/kserve/toolbox/deploy_llmisvc/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@

import yaml

from projects.core.dsl import always, entrypoint, execute_tasks, on_failure, retry, task
from projects.core.dsl import (
EarlyReturn,
always,
entrypoint,
execute_tasks,
on_failure,
retry,
task,
)
from projects.core.dsl.utils import write_text
from projects.core.dsl.utils.k8s import (
oc,
Expand All @@ -28,6 +36,7 @@ def run(
namespace: str,
inference_service_manifest_path: str,
gateway_status_address_name: str = "gateway-external",
dry_run: bool = False,
) -> str:
"""
Deploy an LLMInferenceService and wait for its endpoint.
Expand All @@ -36,42 +45,35 @@ def run(
namespace: Namespace used by llm_d
inference_service_manifest_path: Path to the InferenceService YAML manifest file
gateway_status_address_name: Gateway status address name for endpoint resolution
dry_run: If True, only prepare the manifest without deploying
"""

# Load manifest to extract the service name
manifest = load_yaml(Path(inference_service_manifest_path))
inference_service_name = manifest["metadata"]["name"]

# Pass only the required arguments to avoid including manifest content in logs
task_args = {
"namespace": namespace,
"inference_service_manifest_path": inference_service_manifest_path,
"inference_service_name": inference_service_name,
"gateway_status_address_name": gateway_status_address_name,
}
context = execute_tasks(task_args)
ctx = execute_tasks(locals())

if dry_run:
return ctx.src_manifest_path

# Ensure endpoint_url is available
endpoint_url = getattr(context, "endpoint_url", None)
endpoint_url = getattr(ctx, "endpoint_url", None)
if not endpoint_url:
raise RuntimeError("Failed to resolve gateway endpoint URL after deployment")

# Log service description for visibility
service_description = getattr(context, "service_description", None)
if service_description:
print(f"Deployed: {service_description}")

return endpoint_url


@task
def copy_manifest_to_src(args, ctx):
"""Copy inference service manifest to src directory for inspection and use"""
"""Copy inference service manifest to src directory and extract service name"""
import shutil

# Get the original manifest path
original_path = Path(args.inference_service_manifest_path)

# Load manifest to extract the service name
manifest = load_yaml(original_path)
ctx.inference_service_name = manifest["metadata"]["name"]
ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}"

# Ensure the src directory exists
src_dir = args.artifact_dir / "src"
src_dir.mkdir(parents=True, exist_ok=True)
Expand All @@ -83,14 +85,22 @@ def copy_manifest_to_src(args, ctx):
# Store the src path in context for other tasks to use
ctx.src_manifest_path = str(src_path)

return f"Copied manifest from {original_path} to {src_path}"
return f"Copied manifest from {original_path} to {src_path} (service: {ctx.inference_service_name})"


@task
def check_dry_run(args, ctx):
"""Check if dry-run mode is enabled and return early if so"""
if args.dry_run:
return EarlyReturn(f"Dry-run completed: Prepared manifest for {ctx.inference_service_name}")
return "Proceeding with full deployment"


@task
def delete_existing_service(args, ctx):
"""Delete existing LLMInferenceService"""

name = args.inference_service_name
name = ctx.inference_service_name
oc(
"delete",
"llminferenceservice",
Expand All @@ -100,8 +110,7 @@ def delete_existing_service(args, ctx):
"--ignore-not-found=true",
check=False,
)
ctx.service_name = name
ctx.selector = f"app.kubernetes.io/name={name}"

return f"Deleted existing LLMInferenceService {name}"


Expand All @@ -110,13 +119,11 @@ def delete_existing_service(args, ctx):
def wait_old_pods_gone(args, ctx):
"""Wait for old llm-d pods to disappear"""

# Use service name and selector from context (if set) or create from args
service_name = getattr(ctx, "service_name", args.inference_service_name)
selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}")

pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True)
pods = oc_get_json(
"pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True
)
if not pods or not pods.get("items"):
return f"Old pods gone for {service_name}"
return f"Old pods gone for {ctx.inference_service_name}"
return False # Retry


Expand All @@ -130,7 +137,7 @@ def apply_inference_service(args, ctx):
# Load and apply the manifest from src
manifest = load_yaml(Path(src_manifest_path))
oc_apply(src_manifest_path, manifest)
return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.service_name}"
return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.inference_service_name}"


@on_failure(on_wait_pods_appear_failure)
Expand All @@ -139,13 +146,11 @@ def apply_inference_service(args, ctx):
def wait_pods_appear(args, ctx):
"""Wait for llm-d pods to appear"""

# Use service name and selector from context (if set) or create from args
service_name = getattr(ctx, "service_name", args.inference_service_name)
selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}")

pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True)
pods = oc_get_json(
"pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True
)
if pods and pods.get("items"):
return f"Pods appeared for {service_name}"
return f"Pods appeared for {ctx.inference_service_name}"
return False # Retry


Expand All @@ -154,13 +159,18 @@ def wait_pods_appear(args, ctx):
def capture_llmisv_description(args, ctx):
"""Capture LLMISV description with events and status for failure analysis"""

if args.dry_run:
return "Dry-run, nothing to do"

try:
# Ensure artifacts directory exists
artifacts_dir = args.artifact_dir / "artifacts"
artifacts_dir.mkdir(parents=True, exist_ok=True)

# Use service name from args (available) or context (if set)
service_name = getattr(ctx, "service_name", args.inference_service_name)
# Use LLMInferenceService name from context
service_name = getattr(ctx, "inference_service_name", None)
if not service_name:
return "No service name available"

# Capture LLMISV description
result = oc(
Expand Down Expand Up @@ -188,21 +198,25 @@ def capture_llmisv_description(args, ctx):
def capture_replicaset_description(args, ctx):
"""Capture ReplicaSet description for pod creation failure analysis"""

if args.dry_run:
return "Dry-run, nothing to do"

try:
# Ensure artifacts directory exists
artifacts_dir = args.artifact_dir / "artifacts"
artifacts_dir.mkdir(parents=True, exist_ok=True)

# Use service name from args (available) or context (if set)
service_name = getattr(ctx, "service_name", args.inference_service_name)
selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}")
# Use LLMInferenceService name from context
service_name = getattr(ctx, "inference_service_name", None)
if not service_name:
return "No service name available"

# Get replicasets for the service
rs_result = oc(
"get",
"replicaset",
"-l",
selector,
ctx.selector,
"-n",
args.namespace,
"-o",
Expand All @@ -216,16 +230,18 @@ def capture_replicaset_description(args, ctx):
if rs_result.stdout.strip():
# Describe each replicaset
for rs_name in rs_result.stdout.strip().split("\n"):
if rs_name.strip():
rs_desc_result = oc(
"describe",
rs_name.strip(),
"-n",
args.namespace,
log_stdout=False,
check=False,
)
replicaset_descriptions.append(rs_desc_result.stdout)
if not rs_name.strip():
continue

rs_desc_result = oc(
"describe",
rs_name.strip(),
"-n",
args.namespace,
log_stdout=False,
check=False,
)
replicaset_descriptions.append(rs_desc_result.stdout)

# Save all replicaset descriptions
rs_desc_path = artifacts_dir / "replicaset_description.txt"
Expand All @@ -245,7 +261,7 @@ def capture_replicaset_description(args, ctx):
def query_service_status(args, ctx):
"""Query the status of the LLMInferenceService"""

service_name = getattr(ctx, "service_name", args.inference_service_name)
service_name = ctx.inference_service_name

# Query only the Ready condition status
result = oc(
Expand All @@ -272,7 +288,7 @@ def query_service_status(args, ctx):
def query_service_message(args, ctx):
"""Query detailed message from LLMInferenceService"""

service_name = getattr(ctx, "service_name", args.inference_service_name)
service_name = ctx.inference_service_name

# Query the Ready condition details
result = oc(
Expand Down Expand Up @@ -309,8 +325,7 @@ def query_service_message(args, ctx):
def wait_service_ready(args, ctx):
"""Wait for LLMInferenceService to be ready"""

service_name = getattr(ctx, "service_name", args.inference_service_name)
selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}")
service_name = ctx.inference_service_name

# Query the current status and show diagnostic info
result = oc(
Expand All @@ -329,7 +344,7 @@ def wait_service_ready(args, ctx):
"get",
"pods",
"-l",
selector,
ctx.selector,
"-n",
args.namespace,
log_stdout=True, # Show pod status in logs
Expand Down Expand Up @@ -365,7 +380,7 @@ def resolve_endpoint_task(args, ctx):

endpoint_url = try_resolve_endpoint_url(
namespace=args.namespace,
inference_service_name=args.inference_service_name,
inference_service_name=ctx.inference_service_name,
gateway_status_address_name=args.gateway_status_address_name,
)
if endpoint_url:
Expand All @@ -375,19 +390,6 @@ def resolve_endpoint_task(args, ctx):
return False # Retry


@task
def deploy_llmisvc_task(args, ctx):
"""Deploy the llm_d inference service and resolve its endpoint"""

# All work is done by the individual tasks
service_info = getattr(
ctx, "service_description", f"LLM Service '{args.inference_service_name}'"
)
return (
f"LLMInferenceService deployment completed: {service_info} - Endpoint: {ctx.endpoint_url}"
)


def try_resolve_endpoint_url(
*, namespace: str, inference_service_name: str, gateway_status_address_name: str
) -> str | None:
Expand Down
1 change: 1 addition & 0 deletions projects/llm_d/orchestration/config.d/runtime.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ smoke_request_key: default
benchmark_key: null
job_name: null
namespace_override: null
kserve_dry_run: false
Loading
Loading