Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 26 additions & 11 deletions internal/cmn/schema/dag.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -5820,17 +5820,32 @@
},
"llmProvider": {
"type": "string",
"enum": [
"openai",
"anthropic",
"gemini",
"google",
"openrouter",
"local",
"ollama",
"vllm",
"llama"
]
"anyOf": [
{
"enum": [
"openai",
"openai-codex",
"anthropic",
"gemini",
"google",
"openrouter",
"local",
"ollama",
"vllm",
"llama",
"zai",
"zhipu",
"zhipuai",
"glm",
"opencode"
]
},
{
"pattern": "\\$\\{[^}]+\\}",
"description": "Value reference resolved at runtime, e.g. ${params.PROVIDER} or ${env.LLM_PROVIDER}."
}
],
"description": "LLM provider name. Supports value references such as ${params.PROVIDER} resolved when the step runs."
},
"llmConfig": {
"type": "object",
Expand Down
13 changes: 2 additions & 11 deletions internal/core/spec/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -2352,17 +2352,8 @@ func buildLLM(_ BuildContext, d *dag) (*core.LLMConfig, error) {
cfg := d.LLM

// Validate provider if specified (optional at DAG level)
if cfg.Provider != "" {
validProviders := map[string]bool{
"openai": true, "anthropic": true, "gemini": true,
"openrouter": true, "local": true,
// Aliases for local provider
"ollama": true, "vllm": true, "llama": true,
}
if !validProviders[cfg.Provider] {
return nil, core.NewValidationError("llm.provider", cfg.Provider,
fmt.Errorf("invalid provider: must be one of openai, anthropic, gemini, openrouter, local (or aliases: ollama, vllm, llama)"))
}
if err := validateLLMProvider(cfg.Provider); err != nil {
return nil, core.NewValidationError("llm.provider", cfg.Provider, err)
}

// Get model string or entries (optional at DAG level)
Expand Down
17 changes: 17 additions & 0 deletions internal/core/spec/llm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,23 @@ model:
`,
wantErr: "invalid provider",
},
{
name: "provider reference is resolved at run time",
yaml: `
provider: ${params.PROVIDER}
model: ${params.MODEL}
`,
wantErr: "",
},
{
name: "provider reference in model entry is resolved at run time",
yaml: `
model:
- provider: ${env.LLM_PROVIDER}
name: ${params.MODEL}
`,
wantErr: "",
},
}

for _, tt := range tests {
Expand Down
18 changes: 10 additions & 8 deletions internal/core/spec/spec003_conformance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ steps:
with:
model:
- provider: openai
name: gpt-4o
name: gpt-4o-${params.environment}
base_url: https://${params.environment}.example/v1
api_key_name: OPENAI_API_KEY
system: system ${params.environment}
Expand Down Expand Up @@ -195,6 +195,8 @@ steps:
assert.Equal(t, "system prod", resolved["steps[5].llm.system"])
assert.Equal(t, "https://fallback-prod.example/v1", resolved["steps[5].llm.base_url"])
assert.Equal(t, "https://prod.example/v1", resolved["steps[5].llm.model[0].base_url"])
assert.Equal(t, "openai", resolved["steps[5].llm.model[0].provider"])
assert.Equal(t, "gpt-4o-prod", resolved["steps[5].llm.model[0].name"])
assert.Equal(t, "hello prod", resolved["steps[5].messages[0].content"])

assert.Equal(t, "literal ${params.environment}", resolved["steps[6].run"])
Expand All @@ -205,8 +207,6 @@ steps:
assertSpec003FieldAbsent(t, resolved, "steps[0].id")
assertSpec003FieldAbsent(t, resolved, "steps[0].outputs[0].name")
assertSpec003FieldAbsent(t, resolved, "steps[0].outputs[0].type")
assertSpec003FieldAbsent(t, resolved, "steps[5].llm.model[0].provider")
assertSpec003FieldAbsent(t, resolved, "steps[5].llm.model[0].name")
assertSpec003FieldAbsent(t, resolved, "steps[5].llm.model[0].api_key_name")
assertSpec003FieldAbsent(t, resolved, "steps[5].llm.tools[0]")
}
Expand Down Expand Up @@ -242,10 +242,12 @@ name: root-llm
params:
- name: environment
default: prod
- name: provider
default: openai
llm:
model:
- provider: openai
name: gpt-4o
- provider: ${params.provider}
name: gpt-4o-${params.environment}
base_url: https://${params.environment}.example/v1
system: root system ${params.environment}
base_url: https://fallback-${params.environment}.example/v1
Expand All @@ -258,14 +260,14 @@ steps:
content: hello ${params.environment}
`)

resolved := resolveSpec003Fields(t, dag, cmnvalue.Values{"environment": "prod"})
resolved := resolveSpec003Fields(t, dag, cmnvalue.Values{"environment": "prod", "provider": "openai"})

assert.Equal(t, "root system prod", resolved["steps[0].llm.system"])
assert.Equal(t, "https://fallback-prod.example/v1", resolved["steps[0].llm.base_url"])
assert.Equal(t, "https://prod.example/v1", resolved["steps[0].llm.model[0].base_url"])
assert.Equal(t, "openai", resolved["steps[0].llm.model[0].provider"])
assert.Equal(t, "gpt-4o-prod", resolved["steps[0].llm.model[0].name"])
assert.Equal(t, "hello prod", resolved["steps[0].messages[0].content"])
assertSpec003FieldAbsent(t, resolved, "steps[0].llm.model[0].provider")
assertSpec003FieldAbsent(t, resolved, "steps[0].llm.model[0].name")
}

func TestSpec003ConformanceEscapedDaguReferencesUseBackslashParity(t *testing.T) {
Expand Down
19 changes: 14 additions & 5 deletions internal/core/spec/step.go
Original file line number Diff line number Diff line change
Expand Up @@ -2480,10 +2480,8 @@ func buildStepLLM(ctx StepBuildContext, s *step, result *core.Step) error {
cfg := s.LLM

// Validate provider if specified (for single model config)
if cfg.Provider != "" {
if _, err := llm.ParseProviderType(cfg.Provider); err != nil {
return core.NewValidationError("llm.provider", cfg.Provider, err)
}
if err := validateLLMProvider(cfg.Provider); err != nil {
return core.NewValidationError("llm.provider", cfg.Provider, err)
}

// Model is required when llm config is provided
Expand Down Expand Up @@ -2581,11 +2579,22 @@ func buildWebSearchConfig(cfg *webSearchConfig) *core.WebSearchConfig {
return result
}

// validateLLMProvider reports whether provider names a supported LLM provider.
// An empty provider is accepted; so is one carrying a value reference, since its
// final value is only known once the step runs.
func validateLLMProvider(provider string) error {
if provider == "" || cmnvalue.HasValueReference(provider) {
return nil
}
_, err := llm.ParseProviderType(provider)
return err
}

// convertModelEntries converts types.ModelEntry slice to core.ModelEntry slice with validation.
func convertModelEntries(entries []types.ModelEntry) ([]core.ModelEntry, error) {
models := make([]core.ModelEntry, len(entries))
for i, e := range entries {
if _, err := llm.ParseProviderType(e.Provider); err != nil {
if err := validateLLMProvider(e.Provider); err != nil {
return nil, core.NewValidationError(fmt.Sprintf("llm.model[%d].provider", i), e.Provider, err)
}
models[i] = core.ModelEntry{
Expand Down
4 changes: 4 additions & 0 deletions internal/core/value_fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,13 @@ func (w *referenceFieldWalker) walkLLM(path string, llm *LLMConfig, base Referen
return
}
w.add(base.withPathValue(path+".system", llm.System).withField(cmnvalue.WorkflowField(path + ".system")))
w.add(base.withPathValue(path+".provider", llm.Provider).withField(cmnvalue.ExecutorConfigField(path + ".provider")))
w.add(base.withPathValue(path+".model", llm.Model).withField(cmnvalue.ExecutorConfigField(path + ".model")))
w.add(base.withPathValue(path+".base_url", llm.BaseURL).withField(cmnvalue.ExecutorConfigField(path + ".base_url")))
for i, model := range llm.Models {
modelPath := fmt.Sprintf("%s.model[%d]", path, i)
w.add(base.withPathValue(modelPath+".provider", model.Provider).withField(cmnvalue.ExecutorConfigField(modelPath + ".provider")))
w.add(base.withPathValue(modelPath+".name", model.Name).withField(cmnvalue.ExecutorConfigField(modelPath + ".name")))
w.add(base.withPathValue(modelPath+".base_url", model.BaseURL).withField(cmnvalue.ExecutorConfigField(modelPath + ".base_url")))
}
}
Expand Down
8 changes: 4 additions & 4 deletions internal/core/value_fields_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,19 +213,19 @@ func TestReferenceFieldsEmitsValidationPathSet(t *testing.T) {
"steps[0].container.image",
"steps[0].container.working_dir",
"steps[0].llm.system",
"steps[0].llm.provider",
"steps[0].llm.model",
"steps[0].llm.base_url",
"steps[0].llm.model[0].provider",
"steps[0].llm.model[0].name",
"steps[0].llm.model[0].base_url",
"steps[0].messages[0].content",
"handler_on.init.run",
}, got)
assert.NotContains(t, got, "steps[0].stdout.outputs.fields.image.select")
assert.NotContains(t, got, "steps[0].stdout.outputs.fields.image.path")
assert.NotContains(t, got, "steps[0].output.digest.select")
assert.NotContains(t, got, "steps[0].llm.provider")
assert.NotContains(t, got, "steps[0].llm.model")
assert.NotContains(t, got, "steps[0].llm.api_key_name")
assert.NotContains(t, got, "steps[0].llm.model[0].provider")
assert.NotContains(t, got, "steps[0].llm.model[0].name")
assert.NotContains(t, got, "steps[0].llm.model[0].api_key_name")
assert.NotContains(t, got, "steps[0].llm.tools[0]")
}
60 changes: 53 additions & 7 deletions internal/runtime/builtin/chat/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,16 @@ func newChatExecutor(ctx context.Context, step core.Step) (executor.Executor, er
primaryProvider = cfg.Provider
}

// Parse provider type (required field, validated in spec)
providerType, err := llmpkg.ParseProviderType(primaryProvider)
if err != nil {
return nil, fmt.Errorf("invalid provider: %w", err)
// Parse provider type. A provider carrying a value reference only has a final
// value once the step runs, so leave it unparsed here and let
// createProviderForModel reject an unsupported value after resolution.
var providerType llmpkg.ProviderType
if !cmnvalue.HasValueReference(primaryProvider) {
var err error
providerType, err = llmpkg.ParseProviderType(primaryProvider)
if err != nil {
return nil, err
}
}

// Determine which environment variable to use for API key
Expand Down Expand Up @@ -357,6 +363,43 @@ func evalMessages(ctx context.Context, msgs []exec.LLMMessage) ([]exec.LLMMessag
return result, nil
}

// resolveModels evaluates variable substitution in the provider and model name of
// each entry. base_url is resolved later, in createProviderForModel, once shared
// config has been merged in; api_key_name is not value-resolved at all, since it
// names an environment variable that createProviderForModel reads.
func resolveModels(ctx context.Context, models []core.ModelEntry) ([]core.ModelEntry, error) {
resolved := make([]core.ModelEntry, len(models))
for i, model := range models {
provider, err := runtime.ResolveString(ctx, model.Provider, cmnvalue.WorkflowField("llm.provider"))
if err != nil {
return nil, fmt.Errorf("failed to evaluate provider: %w", err)
}
if provider == "" {
return nil, emptyAfterResolution("provider", model.Provider)
}
name, err := runtime.ResolveString(ctx, model.Name, cmnvalue.WorkflowField("llm.model"))
if err != nil {
return nil, fmt.Errorf("failed to evaluate model: %w", err)
}
if name == "" {
return nil, emptyAfterResolution("model", model.Name)
}
resolved[i] = model
resolved[i].Provider = provider
resolved[i].Name = name
}
return resolved, nil
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

// emptyAfterResolution reports a required LLM field that has no value, naming the
// reference that produced nothing when the field carried one.
func emptyAfterResolution(field, raw string) error {
if raw == "" {
return fmt.Errorf("llm %s is required", field)
}
return fmt.Errorf("llm %s %q resolved to an empty value", field, raw)
}

// maskSecretsForProvider masks secret values in messages before sending to LLM provider.
// This prevents secrets from being leaked to external LLM APIs.
func maskSecretsForProvider(ctx context.Context, msgs []exec.LLMMessage) []exec.LLMMessage {
Expand Down Expand Up @@ -397,7 +440,10 @@ func (e *Executor) Run(ctx context.Context) error {
return err
}

models := e.step.LLM.GetModels()
models, err := resolveModels(ctx, e.step.LLM.GetModels())
if err != nil {
return err
}

// If only one model, use simple path (no fallback)
if len(models) == 1 {
Expand All @@ -416,7 +462,7 @@ func (e *Executor) Run(ctx context.Context) error {
slog.String("model", model.Name),
slog.Int("attemptIndex", i))

err := e.runWithModel(ctx, model, allMessages)
err = e.runWithModel(ctx, model, allMessages)
if err == nil {
return nil // Success
}
Expand Down Expand Up @@ -504,7 +550,7 @@ func (e *Executor) createProviderForModel(ctx context.Context, model core.ModelE
// Parse provider type for this model
providerType, err := llmpkg.ParseProviderType(cfg.Provider)
if err != nil {
return nil, fmt.Errorf("invalid provider: %w", err)
return nil, err
}

// Determine API key env var
Expand Down
Loading