teemow · Sep 25, 2026 · Sep 25, 2026
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- **model-manager reaches the workload clusters' API servers its kserve backends target** (giantswarm/agent-platform#687). A kserve backend cluster-manager registers calls a workload cluster's apiserver as the caller, but model-manager's egress policy had no rule for it, so under the cilium flavour `list_backends` timed out dialing the cluster's API server and every tool naming that backend with it. `modelManager.networkPolicy.workloadClusters` (`fqdns`, `cidrs`, `ports`, the shape of cluster-manager's) follows `clusterManager.networkPolicy.workloadClusters` when empty, the default, and replaces it as a whole when set. It renders in both flavours with a kserve backend (static, or cluster-manager on): names and blocks in the cilium flavour, the blocks or every public destination on the ports in the kubernetes flavour. `make verify-cluster-manager` holds it.
 - **The capz snapshot-store identity also holds Storage Blob Delegator on the storage account** (giantswarm/giantswarm#37705). Substrate 1.1.0's golden tag copies the golden snapshot's objects server-side, and s3proxy signs the copy source with a user delegation SAS; the key is an account-level action the container-scoped Data Contributor cannot grant, so on glean every copy failed with `AuthorizationPermissionMismatch` and no AgentTemplate reached Ready.
 - **The egress gateway reaches the Substrate credential provider (`substrate-atenet-egress` → `k8s-credential-provider` 50051), and the provider has its own policy (ingress from the gateway and the scrape port, egress to the apiserver)** (giantswarm/giantswarm#37705). On the Substrate line from 1.1.0 the gateway asks the provider for every injected credential; without the hop each ask ended in `UpstreamCallTimeout`, the golden boot's git fetch of a private skill got 403 and the AgentTemplate never left `ActorTemplateRetrying`. agentlab renders no network policies, so the gate could not see it.
 

@@ -1494,6 +1494,7 @@ The kagent block is open in the schema, so the template refuses a key under `kag
 | modelManager.networkPolicy.huggingFace.fqdns[3].matchPattern | string | `"*.*.hf.co"` |  |
 | modelManager.networkPolicy.huggingFace.fqdns[4].matchPattern | string | `"*.*.*.hf.co"` |  |
 | modelManager.networkPolicy.huggingFace.cidrs | list | `[]` |  |
+| modelManager.networkPolicy.workloadClusters | object | `{}` |  |
 | modelManager.networkPolicy.egress.fqdns | list | `[]` |  |
 | modelManager.networkPolicy.egress.cidrs | list | `[]` |  |
 | modelManager.networkPolicy.registeredBackends | list | `[]` |  |

@@ -112,6 +112,22 @@ model-manager is off, while this release's modelServing stays off.
 {{- if and (include "agent-platform.modelManager.enabled" .) (or (include "agent-platform.modelServing.enabled" .) (include "agent-platform.modelManager.kserveOn" .)) }}true{{ end -}}
 {{- end -}}
 
+{{/*
+The workload clusters' API servers model-manager's kserve backend calls as the
+caller — a backend cluster-manager registers targets a workload cluster's
+apiserver (modelManager.networkPolicy.workloadClusters: fqdns, cidrs, ports).
+Set, it wins as a whole; empty, the default, it follows
+clusterManager.networkPolicy.workloadClusters: model-manager serves on the
+clusters cluster-manager composes onto. Ports default to 443 and 6443. JSON.
+*/}}
+{{- define "agent-platform.modelManager.workloadClusters" -}}
+{{- $wc := dig "networkPolicy" "workloadClusters" dict .Values.modelManager -}}
+{{- if not $wc -}}
+{{- $wc = dig "networkPolicy" "workloadClusters" dict .Values.clusterManager -}}
+{{- end -}}
+{{- dict "fqdns" (dig "fqdns" list $wc) "cidrs" (dig "cidrs" list $wc) "ports" (dig "ports" (list 443 6443) $wc) | toJson -}}
+{{- end -}}
+
 {{/*
 The namespace the models model-manager serves run in: this release's serving
 namespace with the slice on (model-serving-validate.yaml holds the two equal),

@@ -8,6 +8,7 @@ ConfigMap, invisible to the chart): either way model-manager calls the Hugging
 Face Hub for it (search, fit checks, revision lookups), so the hub egress
 renders whenever components.cluster-manager is on (#494). */ -}}
 {{- $kserveOn := include "agent-platform.modelManager.kserveOn" . }}
+{{- $wc := include "agent-platform.modelManager.workloadClusters" . | fromJson }}
 {{- $musterOn := include "agent-platform.componentEnabled" (dict "root" . "name" "muster") }}
 {{- $mcpOn := and $musterOn (dig "muster" "mcpServer" "enabled" false $chart) }}
 {{- $oauthOn := include "agent-platform.modelManager.oauthEnabled" . }}
@@ -148,6 +149,29 @@ spec:
             - port: "443"
               protocol: TCP
     {{- end }}
+    {{- with $wc.fqdns }}
+    # The workload clusters' API servers a kserve backend targets, by name
+    # (modelManager.networkPolicy.workloadClusters, by default cluster-manager's).
+    - toFQDNs:
+        {{- toYaml . | nindent 8 }}
+      toPorts:
+        - ports:
+            {{- range $wc.ports }}
+            - port: {{ . | toString | quote }}
+              protocol: TCP
+            {{- end }}
+    {{- end }}
+    {{- with $wc.cidrs }}
+    # The workload clusters' API servers by address.
+    - toCIDR:
+        {{- toYaml . | nindent 8 }}
+      toPorts:
+        - ports:
+            {{- range $wc.ports }}
+            - port: {{ . | toString | quote }}
+              protocol: TCP
+            {{- end }}
+    {{- end }}
     {{- end }}
     {{- if $readsServing }}
     # The served models' runtimes, on the workload port: a Ready model's
@@ -417,6 +441,25 @@ spec:
         - port: 443
           protocol: TCP
     {{- end }}
+    {{- if $kserveOn }}
+    # The workload clusters' API servers a kserve backend targets
+    # (modelManager.networkPolicy.workloadClusters, by default cluster-manager's):
+    # vanilla NetworkPolicy has no FQDN selector, so their blocks when set,
+    # else every public destination on their ports.
+    - to:
+        {{- range ($wc.cidrs | default (list "0.0.0.0/0")) }}
+        - ipBlock:
+            cidr: {{ . | quote }}
+            {{- if eq . "0.0.0.0/0" }}
+            except: {{ toYaml $.Values.networkPolicy.kubernetes.worldExcludedCIDRs | nindent 14 }}
+            {{- end }}
+        {{- end }}
+      ports:
+        {{- range $wc.ports }}
+        - port: {{ . | int }}
+          protocol: TCP
+        {{- end }}
+    {{- end }}
     {{- if or $oauthOn (and $kserveOn (not $hf.cidrs)) }}
     - to:
         - ipBlock:

@@ -2355,6 +2355,17 @@ modelManager:  # @schema skipProperties: true; additionalProperties: true
       # egress to exactly these blocks (a mirror, a proxy). The cilium flavor
       # adds them next to the names.
       cidrs: []
+    # The workload clusters' API servers the kserve backend calls as the caller:
+    # a backend cluster-manager registers targets a workload cluster's apiserver
+    # (list_backends, loads and fits all go there). Same shape as
+    # clusterManager.networkPolicy.workloadClusters (fqdns: Cilium FQDN
+    # selectors; cidrs: IP blocks, in the kubernetes flavor these blocks or else
+    # every public destination; ports: default 443 and 6443). Empty, the
+    # default, it follows clusterManager.networkPolicy.workloadClusters, since
+    # model-manager serves on the clusters cluster-manager composes onto; set,
+    # it replaces that value as a whole. Rendered with a kserve backend
+    # (static, or cluster-manager on).
+    workloadClusters: {}
     # Further destinations of the service on 443, whatever the backend — the
     # same knob as agentManager.networkPolicy.egress: a private identity
     # provider, a model mirror.

@@ -1146,6 +1146,7 @@ The map is merged into each component's own `nodeSelector` (`muster.nodeSelector
 | modelManager.networkPolicy.huggingFace.fqdns[3].matchPattern | string | `"*.*.hf.co"` |  |
 | modelManager.networkPolicy.huggingFace.fqdns[4].matchPattern | string | `"*.*.*.hf.co"` |  |
 | modelManager.networkPolicy.huggingFace.cidrs | list | `[]` |  |
+| modelManager.networkPolicy.workloadClusters | object | `{}` |  |
 | modelManager.networkPolicy.egress.fqdns | list | `[]` |  |
 | modelManager.networkPolicy.egress.cidrs | list | `[]` |  |
 | modelManager.networkPolicy.registeredBackends | list | `[]` |  |

@@ -3985,6 +3985,17 @@ modelManager:  # @schema skipProperties: true; additionalProperties: true
       # egress to exactly these blocks (a mirror, a proxy). The cilium flavor
       # adds them next to the names.
       cidrs: []
+    # The workload clusters' API servers the kserve backend calls as the caller:
+    # a backend cluster-manager registers targets a workload cluster's apiserver
+    # (list_backends, loads and fits all go there). Same shape as
+    # clusterManager.networkPolicy.workloadClusters (fqdns: Cilium FQDN
+    # selectors; cidrs: IP blocks, in the kubernetes flavor these blocks or else
+    # every public destination; ports: default 443 and 6443). Empty, the
+    # default, it follows clusterManager.networkPolicy.workloadClusters, since
+    # model-manager serves on the clusters cluster-manager composes onto; set,
+    # it replaces that value as a whole. Rendered with a kserve backend
+    # (static, or cluster-manager on).
+    workloadClusters: {}
     # Further destinations of the service on 443, whatever the backend — the
     # same knob as agentManager.networkPolicy.egress: a private identity
     # provider, a model mirror.

@@ -262,6 +262,24 @@ def main(meta: str, connectivity: str) -> int:
         fail("model-manager's egress opens the Hugging Face Hub with neither a static kserve backend nor cluster-manager")
     ok("model-manager's egress: the Hugging Face Hub by name with cluster-manager on, not without a kserve backend")
 
+    # --- model-manager reaches the workload clusters' API servers its kserve backends target (#687) ---
+    # Nothing set on model-manager: its egress follows clusterManager.networkPolicy.workloadClusters (the
+    # cilium render above sets fqdns api.*.example.com and cidrs 198.51.100.0/24), on the same ports.
+    must_have(mm_egress, ("matchPattern: api.*.example.com\n", "- 198.51.100.0/24\n", '- port: "6443"\n'),
+              "model-manager's egress: the workload clusters derived from cluster-manager's")
+    if "api.*.example.com" in mm_off or "198.51.100.0/24" in mm_off:
+        fail("model-manager's egress opens the workload clusters with neither a static kserve backend nor cluster-manager")
+    own = documents(helm(connectivity, [*CONN, "--set", "networkPolicy.flavor=cilium",
+                                        "--set", f"{WIRING}.networkPolicy.workloadClusters.fqdns[0].matchPattern=api.*.example.com",
+                                        "--set", "modelManager.networkPolicy.workloadClusters.fqdns[0].matchName=api.override.example",
+                                        "--set", "modelManager.networkPolicy.workloadClusters.ports[0]=8443"], ci_values=False))
+    mm_own = own[("CiliumNetworkPolicy", "agent-platform-connectivity-model-manager-egress")]
+    must_have(mm_own, ("matchName: api.override.example\n", '- port: "8443"\n'), "model-manager's egress with its own workloadClusters")
+    if "api.*.example.com" in mm_own:
+        fail("an explicit modelManager.networkPolicy.workloadClusters does not replace cluster-manager's value")
+    must_have(own[("CiliumNetworkPolicy", f"{prefix}-egress")], ("matchPattern: api.*.example.com\n",), "cluster-manager's egress, untouched by model-manager's value")
+    ok("model-manager's egress: the workload clusters' API servers by name and address on 443/6443, derived from cluster-manager's; an explicit value replaces it; none without a kserve backend")
+
     # --- connectivity, kubernetes -----------------------------------------------------
     k8s = documents(helm(connectivity, [*CONN, "--set", "networkPolicy.flavor=kubernetes"], ci_values=False))
     for pol in POLICIES:
@@ -278,7 +296,10 @@ def main(meta: str, connectivity: str) -> int:
     must_have(narrowed, ('cidr: "198.51.100.0/24"\n',), "the narrowed kubernetes egress policy")
     if narrowed.count("cidr: 0.0.0.0/0") != 1:
         fail("kubernetes egress with workload cidrs: the workload rule still opens every public destination")
-    ok("kubernetes: the three policies; the apiserver CIDR; every public destination on the workload ports, narrowed by cidrs; the IdP on 443")
+    mm_narrowed = documents(helm(connectivity, [*CONN, "--set", "networkPolicy.flavor=kubernetes",
+                                                "--set", f"{WIRING}.networkPolicy.workloadClusters.cidrs[0]=198.51.100.0/24"], ci_values=False))[("NetworkPolicy", "agent-platform-connectivity-model-manager-egress")]
+    must_have(mm_narrowed, ('cidr: "198.51.100.0/24"\n', "- port: 6443\n"), "model-manager's kubernetes egress with cluster-manager's workload cidrs")
+    ok("kubernetes: the three policies; the apiserver CIDR; every public destination on the workload ports, narrowed by cidrs; the IdP on 443; model-manager follows the workload cidrs")
 
     # --- the guards -------------------------------------------------------------------------
     for what, flags, fragment in (

@@ -536,6 +536,18 @@ def strip(render: str) -> str:
 API_KEY_LABEL = re.compile(r"^( +)api_key:\n\1  enabled: true\n\1  expression: apiKey\.name\n", re.M)
 
 
+MM_WORKLOAD_CLUSTERS = re.compile(r"^ +workloadClusters: \{\}\n", re.M)
+
+
+def hold_mm_workload_clusters(here: str, there: str) -> tuple:
+    """The two meta renders without model-manager's forwarded, empty workloadClusters
+    (giantswarm/agent-platform#687: it follows cluster-manager's when empty). Dropped
+    once GOLDEN_REF carries it."""
+    if (h := MM_WORKLOAD_CLUSTERS.sub("", here)) != here or MM_WORKLOAD_CLUSTERS.search(there):
+        print("note: #687 hold — model-manager's forwarded, empty networkPolicy.workloadClusters is left out of the golden comparison")
+    return h, MM_WORKLOAD_CLUSTERS.sub("", there)
+
+
 def hold_llm_endpoint(here: str, there: str) -> tuple:
     """The two meta renders with #603's forwarded llmRouting and metric-label keys left out."""
     def strip(render: str) -> str:
@@ -781,6 +793,7 @@ def check_golden(meta: str, connectivity: str) -> None:
                 here, there = hold_otlp_endpoints(here, there)
                 here, there = hold_dataplane_sampling(here, there)
                 here, there = hold_llm_endpoint(here, there)
+                here, there = hold_mm_workload_clusters(here, there)
             else:
                 here, there = hold_dataplane_podmonitor(here, there)
                 here, there = hold_dataplane_tracing(here, there)