fix(operator): detect vMCP volume drift

cmd/thv-operator/controllers/virtualmcpserver_controller.go

@@ -1667,6 +1667,15 @@ func (r *VirtualMCPServerReconciler) deploymentNeedsUpdate(
 		return true
 	}
 
+	_, _, expectedVolumesHash, err := r.buildPodVolumesForVmcp(ctx, vmcp, telemetryCfg, typedWorkloads)
+	if err != nil {
+		log.FromContext(ctx).Error(err, "Failed to build volumes, assuming update needed")
+		return true
+	}
+	if deployment.Annotations[podVolumesHashAnnotation] != expectedVolumesHash {
+		return true
+	}
+
 	// Check if spec.replicas has changed. Only compare when spec.replicas is non-nil;
 	// nil means hands-off mode (HPA or external controller manages replicas) and the live count is authoritative.
 	if vmcp.Spec.Replicas != nil {
@@ -1816,7 +1825,7 @@ func (*VirtualMCPServerReconciler) podTemplateSpecNeedsUpdate(
 // MergeAnnotations otherwise preserves them forever once their source field goes empty (#5817, #5818).
 func mergeDeploymentAnnotations(desired, live map[string]string) map[string]string {
 	merged := ctrlutil.MergeAnnotations(desired, live)
-	for _, key := range []string{imagePullRefsHashAnnotation, podTemplateSpecHashAnnotation} {
+	for _, key := range []string{imagePullRefsHashAnnotation, podTemplateSpecHashAnnotation, podVolumesHashAnnotation} {
 		if _, want := desired[key]; !want {
 			delete(merged, key)
 		}

cmd/thv-operator/controllers/virtualmcpserver_controller_test.go

@@ -1859,6 +1859,10 @@ func TestVirtualMCPServerDeploymentNeedsUpdate(t *testing.T) {
 	expectedLabels, expectedAnnotations := reconciler.buildPodTemplateMetadata(
 		labelsForVirtualMCPServer(vmcp.Name), vmcp, vmcpConfigChecksum,
 	)
+	desiredVolumeMounts, desiredVolumes, desiredVolumesHash, err := reconciler.buildPodVolumesForVmcp(
+		context.Background(), vmcp, nil, nil,
+	)
+	require.NoError(t, err)
 
 	tests := []struct {
 		name           string
@@ -2032,7 +2036,7 @@ func TestVirtualMCPServerDeploymentNeedsUpdate(t *testing.T) {
 			deployment: &appsv1.Deployment{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      labelsForVirtualMCPServer(vmcp.Name),
-					Annotations: make(map[string]string),
+					Annotations: map[string]string{podVolumesHashAnnotation: desiredVolumesHash},
 				},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
@@ -2048,10 +2052,12 @@ func TestVirtualMCPServerDeploymentNeedsUpdate(t *testing.T) {
 									Ports: []corev1.ContainerPort{
 										{ContainerPort: 4483},
 									},
-									Args: reconciler.buildContainerArgsForVmcp(vmcp),
-									Env:  mustBuildEnvVarsForVmcp(reconciler, vmcp),
+									Args:         reconciler.buildContainerArgsForVmcp(vmcp),
+									Env:          mustBuildEnvVarsForVmcp(reconciler, vmcp),
+									VolumeMounts: desiredVolumeMounts,
 								},
 							},
+							Volumes:            desiredVolumes,
 							ServiceAccountName: vmcpServiceAccountName(vmcp.Name),
 						},
 					},
@@ -2097,6 +2103,12 @@ func TestMergeDeploymentAnnotations(t *testing.T) {
 			live:     map[string]string{podTemplateSpecHashAnnotation: "stale-hash"},
 			expected: map[string]string{},
 		},
+		{
+			name:     "prunes stale pod volumes hash annotation when desired no longer wants it",
+			desired:  map[string]string{},
+			live:     map[string]string{podVolumesHashAnnotation: "stale-hash"},
+			expected: map[string]string{},
+		},
 		{
 			name:     "keeps hash annotations desired still wants",
 			desired:  map[string]string{imagePullRefsHashAnnotation: "new-hash", podTemplateSpecHashAnnotation: "new-hash"},

cmd/thv-operator/controllers/virtualmcpserver_deployment.go

@@ -46,6 +46,12 @@ const (
 	// detect every input that influences the deployed PodSpec.ImagePullSecrets.
 	imagePullRefsHashAnnotation = "toolhive.stacklok.io/imagepullsecrets-hash"
 
+	// podVolumesHashAnnotation tracks the SHA256 hash of the desired vMCP
+	// container volume mounts and PodSpec volumes. The hash is stored on the
+	// Deployment so changes to referenced Secrets or ConfigMaps trigger a
+	// rollout without comparing API-server-defaulted live PodSpec fields.
+	podVolumesHashAnnotation = "toolhive.stacklok.io/podvolumes-hash"
+
 	// Log level configuration
 	logLevelDebug = "debug" // Debug log level value
 
@@ -143,7 +149,7 @@ func (r *VirtualMCPServerReconciler) deploymentForVirtualMCPServer(
 
 	// Build deployment components using helper functions
 	args := r.buildContainerArgsForVmcp(vmcp)
-	volumeMounts, volumes, err := r.buildVolumesForVmcp(ctx, vmcp)
+	volumeMounts, volumes, volumesHash, err := r.buildPodVolumesForVmcp(ctx, vmcp, telemetryCfg, typedWorkloads)
 	if err != nil {
 		log.FromContext(ctx).Error(err, "Failed to build volumes for VirtualMCPServer")
 		return nil
@@ -154,31 +160,7 @@ func (r *VirtualMCPServerReconciler) deploymentForVirtualMCPServer(
 		return nil
 	}
 
-	// Add CA bundle volumes for MCPServerEntry backends with caBundleRef
-	caVolumes, caMounts, err := r.buildCABundleVolumesForEntries(ctx, vmcp.Namespace, typedWorkloads)
-	if err != nil {
-		log.FromContext(ctx).Error(err, "Failed to build CA bundle volumes for MCPServerEntries")
-		return nil
-	}
-	volumes = append(volumes, caVolumes...)
-	volumeMounts = append(volumeMounts, caMounts...)
-
-	// Add telemetry CA bundle volumes from the pre-fetched MCPTelemetryConfig
-	if telemetryCfg != nil {
-		telVolumes, telMounts := ctrlutil.AddTelemetryCABundleVolumes(telemetryCfg)
-		volumes = append(volumes, telVolumes...)
-		volumeMounts = append(volumeMounts, telMounts...)
-	}
-
-	// Add embedded auth server volumes if configured (inline config). The matching
-	// env vars are injected by buildEnvVarsForVmcp above so the drift check stays
-	// symmetric with what is built here (see #5616).
-	if vmcp.Spec.AuthServerConfig != nil {
-		authServerVolumes, authServerMounts := ctrlutil.GenerateAuthServerVolumes(vmcp.Spec.AuthServerConfig)
-		volumes = append(volumes, authServerVolumes...)
-		volumeMounts = append(volumeMounts, authServerMounts...)
-	}
-	deploymentLabels, deploymentAnnotations := r.buildDeploymentMetadataForVmcp(ls, vmcp)
+	deploymentLabels, deploymentAnnotations := r.buildDeploymentMetadataForVmcp(ls, vmcp, volumesHash)
 	deploymentTemplateLabels, deploymentTemplateAnnotations := r.buildPodTemplateMetadata(ls, vmcp, vmcpConfigChecksum)
 	podSecurityContext, containerSecurityContext := r.buildSecurityContextsForVmcp(ctx, vmcp)
 	serviceAccountName := r.serviceAccountNameForVmcp(vmcp)
@@ -257,6 +239,81 @@ func (r *VirtualMCPServerReconciler) deploymentForVirtualMCPServer(
 	return dep
 }
 
+// buildPodVolumesForVmcp builds the complete desired volume state for the vmcp
+// container and computes its stable hash in the same pass. Keeping all volume
+// sources here ensures the PodSpec and Deployment annotation use one consistent
+// snapshot of referenced Kubernetes objects.
+func (r *VirtualMCPServerReconciler) buildPodVolumesForVmcp(
+	ctx context.Context,
+	vmcp *mcpv1beta1.VirtualMCPServer,
+	telemetryCfg *mcpv1beta1.MCPTelemetryConfig,
+	typedWorkloads []workloads.TypedWorkload,
+) ([]corev1.VolumeMount, []corev1.Volume, string, error) {
+	volumeMounts, volumes, err := r.buildVolumesForVmcp(ctx, vmcp)
+	if err != nil {
+		return nil, nil, "", err
+	}
+
+	caVolumes, caMounts, err := r.buildCABundleVolumesForEntries(ctx, vmcp.Namespace, typedWorkloads)
+	if err != nil {
+		return nil, nil, "", fmt.Errorf("failed to build CA bundle volumes for MCPServerEntries: %w", err)
+	}
+	volumes = append(volumes, caVolumes...)
+	volumeMounts = append(volumeMounts, caMounts...)
+
+	if telemetryCfg != nil {
+		telVolumes, telMounts := ctrlutil.AddTelemetryCABundleVolumes(telemetryCfg)
+		volumes = append(volumes, telVolumes...)
+		volumeMounts = append(volumeMounts, telMounts...)
+	}
+
+	if vmcp.Spec.AuthServerConfig != nil {
+		authServerVolumes, authServerMounts := ctrlutil.GenerateAuthServerVolumes(vmcp.Spec.AuthServerConfig)
+		volumes = append(volumes, authServerVolumes...)
+		volumeMounts = append(volumeMounts, authServerMounts...)
+	}
+
+	hash, err := podVolumesHash(volumes, volumeMounts)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	return volumeMounts, volumes, hash, nil
+}
+
+// podVolumesHash returns a deterministic hash of the complete desired volume
+// and volume-mount objects. Sorting by identity makes ordering-only changes a
+// no-op while hashing the full Kubernetes structs preserves all drift-relevant
+// source fields, including future VolumeSource additions.
+func podVolumesHash(volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (string, error) {
+	normalizedVolumes := append([]corev1.Volume(nil), volumes...)
+	sort.SliceStable(normalizedVolumes, func(i, j int) bool {
+		return normalizedVolumes[i].Name < normalizedVolumes[j].Name
+	})
+	normalizedMounts := append([]corev1.VolumeMount(nil), volumeMounts...)
+	sort.SliceStable(normalizedMounts, func(i, j int) bool {
+		if normalizedMounts[i].Name != normalizedMounts[j].Name {
+			return normalizedMounts[i].Name < normalizedMounts[j].Name
+		}
+		if normalizedMounts[i].MountPath != normalizedMounts[j].MountPath {
+			return normalizedMounts[i].MountPath < normalizedMounts[j].MountPath
+		}
+		return normalizedMounts[i].SubPath < normalizedMounts[j].SubPath
+	})
+
+	canonical, err := json.Marshal(struct {
+		Volumes      []corev1.Volume
+		VolumeMounts []corev1.VolumeMount
+	}{
+		Volumes:      normalizedVolumes,
+		VolumeMounts: normalizedMounts,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal pod volumes for hashing: %w", err)
+	}
+	hash := sha256.Sum256(canonical)
+	return hex.EncodeToString(hash[:]), nil
+}
+
 // buildContainerArgsForVmcp builds the container arguments for vmcp
 func (*VirtualMCPServerReconciler) buildContainerArgsForVmcp(
 	vmcp *mcpv1beta1.VirtualMCPServer,
@@ -924,6 +981,7 @@ func xaaSecretEnvVars(externalAuthConfig *mcpv1beta1.MCPExternalAuthConfig, conf
 func (r *VirtualMCPServerReconciler) buildDeploymentMetadataForVmcp(
 	baseLabels map[string]string,
 	vmcp *mcpv1beta1.VirtualMCPServer,
+	podVolumesHash ...string,
 ) (map[string]string, map[string]string) {
 	deploymentLabels := baseLabels
 	deploymentAnnotations := make(map[string]string)
@@ -949,6 +1007,10 @@ func (r *VirtualMCPServerReconciler) buildDeploymentMetadataForVmcp(
 		deploymentAnnotations[imagePullRefsHashAnnotation] = hash
 	}
 
+	if len(podVolumesHash) > 0 && podVolumesHash[0] != "" {
+		deploymentAnnotations[podVolumesHashAnnotation] = podVolumesHash[0]
+	}
+
 	// TODO: Add support for ResourceOverrides if needed in the future
 
 	return deploymentLabels, deploymentAnnotations

cmd/thv-operator/controllers/virtualmcpserver_deployment_test.go

@@ -1103,6 +1103,53 @@ func TestDeploymentForVirtualMCPServer_AuthServerConfig_NoUpdateLoop(t *testing.
 		"deploymentNeedsUpdate must not loop on a vMCP with AuthServerConfig (regression #5616)")
 }
 
+// TestDeploymentForVirtualMCPServer_AuthServerSigningKeyVolumeDrift verifies that
+// changing an embedded auth-server signing key Secret reference rolls the vMCP
+// Deployment. The mounted Secret is selected by the PodSpec rather than an env
+// var, so container drift checks alone cannot observe this change.
+func TestDeploymentForVirtualMCPServer_AuthServerSigningKeyVolumeDrift(t *testing.T) {
+	t.Parallel()
+
+	scheme := testutil.NewScheme(t)
+	r := &VirtualMCPServerReconciler{
+		Scheme:           scheme,
+		PlatformDetector: ctrlutil.NewSharedPlatformDetector(),
+	}
+
+	vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default",
+		v1beta1test.WithVMCPGroupRef("test-group"),
+		v1beta1test.WithVMCPAuthServerConfig(&mcpv1beta1.EmbeddedAuthServerConfig{
+			SigningKeySecretRefs: []mcpv1beta1.SecretKeyRef{{Name: "keys-v1", Key: "signing-key.pem"}},
+		}),
+	)
+
+	const cfgChecksum = "test-checksum"
+	initialDeployment := r.deploymentForVirtualMCPServer(t.Context(), vmcp, cfgChecksum, nil, nil)
+	require.NotNil(t, initialDeployment)
+	require.NotEmpty(t, initialDeployment.Annotations["toolhive.stacklok.io/podvolumes-hash"])
+
+	updatedVMCP := vmcp.DeepCopy()
+	updatedVMCP.Spec.AuthServerConfig.SigningKeySecretRefs[0].Name = "keys-v2"
+	updatedDeployment := r.deploymentForVirtualMCPServer(t.Context(), updatedVMCP, cfgChecksum, nil, nil)
+	require.NotNil(t, updatedDeployment)
+
+	assert.NotEqual(t,
+		initialDeployment.Annotations["toolhive.stacklok.io/podvolumes-hash"],
+		updatedDeployment.Annotations["toolhive.stacklok.io/podvolumes-hash"],
+		"changing the signing key Secret reference must change the pod volume hash")
+	assert.True(t, r.deploymentNeedsUpdate(t.Context(), initialDeployment, updatedVMCP, cfgChecksum, nil, nil))
+	assert.False(t, r.deploymentNeedsUpdate(t.Context(), updatedDeployment, updatedVMCP, cfgChecksum, nil, nil))
+
+	var signingKeySecretName string
+	for _, volume := range updatedDeployment.Spec.Template.Spec.Volumes {
+		if volume.Name == ctrlutil.AuthServerKeysVolumePrefix+"0" && volume.Secret != nil {
+			signingKeySecretName = volume.Secret.SecretName
+			break
+		}
+	}
+	assert.Equal(t, "keys-v2", signingKeySecretName)
+}
+
 // TestImagePullSecretsHash verifies the hash helper normalizes order, treats an
 // empty list as the sentinel "" hash, and produces stable hashes across calls.
 func TestImagePullSecretsHash(t *testing.T) {