Add validate_deployments worker job and add ValidateConfig to provisi…

…oner interface (#5221)

admin/deployments.go

@@ -315,7 +315,7 @@ func (s *Service) HibernateDeployments(ctx context.Context) error {
 
 		s.Logger.Info("hibernate: deleting deployment", zap.String("project_id", proj.ID), zap.String("deployment_id", depl.ID))
 
-		err = s.teardownDeployment(ctx, depl)
+		err = s.TeardownDeployment(ctx, depl)
 		if err != nil {
 			s.Logger.Error("hibernate: teardown deployment error", zap.String("project_id", proj.ID), zap.String("deployment_id", depl.ID), zap.Error(err), observability.ZapCtx(ctx))
 			continue
@@ -366,7 +366,7 @@ func (s *Service) OpenRuntimeClient(host, audience string) (*client.Client, erro
 	return rt, nil
 }
 
-func (s *Service) teardownDeployment(ctx context.Context, depl *database.Deployment) error {
+func (s *Service) TeardownDeployment(ctx context.Context, depl *database.Deployment) error {
 	// Delete the deployment
 	err := s.DB.DeleteDeployment(ctx, depl.ID)
 	if err != nil {

admin/projects.go

@@ -100,7 +100,7 @@ func (s *Service) CreateProject(ctx context.Context, org *database.Organization,
 		Annotations:          proj.Annotations,
 	})
 	if err != nil {
-		err2 := s.teardownDeployment(ctx, depl)
+		err2 := s.TeardownDeployment(ctx, depl)
 		err3 := s.DB.DeleteProject(ctx, proj.ID)
 		return nil, multierr.Combine(err, err2, err3)
 	}
@@ -119,7 +119,7 @@ func (s *Service) TeardownProject(ctx context.Context, p *database.Project) erro
 	}
 
 	for _, d := range ds {
-		err := s.teardownDeployment(ctx, d)
+		err := s.TeardownDeployment(ctx, d)
 		if err != nil {
 			return err
 		}
@@ -285,13 +285,13 @@ func (s *Service) TriggerRedeploy(ctx context.Context, proj *database.Project, p
 		Annotations:          proj.Annotations,
 	})
 	if err != nil {
-		err2 := s.teardownDeployment(ctx, newDepl)
+		err2 := s.TeardownDeployment(ctx, newDepl)
 		return nil, multierr.Combine(err, err2)
 	}
 
 	// Delete old prod deployment if exists
 	if prevDepl != nil {
-		err = s.teardownDeployment(ctx, prevDepl)
+		err = s.TeardownDeployment(ctx, prevDepl)
 		if err != nil {
 			s.Logger.Error("trigger redeploy: could not teardown old deployment", zap.String("deployment_id", prevDepl.ID), zap.Error(err), observability.ZapCtx(ctx))
 		}

admin/provisioner/kubernetes.go

@@ -3,9 +3,12 @@ package provisioner
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
@@ -44,9 +47,10 @@ type KubernetesTemplatePaths struct {
 }
 
 type KubernetesProvisioner struct {
-	Spec      *KubernetesSpec
-	clientset *kubernetes.Clientset
-	templates *template.Template
+	Spec              *KubernetesSpec
+	clientset         *kubernetes.Clientset
+	templates         *template.Template
+	templatesChecksum string
 }
 
 type TemplateData struct {
@@ -94,18 +98,33 @@ func NewKubernetes(spec json.RawMessage) (*KubernetesProvisioner, error) {
 	delete(funcMap, "env")
 	delete(funcMap, "expandenv")
 
-	// Parse the template definitions
-	templates := template.Must(template.New("").Funcs(funcMap).ParseFiles(
+	// Define template files
+	templateFiles := []string{
 		ksp.TemplatePaths.HTTPIngress,
 		ksp.TemplatePaths.GRPCIngress,
 		ksp.TemplatePaths.Service,
 		ksp.TemplatePaths.StatefulSet,
-	))
+	}
+
+	// Parse the template definitions
+	templates := template.Must(template.New("").Funcs(funcMap).ParseFiles(templateFiles...))
+
+	// Calculate the combined sha256 sum of all the template files
+	h := sha256.New()
+	for _, f := range templateFiles {
+		d, err := os.ReadFile(f)
+		if err != nil {
+			return nil, err
+		}
+		h.Write(d)
+	}
+	templatesChecksum := hex.EncodeToString(h.Sum(nil))
 
 	return &KubernetesProvisioner{
-		Spec:      ksp,
-		clientset: clientset,
-		templates: templates,
+		Spec:              ksp,
+		clientset:         clientset,
+		templates:         templates,
+		templatesChecksum: templatesChecksum,
 	}, nil
 }
 
@@ -165,6 +184,7 @@ func (p *KubernetesProvisioner) Provision(ctx context.Context, opts *ProvisionOp
 
 	// Create statefulset
 	sts.ObjectMeta.Name = names.StatefulSet
+	sts.ObjectMeta.Annotations["checksum/templates"] = p.templatesChecksum
 	p.addCommonLabels(opts.ProvisionID, sts.ObjectMeta.Labels)
 	_, err = p.clientset.AppsV1().StatefulSets(p.Spec.Namespace).Create(ctx, sts, metav1.CreateOptions{})
 	if err != nil {
@@ -308,6 +328,28 @@ func (p *KubernetesProvisioner) CheckCapacity(ctx context.Context) error {
 	return nil
 }
 
+func (p *KubernetesProvisioner) ValidateConfig(ctx context.Context, provisionID string) (bool, error) {
+	// Get Kubernetes resource names
+	names := p.getResourceNames(provisionID)
+
+	// Get the statefulset
+	sts, err := p.clientset.AppsV1().StatefulSets(p.Spec.Namespace).Get(ctx, names.StatefulSet, metav1.GetOptions{})
+	if err != nil {
+		return false, err
+	}
+
+	// Compare the provisioned templates checksum with the current one
+	if sts.ObjectMeta.Annotations["checksum/templates"] != p.templatesChecksum {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (p *KubernetesProvisioner) Type() string {
+	return "kubernetes"
+}
+
 func (p *KubernetesProvisioner) getResourceNames(provisionID string) ResourceNames {
 	return ResourceNames{
 		StatefulSet: fmt.Sprintf("runtime-%s", provisionID),

admin/provisioner/provisioner.go

@@ -15,6 +15,8 @@ type Provisioner interface {
 	AwaitReady(ctx context.Context, provisionID string) error
 	Update(ctx context.Context, provisionID string, newVersion string) error
 	CheckCapacity(ctx context.Context) error
+	ValidateConfig(ctx context.Context, provisionID string) (bool, error)
+	Type() string
 }
 
 type ProvisionOptions struct {

admin/provisioner/static.go

@@ -129,3 +129,12 @@ func (p *StaticProvisioner) Update(ctx context.Context, provisionID, newVersion
 	// No-op
 	return nil
 }
+
+func (p *StaticProvisioner) ValidateConfig(ctx context.Context, provisionID string) (bool, error) {
+	// No-op
+	return true, nil
+}
+
+func (p *StaticProvisioner) Type() string {
+	return "static"
+}