feat: automatically use HEAD branch for scm-engine config if MR is ou…

…t of sync/behind HEAD

cmd/gitlab_evaluate.go

@@ -10,8 +10,10 @@ import (
 )
 
 func Evaluate(cCtx *cli.Context) error {
-	ctx := state.WithProjectID(cCtx.Context, cCtx.String(FlagSCMProject))
+	ctx := cCtx.Context
 	ctx = state.WithCommitSHA(ctx, cCtx.String(FlagCommitSHA))
+	ctx = state.WithConfigFilePath(ctx, cCtx.String(FlagConfigFile))
+	ctx = state.WithProjectID(ctx, cCtx.String(FlagSCMProject))
 	ctx = state.WithToken(ctx, cCtx.String(FlagAPIToken))
 	ctx = state.WithUpdatePipeline(ctx, cCtx.Bool(FlagUpdatePipeline), cCtx.String(FlagUpdatePipelineURL))
 

cmd/gitlab_server.go

@@ -22,11 +22,12 @@ func Server(cCtx *cli.Context) error {
 	var wg sync.WaitGroup
 
 	// Setup context configuration
-	ctx := state.WithUpdatePipeline(cCtx.Context, cCtx.Bool(FlagUpdatePipeline), cCtx.String(FlagUpdatePipelineURL))
+	ctx := cCtx.Context
+	ctx = state.WithConfigFilePath(ctx, cCtx.String(FlagConfigFile))
+	ctx = state.WithUpdatePipeline(ctx, cCtx.Bool(FlagUpdatePipeline), cCtx.String(FlagUpdatePipelineURL))
 
 	// Add logging context key/value pairs
 	ctx = slogctx.With(ctx, slog.String("gitlab_url", cCtx.String(FlagSCMBaseURL)))
-	ctx = slogctx.With(ctx, slog.String("config_file", cCtx.String(FlagConfigFile)))
 	ctx = slogctx.With(ctx, slog.Duration("server_timeout", cCtx.Duration(FlagServerTimeout)))
 
 	//
@@ -54,7 +55,7 @@ func Server(cCtx *cli.Context) error {
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /_status", GitLabStatusHandler)
-	mux.HandleFunc("POST /gitlab", GitLabWebhookHandler(ctx, cCtx.String(FlagWebhookSecret), cCtx.String(FlagConfigFile)))
+	mux.HandleFunc("POST /gitlab", GitLabWebhookHandler(ctx, cCtx.String(FlagWebhookSecret)))
 
 	server := &http.Server{
 		Addr:         listenAddr,

cmd/gitlab_server_handlers.go

@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/jippi/scm-engine/pkg/config"
 	"github.com/jippi/scm-engine/pkg/state"
 	slogctx "github.com/veqryn/slog-context"
 )
@@ -25,7 +24,7 @@ func GitLabStatusHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("scm-engine status: OK\n\nNOTE: this is a static 'OK', no actual checks are being made"))
 }
 
-func GitLabWebhookHandler(ctx context.Context, ourSecret, configFilePath string) http.HandlerFunc {
+func GitLabWebhookHandler(ctx context.Context, webhookSecret string) http.HandlerFunc {
 	// Initialize GitLab client
 	client, err := getClient(ctx)
 	if err != nil {
@@ -36,9 +35,9 @@ func GitLabWebhookHandler(ctx context.Context, ourSecret, configFilePath string)
 		ctx := r.Context()
 
 		// Check if the webhook secret is set (and if its matching)
-		if len(ourSecret) > 0 {
+		if len(webhookSecret) > 0 {
 			theirSecret := r.Header.Get("X-Gitlab-Token")
-			if ourSecret != theirSecret {
+			if webhookSecret != theirSecret {
 				errHandler(ctx, w, http.StatusForbidden, errors.New("Missing or invalid X-Gitlab-Token header"))
 
 				return
@@ -104,22 +103,6 @@ func GitLabWebhookHandler(ctx context.Context, ourSecret, configFilePath string)
 
 		slogctx.Info(ctx, "GET /gitlab webhook")
 
-		// Get the remote config file
-		file, err := client.MergeRequests().GetRemoteConfig(ctx, configFilePath, gitSha)
-		if err != nil {
-			errHandler(ctx, w, http.StatusOK, fmt.Errorf("could not read remote config file: %w", err))
-
-			return
-		}
-
-		// Parse the file
-		cfg, err := config.ParseFile(file)
-		if err != nil {
-			errHandler(ctx, w, http.StatusOK, fmt.Errorf("could not parse config file: %w", err))
-
-			return
-		}
-
 		// Decode request payload into 'any' so we have all the details
 		var fullEventPayload any
 		if err := json.NewDecoder(bytes.NewReader(body)).Decode(&fullEventPayload); err != nil {
@@ -129,7 +112,7 @@ func GitLabWebhookHandler(ctx context.Context, ourSecret, configFilePath string)
 		}
 
 		// Process the MR
-		if err := ProcessMR(ctx, client, cfg, fullEventPayload); err != nil {
+		if err := ProcessMR(ctx, client, nil, fullEventPayload); err != nil {
 			errHandler(ctx, w, http.StatusOK, err)
 
 			return

cmd/shared.go

@@ -33,6 +33,7 @@ func getClient(ctx context.Context) (scm.Client, error) {
 func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event any) (err error) {
 	// Attach unique eval id to the logs so they are easy to filter on later
 	ctx = slogctx.With(ctx, slog.String("eval_id", sid.MustGenerate()))
+	ctx = slogctx.With(ctx, slog.String("config_source_branch", "merge_request_branch"))
 
 	defer state.LockForProcessing(ctx)()
 
@@ -51,12 +52,9 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 		return fmt.Errorf("failed to update pipeline monitor: %w", err)
 	}
 
-	slogctx.Info(ctx, "Processing MR")
-
-	remoteLabels, err := client.Labels().List(ctx)
-	if err != nil {
-		return err
-	}
+	//
+	// Create and validate the evaluation context
+	//
 
 	slogctx.Info(ctx, "Creating evaluation context")
 
@@ -72,10 +70,50 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 	}
 
 	evalContext.SetWebhookEvent(event)
+
 	// Add our "ctx" to evalContext so Expr-Lang functions can reference them
 	// when they need to read our "cfg"
 	evalContext.SetContext(ctx)
 
+	//
+	// (Optional) Download the .scm-engine.yml configuration file from the GitLab HTTP API
+	//
+
+	var (
+		configShouldBeDownloaded = cfg == nil
+		configSourceRef          = state.CommitSHA(ctx)
+	)
+
+	// If the current branch is not in a state where the config file can be trusted,
+	// we instead use the HEAD version of the file
+	if !evalContext.CanUseConfigurationFileFromChange(ctx) {
+		configShouldBeDownloaded = true
+		configSourceRef = "HEAD"
+
+		// Update the logger with new value
+		ctx = slogctx.With(ctx, slog.String("config_source_branch", configSourceRef))
+	}
+
+	// Download and parse the configuration file if necessary
+	if configShouldBeDownloaded {
+		slogctx.Debug(ctx, "Downloading scm-engine configuration from ref")
+
+		file, err := client.MergeRequests().GetRemoteConfig(ctx, state.ConfigFilePath(ctx), configSourceRef)
+		if err != nil {
+			return fmt.Errorf("could not read remote config file: %w", err)
+		}
+
+		// Parse the file
+		cfg, err = config.ParseFile(file)
+		if err != nil {
+			return fmt.Errorf("could not parse config file: %w", err)
+		}
+	}
+
+	//
+	// Do the actual context evaluation
+	//
+
 	slogctx.Info(ctx, "Evaluating context")
 
 	labels, actions, err := cfg.Evaluate(ctx, evalContext)
@@ -85,8 +123,17 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 
 	slogctx.Debug(ctx, "Evaluation complete", slog.Int("number_of_labels", len(labels)), slog.Int("number_of_actions", len(actions)))
 
+	//
+	// Post-evaluation sync of labels
+	//
+
 	slogctx.Info(ctx, "Sync labels")
 
+	remoteLabels, err := client.Labels().List(ctx)
+	if err != nil {
+		return err
+	}
+
 	if err := syncLabels(ctx, client, remoteLabels, labels); err != nil {
 		return err
 	}
@@ -104,6 +151,10 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 		}
 	}
 
+	//
+	// Post-evaluation sync of actions
+	//
+
 	update := &scm.UpdateMergeRequestOptions{
 		AddLabels:    &add,
 		RemoveLabels: &remove,
@@ -115,6 +166,10 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 		return err
 	}
 
+	//
+	// Update the Merge Request with the outcome of labels and actions
+	//
+
 	slogctx.Info(ctx, "Updating Merge Request")
 
 	return updateMergeRequest(ctx, client, update)

pkg/scm/github/context.go

@@ -92,3 +92,7 @@ func (c *Context) SetContext(ctx context.Context) {
 func (c *Context) GetDescription() string {
 	return c.PullRequest.Body
 }
+
+func (c *Context) CanUseConfigurationFileFromChange(ctx context.Context) bool {
+	return true
+}

pkg/scm/gitlab/context.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hasura/go-graphql-client"
 	"github.com/jippi/scm-engine/pkg/scm"
 	"github.com/jippi/scm-engine/pkg/state"
+	slogctx "github.com/veqryn/slog-context"
 	"golang.org/x/oauth2"
 )
 
@@ -105,3 +106,23 @@ func (c *Context) GetDescription() string {
 
 	return *c.MergeRequest.Description
 }
+
+func (c *Context) CanUseConfigurationFileFromChange(ctx context.Context) bool {
+	// If the Merge Request has diverged from HEAD we can't trust the configuration
+	if c.MergeRequest.DivergedFromTargetBranch {
+		slogctx.Warn(ctx, "The Merge Request branch has diverged from HEAD; will use the scm-engine config from HEAD instead")
+
+		return false
+	}
+
+	// If the Merge Request is not up to date with HEAD we can't trust the configuration
+	if c.MergeRequest.ShouldBeRebased {
+		slogctx.Warn(ctx, "The Merge Request branch is not up to date with HEAD; will use the scm-engine config from HEAD instead")
+
+		return false
+	}
+
+	slogctx.Info(ctx, "The Merge Request branch is up to date with HEAD; will use the scm-engine config from the branch")
+
+	return true
+}

pkg/scm/interfaces.go

@@ -28,8 +28,9 @@ type MergeRequestClient interface {
 }
 
 type EvalContext interface {
+	GetDescription() string
 	IsValid() bool
-	SetWebhookEvent(in any)
 	SetContext(ctx context.Context)
-	GetDescription() string
+	SetWebhookEvent(in any)
+	CanUseConfigurationFileFromChange(ctx context.Context) bool
 }

pkg/state/context.go

@@ -11,15 +11,17 @@ import (
 type contextKey uint
 
 const (
-	projectID contextKey = iota
+	_ contextKey = iota
+	baseURL
+	commitSha
 	dryRun
 	mergeRequestID
-	commitSha
-	updatePipeline
-	updatePipelineURL
+	projectID
 	provider
 	token
-	baseURL
+	configFilePath
+	updatePipeline
+	updatePipelineURL
 )
 
 func ProjectID(ctx context.Context) string {
@@ -30,6 +32,10 @@ func CommitSHA(ctx context.Context) string {
 	return ctx.Value(commitSha).(string) //nolint:forcetypeassert
 }
 
+func ConfigFilePath(ctx context.Context) string {
+	return ctx.Value(configFilePath).(string) //nolint:forcetypeassert
+}
+
 func BaseURL(ctx context.Context) string {
 	return ctx.Value(baseURL).(string) //nolint:forcetypeassert
 }
@@ -64,6 +70,13 @@ func WithProjectID(ctx context.Context, value string) context.Context {
 	return ctx
 }
 
+func WithConfigFilePath(ctx context.Context, value string) context.Context {
+	ctx = slogctx.With(ctx, slog.String("config_file_path", value))
+	ctx = context.WithValue(ctx, configFilePath, value)
+
+	return ctx
+}
+
 func WithDryRun(ctx context.Context, dry bool) context.Context {
 	ctx = slogctx.With(ctx, slog.Bool("dry_run", dry))
 	ctx = context.WithValue(ctx, dryRun, dry)