feat(api): manage more techno + purge workflow version + update plugin

contrib/grpcplugins/action/artifactory-plugins-lib/artifactory-plugins-lib.go

@@ -339,7 +339,7 @@ func ReleaseArtifactoryRunResult(ctx context.Context, c *actionplugin.Common, re
 }
 
 func getReleaseVersion(jobContext sdk.WorkflowRunJobsContext) string {
-	return strings.ReplaceAll(jobContext.Git.SemverCurrent, "+", "-")
+	return strings.ReplaceAll(jobContext.CDS.Version, "+", "-")
 }
 
 func getBuildInfoAndReleaseName(buildInfo, projectKey, vcs, repo, workflowName string) (string, string) {

contrib/grpcplugins/action/artifactoryRelease/artifactoryRelease.yml

@@ -15,7 +15,7 @@ inputs:
     type: string
     description: Properties to add on the promoted artifacts
   releaseNotes:
-    default: Release ${{git.semver.current}}
+    default: Release ${{cds.version}}
     type: string
     description: Notes associated to your Release.
     required: true

contrib/grpcplugins/action/debianPush/main.go

@@ -195,7 +195,7 @@ func (p *debianPushPlugin) UploadArtifactoryDebianPackage(ctx context.Context, o
 	}
 
 	debInfo := fmt.Sprintf("%s;%s;%s", distribLayout, componentLayout, archLayout)
-	cdsInfo := fmt.Sprintf("cds_version=%s;cds_workflow=%s", opts.jobContext.Git.SemverCurrent, opts.jobContext.CDS.Workflow)
+	cdsInfo := fmt.Sprintf("cds_version=%s;cds_workflow=%s", opts.jobContext.CDS.Version, opts.jobContext.CDS.Workflow)
 	maturity := opts.jobContext.Integrations.ArtifactManager.Get(sdk.ArtifactoryConfigPromotionLowMaturity)
 	path := fmt.Sprintf("/pool/%s;%s;%s;deb.release.origin=%s;deb.release.label=%s",
 		fileName, debInfo, cdsInfo, opts.origin, opts.label)

contrib/grpcplugins/action/deployArsenal/deployArsenal.yml

@@ -19,7 +19,7 @@ inputs:
   version:
     type: text
     description: "The version to deploy"
-    default: ${{git.semver_current}}
+    default: ${{cds.version}}
   alternative-config:
     type: text
     description: Alternative configuration to push with the deployment

contrib/grpcplugins/action/deployArsenal/main.go

@@ -43,7 +43,7 @@ const deployData = `{
 		"CDS_GIT_BRANCH": "${{git.ref_name}}",
 		"CDS_WORKFLOW": "${{cds.workflow}}",
 		"CDS_PROJECT": "${{cds.project_key}}",
-		"CDS_VERSION": "${{git.semver_current}}",
+		"CDS_VERSION": "${{cds.version}}",
 		"CDS_GIT_REPOSITORY": "${{git.repository}}",
 		"CDS_GIT_HASH": "${{git.Sha}}"
 	}

contrib/grpcplugins/action/helmPush/helmPush.yml

@@ -10,7 +10,7 @@ inputs:
     description: Relative path to chart folder to be published	
     required: true
   chartVersion:
-    default: ${{git.semver_current}}
+    default: ${{cds.version}}
     type: string
     description: |-
       (Optional) Override the chart version before push to registry.

contrib/grpcplugins/grpcplugins.go

@@ -721,7 +721,7 @@ func UploadRunResult(ctx context.Context, actplugin *actionplugin.Common, jobCon
 			strings.ToLower(jobContext.Git.Repository),
 			jobContext.CDS.ProjectKey,
 			jobContext.CDS.Workflow,
-			jobContext.Git.SemverCurrent)
+			jobContext.CDS.Version)
 
 		response.RunResult.ArtifactManagerMetadata = &sdk.V2WorkflowRunResultArtifactManagerMetadata{}
 		response.RunResult.ArtifactManagerMetadata.Set("repository", repository) // This is the virtual repository

engine/api/api.go

@@ -241,10 +241,12 @@ type Configuration struct {
 		WorkerModelDockerImageWhiteList []string         `toml:"workerModelDockerImageWhiteList" comment:"White list for docker image worker model " json:"workerModelDockerImageWhiteList" commented:"true"`
 	} `toml:"workflow" comment:"######################\n 'Workflow' global configuration \n######################" json:"workflow"`
 	WorkflowV2 struct {
-		JobSchedulingTimeout   int64  `toml:"jobSchedulingTimeout" comment:"Timeout delay for job scheduling (in seconds)" json:"jobSchedulingTimeout" default:"600"`
-		RunRetentionScheduling int64  `toml:"runRetentionScheduling" comment:"Time in minute between 2 run of the workflow run purge" json:"runRetentionScheduling" default:"15"`
-		WorkflowRunRetention   int64  `toml:"workflowRunRetention" comment:"Workflow run retention in days" json:"workflowRunRetention" default:"90"`
-		LibraryProjectKey      string `toml:"libraryProjectKey" comment:"Library project key" json:"libraryProjectKey" commented:"true"`
+		JobSchedulingTimeout       int64  `toml:"jobSchedulingTimeout" comment:"Timeout delay for job scheduling (in seconds)" json:"jobSchedulingTimeout" default:"600"`
+		RunRetentionScheduling     int64  `toml:"runRetentionScheduling" comment:"Time in minute between 2 run of the workflow run purge" json:"runRetentionScheduling" default:"15"`
+		WorkflowRunRetention       int64  `toml:"workflowRunRetention" comment:"Workflow run retention in days" json:"workflowRunRetention" default:"90"`
+		LibraryProjectKey          string `toml:"libraryProjectKey" comment:"Library project key" json:"libraryProjectKey" commented:"true"`
+		VersionRetentionScheduling int64  `toml:"versionRetentionScheduling" comment:"Time in minute between 2 run of the workflow version purge" json:"versionRetentionScheduling" default:"60"`
+		VersionRetention           int64  `toml:"versionRetention" comment:"Number of Workflow version CDS keep" json:"versionRetention" commented:"true"`
 	} `toml:"workflowv2" comment:"######################\n 'Workflow V2' global configuration \n######################" json:"workflowv2"`
 	Entity struct {
 		RoutineDelay int64  `toml:"routineDelay" comment:"Delay in minutes between to run of entities purge" json:"routineDelay" default:"15"`
@@ -508,6 +510,12 @@ func (a *API) Serve(ctx context.Context) error {
 	if a.Config.Entity.Retention == "" {
 		a.Config.Entity.Retention = "24h"
 	}
+	if a.Config.WorkflowV2.VersionRetentionScheduling == 0 {
+		a.Config.WorkflowV2.VersionRetentionScheduling = 60
+	}
+	if a.Config.WorkflowV2.VersionRetention == 0 {
+		a.Config.WorkflowV2.VersionRetention = 25
+	}
 	entityRetention, err := time.ParseDuration(a.Config.Entity.Retention)
 	if err != nil {
 		return sdk.WrapError(err, "wrong entity retention %s, bad format.", a.Config.Entity.Retention)
@@ -958,6 +966,9 @@ func (a *API) Serve(ctx context.Context) error {
 	a.GoRoutines.RunWithRestart(ctx, "project.CleanAsCodeEntities", func(ctx context.Context) {
 		a.cleanProjectEntities(ctx, entityRetention)
 	})
+	a.GoRoutines.RunWithRestart(ctx, "project.CleanWorkflowVersion", func(ctx context.Context) {
+		a.cleanWorkflowVersion(ctx)
+	})
 
 	a.GoRoutines.RunWithRestart(ctx, "worker.DeleteDisabledWorkers", func(ctx context.Context) {
 		DeleteDisabledWorkers(ctx, a.Cache, a.mustDB)

engine/api/v2_project_clean_ascode.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/go-gorp/gorp"
 	"github.com/rockbears/log"
 
@@ -33,6 +35,156 @@ type EntitiesCleaner struct {
 	retention time.Duration
 }
 
+func (a *API) cleanWorkflowVersion(ctx context.Context) {
+	ticker := time.NewTicker(time.Duration(a.Config.WorkflowV2.VersionRetentionScheduling) * time.Minute)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if ctx.Err() != nil {
+				log.Error(ctx, "%v", ctx.Err())
+			}
+			return
+		case <-ticker.C:
+			workflows, err := workflow_v2.LoadDistinctWorkflowVersionByWorkflow(ctx, a.mustDB())
+			if err != nil {
+				log.ErrorWithStackTrace(ctx, err)
+				continue
+			}
+			inputChan := make(chan workflow_v2.V2WorkflowVersionWorkflowShort, len(workflows))
+			resultChan := make(chan bool)
+			for w := 0; w < 10; w++ {
+				a.GoRoutines.Exec(ctx, "cleanWorkflowVersion-"+strconv.Itoa(w), func(ctx context.Context) {
+					for w := range inputChan {
+						if err := workerCleanWorkflowVersion(ctx, a.mustDB(), a.Cache, w, a.Config.WorkflowV2.VersionRetention); err != nil {
+							log.ErrorWithStackTrace(ctx, err)
+						}
+						resultChan <- true
+					}
+				})
+			}
+			for _, w := range workflows {
+				inputChan <- w
+			}
+			close(inputChan)
+			for r := 0; r < len(workflows); r++ {
+				<-resultChan
+			}
+		}
+	}
+}
+
+func workerCleanWorkflowVersion(ctx context.Context, db *gorp.DbMap, store cache.Store, w workflow_v2.V2WorkflowVersionWorkflowShort, nbVersionToKeep int64) error {
+	ctx = context.WithValue(ctx, cdslog.Action, "workerCleanWorkflowVersion")
+	ctx = context.WithValue(ctx, "action_metadata_project_key", w.ProjectKey)
+	ctx = context.WithValue(ctx, cdslog.VCSServer, w.WorkflowVCS)
+	ctx = context.WithValue(ctx, cdslog.Repository, w.WorkflowRepository)
+	ctx = context.WithValue(ctx, cdslog.Workflow, w.WorkflowName)
+
+	log.Info(ctx, "Clean workflow version for "+w.String())
+	lockKey := cache.Key("workflow", "version", w.String())
+	locked, err := store.Lock(lockKey, 5*time.Minute, 500, 1)
+	if err != nil {
+		return err
+	}
+	if !locked {
+		return nil
+	}
+	defer store.Unlock(lockKey)
+	if err := cleanWorkflowVersion(ctx, db, store, w, nbVersionToKeep); err != nil {
+		return err
+	}
+	return nil
+}
+
+func cleanWorkflowVersion(ctx context.Context, db *gorp.DbMap, store cache.Store, w workflow_v2.V2WorkflowVersionWorkflowShort, nbVersionToKeep int64) error {
+	allVersions, err := workflow_v2.LoadAllVerionsByWorkflow(ctx, db, w.ProjectKey, w.WorkflowVCS, w.WorkflowRepository, w.WorkflowName)
+	if err != nil {
+		return err
+	}
+	if len(allVersions) < int(nbVersionToKeep) {
+		return nil
+	}
+
+	versions := make([]*semver.Version, 0, len(allVersions))
+	for _, v := range allVersions {
+		sVer, _ := semver.NewVersion(v.Version)
+		versions = append(versions, sVer)
+	}
+	sort.Slice(versions, func(i, j int) bool {
+		return versions[i].LessThan(versions[j])
+	})
+
+	var versionsToClean []*semver.Version
+	cleanAll := false
+	// Check if the workflow still exists on default branch
+	vcsProject, err := vcs.LoadVCSByProject(ctx, db, w.ProjectKey, w.WorkflowVCS)
+	if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+		return err
+	}
+	// If vcs doesn't exist, cleann all
+	if err != nil && sdk.ErrorIs(err, sdk.ErrNotFound) {
+		log.Info(ctx, "vcs %s doesn't exist anymore. Cleaning all versions", w.WorkflowVCS)
+		cleanAll = true
+	}
+
+	if !cleanAll {
+		repository, err := repository.LoadRepositoryByName(ctx, db, vcsProject.ID, w.WorkflowRepository)
+		if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+			return err
+		}
+		if err != nil && sdk.ErrorIs(err, sdk.ErrNotFound) {
+			log.Info(ctx, "repository %s doesn't exist anymore. Cleaning all versions", w.WorkflowRepository)
+			cleanAll = true
+		}
+
+		if !cleanAll {
+			vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, store, w.ProjectKey, w.WorkflowVCS)
+			if err != nil {
+				return err
+			}
+			defaultBranch, err := vcsClient.Branch(ctx, w.WorkflowRepository, sdk.VCSBranchFilters{Default: true})
+			if err != nil {
+				return err
+			}
+			_, err = entity.LoadByRefTypeNameCommit(ctx, db, repository.ID, defaultBranch.ID, sdk.EntityTypeWorkflow, w.WorkflowName, "HEAD")
+			if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+				return err
+			}
+			if err != nil && sdk.ErrorIs(err, sdk.ErrNotFound) {
+				log.Info(ctx, "workflow %s doesn't exist anymore. Cleaning all versions", w.WorkflowName)
+				cleanAll = true
+			}
+		}
+	}
+
+	if cleanAll {
+		versionsToClean = versions
+	} else {
+		versionsToClean = versions[0 : len(versions)-int(nbVersionToKeep)]
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return sdk.WithStack(err)
+	}
+	defer tx.Rollback()
+	for _, v := range versionsToClean {
+		wkfVersion, err := workflow_v2.LoadWorkflowVersion(ctx, tx, w.ProjectKey, w.WorkflowVCS, w.WorkflowRepository, w.WorkflowName, v.String())
+		if err != nil {
+			log.ErrorWithStackTrace(ctx, err)
+			continue
+		}
+		if err := workflow_v2.DeleteWorkflowVersion(ctx, tx, wkfVersion); err != nil {
+			log.ErrorWithStackTrace(ctx, err)
+			continue
+		}
+		log.Info(ctx, "version %s deleted for %s", v.String(), w.String())
+	}
+	return sdk.WithStack(tx.Commit())
+}
+
 func (a *API) cleanProjectEntities(ctx context.Context, entityRetention time.Duration) {
 	ticker := time.NewTicker(time.Duration(a.Config.Entity.RoutineDelay) * time.Minute)
 	defer ticker.Stop()