Skip to content

Commit 65ed2b5

Browse files
motatoesmotatoes
authored
always load digger.yml from branch, not from database repo (#1116)
* always load digger.yml from repo branch, not from database repo
1 parent 5dbcbfd commit 65ed2b5

File tree

5 files changed

+44
-68
lines changed

5 files changed

+44
-68
lines changed

‎backend/controllers/github.go‎

Lines changed: 25 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -391,13 +391,14 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) e
391391
log.Printf("Error getting github service: %v", err)
392392
return fmt.Errorf("error getting github service")
393393
}
394-
utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) {
394+
utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error {
395395
dat, err := os.ReadFile(path.Join(dir, "digger.yml"))
396396
//TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that
397397
if err != nil {
398398
log.Printf("ERROR fetching digger.yml file: %v", err)
399399
}
400400
models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo)
401+
return nil
401402
})
402403
}
403404

@@ -412,7 +413,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
412413
cloneURL := *payload.Repo.CloneURL
413414
prNumber := *payload.PullRequest.Number
414415

415-
ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
416+
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
416417
if err != nil {
417418
log.Printf("getDiggerConfig error: %v", err)
418419
return fmt.Errorf("error getting digger config")
@@ -466,14 +467,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
466467
impactedJobsMap[j.ProjectName] = j
467468
}
468469

469-
repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
470-
if err != nil {
471-
log.Printf("GetRepoByInstallationId error: %v", err)
472-
utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: GetRepoByInstallationId error: %v", err))
473-
return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err)
474-
}
475470
batchType := getBatchType(jobsForImpactedProjects)
476-
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType)
471+
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
477472
if err != nil {
478473
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
479474
utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -490,54 +485,39 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
490485
return nil
491486
}
492487

493-
func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (*dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
488+
func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
494489
ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
495490
if err != nil {
496491
log.Printf("Error getting github service: %v", err)
497-
return nil, nil, nil, nil, fmt.Errorf("error getting github service")
492+
return "", nil, nil, nil, nil, fmt.Errorf("error getting github service")
498493
}
499494
var prBranch string
500495
prBranch, err = ghService.GetBranchName(prNumber)
501496
if err != nil {
502497
log.Printf("Error getting branch name: %v", err)
503-
return nil, nil, nil, nil, fmt.Errorf("error getting branch name")
504-
}
505-
506-
repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
507-
if err != nil {
508-
return nil, nil, nil, nil, err
509-
}
510-
if err != nil {
511-
log.Printf("Error getting repo: %v", err)
512-
return nil, nil, nil, nil, fmt.Errorf("error getting repo")
513-
}
514-
515-
configYaml, err := dg_configuration.LoadDiggerConfigYamlFromString(repo.DiggerConfig)
516-
if err != nil {
517-
log.Printf("Error loading digger config: %v", err)
518-
return nil, nil, nil, nil, fmt.Errorf("error loading digger config")
498+
return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name")
519499
}
520500

521-
log.Printf("Digger config loadded successfully\n")
522-
523-
if configYaml.GenerateProjectsConfig != nil {
524-
err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) {
525-
dg_configuration.HandleYamlProjectGeneration(configYaml, dir)
526-
})
501+
var config *dg_configuration.DiggerConfig
502+
var diggerYmlStr string
503+
var dependencyGraph graph.Graph[string, dg_configuration.Project]
504+
err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) error {
505+
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
506+
diggerYmlStr = string(diggerYmlBytes)
507+
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir)
527508
if err != nil {
528-
log.Printf("Error generating projects: %v", err)
529-
return nil, nil, nil, nil, fmt.Errorf("error generating projects")
509+
log.Printf("Error loading digger config: %v", err)
510+
return err
530511
}
531-
}
532-
533-
config, dependencyGraph, err := loadDiggerConfig(configYaml)
534-
512+
return nil
513+
})
535514
if err != nil {
536-
log.Printf("Error loading digger config: %v", err)
537-
return nil, nil, nil, nil, fmt.Errorf("error loading digger config")
515+
log.Printf("Error generating projects: %v", err)
516+
return "", nil, nil, nil, nil, fmt.Errorf("error generating projects")
538517
}
539-
log.Printf("Digger config parsed successfully\n")
540-
return ghService, config, dependencyGraph, &prBranch, nil
518+
519+
log.Printf("Digger config loadded successfully\n")
520+
return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, nil
541521
}
542522

543523
func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
@@ -581,7 +561,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
581561
return nil
582562
}
583563

584-
ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
564+
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
585565
if err != nil {
586566
log.Printf("getDiggerConfig error: %v", err)
587567
return fmt.Errorf("error getting digger config")
@@ -649,13 +629,8 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
649629
impactedProjectsJobMap[j.ProjectName] = j
650630
}
651631

652-
repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
653-
if err != nil {
654-
log.Printf("GetRepoByInstallationId error: %v", err)
655-
return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err)
656-
}
657632
batchType := getBatchType(jobs)
658-
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType)
633+
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
659634
if err != nil {
660635
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
661636
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

‎backend/templates/repo_add.tmpl‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,13 @@
99
<div class="card-body">
1010

1111
{{template "notifications" . }}
12-
<form method="POST">
13-
<div class="row">
14-
<div class="col">
15-
<div class="mb-3"><label class="form-label" ><strong>Digger Config</strong></label>
16-
<textarea class="form-control prism-live language-yaml" type="text" id="diggerconfig" name="diggerconfig">{{.Repo.DiggerConfig}}</textarea>
17-
</div>
12+
<div class="row">
13+
<div class="col">
14+
<div class="mb-3"><label class="form-label" ><strong>Digger Config</strong></label>
15+
<textarea class="form-control prism-live language-yaml" type="text" id="diggerconfig" name="diggerconfig" readonly>{{.Repo.DiggerConfig}}</textarea>
1816
</div>
1917
</div>
20-
<div class="mb-3"><button class="btn btn-primary btn-sm" type="submit">Update</button></div>
21-
</form>
18+
</div>
2219
</div>
2320
</div>
2421
</div>

‎backend/utils/github.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func createTempDir() string {
2727
return tempDir
2828
}
2929

30-
type action func(string)
30+
type action func(string) error
3131

3232
func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error {
3333
dir := createTempDir()
@@ -50,10 +50,14 @@ func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action
5050
log.Printf("PlainClone error: %v\n", err)
5151
return err
5252
}
53+
defer os.RemoveAll(dir)
5354

54-
action(dir)
55+
err = action(dir)
56+
if err != nil {
57+
log.Printf("error performing action: %v", err)
58+
return err
59+
}
5560

56-
defer os.RemoveAll(dir)
5761
return nil
5862

5963
}

‎backend/utils/github_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,21 @@ func init() {
1313
}
1414

1515
func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) {
16-
f := func(d string) {}
16+
f := func(d string) error { return nil }
1717
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "invalid-token", f)
1818
assert.NotNil(t, err)
1919
}
2020

2121
func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) {
2222
token := os.Getenv("GITHUB_PAT_TOKEN")
23-
f := func(d string) {}
23+
f := func(d string) error { return nil }
2424
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", token, f)
2525
assert.Nil(t, err)
2626
}
2727

2828
func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) {
2929
token := os.Getenv("GITHUB_PAT_TOKEN")
30-
f := func(d string) {}
30+
f := func(d string) error { return nil }
3131
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", token, f)
3232
assert.NotNil(t, err)
3333
}

‎go.work.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -510,8 +510,6 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf
510510
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
511511
github.com/alecthomas/kingpin/v2 v2.3.1 h1:ANLJcKmQm4nIaog7xdr/id6FM6zm5hHnfZrvtKPxqGg=
512512
github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
513-
github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
514-
github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
515513
github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
516514
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
517515
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
@@ -801,6 +799,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
801799
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI=
802800
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk=
803801
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM=
802+
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
803+
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
804804
github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw=
805805
github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA=
806806
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
@@ -955,7 +955,6 @@ github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiN
955955
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
956956
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
957957
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
958-
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
959958
github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs=
960959
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
961960
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
@@ -981,7 +980,6 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
981980
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
982981
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
983982
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
984-
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
985983
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
986984
github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
987985
github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
@@ -1005,6 +1003,8 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbm
10051003
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
10061004
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
10071005
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
1006+
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
1007+
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
10081008
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
10091009
github.com/stuart-warren/yamlfmt v0.1.2/go.mod h1:X5TuPH+hf4O0U1KBvNqygvHbvAnoi9Wyl9BbtPv8SZk=
10101010
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=

0 commit comments

Comments
 (0)