Skip to content

Commit 2efa7e5

Browse files
SpartakovicSpartakovic
authored
feat: add plan storage to enable different types of storages (#177)
* feat: add plan storage to enable different types of storages
1 parent 6136c96 commit 2efa7e5

File tree

11 files changed

+341
-148
lines changed

11 files changed

+341
-148
lines changed

‎action.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ inputs:
4949
description: Terraform version
5050
required: false
5151
default: v1.4.5
52+
upload-plan-destination:
53+
description: Destination to upload the plan to. gcp and github are currently supported
54+
required: false
5255

5356
outputs:
5457
output:
@@ -73,7 +76,7 @@ runs:
7376

7477
- name: Validate Input Configuration for Google
7578
run: |
76-
if [[ -z "${{ toJSON(inputs.google-auth-credentials) }}" && -z "${{ inputs.google-workload-identity-provider }}" ]]; then
79+
if [[ -z ${{ toJSON(inputs.google-auth-credentials) }} && -z "${{ inputs.google-workload-identity-provider }}" ]]; then
7780
echo "Either 'google-auth-credentials' or 'google-workload-identity-provider' input must be specified with 'setup-google-cloud'"
7881
elif [[ ! -z "${{ inputs.google-workload-identity-provider }}" && -z "${{ inputs.google-service-account }}" ]]; then
7982
echo "'google-service-account' input must be specified with 'google-workload-identity-provider'"
@@ -87,7 +90,7 @@ runs:
8790
- name: Set up Google Auth Using A Service Account Key
8891
uses: google-github-actions/auth@v1
8992
with:
90-
credentials_json: ${{ inputs.google-auth-credentials }}
93+
credentials_json: '${{ inputs.google-auth-credentials }}'
9194
if: ${{ inputs.setup-google-cloud == 'true' && inputs.google-auth-credentials != '' }}
9295

9396
- name: Set up Google Auth Using Workload Identity Federation
@@ -127,6 +130,8 @@ runs:
127130
- name: build and run digger
128131
if: ${{ !startsWith(github.action_ref, 'v') }}
129132
shell: bash
133+
env:
134+
PLAN_UPLOAD_DESTINATION: ${{ inputs.upload-plan-destination }}
130135
run: |
131136
cd ${{ github.action_path }}
132137
go build -o digger ./cmd/digger
@@ -138,6 +143,7 @@ runs:
138143
if: ${{ startsWith(github.action_ref, 'v') }}
139144
env:
140145
actionref: ${{ github.action_ref }}
146+
PLAN_UPLOAD_DESTINATION: ${{ inputs.upload-plan-destination }}
141147
id: digger
142148
shell: bash
143149
run: |
@@ -161,6 +167,7 @@ runs:
161167
name: ${{ steps.artifact.outputs.artifact }}
162168
path: '${{ github.workspace }}/**/*.tfplan'
163169
retention-days: 14
170+
if: ${{ inputs.upload-plan-destination == 'github' }}
164171

165172
branding:
166173
icon: globe

‎cmd/digger/main.go‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package main
22

33
import (
4+
"context"
45
"digger/pkg/configuration"
56
"digger/pkg/digger"
6-
"digger/pkg/github"
7+
"digger/pkg/gcp"
8+
dg_github "digger/pkg/github"
79
"digger/pkg/models"
810
"digger/pkg/utils"
911
"fmt"
12+
"github.com/google/go-github/v51/github"
1013
"log"
1114
"os"
1215
"strings"
@@ -64,7 +67,7 @@ func main() {
6467
eventName := parsedGhContext.EventName
6568
splitRepositoryName := strings.Split(parsedGhContext.Repository, "/")
6669
repoOwner, repositoryName := splitRepositoryName[0], splitRepositoryName[1]
67-
githubPrService := github.NewGithubPullRequestService(ghToken, repositoryName, repoOwner)
70+
githubPrService := dg_github.NewGithubPullRequestService(ghToken, repositoryName, repoOwner)
6871

6972
impactedProjects, prNumber, err := digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
7073
if err != nil {
@@ -85,7 +88,9 @@ func main() {
8588
println("GitHub event converted to commands successfully")
8689
logCommands(commandsToRunPerProject)
8790

88-
allAppliesSuccess, err := digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, "")
91+
planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, prNumber)
92+
93+
allAppliesSuccess, err := digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, "")
8994
if err != nil {
9095
reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to run commands. %s", err), 8)
9196
}
@@ -107,6 +112,35 @@ func main() {
107112

108113
}
109114

115+
func newPlanStorage(ghToken string, repoOwner string, repositoryName string, prNumber int) utils.PlanStorage {
116+
var planStorage utils.PlanStorage
117+
118+
if os.Getenv("PLAN_UPLOAD_DESTINATION") == "github" {
119+
zipManager := utils.Zipper{}
120+
planStorage = &utils.GithubPlanStorage{
121+
Client: github.NewTokenClient(context.Background(), ghToken),
122+
Owner: repoOwner,
123+
RepoName: repositoryName,
124+
PullRequestNumber: prNumber,
125+
ZipManager: zipManager,
126+
}
127+
} else if os.Getenv("PLAN_UPLOAD_DESTINATION") == "gcp" {
128+
ctx, client := gcp.GetGoogleStorageClient()
129+
130+
bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_BUCKET"))
131+
if bucketName == "" {
132+
reportErrorAndExit(repoOwner, fmt.Sprintf("GOOGLE_STORAGE_BUCKET is not defined"), 9)
133+
}
134+
bucket := client.Bucket(bucketName)
135+
planStorage = &utils.PlanStorageGcp{
136+
Client: client,
137+
Bucket: bucket,
138+
Context: ctx,
139+
}
140+
}
141+
return planStorage
142+
}
143+
110144
func logImpactedProjects(projects []configuration.Project, prNumber int) {
111145
logMessage := fmt.Sprintf("Following projects are impacted by pull request #%d\n", prNumber)
112146
for _, p := range projects {

‎cmd/digger/main_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -873,10 +873,12 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
873873
diggerConfig := configuration.DiggerConfig{}
874874
lock := &utils.MockLock{}
875875
prManager := &utils.MockPullRequestManager{ChangedFiles: []string{"dev/test.tf"}}
876+
planStorage := &utils.MockPlanStorage{}
877+
876878
impactedProjects, prNumber, err := digger.ProcessGitHubEvent(ghEvent, &diggerConfig, prManager)
877879

878880
commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
879-
_, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, "")
881+
_, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, planStorage, "")
880882

881883
assert.NoError(t, err)
882884
if err != nil {
@@ -895,10 +897,11 @@ func TestGitHubNewCommentContext(t *testing.T) {
895897
diggerConfig := configuration.DiggerConfig{}
896898
lock := &utils.MockLock{}
897899
prManager := &utils.MockPullRequestManager{ChangedFiles: []string{"dev/test.tf"}}
900+
planStorage := &utils.MockPlanStorage{}
898901
impactedProjects, prNumber, err := digger.ProcessGitHubEvent(ghEvent, &diggerConfig, prManager)
899902

900903
commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
901-
_, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, "")
904+
_, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, planStorage, "")
902905

903906
assert.NoError(t, err)
904907
if err != nil {

‎pkg/digger/digger.go‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func ProcessGitHubEvent(ghEvent models.Event, diggerConfig *configuration.Digger
5454
return impactedProjects, prNumber, nil
5555
}
5656

57-
func RunCommandsPerProject(commandsPerProject []ProjectCommand, repoOwner string, repoName string, eventName string, prNumber int, prManager github.PullRequestManager, lock utils.Lock, workingDir string) (bool, error) {
57+
func RunCommandsPerProject(commandsPerProject []ProjectCommand, repoOwner string, repoName string, eventName string, prNumber int, prManager github.PullRequestManager, lock utils.Lock, planStorage utils.PlanStorage, workingDir string) (bool, error) {
5858
allAppliesSuccess := true
5959
appliesPerProject := make(map[string]bool)
6060
for _, projectCommands := range commandsPerProject {
@@ -77,17 +77,16 @@ func RunCommandsPerProject(commandsPerProject []ProjectCommand, repoOwner string
7777
}
7878

7979
commandRunner := CommandRunner{}
80-
zipManager := &utils.Zipper{}
8180

8281
diggerExecutor := DiggerExecutor{
8382
projectCommands.ProjectName,
8483
projectCommands.ApplyStage,
8584
projectCommands.PlanStage,
8685
commandRunner,
87-
zipManager,
8886
terraformExecutor,
8987
prManager,
9088
projectLock,
89+
planStorage,
9190
}
9291
switch command {
9392
case "digger plan":
@@ -299,10 +298,10 @@ type DiggerExecutor struct {
299298
applyStage *configuration.Stage
300299
planStage *configuration.Stage
301300
commandRunner CommandRun
302-
zipManager utils.Zip
303301
terraformExecutor terraform.TerraformExecutor
304302
prManager github.PullRequestManager
305303
lock utils.ProjectLock
304+
planStorage utils.PlanStorage
306305
}
307306

308307
type CommandRun interface {
@@ -372,6 +371,12 @@ func (d DiggerExecutor) Plan(prNumber int) error {
372371
if err != nil {
373372
return fmt.Errorf("error executing plan: %v", err)
374373
}
374+
if d.planStorage != nil {
375+
err = d.planStorage.StorePlan(d.planFileName())
376+
if err != nil {
377+
return fmt.Errorf("error storing plan: %v", err)
378+
}
379+
}
375380
plan := cleanupTerraformPlan(isNonEmptyPlan, err, stdout, stderr)
376381
comment := utils.GetTerraformOutputAsCollapsibleComment("Plan for **"+d.lock.LockId()+"**", plan)
377382
d.prManager.PublishComment(prNumber, comment)
@@ -389,24 +394,13 @@ func (d DiggerExecutor) Plan(prNumber int) error {
389394
}
390395

391396
func (d DiggerExecutor) Apply(prNumber int) error {
392-
plansFilename, err := d.prManager.DownloadLatestPlans(prNumber)
393-
394-
if err != nil {
395-
return fmt.Errorf("error downloading plan: %v", err)
396-
}
397-
398-
if plansFilename == "" {
399-
return fmt.Errorf("no plans found for this PR")
400-
}
401-
402-
plansFilename, err = d.zipManager.GetFileFromZip(plansFilename, d.planFileName())
403-
404-
if err != nil {
405-
return fmt.Errorf("error extracting plan: %v", err)
406-
}
407-
408-
if plansFilename == "" {
409-
return fmt.Errorf("no plans found for this project")
397+
var plansFilename *string
398+
if d.planStorage != nil {
399+
var err error
400+
plansFilename, err = d.planStorage.RetrievePlan(d.planFileName())
401+
if err != nil {
402+
return fmt.Errorf("error retrieving plan: %v", err)
403+
}
410404
}
411405

412406
isMergeable, _, err := d.prManager.IsMergeable(prNumber)

‎pkg/digger/digger_test.go‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@ func (m *MockTerraformExecutor) Init(params []string) (string, string, error) {
3535
return "", "", nil
3636
}
3737

38-
func (m *MockTerraformExecutor) Apply(params []string, plan string) (string, string, error) {
39-
m.Commands = append(m.Commands, RunInfo{"Apply", strings.Join(params, " ") + " " + plan, time.Now()})
38+
func (m *MockTerraformExecutor) Apply(params []string, plan *string) (string, string, error) {
39+
if plan != nil {
40+
params = append(params, *plan)
41+
}
42+
m.Commands = append(m.Commands, RunInfo{"Apply", strings.Join(params, " "), time.Now()})
4043
return "", "", nil
4144
}
4245

@@ -116,14 +119,27 @@ func (m *MockZipper) GetFileFromZip(zipFile string, filename string) (string, er
116119
return "plan", nil
117120
}
118121

122+
type MockPlanStorage struct {
123+
Commands []RunInfo
124+
}
125+
126+
func (m *MockPlanStorage) StorePlan(planFileName string) error {
127+
m.Commands = append(m.Commands, RunInfo{"StorePlan", planFileName, time.Now()})
128+
return nil
129+
}
130+
131+
func (m *MockPlanStorage) RetrievePlan(planFileName string) (*string, error) {
132+
m.Commands = append(m.Commands, RunInfo{"RetrievePlan", planFileName, time.Now()})
133+
return nil, nil
134+
}
135+
119136
func TestCorrectCommandExecutionWhenApplying(t *testing.T) {
120137

121138
commandRunner := &MockCommandRunner{}
122139
terraformExecutor := &MockTerraformExecutor{}
123140
prManager := &MockPRManager{}
124141
lock := &MockProjectLock{}
125-
zipper := &MockZipper{}
126-
142+
planStorage := &MockPlanStorage{}
127143
executor := DiggerExecutor{
128144
applyStage: &configuration.Stage{
129145
Steps: []configuration.Step{
@@ -145,18 +161,18 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) {
145161
},
146162
},
147163
planStage: &configuration.Stage{},
148-
zipManager: zipper,
149164
commandRunner: commandRunner,
150165
terraformExecutor: terraformExecutor,
151166
prManager: prManager,
152167
lock: lock,
168+
planStorage: planStorage,
153169
}
154170

155171
executor.Apply(1)
156172

157-
commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock)
173+
commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage)
158174

159-
assert.Equal(t, []string{"DownloadLatestPlans 1", "IsMergeable 1", "Lock 1", "Init ", "Apply plan", "LockId ", "PublishComment 1 <details>\n <summary>Apply for ****</summary>\n\n ```terraform\n\n ```\n</details>", "Unlock 1", "Run echo", "LockId "}, commandStrings)
175+
assert.Equal(t, []string{"RetrievePlan .tfplan", "IsMergeable 1", "Lock 1", "Init ", "Apply ", "LockId ", "PublishComment 1 <details>\n <summary>Apply for ****</summary>\n\n ```terraform\n\n ```\n</details>", "Unlock 1", "Run echo", "LockId "}, commandStrings)
160176
}
161177

162178
func TestCorrectCommandExecutionWhenPlanning(t *testing.T) {
@@ -165,6 +181,7 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) {
165181
terraformExecutor := &MockTerraformExecutor{}
166182
prManager := &MockPRManager{}
167183
lock := &MockProjectLock{}
184+
planStorage := &MockPlanStorage{}
168185

169186
executor := DiggerExecutor{
170187
applyStage: &configuration.Stage{},
@@ -191,16 +208,17 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) {
191208
terraformExecutor: terraformExecutor,
192209
prManager: prManager,
193210
lock: lock,
211+
planStorage: planStorage,
194212
}
195213

196214
executor.Plan(1)
197215

198-
commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock)
216+
commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage)
199217

200-
assert.Equal(t, []string{"Lock 1", "Init ", "Plan -out .tfplan", "LockId ", "PublishComment 1 <details>\n <summary>Plan for ****</summary>\n\n ```terraform\n\n ```\n</details>", "Run echo", "LockId "}, commandStrings)
218+
assert.Equal(t, []string{"Lock 1", "Init ", "Plan -out .tfplan", "StorePlan .tfplan", "LockId ", "PublishComment 1 <details>\n <summary>Plan for ****</summary>\n\n ```terraform\n\n ```\n</details>", "Run echo", "LockId "}, commandStrings)
201219
}
202220

203-
func allCommandsInOrderWithParams(terraformExecutor *MockTerraformExecutor, commandRunner *MockCommandRunner, prManager *MockPRManager, lock *MockProjectLock) []string {
221+
func allCommandsInOrderWithParams(terraformExecutor *MockTerraformExecutor, commandRunner *MockCommandRunner, prManager *MockPRManager, lock *MockProjectLock, planStorage *MockPlanStorage) []string {
204222
var commands []RunInfo
205223
for _, command := range terraformExecutor.Commands {
206224
commands = append(commands, command)
@@ -214,6 +232,9 @@ func allCommandsInOrderWithParams(terraformExecutor *MockTerraformExecutor, comm
214232
for _, command := range lock.Commands {
215233
commands = append(commands, command)
216234
}
235+
for _, command := range planStorage.Commands {
236+
commands = append(commands, command)
237+
}
217238

218239
sort.Slice(commands, func(i, j int) bool {
219240
return commands[i].Timestamp.Before(commands[j].Timestamp)

0 commit comments

Comments
 (0)