diff --git a/tfe/resource_tfe_workspace_run.go b/tfe/resource_tfe_workspace_run.go index 20d9e0212..2c99c0b0c 100644 --- a/tfe/resource_tfe_workspace_run.go +++ b/tfe/resource_tfe_workspace_run.go @@ -32,13 +32,14 @@ var planPendingStatuses = map[tfe.RunStatus]bool{ } var applyPendingStatuses = map[tfe.RunStatus]bool{ - tfe.RunConfirmed: true, - tfe.RunApplyQueued: true, - tfe.RunApplying: true, - tfe.RunQueuing: true, - tfe.RunFetching: true, - tfe.RunQueuingApply: true, - tfe.RunPreApplyRunning: true, + tfe.RunConfirmed: true, + tfe.RunApplyQueued: true, + tfe.RunApplying: true, + tfe.RunQueuing: true, + tfe.RunFetching: true, + tfe.RunQueuingApply: true, + tfe.RunPreApplyRunning: true, + tfe.RunPreApplyCompleted: true, } var applyDoneStatuses = map[tfe.RunStatus]bool{ @@ -54,15 +55,16 @@ var confirmationPendingStatuses = map[tfe.RunStatus]bool{ } var confirmationDoneStatuses = map[tfe.RunStatus]bool{ - tfe.RunConfirmed: true, - tfe.RunApplyQueued: true, - tfe.RunApplying: true, - tfe.RunPrePlanCompleted: true, - tfe.RunPrePlanRunning: true, - tfe.RunQueuingApply: true, + tfe.RunConfirmed: true, + tfe.RunApplyQueued: true, + tfe.RunApplying: true, + tfe.RunPrePlanCompleted: true, + tfe.RunPrePlanRunning: true, + tfe.RunQueuingApply: true, + tfe.RunPreApplyCompleted: true, } -var policyOverridenStatuses = map[tfe.RunStatus]bool{ +var policyOverriddenStatuses = map[tfe.RunStatus]bool{ tfe.RunPolicyChecked: true, tfe.RunConfirmed: true, tfe.RunApplyQueued: true, diff --git a/tfe/workspace_run_helpers.go b/tfe/workspace_run_helpers.go index 8d2a6e751..35870adf4 100644 --- a/tfe/workspace_run_helpers.go +++ b/tfe/workspace_run_helpers.go @@ -15,23 +15,9 @@ import ( ) func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun bool, currentRetryAttempts int) error { - var runArgs map[string]interface{} - - if isDestroyRun { - // destroy block is optional, if it is not set then destroy action is noop for a destroy type run - destroyArgs, ok := d.GetOk("destroy") - if !ok { - return nil - } - runArgs = destroyArgs.([]interface{})[0].(map[string]interface{}) - } else { - createArgs, ok := d.GetOk("apply") - if !ok { - // apply block is optional, if it is not set then set a random ID to allow for consistent result after apply ops - d.SetId(fmt.Sprintf("%d", rand.New(rand.NewSource(time.Now().UnixNano())).Int())) - return nil - } - runArgs = createArgs.([]interface{})[0].(map[string]interface{}) + runArgs := getRunArgs(d, isDestroyRun) + if runArgs == nil { + return nil } retryBOMin := runArgs["retry_backoff_min"].(int) @@ -63,39 +49,7 @@ func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun b waitForRun := runArgs["wait_for_run"].(bool) manualConfirm := runArgs["manual_confirm"].(bool) - // In fire-and-forget mode (waitForRun=false), autoapply is set to !manualConfirm - // This should be intuitive, as "manual confirm" is the opposite of "auto apply" - // - // In apply-and-wait mode (waitForRun=true), autoapply is set to false to give the tfe_workspace_run resource full control of run confirmation. - autoApply := false - if !waitForRun { - autoApply = !manualConfirm - } - - runConfig := tfe.RunCreateOptions{ - Workspace: ws, - IsDestroy: tfe.Bool(isDestroyRun), - Message: tfe.String(fmt.Sprintf( - "Triggered by tfe_workspace_run resource via terraform-provider-tfe on %s", - time.Now().Format(time.UnixDate), - )), - AutoApply: tfe.Bool(autoApply), - } - log.Printf("[DEBUG] Create run for workspace: %s", ws.ID) - run, err := config.Client.Runs.Create(ctx, runConfig) - if err != nil { - return fmt.Errorf( - "error creating run for workspace %s: %w", ws.ID, err) - } - - if run == nil { - log.Printf("[ERROR] The client returned both a nil run and nil error, this should not happen") - return fmt.Errorf( - "the client returned both a nil run and nil error for workspace %s, this should not happen", ws.ID) - } - - log.Printf("[DEBUG] Run %s created for workspace %s", run.ID, ws.ID) - hasPostPlanTaskStage, err := readPostPlanTaskStageInRun(meta, run.ID) + run, err := createRun(config.Client, waitForRun, manualConfirm, isDestroyRun, ws) if err != nil { return err } @@ -107,12 +61,17 @@ func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun b } isPlanOp := true - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, planPendingStatuses, isPlanComplete) + hasPostPlanTaskStage, err := readPostPlanTaskStageInRun(config.Client, run.ID) + if err != nil { + return err + } + + run, err = awaitRun(config.Client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, planPendingStatuses, isPlanComplete) if err != nil { return err } - if run.Status == tfe.RunErrored || run.Status == tfe.RunStatus(tfe.PolicySoftFailed) { + if (run.Status == tfe.RunErrored) || (run.Status == tfe.RunStatus(tfe.PolicySoftFailed)) { if retry && currentRetryAttempts < retryMaxAttempts { currentRetryAttempts++ log.Printf("[INFO] Run errored during plan, retrying run, retry count: %d", currentRetryAttempts) @@ -124,14 +83,14 @@ func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun b if run.Status == tfe.RunPolicyOverride { log.Printf("[INFO] Policy check soft-failed, awaiting manual override for run %q", run.ID) - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, policyOverridePendingStatuses, isManuallyOverriden) + run, err = awaitRun(config.Client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, policyOverridePendingStatuses, isManuallyOverriden) if err != nil { return err } } if !run.HasChanges && !run.AllowEmptyApply { - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatuses, isPlannedAndFinished) + run, err = awaitRun(config.Client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatuses, isPlannedAndFinished) if err != nil { return err } @@ -144,42 +103,114 @@ func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun b } // wait for run to be comfirmable before attempting to confirm - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatuses, isConfirmable) + run, err = awaitRun(config.Client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatuses, isConfirmable) if err != nil { return err } + + err = confirmRun(config.Client, manualConfirm, isPlanOp, hasPostPlanTaskStage, run, ws) + if err != nil { + return err + } + + isPlanOp = false + run, err = awaitRun(config.Client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, applyPendingStatuses, isCompleted) + if err != nil { + return err + } + + return completeOrRetryRun(meta, run, d, retry, currentRetryAttempts, retryMaxAttempts, isDestroyRun) +} + +func getRunArgs(d *schema.ResourceData, isDestroyRun bool) map[string]interface{} { + var runArgs map[string]interface{} + + if isDestroyRun { + // destroy block is optional, if it is not set then destroy action is noop for a destroy type run + destroyArgs, ok := d.GetOk("destroy") + if !ok { + return nil + } + runArgs = destroyArgs.([]interface{})[0].(map[string]interface{}) + } else { + createArgs, ok := d.GetOk("apply") + if !ok { + // apply block is optional, if it is not set then set a random ID to allow for consistent result after apply ops + d.SetId(fmt.Sprintf("%d", rand.New(rand.NewSource(time.Now().UnixNano())).Int())) + return nil + } + runArgs = createArgs.([]interface{})[0].(map[string]interface{}) + } + + return runArgs +} + +func createRun(client *tfe.Client, waitForRun bool, manualConfirm bool, isDestroyRun bool, ws *tfe.Workspace) (*tfe.Run, error) { + // In fire-and-forget mode (waitForRun=false), autoapply is set to !manualConfirm + // This should be intuitive, as "manual confirm" is the opposite of "auto apply" + // + // In apply-and-wait mode (waitForRun=true), autoapply is set to false to give the tfe_workspace_run resource full control of run confirmation. + autoApply := false + if !waitForRun { + autoApply = !manualConfirm + } + + runConfig := tfe.RunCreateOptions{ + Workspace: ws, + IsDestroy: tfe.Bool(isDestroyRun), + Message: tfe.String(fmt.Sprintf( + "Triggered by tfe_workspace_run resource via terraform-provider-tfe on %s", + time.Now().Format(time.UnixDate), + )), + AutoApply: tfe.Bool(autoApply), + } + log.Printf("[DEBUG] Create run for workspace: %s", ws.ID) + run, err := client.Runs.Create(ctx, runConfig) + if err != nil { + return nil, fmt.Errorf( + "error creating run for workspace %s: %w", ws.ID, err) + } + + if run == nil { + log.Printf("[ERROR] The client returned both a nil run and nil error, this should not happen") + return nil, fmt.Errorf( + "the client returned both a nil run and nil error for workspace %s, this should not happen", ws.ID) + } + + log.Printf("[DEBUG] Run %s created for workspace %s", run.ID, ws.ID) + return run, nil +} + +func confirmRun(client *tfe.Client, manualConfirm bool, isPlanOp bool, hasPostPlanTaskStage bool, run *tfe.Run, ws *tfe.Workspace) error { // if human approval is required, an apply will auto kick off when run is manually approved if manualConfirm { confirmationPendingStatus := map[tfe.RunStatus]bool{} confirmationPendingStatus[run.Status] = true log.Printf("[INFO] Plan complete, waiting for manual confirm before proceeding run %q", run.ID) - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatus, isConfirmed) + _, err := awaitRun(client, run.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, confirmationPendingStatus, isConfirmed) if err != nil { return err } } else { // if human approval is NOT required, go ahead and kick off an apply log.Printf("[INFO] Plan complete, confirming an apply for run %q", run.ID) - err = config.Client.Runs.Apply(ctx, run.ID, tfe.RunApplyOptions{ + err := client.Runs.Apply(ctx, run.ID, tfe.RunApplyOptions{ Comment: tfe.String(fmt.Sprintf("Run confirmed by tfe_workspace_run resource via terraform-provider-tfe on %s", time.Now().Format(time.UnixDate))), }) if err != nil { - refreshed, fetchErr := config.Client.Runs.Read(ctx, run.ID) + refreshed, fetchErr := client.Runs.Read(ctx, run.ID) if fetchErr != nil { err = fmt.Errorf("%w\n additionally, got an error while reading the run: %s", err, fetchErr.Error()) } return fmt.Errorf("run errored while applying run %s (waited til status %s, currently status %s): %w", run.ID, run.Status, refreshed.Status, err) } } + return nil +} - isPlanOp = false - run, err = awaitRun(meta, run.ID, ws.ID, ws.Organization.Name, isPlanOp, hasPostPlanTaskStage, applyPendingStatuses, isCompleted) - if err != nil { - return err - } - +func completeOrRetryRun(meta interface{}, run *tfe.Run, d *schema.ResourceData, retry bool, currentRetryAttempts int, retryMaxAttempts int, isDestroyRun bool) error { switch run.Status { case tfe.RunApplied: log.Printf("[INFO] Apply complete for run %q", run.ID) @@ -198,17 +229,15 @@ func createWorkspaceRun(d *schema.ResourceData, meta interface{}, isDestroyRun b } } -func awaitRun(meta interface{}, runID string, wsID string, organization string, isPlanOp bool, +func awaitRun(client *tfe.Client, runID string, organization string, isPlanOp bool, hasPostPlanTaskStage bool, runPendingStatus map[tfe.RunStatus]bool, isDone func(*tfe.Run, bool) bool) (*tfe.Run, error) { - config := meta.(ConfiguredClient) - for i := 0; ; i++ { select { case <-ctx.Done(): return nil, fmt.Errorf("context canceled: %w", ctx.Err()) case <-time.After(backoff(backoffMin, backoffMax, i)): log.Printf("[DEBUG] Polling run %s", runID) - run, err := config.Client.Runs.Read(ctx, runID) + run, err := client.Runs.Read(ctx, runID) if err != nil { log.Printf("[ERROR] Could not read run %s: %v", runID, err) continue @@ -221,64 +250,7 @@ func awaitRun(meta interface{}, runID string, wsID string, organization string, log.Printf("[INFO] Run %s has reached a terminal state: %s", runID, run.Status) return run, nil case runIsInProgress: - log.Printf("[DEBUG] Reading workspace %s", wsID) - ws, err := config.Client.Workspaces.ReadByID(ctx, wsID) - if err != nil { - log.Printf("[ERROR] Unable to read workspace %s: %v", wsID, err) - continue - } - - // if the workspace is locked and the current run has not started, assume that workspace was locked for other purposes. - // display a message to indicate that the workspace is waiting to be manually unlocked before the run can proceed - if ws.Locked && ws.CurrentRun != nil { - currentRun, err := config.Client.Runs.Read(ctx, ws.CurrentRun.ID) - if err != nil { - log.Printf("[ERROR] Unable to read current run %s: %v", ws.CurrentRun.ID, err) - continue - } - - if currentRun.Status == tfe.RunPending { - log.Printf("[INFO] Waiting for manually locked workspace to be unlocked") - continue - } - } - - // if this run is the current run in it's workspace, display it's position in the organization queue - if ws.CurrentRun != nil && ws.CurrentRun.ID == runID { - runPositionInOrg, err := readRunPositionInOrgQueue(meta, runID, organization) - if err != nil { - log.Printf("[ERROR] Unable to read run position in organization queue %v", err) - continue - } - - orgCapacity, err := config.Client.Organizations.ReadCapacity(ctx, organization) - if err != nil { - log.Printf("[ERROR] Unable to read capacity for organization %s: %v", organization, err) - continue - } - if runPositionInOrg > 0 { - log.Printf("[INFO] Waiting for %d queued run(s) before starting run", runPositionInOrg-orgCapacity.Running) - continue - } - } - - // if this run is not the current run in it's workspace, display it's position in the workspace queue - runPositionInWorkspace, err := readRunPositionInWorkspaceQueue(meta, runID, wsID, isPlanOp, ws.CurrentRun) - if err != nil { - log.Printf("[ERROR] Unable to read run position in workspace queue %v", err) - continue - } - - if runPositionInWorkspace > 0 { - log.Printf( - "[INFO] Waiting for %d run(s) to finish in workspace %s before being queued...", - runPositionInWorkspace, - ws.Name, - ) - continue - } - - log.Printf("[INFO] Waiting for run %s, status is %s", runID, run.Status) + logRunProgress(client, organization, isPlanOp, run) case run.Status == tfe.RunCanceled: log.Printf("[INFO] Run %s has been canceled, status is %s", runID, run.Status) return nil, fmt.Errorf("run %s has been canceled, status is %s", runID, run.Status) @@ -290,13 +262,73 @@ func awaitRun(meta interface{}, runID string, wsID string, organization string, } } -func readRunPositionInOrgQueue(meta interface{}, runID string, organization string) (int, error) { - config := meta.(ConfiguredClient) +func logRunProgress(client *tfe.Client, organization string, isPlanOp bool, run *tfe.Run) { + log.Printf("[DEBUG] Reading workspace %s", run.Workspace.ID) + ws, err := client.Workspaces.ReadByID(ctx, run.Workspace.ID) + if err != nil { + log.Printf("[ERROR] Unable to read workspace %s: %v", run.Workspace.ID, err) + return + } + + // if the workspace is locked and the current run has not started, assume that workspace was locked for other purposes. + // display a message to indicate that the workspace is waiting to be manually unlocked before the run can proceed + if ws.Locked && ws.CurrentRun != nil { + currentRun, err := client.Runs.Read(ctx, ws.CurrentRun.ID) + if err != nil { + log.Printf("[ERROR] Unable to read current run %s: %v", ws.CurrentRun.ID, err) + return + } + + if currentRun.Status == tfe.RunPending { + log.Printf("[INFO] Waiting for manually locked workspace to be unlocked") + return + } + } + + // if this run is the current run in it's workspace, display it's position in the organization queue + if ws.CurrentRun != nil && ws.CurrentRun.ID == run.ID { + runPositionInOrg, err := readRunPositionInOrgQueue(client, run.ID, organization) + if err != nil { + log.Printf("[ERROR] Unable to read run position in organization queue %v", err) + return + } + + orgCapacity, err := client.Organizations.ReadCapacity(ctx, organization) + if err != nil { + log.Printf("[ERROR] Unable to read capacity for organization %s: %v", organization, err) + return + } + if runPositionInOrg > 0 { + log.Printf("[INFO] Waiting for %d queued run(s) before starting run", runPositionInOrg-orgCapacity.Running) + return + } + } + + // if this run is not the current run in it's workspace, display it's position in the workspace queue + runPositionInWorkspace, err := readRunPositionInWorkspaceQueue(client, run.ID, ws.ID, isPlanOp, ws.CurrentRun) + if err != nil { + log.Printf("[ERROR] Unable to read run position in workspace queue %v", err) + return + } + + if runPositionInWorkspace > 0 { + log.Printf( + "[INFO] Waiting for %d run(s) to finish in workspace %s before being queued...", + runPositionInWorkspace, + ws.Name, + ) + return + } + + log.Printf("[INFO] Waiting for run %s, status is %s", run.ID, run.Status) +} + +func readRunPositionInOrgQueue(client *tfe.Client, runID string, organization string) (int, error) { position := 0 options := tfe.ReadRunQueueOptions{} for { - runQueue, err := config.Client.Organizations.ReadRunQueue(ctx, organization, options) + runQueue, err := client.Organizations.ReadRunQueue(ctx, organization, options) if err != nil { return position, fmt.Errorf("unable to read run queue for organization %s: %w", organization, err) } @@ -318,14 +350,13 @@ func readRunPositionInOrgQueue(meta interface{}, runID string, organization stri return position, nil } -func readRunPositionInWorkspaceQueue(meta interface{}, runID string, wsID string, isPlanOp bool, currentRun *tfe.Run) (int, error) { - config := meta.(ConfiguredClient) +func readRunPositionInWorkspaceQueue(client *tfe.Client, runID string, wsID string, isPlanOp bool, currentRun *tfe.Run) (int, error) { position := 0 options := tfe.RunListOptions{} found := false for { - runList, err := config.Client.Runs.List(ctx, wsID, &options) + runList, err := client.Runs.List(ctx, wsID, &options) if err != nil { return position, fmt.Errorf("unable to read run list for workspace %s: %w", wsID, err) } @@ -377,13 +408,12 @@ func backoff(min, max float64, iter int) time.Duration { return time.Duration(backoff) * time.Millisecond } -func readPostPlanTaskStageInRun(meta interface{}, runID string) (bool, error) { - config := meta.(ConfiguredClient) +func readPostPlanTaskStageInRun(client *tfe.Client, runID string) (bool, error) { hasPostPlanTaskStage := false options := tfe.TaskStageListOptions{} for { - taskStages, err := config.Client.TaskStages.List(ctx, runID, &options) + taskStages, err := client.TaskStages.List(ctx, runID, &options) if err != nil { return hasPostPlanTaskStage, fmt.Errorf("[ERROR] Could not read task stages for run %s: %v", runID, err) } @@ -447,7 +477,7 @@ func isPlanComplete(run *tfe.Run, hasPostPlanTaskStage bool) bool { } func isManuallyOverriden(run *tfe.Run, hasPostPlanTaskStage bool) bool { - _, found := policyOverridenStatuses[run.Status] + _, found := policyOverriddenStatuses[run.Status] return found } diff --git a/tfe/workspace_run_helpers_test.go b/tfe/workspace_run_helpers_test.go index 8c24c592e..b1ecf3bb7 100644 --- a/tfe/workspace_run_helpers_test.go +++ b/tfe/workspace_run_helpers_test.go @@ -81,7 +81,6 @@ func MockRunsListForWorkspaceQueue(t *testing.T, client *tfe.Client, workspaceID } func TestReadRunPositionInWorkspaceQueue(t *testing.T) { - defaultOrganization := "my-org" client := testTfeClient(t, testClientOptions{}) MockRunsListForWorkspaceQueue(t, client, "ws-1", "ws-2") @@ -114,7 +113,7 @@ func TestReadRunPositionInWorkspaceQueue(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { position, err := readRunPositionInWorkspaceQueue( - ConfiguredClient{Client: client, Organization: defaultOrganization}, + client, testCase.currentRunID, testCase.workspace, false, @@ -224,7 +223,7 @@ func TestReadRunPositionInOrgQueue(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { position, err := readRunPositionInOrgQueue( - ConfiguredClient{Client: client, Organization: defaultOrganization}, + client, "run-02", testCase.orgName, )