helper/resource: Clarify configuration step error messages

.changes/unreleased/BUG FIXES-20231205-142936.yaml

@@ -0,0 +1,6 @@
+kind: BUG FIXES
+body: 'helper/resource: Clarified error messaging from testing failures, especially
+  when using `TestStep.PlanOnly: true`'
+time: 2023-12-05T14:29:36.098289-05:00
+custom:
+  Issue: "238"

.changes/unreleased/NOTES-20231205-143152.yaml

@@ -0,0 +1,10 @@
+kind: NOTES
+body: 'helper/resource: Error messages generated by the testing logic, which were
+  updated for clarity in this release, are not protected by compatibility promises.
+  While testing logic errors are usable in certain scenarios with `ErrorCheck`
+  and `ExpectError` functionality, error messaging checks should be based on
+  provider-controlled messaging or when appropriate to use other testing
+  features such as `ExpectNonEmptyPlan` instead.'
+time: 2023-12-05T14:31:52.574105-05:00
+custom:
+  Issue: "238"

helper/resource/testing_new_config.go

@@ -221,7 +221,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 		return wd.CreatePlan(ctx, opts...)
 	}, wd, providers)
 	if err != nil {
-		return fmt.Errorf("Error running post-apply plan: %w", err)
+		if step.PlanOnly {
+			return fmt.Errorf("Error running non-refresh plan: %w", err)
+		}
+
+		return fmt.Errorf("Error running post-apply non-refresh plan: %w", err)
 	}
 
 	var plan *tfjson.Plan
@@ -231,13 +235,21 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 		return err
 	}, wd, providers)
 	if err != nil {
-		return fmt.Errorf("Error retrieving post-apply plan: %w", err)
+		if step.PlanOnly {
+			return fmt.Errorf("Error reading saved non-refresh plan: %w", err)
+		}
+
+		return fmt.Errorf("Error reading saved post-apply non-refresh plan: %w", err)
 	}
 
 	// Run post-apply, pre-refresh plan checks
 	if len(step.ConfigPlanChecks.PostApplyPreRefresh) > 0 {
 		err = runPlanChecks(ctx, t, plan, step.ConfigPlanChecks.PostApplyPreRefresh)
 		if err != nil {
+			if step.PlanOnly {
+				return fmt.Errorf("Non-refresh plan checks(s) failed:\n%w", err)
+			}
+
 			return fmt.Errorf("Post-apply, pre-refresh plan check(s) failed:\n%w", err)
 		}
 	}
@@ -250,9 +262,14 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 			return err
 		}, wd, providers)
 		if err != nil {
-			return fmt.Errorf("Error retrieving formatted plan output: %w", err)
+			return fmt.Errorf("Error reading saved human-readable non-refresh plan output: %w", err)
+		}
+
+		if step.PlanOnly {
+			return fmt.Errorf("The non-refresh plan was not empty.\nstdout:\n\n%s", stdout)
 		}
-		return fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", stdout)
+
+		return fmt.Errorf("After applying this test step, the non-refresh plan was not empty.\nstdout:\n\n%s", stdout)
 	}
 
 	// do another plan
@@ -268,7 +285,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 		return wd.CreatePlan(ctx, opts...)
 	}, wd, providers)
 	if err != nil {
-		return fmt.Errorf("Error running second post-apply plan: %w", err)
+		if step.PlanOnly {
+			return fmt.Errorf("Error running refresh plan: %w", err)
+		}
+
+		return fmt.Errorf("Error running post-apply refresh plan: %w", err)
 	}
 
 	err = runProviderCommand(ctx, t, func() error {
@@ -277,14 +298,18 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 		return err
 	}, wd, providers)
 	if err != nil {
-		return fmt.Errorf("Error retrieving second post-apply plan: %w", err)
+		if step.PlanOnly {
+			return fmt.Errorf("Error reading refresh plan: %w", err)
+		}
+
+		return fmt.Errorf("Error reading post-apply refresh plan: %w", err)
 	}
 
 	// Run post-apply, post-refresh plan checks
 	if len(step.ConfigPlanChecks.PostApplyPostRefresh) > 0 {
 		err = runPlanChecks(ctx, t, plan, step.ConfigPlanChecks.PostApplyPostRefresh)
 		if err != nil {
-			return fmt.Errorf("Post-apply, post-refresh plan check(s) failed:\n%w", err)
+			return fmt.Errorf("Post-apply refresh plan check(s) failed:\n%w", err)
 		}
 	}
 
@@ -297,11 +322,16 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 			return err
 		}, wd, providers)
 		if err != nil {
-			return fmt.Errorf("Error retrieving formatted second plan output: %w", err)
+			return fmt.Errorf("Error reading human-readable refresh plan output: %w", err)
+		}
+
+		if step.PlanOnly {
+			return fmt.Errorf("The refresh plan was not empty.\nstdout\n\n%s", stdout)
 		}
-		return fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", stdout)
+
+		return fmt.Errorf("After applying this test step, the refresh plan was not empty.\nstdout\n\n%s", stdout)
 	} else if step.ExpectNonEmptyPlan && planIsEmpty(plan, helper.TerraformVersion()) {
-		return errors.New("Expected a non-empty plan, but got an empty plan")
+		return errors.New("Expected a non-empty plan, but got an empty refresh plan")
 	}
 
 	// ID-ONLY REFRESH

helper/resource/testing_new_config_test.go

@@ -79,7 +79,7 @@ func Test_ExpectNonEmptyPlan_EmptyPlanError(t *testing.T) {
 			{
 				Config:             `resource "terraform_data" "test" {}`,
 				ExpectNonEmptyPlan: true,
-				ExpectError:        regexp.MustCompile("Expected a non-empty plan, but got an empty plan"),
+				ExpectError:        regexp.MustCompile("Expected a non-empty plan, but got an empty refresh plan"),
 			},
 		},
 	})
@@ -254,7 +254,7 @@ func Test_NonEmptyPlan_PreRefresh_Error(t *testing.T) {
 					},
 				},
 				ExpectNonEmptyPlan: false, // intentional
-				ExpectError:        regexp.MustCompile("After applying this test step, the plan was not empty."),
+				ExpectError:        regexp.MustCompile("After applying this test step, the non-refresh plan was not empty."),
 			},
 		},
 	})
@@ -331,7 +331,7 @@ func Test_NonEmptyPlan_PostRefresh_Error(t *testing.T) {
 					},
 				},
 				ExpectNonEmptyPlan: false, // intentional
-				ExpectError:        regexp.MustCompile("After applying this test step and performing a `terraform refresh`, the plan was not empty."),
+				ExpectError:        regexp.MustCompile("After applying this test step, the refresh plan was not empty."),
 			},
 		},
 	})

helper/resource/testing_new_refresh_state_test.go

@@ -168,7 +168,7 @@ func Test_RefreshState_ExpectNonEmptyPlan(t *testing.T) {
 					input = timestamp()
 				}`,
 				ExpectNonEmptyPlan: false, // intentional
-				ExpectError:        regexp.MustCompile("After applying this test step, the plan was not empty."),
+				ExpectError:        regexp.MustCompile("After applying this test step, the non-refresh plan was not empty."),
 			},
 			{
 				RefreshState:       true,
@@ -196,7 +196,7 @@ func Test_RefreshState_NonEmptyPlan_Error(t *testing.T) {
 					input = timestamp()
 				}`,
 				ExpectNonEmptyPlan: false, // intentional
-				ExpectError:        regexp.MustCompile("After applying this test step, the plan was not empty."),
+				ExpectError:        regexp.MustCompile("After applying this test step, the non-refresh plan was not empty."),
 			},
 			{
 				RefreshState:       true,