PSS : Add fs and inmem state storage implementationd to the builtin simplev6 provider, update grpcwrap package, use PSS implementation in E2E test

.github/workflows/checks.yml

@@ -108,7 +108,7 @@ jobs:
 
       - name: "End-to-end tests"
         run: |
-          TF_ACC=1 go test -v ./internal/command/e2etest
+          TF_TEST_EXPERIMENTS=1 TF_ACC=1 go test -v ./internal/command/e2etest
 
   consistency-checks:
     name: "Code Consistency Checks"

internal/backend/pluggable/chunks.go

@@ -0,0 +1,18 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package pluggable
+
+const (
+	// DefaultStateStoreChunkSize is the default chunk size proposed
+	// to the provider.
+	// This can be tweaked but should provide reasonable performance
+	// trade-offs for average network conditions and state file sizes.
+	DefaultStateStoreChunkSize int64 = 8 << 20 // 8 MB
+
+	// MaxStateStoreChunkSize is the highest chunk size provider may choose
+	// which we still consider reasonable/safe.
+	// This reflects terraform-plugin-go's max. RPC message size of 256MB
+	// and leaves plenty of space for other variable data like diagnostics.
+	MaxStateStoreChunkSize int64 = 128 << 20 // 128 MB
+)

internal/backend/pluggable/pluggable.go

@@ -105,6 +105,9 @@ func (p *Pluggable) Configure(config cty.Value) tfdiags.Diagnostics {
 	req := providers.ConfigureStateStoreRequest{
 		TypeName: p.typeName,
 		Config:   config,
+		Capabilities: providers.StateStoreClientCapabilities{
+			ChunkSize: DefaultStateStoreChunkSize,
+		},
 	}
 	resp := p.provider.ConfigureStateStore(req)
 	return resp.Diagnostics

internal/backend/remote-state/inmem/backend_test.go

@@ -65,7 +65,7 @@ func TestBackendLocked(t *testing.T) {
 	backend.TestBackendStateLocks(t, b1, b2)
 }
 
-// use the this backen to test the remote.State implementation
+// use this backend to test the remote.State implementation
 func TestRemoteState(t *testing.T) {
 	defer Reset()
 	b := backend.TestBackendConfig(t, New(), hcl.EmptyBody())

internal/builtin/providers/terraform/provider.go

@@ -16,6 +16,8 @@ import (
 // Provider is an implementation of providers.Interface
 type Provider struct{}
 
+var _ providers.Interface = &Provider{}
+
 // NewProvider returns a new terraform provider
 func NewProvider() providers.Interface {
 	return &Provider{}

internal/command/apply.go

@@ -219,19 +219,18 @@ func (c *ApplyCommand) PrepareBackend(planFile *planfile.WrappedPlanFile, args *
 			))
 			return nil, diags
 		}
+		// TODO: Update BackendForLocalPlan to use state storage, and plan to be able to contain State Store config details
 		be, beDiags = c.BackendForLocalPlan(plan.Backend)
 	} else {
 		// Both new plans and saved cloud plans load their backend from config.
-		backendConfig, configDiags := c.loadBackendConfig(".")
-		diags = diags.Append(configDiags)
-		if configDiags.HasErrors() {
+		mod, mDiags := c.loadSingleModule(".")
+		if mDiags.HasErrors() {
+			diags = diags.Append(mDiags)
 			return nil, diags
 		}
 
-		be, beDiags = c.Backend(&BackendOpts{
-			BackendConfig: backendConfig,
-			ViewType:      viewType,
-		})
+		// Load the backend
+		be, beDiags = c.backend(mod)
 	}
 
 	diags = diags.Append(beDiags)

internal/command/arguments/init.go

@@ -146,7 +146,7 @@ func ParseInit(args []string, experimentsEnabled bool) (*Init, tfdiags.Diagnosti
 			diags = diags.Append(tfdiags.Sourceless(
 				tfdiags.Error,
 				"Cannot use -enable-pluggable-state-storage-experiment flag without experiments enabled",
-				"Terraform cannot use the-enable-pluggable-state-storage-experiment flag (or TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable) unless experiments are enabled.",
+				"Terraform cannot use the -enable-pluggable-state-storage-experiment flag (or TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable) unless experiments are enabled.",
 			))
 		}
 		if !init.CreateDefaultWorkspace {

internal/command/console.go

@@ -53,17 +53,14 @@ func (c *ConsoleCommand) Run(args []string) int {
 
 	var diags tfdiags.Diagnostics
 
-	backendConfig, backendDiags := c.loadBackendConfig(configPath)
-	diags = diags.Append(backendDiags)
+	mod, diags := c.loadSingleModule(configPath)
 	if diags.HasErrors() {
 		c.showDiagnostics(diags)
 		return 1
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-	})
+	b, backendDiags := c.backend(mod)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/e2etest/primary_test.go

@@ -4,6 +4,7 @@
 package e2etest
 
 import (
+	"os"
 	"path/filepath"
 	"reflect"
 	"sort"
@@ -12,7 +13,9 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/terraform/internal/e2e"
+	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/internal/plans"
+	"github.com/hashicorp/terraform/internal/states/statefile"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -230,3 +233,149 @@ func TestPrimaryChdirOption(t *testing.T) {
 		t.Errorf("incorrect destroy tally; want 0 destroyed:\n%s", stdout)
 	}
 }
+
+// Requires TF_TEST_EXPERIMENTS to be set in the environment
+func TestPrimary_stateStore(t *testing.T) {
+	if v := os.Getenv("TF_TEST_EXPERIMENTS"); v == "" {
+		t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENTS=1")
+	}
+
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
+	t.Parallel()
+
+	tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store-fs")
+
+	// In order to test integration with PSS we need a provider plugin implementing a state store.
+	// Here will build the simple6 (built with protocol v6) provider, which implements PSS.
+	simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6")
+	simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider)
+
+	// Move the provider binaries into a directory that we will point terraform
+	// to using the -plugin-dir cli flag.
+	platform := getproviders.CurrentPlatform.String()
+	hashiDir := "cache/registry.terraform.io/hashicorp/"
+	if err := os.MkdirAll(tf.Path(hashiDir, "simple6/0.0.1/", platform), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Rename(simple6ProviderExe, tf.Path(hashiDir, "simple6/0.0.1/", platform, "terraform-provider-simple6")); err != nil {
+		t.Fatal(err)
+	}
+
+	//// INIT
+	stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color")
+	if err != nil {
+		t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Terraform created an empty state file for the default workspace") {
+		t.Errorf("notice about creating the default workspace is missing from init output:\n%s", stdout)
+	}
+
+	//// PLAN
+	// No separate plan step; this test lets the apply make a plan.
+
+	//// APPLY
+	stdout, stderr, err = tf.Run("apply", "-auto-approve", "-no-color")
+	if err != nil {
+		t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") {
+		t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout)
+	}
+
+	// Check the statefile saved by the fs state store.
+	path := "terraform.tfstate.d/default/terraform.tfstate"
+	f, err := tf.OpenFile(path)
+	if err != nil {
+		t.Fatalf("unexpected error opening state file %s: %s\nstderr:\n%s", path, err, stderr)
+	}
+	defer f.Close()
+
+	stateFile, err := statefile.Read(f)
+	if err != nil {
+		t.Fatalf("unexpected error reading statefile %s: %s\nstderr:\n%s", path, err, stderr)
+	}
+
+	r := stateFile.State.RootModule().Resources
+	if len(r) != 1 {
+		t.Fatalf("expected state to include one resource, but got %d", len(r))
+	}
+	if _, ok := r["terraform_data.my-data"]; !ok {
+		t.Fatalf("expected state to include terraform_data.my-data but it's missing")
+	}
+}
+
+// Requires TF_TEST_EXPERIMENTS to be set in the environment
+func TestPrimary_stateStore_inMem(t *testing.T) {
+	if v := os.Getenv("TF_TEST_EXPERIMENTS"); v == "" {
+		t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENTS=1")
+	}
+
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
+	t.Parallel()
+
+	tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store-inmem")
+
+	// In order to test integration with PSS we need a provider plugin implementing a state store.
+	// Here will build the simple6 (built with protocol v6) provider, which implements PSS.
+	simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6")
+	simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider)
+
+	// Move the provider binaries into a directory that we will point terraform
+	// to using the -plugin-dir cli flag.
+	platform := getproviders.CurrentPlatform.String()
+	hashiDir := "cache/registry.terraform.io/hashicorp/"
+	if err := os.MkdirAll(tf.Path(hashiDir, "simple6/0.0.1/", platform), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Rename(simple6ProviderExe, tf.Path(hashiDir, "simple6/0.0.1/", platform, "terraform-provider-simple6")); err != nil {
+		t.Fatal(err)
+	}
+
+	//// INIT
+	//
+	// Note - the inmem PSS implementation means that the default workspace state created during init
+	// is lost as soon as the command completes.
+	stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color")
+	if err != nil {
+		t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Terraform created an empty state file for the default workspace") {
+		t.Errorf("notice about creating the default workspace is missing from init output:\n%s", stdout)
+	}
+
+	//// PLAN
+	// No separate plan step; this test lets the apply make a plan.
+
+	//// APPLY
+	//
+	// Note - the inmem PSS implementation means that writing to the default workspace during apply
+	// is creating the default state file for the first time.
+	stdout, stderr, err = tf.Run("apply", "-auto-approve", "-no-color")
+	if err != nil {
+		t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") {
+		t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout)
+	}
+
+	// We cannot inspect state or perform a destroy here, as the state isn't persisted between steps
+	// when we use the simple6_inmem state store.
+}

internal/command/e2etest/testdata/full-workflow-with-state-store-fs/main.tf

@@ -0,0 +1,23 @@
+terraform {
+  required_providers {
+    simple6 = {
+      source = "registry.terraform.io/hashicorp/simple6"
+    }
+  }
+
+  state_store "simple6_fs" {
+    provider "simple6" {}
+  }
+}
+
+variable "name" {
+  default = "world"
+}
+
+resource "terraform_data" "my-data" {
+  input = "hello ${var.name}"
+}
+
+output "greeting" {
+  value = resource.terraform_data.my-data.output
+}

internal/command/e2etest/testdata/full-workflow-with-state-store-inmem/main.tf

@@ -0,0 +1,23 @@
+terraform {
+  required_providers {
+    simple6 = {
+      source = "registry.terraform.io/hashicorp/simple6"
+    }
+  }
+
+  state_store "simple6_inmem" {
+    provider "simple6" {}
+  }
+}
+
+variable "name" {
+  default = "world"
+}
+
+resource "terraform_data" "my-data" {
+  input = "hello ${var.name}"
+}
+
+output "greeting" {
+  value = resource.terraform_data.my-data.output
+}

internal/command/graph.go

@@ -68,17 +68,14 @@ func (c *GraphCommand) Run(args []string) int {
 
 	var diags tfdiags.Diagnostics
 
-	backendConfig, backendDiags := c.loadBackendConfig(configPath)
-	diags = diags.Append(backendDiags)
+	mod, diags := c.loadSingleModule(configPath)
 	if diags.HasErrors() {
 		c.showDiagnostics(diags)
 		return 1
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-	})
+	b, backendDiags := c.backend(mod)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/import.go

@@ -164,9 +164,7 @@ func (c *ImportCommand) Run(args []string) int {
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: config.Module.Backend,
-	})
+	b, backendDiags := c.backend(config.Module)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)