Support !terraform.state on GCS Backends (#1393)

* feat: implement GCS backend support for YAML function

* feat: improve GCS backend with performance optimizations and unified Google Cloud auth

Performance Improvements:
- Add client caching to avoid recreating GCS clients for repeated operations
- Implement retry logic with exponential backoff (up to 3 attempts)
- Extend timeouts to 30 seconds to match S3 backend performance
- Add comprehensive debug logging for better troubleshooting

Unified Google Cloud Authentication:
- Create internal/gcp package for unified authentication across all GCP services
- Support both JSON credentials and file paths consistently
- Leverage Google's Application Default Credentials (ADC) automatically
- Unify authentication between GCS backend and Secret Manager store
- Remove duplicate authentication logic and improve maintainability

Enhanced Error Handling:
- Improve resource cleanup (explicit close instead of defer in loops)
- Add detailed error context and logging
- Better handling of transient failures and missing files

Test Coverage:
- Add comprehensive test suite for unified authentication (4 test functions)
- Enhance GCS backend tests with retry and caching scenarios
- Maintain 100% backward compatibility with existing configurations

This brings GCS backend performance and reliability in line with S3 backend
while establishing a foundation for consistent Google Cloud integrations.

* fix: address CodeRabbitAI feedback for GCS backend implementation

Critical fixes:
- Fix indentation issue in test assertions
- Replace len() with SHA256 hash for cache key security
- Fix context leak in retry loop by removing defer from loop

Code quality improvements:
- Move mock client creation outside test loop for performance
- Standardize test function naming (Test_* -> Test*)
- Use consistent assert.Equal() style instead of manual error checks
- Optimize resource management and prevent cache collisions

All tests pass with improved performance and security.

* docs: fix misleading GCS service account impersonation claims

- Remove impersonate_service_account from example configurations
- Update authentication methods to remove impersonation reference
- Add clear limitation notice that impersonation is not yet implemented
- Remove commented impersonation example in configuration

This ensures users understand current limitations and prevents
confusion about unsupported features.

* [autofix.ci] apply automated fixes

* Use constants instead of strings for backend type

* use errors package

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Andriy Knysh <aknysh@users.noreply.github.com>
Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <erik@cloudposse.com>

Author: 4 people

errors/errors.go

@@ -70,10 +70,14 @@ var (
 	ErrEvaluateTerraformBackendVariable = errors.New("failed to evaluate terraform backend variable")
 	ErrUnsupportedBackendType           = errors.New("unsupported backend type")
 	ErrProcessTerraformStateFile        = errors.New("error processing terraform state file")
-
-	ErrLoadAwsConfig    = errors.New("failed to load AWS config")
-	ErrGetObjectFromS3  = errors.New("failed to get object from S3")
-	ErrReadS3ObjectBody = errors.New("failed to read S3 object body")
+	ErrLoadAwsConfig                    = errors.New("failed to load AWS config")
+	ErrGetObjectFromS3                  = errors.New("failed to get object from S3")
+	ErrReadS3ObjectBody                 = errors.New("failed to read S3 object body")
+	ErrCreateGCSClient                  = errors.New("failed to create GCS client")
+	ErrGetObjectFromGCS                 = errors.New("failed to get object from GCS")
+	ErrReadGCSObjectBody                = errors.New("failed to read GCS object body")
+	ErrGCSBucketRequired                = errors.New("bucket is required for gcs backend")
+	ErrInvalidBackendConfig             = errors.New("invalid backend configuration")
 
 	// Azure Blob Storage specific errors.
 	ErrGetBlobFromAzure       = errors.New("failed to get blob from Azure Blob Storage")

go.mod

@@ -7,6 +7,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
+	errUtils "github.com/cloudposse/atmos/errors"
 	cfg "github.com/cloudposse/atmos/pkg/config"
 	log "github.com/cloudposse/atmos/pkg/logger"
 	"github.com/cloudposse/atmos/pkg/perf"
@@ -81,12 +82,19 @@ func ExecuteTerraformGenerateBackendCmd(cmd *cobra.Command, args []string) error
 	log.Debug("Component backend", "config", componentBackendConfig)
 
 	// Check if the `backend` section has `workspace_key_prefix` when `backend_type` is `s3`
-	if info.ComponentBackendType == "s3" {
+	if info.ComponentBackendType == cfg.BackendTypeS3 {
 		if _, ok := info.ComponentBackendSection["workspace_key_prefix"].(string); !ok {
 			return fmt.Errorf("backend config for the '%s' component is missing 'workspace_key_prefix'", component)
 		}
 	}
 
+	// Check if the `backend` section has `bucket` when `backend_type` is `gcs`
+	if info.ComponentBackendType == cfg.BackendTypeGCS {
+		if _, ok := info.ComponentBackendSection["bucket"].(string); !ok {
+			return errUtils.ErrGCSBucketRequired
+		}
+	}
+
 	// Write the backend config to a file
 	backendFilePath := filepath.Join(
 		atmosConfig.BasePath,

internal/exec/terraform_generate_backend.go

@@ -0,0 +1,66 @@
+package gcp
+
+import (
+	"strings"
+
+	"google.golang.org/api/option"
+)
+
+// AuthOptions contains configuration for Google Cloud authentication.
+type AuthOptions struct {
+	// Credentials can be either:
+	// - JSON content (if starts with "{")
+	// - File path to service account JSON file
+	// - Empty string to use Application Default Credentials (ADC)
+	Credentials string
+
+	// TODO: Add support for service account impersonation
+	// ImpersonateServiceAccount string
+}
+
+// GetClientOptions returns Google Cloud client options based on the provided authentication configuration.
+// This function provides unified authentication handling across all Google Cloud services in Atmos.
+//
+// Authentication precedence:
+// 1. Explicit credentials (JSON content or file path)
+// 2. Application Default Credentials (ADC) which automatically handles:
+//   - GOOGLE_APPLICATION_CREDENTIALS environment variable
+//   - Compute Engine metadata service
+//   - Cloud Shell credentials
+//   - gcloud user credentials (from `gcloud auth application-default login`)
+//   - Workload Identity (in GKE)
+func GetClientOptions(opts AuthOptions) []option.ClientOption {
+	var clientOpts []option.ClientOption
+
+	if opts.Credentials != "" {
+		// Determine if credentials are JSON content or file path
+		if strings.HasPrefix(strings.TrimSpace(opts.Credentials), "{") {
+			// JSON content
+			clientOpts = append(clientOpts, option.WithCredentialsJSON([]byte(opts.Credentials)))
+		} else {
+			// File path
+			clientOpts = append(clientOpts, option.WithCredentialsFile(opts.Credentials))
+		}
+	}
+	// If no explicit credentials, Google Cloud client libraries will automatically use ADC
+
+	return clientOpts
+}
+
+// GetCredentialsFromBackend extracts credentials from a Terraform backend configuration.
+// This is used by the GCS Terraform backend.
+func GetCredentialsFromBackend(backend map[string]any) string {
+	if credentials, ok := backend["credentials"].(string); ok {
+		return credentials
+	}
+	return ""
+}
+
+// GetCredentialsFromStore extracts credentials from a store configuration.
+// This is used by the Google Secret Manager store.
+func GetCredentialsFromStore(credentials *string) string {
+	if credentials != nil {
+		return *credentials
+	}
+	return ""
+}

internal/gcp/auth.go

@@ -0,0 +1,178 @@
+package gcp
+
+import (
+	"testing"
+
+	"google.golang.org/api/option"
+)
+
+func TestGetClientOptions(t *testing.T) {
+	tests := []struct {
+		name     string
+		opts     AuthOptions
+		expected int // number of client options returned
+	}{
+		{
+			name: "no credentials - use ADC",
+			opts: AuthOptions{
+				Credentials: "",
+			},
+			expected: 0, // ADC uses no explicit options
+		},
+		{
+			name: "JSON credentials",
+			opts: AuthOptions{
+				Credentials: `{"type": "service_account", "project_id": "test"}`,
+			},
+			expected: 1, // WithCredentialsJSON
+		},
+		{
+			name: "file path credentials",
+			opts: AuthOptions{
+				Credentials: "/path/to/service-account.json",
+			},
+			expected: 1, // WithCredentialsFile
+		},
+		{
+			name: "JSON with whitespace",
+			opts: AuthOptions{
+				Credentials: `  {"type": "service_account"}  `,
+			},
+			expected: 1, // WithCredentialsJSON
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			clientOpts := GetClientOptions(tt.opts)
+			if len(clientOpts) != tt.expected {
+				t.Errorf("GetClientOptions() returned %d options, expected %d", len(clientOpts), tt.expected)
+			}
+		})
+	}
+}
+
+func TestGetCredentialsFromBackend(t *testing.T) {
+	tests := []struct {
+		name     string
+		backend  map[string]any
+		expected string
+	}{
+		{
+			name: "credentials present",
+			backend: map[string]any{
+				"credentials": "/path/to/creds.json",
+				"bucket":      "test-bucket",
+			},
+			expected: "/path/to/creds.json",
+		},
+		{
+			name: "JSON credentials",
+			backend: map[string]any{
+				"credentials": `{"type": "service_account"}`,
+			},
+			expected: `{"type": "service_account"}`,
+		},
+		{
+			name: "no credentials",
+			backend: map[string]any{
+				"bucket": "test-bucket",
+			},
+			expected: "",
+		},
+		{
+			name:     "empty backend",
+			backend:  map[string]any{},
+			expected: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := GetCredentialsFromBackend(tt.backend)
+			if result != tt.expected {
+				t.Errorf("GetCredentialsFromBackend() = %v, expected %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestGetCredentialsFromStore(t *testing.T) {
+	tests := []struct {
+		name        string
+		credentials *string
+		expected    string
+	}{
+		{
+			name:        "credentials present",
+			credentials: stringPtr("/path/to/creds.json"),
+			expected:    "/path/to/creds.json",
+		},
+		{
+			name:        "JSON credentials",
+			credentials: stringPtr(`{"type": "service_account"}`),
+			expected:    `{"type": "service_account"}`,
+		},
+		{
+			name:        "empty credentials",
+			credentials: stringPtr(""),
+			expected:    "",
+		},
+		{
+			name:        "nil credentials",
+			credentials: nil,
+			expected:    "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := GetCredentialsFromStore(tt.credentials)
+			if result != tt.expected {
+				t.Errorf("GetCredentialsFromStore() = %v, expected %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestClientOptionsCreation(t *testing.T) {
+	// Test that we can actually create valid client options without errors
+	tests := []struct {
+		name        string
+		credentials string
+		expectType  string
+	}{
+		{
+			name:        "JSON credentials create WithCredentialsJSON option",
+			credentials: `{"type": "service_account", "project_id": "test"}`,
+			expectType:  "JSON",
+		},
+		{
+			name:        "file path creates WithCredentialsFile option",
+			credentials: "/path/to/service-account.json",
+			expectType:  "File",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			opts := GetClientOptions(AuthOptions{
+				Credentials: tt.credentials,
+			})
+
+			// We can't easily inspect the actual option type without reflection,
+			// but we can verify that an option was created
+			if len(opts) != 1 {
+				t.Errorf("Expected 1 client option, got %d", len(opts))
+			}
+
+			// Verify the option is of type option.ClientOption
+			var _ option.ClientOption = opts[0]
+		})
+	}
+}
+
+// Helper function to create string pointers for tests
+func stringPtr(s string) *string {
+	return &s
+}