Add proxy support for GCS buckets

Signed-off-by: Matheus Pimenta <matheuscscp@gmail.com>

Author: matheuscscp

api/v1beta2/bucket_types.go

@@ -103,7 +103,7 @@ type BucketSpec struct {
 	// ProxySecretRef specifies the Secret containing the proxy configuration
 	// to use while communicating with the Bucket server.
 	//
-	// Only supported for the generic provider.
+	// Only supported for the `generic` and `gcp` providers.
 	// +optional
 	ProxySecretRef *meta.LocalObjectReference `json:"proxySecretRef,omitempty"`
 

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -397,7 +397,7 @@ spec:
                   to use while communicating with the Bucket server.
 
 
-                  Only supported for the generic provider.
+                  Only supported for the `generic` and `gcp` providers.
                 properties:
                   name:
                     description: Name of the referent.

docs/api/v1beta2/source.md

@@ -202,7 +202,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference
 <em>(Optional)</em>
 <p>ProxySecretRef specifies the Secret containing the proxy configuration
 to use while communicating with the Bucket server.</p>
-<p>Only supported for the generic provider.</p>
+<p>Only supported for the <code>generic</code> and <code>gcp</code> providers.</p>
 </td>
 </tr>
 <tr>
@@ -1568,7 +1568,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference
 <em>(Optional)</em>
 <p>ProxySecretRef specifies the Secret containing the proxy configuration
 to use while communicating with the Bucket server.</p>
-<p>Only supported for the generic provider.</p>
+<p>Only supported for the <code>generic</code> and <code>gcp</code> providers.</p>
 </td>
 </tr>
 <tr>

docs/spec/v1beta2/buckets.md

@@ -837,7 +837,7 @@ The Secret can contain three keys:
 - `password`, to specify the password to use if the proxy server is protected by
    basic authentication. This is an optional key.
 
-This API is only supported for the `generic` [provider](#provider).
+This API is only supported for the `generic` and `gcp` [providers](#provider).
 
 Example:
 

go.mod

@@ -60,6 +60,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/crypto v0.22.0
+	golang.org/x/oauth2 v0.19.0
 	golang.org/x/sync v0.7.0
 	google.golang.org/api v0.177.0
 	gotest.tools v2.2.0+incompatible
@@ -360,7 +361,6 @@ require (
 	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/oauth2 v0.19.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect

internal/controller/bucket_controller.go

@@ -431,6 +431,12 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
 		// Return error as the world as observed may change
 		return sreconcile.ResultEmpty, e
 	}
+	proxyURL, err := r.getProxyURL(ctx, obj)
+	if err != nil {
+		e := serror.NewGeneric(err, sourcev1.AuthenticationFailedReason)
+		conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
+		return sreconcile.ResultEmpty, e
+	}
 
 	// Construct provider client
 	var provider BucketProvider
@@ -441,7 +447,14 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, "%s", e)
 			return sreconcile.ResultEmpty, e
 		}
-		if provider, err = gcp.NewClient(ctx, secret); err != nil {
+		var opts []gcp.Option
+		if secret != nil {
+			opts = append(opts, gcp.WithSecret(secret))
+		}
+		if proxyURL != nil {
+			opts = append(opts, gcp.WithProxyURL(proxyURL))
+		}
+		if provider, err = gcp.NewClient(ctx, opts...); err != nil {
 			e := serror.NewGeneric(err, "ClientError")
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, "%s", e)
 			return sreconcile.ResultEmpty, e
@@ -469,12 +482,6 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, "%s", e)
 			return sreconcile.ResultEmpty, e
 		}
-		proxyURL, err := r.getProxyURL(ctx, obj)
-		if err != nil {
-			e := serror.NewGeneric(err, sourcev1.AuthenticationFailedReason)
-			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
-			return sreconcile.ResultEmpty, e
-		}
 		var opts []minio.Option
 		if secret != nil {
 			opts = append(opts, minio.WithSecret(secret))

pkg/gcp/gcp.go

@@ -21,13 +21,17 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 
 	gcpstorage "cloud.google.com/go/storage"
 	"github.com/go-logr/logr"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
+	htransport "google.golang.org/api/transport/http"
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
@@ -48,24 +52,64 @@ type GCSClient struct {
 	*gcpstorage.Client
 }
 
-// NewClient creates a new GCP storage client. The Client will automatically look for  the Google Application
+// Option is a functional option for configuring the GCS client.
+type Option func(*options)
+
+// WithSecret sets the secret to use for authenticating with GCP.
+func WithSecret(secret *corev1.Secret) Option {
+	return func(o *options) {
+		o.secret = secret
+	}
+}
+
+// WithProxyURL sets the proxy URL to use for the GCS client.
+func WithProxyURL(proxyURL *url.URL) Option {
+	return func(o *options) {
+		o.proxyURL = proxyURL
+	}
+}
+
+type options struct {
+	secret   *corev1.Secret
+	proxyURL *url.URL
+}
+
+// NewClient creates a new GCP storage client. The Client will automatically look for the Google Application
 // Credential environment variable or look for the Google Application Credential file.
-func NewClient(ctx context.Context, secret *corev1.Secret) (*GCSClient, error) {
-	c := &GCSClient{}
+func NewClient(ctx context.Context, opts ...Option) (*GCSClient, error) {
+	var o options
+	for _, opt := range opts {
+		opt(&o)
+	}
+	secret := o.secret
+	proxyURL := o.proxyURL
+
+	var creds *google.Credentials
+	var err error
 	if secret != nil {
-		client, err := gcpstorage.NewClient(ctx, option.WithCredentialsJSON(secret.Data["serviceaccount"]))
-		if err != nil {
-			return nil, err
-		}
-		c.Client = client
+		creds, err = google.CredentialsFromJSON(ctx, secret.Data["serviceaccount"], gcpstorage.ScopeReadOnly)
 	} else {
-		client, err := gcpstorage.NewClient(ctx)
-		if err != nil {
-			return nil, err
-		}
-		c.Client = client
+		creds, err = google.FindDefaultCredentials(ctx, gcpstorage.ScopeReadOnly)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get Google credentials: %w", err)
+	}
+
+	baseTransport := http.DefaultTransport.(*http.Transport).Clone()
+	if proxyURL != nil {
+		baseTransport.Proxy = http.ProxyURL(proxyURL)
 	}
-	return c, nil
+	transport, err := htransport.NewTransport(ctx, baseTransport, option.WithCredentials(creds))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Google HTTP transport: %w", err)
+	}
+
+	client, err := gcpstorage.NewClient(ctx, option.WithHTTPClient(&http.Client{Transport: transport}))
+	if err != nil {
+		return nil, err
+	}
+
+	return &GCSClient{Client: client}, nil
 }
 
 // ValidateSecret validates the credential secret. The provided Secret may

pkg/gcp/gcp_test.go

@@ -140,9 +140,9 @@ func TestMain(m *testing.M) {
 }
 
 func TestNewClientWithSecretErr(t *testing.T) {
-	gcpClient, err := NewClient(context.Background(), secret.DeepCopy())
+	gcpClient, err := NewClient(context.Background(), WithSecret(secret.DeepCopy()))
 	t.Log(err)
-	assert.Error(t, err, "dialing: invalid character 'e' looking for beginning of value")
+	assert.Error(t, err, "failed to get Google credentials: invalid character 'e' looking for beginning of value")
 	assert.Assert(t, gcpClient == nil)
 }