Adding GCS output and fixing linting issues for few outputs (#46)

* Adding GCS output

* fixing linting issues

* adding comments and checking some more returned errors

* update changelog

* update test_go with environment variable again

* adding possibility to connect to public bucket (without credentials) and removing environment variables

* go mod tidy

* update readme

CHANGELOG.md

@@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Add GCS output support: [#46](https://github.com/elastic/stream/pull/46)
 - Added support for azure blob storage output: [#46](https://github.com/elastic/stream/pull/46)
 
 ## [0.8.0]

README.md

@@ -12,6 +12,8 @@ stream is a test utility for streaming data via:
 - Kafka
 - [Lumberjack](#lumberjack-output-reference)
 - HTTP Mock Server
+- Azure Blob Storage
+- Google Cloud Storage
 
 Input data can be read from:
 
@@ -113,3 +115,16 @@ By default, Lumberjack batches contain one event with a `message` field.
 
 If `--lumberjack-parse-json` is used then the input data is parsed as JSON
 and the resulting data is sent as a batch.
+
+## GCS Output Reference
+
+The GCS output is used to collect data from the configured source, create a GCS bucket, and populate it with the incoming data.
+When specifying a (`--addr`) which should be a combination of both host and port, usually pointing to a locally running emulator,
+the client will be overriding the configured API endpoint, which defaults to the public google storage API, towards the emulator instead.
+The emulator does not require authentication.
+
+### Options
+
+- `gcs-bucket`: The name of the GCS bucket that should be created, should not already exist.
+- `gcs-object`: The name of the GCS object that will be populated with the collected data, using the configured GCS bucket.
+- `gcs-projectid`: The related projectID used when creating the bucket, this is required to be changed from the default value when not using an emulator.

command/root.go

@@ -28,6 +28,7 @@ import (
 	// Register outputs.
 	_ "github.com/elastic/stream/pkg/output/azureblobstorage"
 	_ "github.com/elastic/stream/pkg/output/gcppubsub"
+	_ "github.com/elastic/stream/pkg/output/gcs"
 	_ "github.com/elastic/stream/pkg/output/kafka"
 	_ "github.com/elastic/stream/pkg/output/lumberjack"
 	_ "github.com/elastic/stream/pkg/output/tcp"
@@ -87,6 +88,11 @@ func ExecuteContext(ctx context.Context) error {
 	// Kafka Pubsub output flags.
 	rootCmd.PersistentFlags().StringVar(&opts.KafkaOptions.Topic, "kafka-topic", "test", "Kafka topic name")
 
+	// GCS output flags.
+	rootCmd.PersistentFlags().StringVar(&opts.GcsOptions.Bucket, "gcs-bucket", "testbucket", "GCS Bucket name")
+	rootCmd.PersistentFlags().StringVar(&opts.GcsOptions.Object, "gcs-object", "testobject", "GCS Object name")
+	rootCmd.PersistentFlags().StringVar(&opts.GcsOptions.ProjectID, "gcs-projectid", "testproject", "GCS Project name")
+
 	// Lumberjack output flags.
 	rootCmd.PersistentFlags().BoolVar(&opts.LumberjackOptions.ParseJSON, "lumberjack-parse-json", false, "Parse the input data as JSON and send the structured data as a Lumberjack batch.")
 

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	cloud.google.com/go/pubsub v1.25.1
+	cloud.google.com/go/storage v1.28.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
 	github.com/Shopify/sarama v1.36.0
 	github.com/elastic/go-concert v0.2.0
@@ -19,14 +20,15 @@ require (
 	go.uber.org/zap v1.23.0
 	golang.org/x/sys v0.0.0-20220913175220-63ea55921009
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
-	google.golang.org/api v0.95.0
+	google.golang.org/api v0.102.0
 	gotest.tools v2.2.0+incompatible
 )
 
 require (
 	cloud.google.com/go v0.104.0 // indirect
-	cloud.google.com/go/compute v1.9.0 // indirect
-	cloud.google.com/go/iam v0.4.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.5.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
@@ -48,8 +50,9 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
@@ -79,13 +82,14 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
-	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
-	golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5 // indirect
-	google.golang.org/grpc v1.49.0 // indirect
+	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
+	google.golang.org/grpc v1.50.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect