From 7ab7c805d89228de58f5e9f9f2e74965841819de Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 11:01:03 -0700 Subject: [PATCH] feat: Add ZSTD compression as an option for Arrow. (#165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add ZSTD compression as an option for Arrow. Committer: @emkornfield PiperOrigin-RevId: 374220891 Source-Link: https://github.com/googleapis/googleapis/commit/23efea9fc7bedfe53b24295ed84b5f873606edcb Source-Link: https://github.com/googleapis/googleapis-gen/commit/79c15da3a71c276e23aa2746f9fa243741763179 * 🦉 Updates from OwlBot Co-authored-by: Owl Bot --- .../cloud/bigquery/storage/v1/arrow.proto | 3 +++ .../cloud/bigquery/storage/v1/storage.proto | 6 ++--- .../cloud/bigquery/storage/v1/stream.proto | 22 +++++++++---------- .../protos/protos.d.ts | 3 ++- .../protos/protos.js | 7 ++++++ .../protos/protos.json | 3 ++- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/arrow.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/arrow.proto index 4b240f52139..514b77e617c 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/arrow.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/arrow.proto @@ -52,6 +52,9 @@ message ArrowSerializationOptions { // LZ4 Frame (https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md) LZ4_FRAME = 1; + + // Zstandard compression. + ZSTD = 2; } // The compression codec to use for Arrow buffers in serialized record diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto index a5fa2b9eb86..8ac56375044 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/storage.proto @@ -69,8 +69,7 @@ service BigQueryRead { post: "/v1/{read_session.table=projects/*/datasets/*/tables/*}" body: "*" }; - option (google.api.method_signature) = - "parent,read_session,max_stream_count"; + option (google.api.method_signature) = "parent,read_session,max_stream_count"; } // Reads rows from the stream in the format prescribed by the ReadSession. @@ -99,8 +98,7 @@ service BigQueryRead { // original, primary, and residual, that original[0-j] = primary[0-j] and // original[j-n] = residual[0-m] once the streams have been read to // completion. - rpc SplitReadStream(SplitReadStreamRequest) - returns (SplitReadStreamResponse) { + rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/sessions/*/streams/*}" }; diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto index 28b2ac1bbf3..291cf4115bc 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto @@ -75,10 +75,9 @@ message ReadSession { // Restricted to a maximum length for 1 MB. string row_restriction = 2; - // Optional. Options specific to the Apache Arrow output format. oneof output_format_serialization_options { - ArrowSerializationOptions arrow_serialization_options = 3 - [(google.api.field_behavior) = OPTIONAL]; + // Optional. Options specific to the Apache Arrow output format. + ArrowSerializationOptions arrow_serialization_options = 3 [(google.api.field_behavior) = OPTIONAL]; } } @@ -86,12 +85,10 @@ message ReadSession { // `projects/{project_id}/locations/{location}/sessions/{session_id}`. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Output only. Time at which the session becomes invalid. After this time, - // subsequent requests to read this Session will return errors. The - // expire_time is automatically assigned and currently cannot be specified or - // updated. - google.protobuf.Timestamp expire_time = 2 - [(google.api.field_behavior) = OUTPUT_ONLY]; + // Output only. Time at which the session becomes invalid. After this time, subsequent + // requests to read this Session will return errors. The expire_time is + // automatically assigned and currently cannot be specified or updated. + google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Immutable. Data format of the output data. DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE]; @@ -111,11 +108,12 @@ message ReadSession { // `projects/{project_id}/datasets/{dataset_id}/tables/{table_id}` string table = 6 [ (google.api.field_behavior) = IMMUTABLE, - (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" } + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } ]; - // Optional. Any modifiers which are applied when reading from the specified - // table. + // Optional. Any modifiers which are applied when reading from the specified table. TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. Read options for this session (e.g. column selection, filters). diff --git a/packages/google-cloud-bigquery-storage/protos/protos.d.ts b/packages/google-cloud-bigquery-storage/protos/protos.d.ts index 4ab3a82b13b..fea18b2d33b 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.d.ts +++ b/packages/google-cloud-bigquery-storage/protos/protos.d.ts @@ -310,7 +310,8 @@ export namespace google { /** CompressionCodec enum. */ enum CompressionCodec { COMPRESSION_UNSPECIFIED = 0, - LZ4_FRAME = 1 + LZ4_FRAME = 1, + ZSTD = 2 } } diff --git a/packages/google-cloud-bigquery-storage/protos/protos.js b/packages/google-cloud-bigquery-storage/protos/protos.js index a948d7dd4df..4ead87bfb66 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.js +++ b/packages/google-cloud-bigquery-storage/protos/protos.js @@ -640,6 +640,7 @@ return "bufferCompression: enum value expected"; case 0: case 1: + case 2: break; } return null; @@ -666,6 +667,10 @@ case 1: message.bufferCompression = 1; break; + case "ZSTD": + case 2: + message.bufferCompression = 2; + break; } return message; }; @@ -707,11 +712,13 @@ * @enum {number} * @property {number} COMPRESSION_UNSPECIFIED=0 COMPRESSION_UNSPECIFIED value * @property {number} LZ4_FRAME=1 LZ4_FRAME value + * @property {number} ZSTD=2 ZSTD value */ ArrowSerializationOptions.CompressionCodec = (function() { var valuesById = {}, values = Object.create(valuesById); values[valuesById[0] = "COMPRESSION_UNSPECIFIED"] = 0; values[valuesById[1] = "LZ4_FRAME"] = 1; + values[valuesById[2] = "ZSTD"] = 2; return values; })(); diff --git a/packages/google-cloud-bigquery-storage/protos/protos.json b/packages/google-cloud-bigquery-storage/protos/protos.json index 5ed51835500..6a76834bc84 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.json +++ b/packages/google-cloud-bigquery-storage/protos/protos.json @@ -51,7 +51,8 @@ "CompressionCodec": { "values": { "COMPRESSION_UNSPECIFIED": 0, - "LZ4_FRAME": 1 + "LZ4_FRAME": 1, + "ZSTD": 2 } } }