Skip to content

Commit

Permalink
feat: Add ZSTD compression as an option for Arrow. (#165)
Browse files Browse the repository at this point in the history
* feat: Add ZSTD compression as an option for Arrow.

Committer: @emkornfield
PiperOrigin-RevId: 374220891

Source-Link: googleapis/googleapis@23efea9

Source-Link: googleapis/googleapis-gen@79c15da

* 🦉 Updates from OwlBot

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
gcf-owl-bot[bot] and gcf-owl-bot[bot] authored May 17, 2021
1 parent 9085340 commit 7ab7c80
Show file tree
Hide file tree
Showing 6 changed files with 26 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ message ArrowSerializationOptions {

// LZ4 Frame (https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md)
LZ4_FRAME = 1;

// Zstandard compression.
ZSTD = 2;
}

// The compression codec to use for Arrow buffers in serialized record
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ service BigQueryRead {
post: "/v1/{read_session.table=projects/*/datasets/*/tables/*}"
body: "*"
};
option (google.api.method_signature) =
"parent,read_session,max_stream_count";
option (google.api.method_signature) = "parent,read_session,max_stream_count";
}

// Reads rows from the stream in the format prescribed by the ReadSession.
Expand Down Expand Up @@ -99,8 +98,7 @@ service BigQueryRead {
// original, primary, and residual, that original[0-j] = primary[0-j] and
// original[j-n] = residual[0-m] once the streams have been read to
// completion.
rpc SplitReadStream(SplitReadStreamRequest)
returns (SplitReadStreamResponse) {
rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/sessions/*/streams/*}"
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,23 +75,20 @@ message ReadSession {
// Restricted to a maximum length for 1 MB.
string row_restriction = 2;

// Optional. Options specific to the Apache Arrow output format.
oneof output_format_serialization_options {
ArrowSerializationOptions arrow_serialization_options = 3
[(google.api.field_behavior) = OPTIONAL];
// Optional. Options specific to the Apache Arrow output format.
ArrowSerializationOptions arrow_serialization_options = 3 [(google.api.field_behavior) = OPTIONAL];
}
}

// Output only. Unique identifier for the session, in the form
// `projects/{project_id}/locations/{location}/sessions/{session_id}`.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Time at which the session becomes invalid. After this time,
// subsequent requests to read this Session will return errors. The
// expire_time is automatically assigned and currently cannot be specified or
// updated.
google.protobuf.Timestamp expire_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Time at which the session becomes invalid. After this time, subsequent
// requests to read this Session will return errors. The expire_time is
// automatically assigned and currently cannot be specified or updated.
google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

// Immutable. Data format of the output data.
DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE];
Expand All @@ -111,11 +108,12 @@ message ReadSession {
// `projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`
string table = 6 [
(google.api.field_behavior) = IMMUTABLE,
(google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" }
(google.api.resource_reference) = {
type: "bigquery.googleapis.com/Table"
}
];

// Optional. Any modifiers which are applied when reading from the specified
// table.
// Optional. Any modifiers which are applied when reading from the specified table.
TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL];

// Optional. Read options for this session (e.g. column selection, filters).
Expand Down
3 changes: 2 additions & 1 deletion packages/google-cloud-bigquery-storage/protos/protos.d.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions packages/google-cloud-bigquery-storage/protos/protos.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion packages/google-cloud-bigquery-storage/protos/protos.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 7ab7c80

Please sign in to comment.