From e36fa87f3bc0c7c0c80051f388cf6a18a71549d9 Mon Sep 17 00:00:00 2001 From: Grant Timmerman <744973+grant@users.noreply.github.com> Date: Tue, 13 Apr 2021 22:12:14 -0500 Subject: [PATCH] feat!: use byte types for bytes proto fields (#58) * feat!: use byte types for bytes proto fields Signed-off-by: Grant Timmerman * docs: update pubsub sample with automatic base64 decoding Signed-off-by: Grant Timmerman * docs: update readme with features section Signed-off-by: Grant Timmerman * docs: update documentation based on feedback from Adam Signed-off-by: Grant Timmerman --- CONTRIBUTING.md | 16 +++- README.md | 19 ++--- cloud/cloudbuild/v1/build_event_data.go | 4 +- cloud/firestore/v1/document_event_data.go | 6 +- cloud/pubsub/v1/message_published_data.go | 2 +- cloud/scheduler/v1/scheduler_job_data.go | 2 +- samples/main.go | 9 +-- tools/src/postgen-64types.ts | 2 +- tools/src/postgen-bytetypes.ts | 89 +++++++++++++++++++++++ tools/src/postgen.ts | 34 +++++---- 10 files changed, 142 insertions(+), 41 deletions(-) create mode 100644 tools/src/postgen-bytetypes.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ed79fa..511b41c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,11 +29,19 @@ Guidelines](https://opensource.google/conduct/). ## Generating the Library -**Note**: Before generating the package, set up Node.js 12+. +### Prerequisites -To generate this package, run +- Clone this repo +- Clone `https://github.com/googleapis/google-cloudevents` in the same directory as this repo +- Install Node.js 12+ +- Install the `qt` CLI globally: https://github.com/googleapis/google-cloudevents/tree/master/tools/quicktype-wrapper + +### Generate + +To generate this package, run the following script: ``` sh -chmod +x ./tools/gen.sh ./tools/gen.sh -``` \ No newline at end of file +``` + +This will generate the source code for this repo. diff --git a/README.md b/README.md index cb7044c..37c69b0 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,14 @@ [![GoDoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://pkg.go.dev/mod/github.com/googleapis/google-cloudevents-go) [![unstable](http://badges.github.io/stability-badges/dist/unstable.svg)](http://github.com/badges/stability-badges) +This library provides Go types for Google CloudEvent data. -This library provides classes of common event types used with Google services. +## Features + +- Simple import and interface +- Inline documentation for Go structs +- 64 bit number parsing +- Automatic decoding of base64 data ## Installation @@ -17,13 +23,12 @@ go get -u github.com/googleapis/google-cloudevents-go ## Example Usage -Here's an exmaple of using this library with an event from Cloud Pub/Sub with data `MessagePublishedData`. +Example event of type `MessagePublishedData` from Cloud Pub/Sub. ```go package main import ( - "encoding/base64" "encoding/json" "fmt" @@ -36,7 +41,7 @@ func main() { "attributes": { "key": "value" }, - "data": "Q2xvdWQgUHViL1N1Yg==", + "data": "SGVsbG8sIFdvcmxkIQ==", "messageId": "136969346945" }, "subscription": "projects/myproject/subscriptions/mysubscription" @@ -47,11 +52,7 @@ func main() { if err != nil { panic(err) } - s, err := base64.URLEncoding.DecodeString(*e.Message.Data) - if err != nil { - panic(err) - } - fmt.Printf("%+s\n", s) + fmt.Printf("%+s\n", e.Message.Data) // Hello, World! } ``` diff --git a/cloud/cloudbuild/v1/build_event_data.go b/cloud/cloudbuild/v1/build_event_data.go index ffb7dc9..c97b666 100644 --- a/cloud/cloudbuild/v1/build_event_data.go +++ b/cloud/cloudbuild/v1/build_event_data.go @@ -192,8 +192,8 @@ type FileHashValue struct { // FileHashElement: Container message for hash values. type FileHashElement struct { - Type *Type `json:"type"` // The type of hash that was performed. - Value *string `json:"value,omitempty"` // The hash value. + Type *Type `json:"type"` // The type of hash that was performed. + Value []byte `json:"value,omitempty"` // The hash value. } // ResolvedRepoSourceClass: A copy of the build's `source.repo_source`, if exists, with any diff --git a/cloud/firestore/v1/document_event_data.go b/cloud/firestore/v1/document_event_data.go index 881a62a..40837cd 100644 --- a/cloud/firestore/v1/document_event_data.go +++ b/cloud/firestore/v1/document_event_data.go @@ -36,7 +36,7 @@ type OldValue struct { type OldValueField struct { ArrayValue *ArrayValue `json:"arrayValue,omitempty"` // An array value.; ; Cannot directly contain another array value, though can contain an; map which contains another array. BooleanValue *bool `json:"booleanValue,omitempty"` // A boolean value. - BytesValue *string `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. + BytesValue []byte `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. DoubleValue *float64 `json:"doubleValue,omitempty"` // A double value. GeoPointValue *GeoPointValue `json:"geoPointValue,omitempty"` // A geo point value representing a point on the surface of Earth. IntegerValue *int64 `json:"integerValue,string,omitempty"` // An integer value. @@ -51,7 +51,7 @@ type OldValueField struct { type MapValueField struct { ArrayValue *ArrayValue `json:"arrayValue,omitempty"` // An array value.; ; Cannot directly contain another array value, though can contain an; map which contains another array. BooleanValue *bool `json:"booleanValue,omitempty"` // A boolean value. - BytesValue *string `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. + BytesValue []byte `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. DoubleValue *float64 `json:"doubleValue,omitempty"` // A double value. GeoPointValue *GeoPointValue `json:"geoPointValue,omitempty"` // A geo point value representing a point on the surface of Earth. IntegerValue *int64 `json:"integerValue,string,omitempty"` // An integer value. @@ -71,7 +71,7 @@ type MapValue struct { type ValueElement struct { ArrayValue *ArrayValue `json:"arrayValue,omitempty"` // An array value.; ; Cannot directly contain another array value, though can contain an; map which contains another array. BooleanValue *bool `json:"booleanValue,omitempty"` // A boolean value. - BytesValue *string `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. + BytesValue []byte `json:"bytesValue,omitempty"` // A bytes value.; ; Must not exceed 1 MiB - 89 bytes.; Only the first 1,500 bytes are considered by queries. DoubleValue *float64 `json:"doubleValue,omitempty"` // A double value. GeoPointValue *GeoPointValue `json:"geoPointValue,omitempty"` // A geo point value representing a point on the surface of Earth. IntegerValue *int64 `json:"integerValue,string,omitempty"` // An integer value. diff --git a/cloud/pubsub/v1/message_published_data.go b/cloud/pubsub/v1/message_published_data.go index 234897e..7e10457 100644 --- a/cloud/pubsub/v1/message_published_data.go +++ b/cloud/pubsub/v1/message_published_data.go @@ -23,7 +23,7 @@ type MessagePublishedData struct { // Message: The message that was published. type Message struct { Attributes map[string]string `json:"attributes,omitempty"` // Attributes for this message. - Data *string `json:"data,omitempty"` // The binary data in the message. + Data []byte `json:"data,omitempty"` // The binary data in the message. MessageID *string `json:"messageId,omitempty"` // ID of this message, assigned by the server when the message is published.; Guaranteed to be unique within the topic. PublishTime *string `json:"publishTime,omitempty"` // The time at which the message was published, populated by the server when; it receives the `Publish` call. } diff --git a/cloud/scheduler/v1/scheduler_job_data.go b/cloud/scheduler/v1/scheduler_job_data.go index 22161df..2bfa7d0 100644 --- a/cloud/scheduler/v1/scheduler_job_data.go +++ b/cloud/scheduler/v1/scheduler_job_data.go @@ -16,5 +16,5 @@ package scheduler // SchedulerJobData: Scheduler job data. type SchedulerJobData struct { - CustomData *string `json:"customData,omitempty"` // The custom data the user specified when creating the scheduler source. + CustomData []byte `json:"customData,omitempty"` // The custom data the user specified when creating the scheduler source. } diff --git a/samples/main.go b/samples/main.go index e557c5e..fbfd9a2 100644 --- a/samples/main.go +++ b/samples/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/base64" "encoding/json" "fmt" @@ -14,7 +13,7 @@ func main() { "attributes": { "key": "value" }, - "data": "Q2xvdWQgUHViL1N1Yg==", + "data": "SGVsbG8sIFdvcmxkIQ==", "messageId": "136969346945" }, "subscription": "projects/myproject/subscriptions/mysubscription" @@ -25,9 +24,5 @@ func main() { if err != nil { panic(err) } - s, err := base64.URLEncoding.DecodeString(*e.Message.Data) - if err != nil { - panic(err) - } - fmt.Printf("%+s\n", s) + fmt.Printf("%+s\n", e.Message.Data) } diff --git a/tools/src/postgen-64types.ts b/tools/src/postgen-64types.ts index 91c3b2e..d53e66c 100644 --- a/tools/src/postgen-64types.ts +++ b/tools/src/postgen-64types.ts @@ -25,7 +25,7 @@ * as JSON schema does not have a representation for 64 bit numbers serialized as strings. * @see https://github.com/json-schema-org/json-schema-spec/issues/361 * @param {string} golangFile - * @returns {string} The golang file with fixed 64 + * @returns {string} The golang file with fixed 64 bit number fields */ export const fix64BitNumberFields = (golangFile: string) => { const lines = golangFile.split('\n'); diff --git a/tools/src/postgen-bytetypes.ts b/tools/src/postgen-bytetypes.ts new file mode 100644 index 0000000..7c59f5a --- /dev/null +++ b/tools/src/postgen-bytetypes.ts @@ -0,0 +1,89 @@ +const pascalcase = require('pascalcase'); + +/** + * This file converts string types with: + * - byte data + * to + * - byte types + * + * ## Example + * ### Before + * type Message struct { + * Data *string `json:"data,omitempty"` // The binary data in the message. + * } + * + * ### After + * type Message struct { + * Data []byte `json:"data,omitempty"` // The binary data in the message. + * } + * + * Also handles repeated proto fields. + * + * --- + * + * Does this by inspecting the proto, and modifing the source where appropriate. + * + * This modification currently cannot be done at the JSON schema + Quicktype level, + * as JSON schema + Quicktype does create base64 formatted strings. + * @see https://github.com/quicktype/quicktype/issues/138 + * @param {string} golangFile The golang file source code + * @param {string} protoFile The proto source + * @returns {string} The golang file with fixed base64 fields + */ +export const fixByteFields = (golangFile: string, protoFile: string) => { + // A proto line with bytes + type LineWithByte = { + field: string; + isRepeated: boolean; + }; + + // Get all proto lines with the "bytes" type: + const protoLines = protoFile.split('\n'); + const protoLinesWithBytes: LineWithByte[] = []; + protoLines.map((line: string) => { + /** + * Include lines from our proto files. + * The two spaces in front are part of the file identifiers as we do not do AST parsing. + * An eventual solution to this would be to fix the byte fields upstream in the jsonschema + * and quicktype generators. + * @example repeated bytes build_step_outputs = 6; + * @example bytes data = 1; + * @example bytes custom_data = 1; + */ + const isBytes = line.includes(' bytes '); + const isRepeatedBytes = line.includes(' repeated bytes '); + if (isBytes || isRepeatedBytes) { + // The field is right before the " = " sign: + const fieldTokens = line.split(' = ')[0].split(' '); + const field = pascalcase(fieldTokens[fieldTokens.length - 1]); + + // Add the line to the array + protoLinesWithBytes.push({ + field, + isRepeated: isRepeatedBytes, + }); + } + }); + + // For all proto "bytes" fields, + // Change the type to []byte instead of *string. + let updatedGolangFile = golangFile; + protoLinesWithBytes.forEach((byteLine: LineWithByte) => { + // Replace the golang field with the same name + updatedGolangFile = golangFile.split('\n').map((golangLine: string, i: number) => { + // Match on the exact field (with tab and space) + const lineIncludesFieldname = golangLine.includes(` ${byteLine.field} `); + if (lineIncludesFieldname) { + if (byteLine.isRepeated) { + // If repeated proto bytes, replace the type "[]string" to "[][]byte" + return golangLine.replace('[]string', '[][]byte'); + } else { + // If non-repeated proto bytes, replace the type "*string" to "[]byte" + return golangLine.replace('*string', '[]byte'); + } + } + return golangLine; + }).join('\n'); + }); + return updatedGolangFile; +}; diff --git a/tools/src/postgen.ts b/tools/src/postgen.ts index a7ae0eb..ff4459e 100644 --- a/tools/src/postgen.ts +++ b/tools/src/postgen.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import {fix64BitNumberFields} from './postgen-64types'; -const pascalcase = require('pascalcase'); +import {fixByteFields} from './postgen-bytetypes'; const recursive = require("recursive-readdir"); /** @@ -9,19 +9,24 @@ const recursive = require("recursive-readdir"); * * Prerequisites: * - A `cloud/` folder with `*Data.go` files within subfolders. + * - A `firebase/` folder with `*Data.go` files within subfolders. + * - The cloned Google CloudEvents repo in `../google-cloudevents` * * After gen, this postgen scripts modififies the files this way: - * - Add package based on folder + version (i.e. "pubsubv1") + * - Add package based on folder (i.e. "pubsub") + * - Fixes 64 bit types on the structs. + * - Adds the correct Go package. + * - Fixes map of strings interfaces. + * - Fixes godoc string prefixes. + * - Fixes []byte fields. */ // The abs path of the repo root const REPO_ROOT = path.dirname(process.cwd()); +const PROTO_ROOT = path.resolve(REPO_ROOT, '..', 'google-cloudevents', 'proto'); /** - * Runs post-gen processing on the generated files: - * - Fixes 64 bit types on the structs. - * - Adds the correct Go package. - * - Adds JSON Unmarshal and Marshal functions. + * Runs post-gen processing on the generated files. */ async function main() { const filePaths: string[] = [ @@ -31,18 +36,21 @@ async function main() { // For each schema filePaths.forEach(filePath => { - // Read file + // Get relative path info + const relativePath = filePath.substr(REPO_ROOT.length + 1); + + // Get proto file + const relativeProtoPath = `google/events/${relativePath.split('/').slice(0, -1).join('/')}/data.proto`; + const protoPath = path.join(PROTO_ROOT, relativeProtoPath); + const protoFile = fs.readFileSync(protoPath).toString(); + + // Read file and apply fixes const [typeFileContent] = [fs.readFileSync(filePath).toString()] .map(fix64BitNumberFields) + .map((s: string) => fixByteFields(s, protoFile)) .map(fixMapStringToInterface) .map(fixTypeGoDocPrefix) ; - - // Get relative path info - const relativePath = filePath.substr(REPO_ROOT.length + 1); - - // Get Data field name from file path, in PascalCase - const dataField = pascalcase(path.parse(path.basename(filePath)).name); // Get package info //