From 82e971b95f34e445d1eb51ff202be9441ec4c906 Mon Sep 17 00:00:00 2001 From: Adam Ross Date: Tue, 4 Oct 2022 14:38:15 -0700 Subject: [PATCH] feat: Tooling to generate types from protos (#115) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam Ross --- .gitignore | 1 + CONTRIBUTING.md | 37 ++++++++---- README.md | 6 +- generate-code.sh | 120 +++++++++++++++++++++++++++++++++++++++ tools/setup-generator.sh | 87 ++++++++++++++++++++++++++++ 5 files changed, 238 insertions(+), 13 deletions(-) create mode 100644 generate-code.sh create mode 100644 tools/setup-generator.sh diff --git a/.gitignore b/.gitignore index 688aa61..fcb26f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ tools/build/** +tmp/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 511b41c..483c297 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,19 +29,36 @@ Guidelines](https://opensource.google/conduct/). ## Generating the Library -### Prerequisites +```sh +git clone https://github.com/googleapis/google-cloudevents-go +cd google-cloudevents-go +sh ./tools/setup-generator.sh +export GENERATE_DATA_SOURCE=tmp/google-cloudevents +export GENERATE_PROTOC_PATH=tmp/protobuf/bin/protoc +sh ./generate-code.sh +``` + +### Generating the Library (Locally modified protos) + +The "data source" for code generation is a collection of protos maintained in +[googleapis/google-cloudevets](https://github.com/googleapis/google-cloudevents). -- Clone this repo -- Clone `https://github.com/googleapis/google-cloudevents` in the same directory as this repo -- Install Node.js 12+ -- Install the `qt` CLI globally: https://github.com/googleapis/google-cloudevents/tree/master/tools/quicktype-wrapper +If you have a local copy of this repository, such as for trying modifications to +those protos, you can use these instructions to use that copy instead of +retrieving a new clone. -### Generate +```sh +cd /path/to/shared/repositories +git clone https://github.com/googleapis/google-cloudevents --depth 1 -To generate this package, run the following script: +cd path/to/the/project +git clone https://github.com/googleapis/google-cloudevents-go -``` sh -./tools/gen.sh +# Configure this before running the setup script to reuse the repository. +export GENERATE_DATA_SOURCE=/path/to/shared/repositories/google-cloudevents +sh ./tools/setup-generator.sh +export GENERATE_PROTOC_PATH=tmp/protobuf/bin/protoc +sh ./generate-code.sh ``` -This will generate the source code for this repo. +A similar "skip" does not exist for the protobuf library. diff --git a/README.md b/README.md index 7c61755..a471d5c 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ This library has been unstable. We are working to make it production ready. For more details, see #113. -You can access the previous development version of this library at its [latest -commit](https://github.com/googleapis/google-cloudevents-go/tree/eabc4f975145db6a8a482109a9ec11ee8724dfb5/) +You can access the previous development version of this library at its +[latest commit](https://github.com/googleapis/google-cloudevents-go/tree/eabc4f975145db6a8a482109a9ec11ee8724dfb5/) # Google CloudEvents - Go @@ -22,7 +22,7 @@ This library provides Go types for Google CloudEvent data. ## Installation -**Note**: This library requires Go 1.11+. +**Note**: This library requires Go 1.12+ and is tested at Go 1.17+ To install this package, run: diff --git a/generate-code.sh b/generate-code.sh new file mode 100644 index 0000000..4415abc --- /dev/null +++ b/generate-code.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Copyright 2022, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates code from https://github.com/googleapis/google-cloudevents: +# - Types from protobuf messages +# +# Configuration: +# - GENERATE_DATA_SOURCE: Path to google-cloudevent repo. +# - GENERATE_PROTOC_PATH: Path to protobuf tool. +# +# Usage: +# - sh ./build.sh +# - GENERATE_DATA_SOURCE=tmp/google-cloudevents GENERATE_PROTOC_PATH=protoc sh ./build.sh + +set -e + +# Output Utilities +_heading() { + echo + echo "$(tput bold)${1}$(tput sgr0)" +} + +name=$(basename "${BASH_SOURCE[0]}") +library_version=$(git rev-parse --short HEAD) +library_date=$(git show -s --format=%ci "${library_version}") +echo "google-cloudevents-go > ${name} (${library_version} on ${library_date})" + +# Required configuration. +if [[ -z "${GENERATE_DATA_SOURCE}" ]]; then + echo + echo "Environment variable 'GENERATE_DATE_SOURCE' not found." + echo "Please run 'sh tools/setup-generator.sh' and follow the instructions." + exit 1 +fi + +if [[ -z "${GENERATE_PROTOC_PATH}" ]]; then + echo + echo "Environment variable 'GENERATE_PROTOC_PATH' not found." + echo "Please run 'sh tools/setup_generator.sh' and follow the instructions." + exit 1 +fi + +# Derive proto repo metadata. +data_version=$(git -C "${GENERATE_DATA_SOURCE}" rev-parse --short HEAD) +data_date=$(git -C "${GENERATE_DATA_SOURCE}" show -s --format=%ci "${data_version}") +# Derive proto lookup paths. +src_dir="${GENERATE_DATA_SOURCE}/proto/google/events" +googleapis_dir="${GENERATE_DATA_SOURCE}/third_party/googleapis" +# Prepare dependencies for build & generation. +monitored_proto="google/api/monitored_resource.proto" + +_heading "Preparing to generate library..." +echo "- Schema Source Repository: \t${GENERATE_DATA_SOURCE} (${data_version} on ${data_date})" +echo "- Proto Source Directory: \t${src_dir}" +echo "- Shared googleapis Protos: \t${googleapis_dir}" + +# Manifest file with details about how code was most recently generated. +# Useful when troubleshooting without build logs. +cat > generated.txt < ${code_dest}data${version}" + + $GENERATE_PROTOC_PATH --go_out=. \ + --go_opt="M${proto_src}"="${code_dest}data${version};${product}data${version}" \ + --go_opt="M${monitored_proto}"="github.com/googleapis/google-cloudevents-go/shared/google;google" \ + --proto_path="${src_dir}" \ + --proto_path="${googleapis_dir}" \ + "${proto_src}" +} + +_heading "Generating data type code in Go..." +for i in $(find "${GENERATE_DATA_SOURCE}/proto" -type f -name data.proto); do + _generateData "$i" +done diff --git a/tools/setup-generator.sh b/tools/setup-generator.sh new file mode 100644 index 0000000..229dcc5 --- /dev/null +++ b/tools/setup-generator.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Copyright 2022, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup environment for code generation: +# - install protobuf tools in a local temp directory +# - clone google-cloudevents repo if needed +set -e + +name=$(basename $BASH_SOURCE) +library_version=$(git rev-parse --short HEAD) +library_date=$(git show -s --format=%ci "${library_version}") +echo "google-cloudevents-go > ${name} (${library_version} on ${library_date})" +echo + +# Create a location for local tool installation. +echo "- Removing existing tmp/ directory" +rm -rf tmp +mkdir tmp + +# Protobuf Setup +PROTOBUF_VERSION=21.6 +# protoc is a native application, so we need to download different zip files +# and use different binaries depending on the OS. +echo "- Determining OS type" +case "$OSTYPE" in + linux*) + PROTOBUF_PLATFORM=linux-x86_64 + PROTOC=tmp/protobuf/bin/protoc + ;; + win* | msys* | cygwin*) + PROTOBUF_PLATFORM=win64 + PROTOC=tmp/protobuf/bin/protoc.exe + ;; + darwin*) + PROTOBUF_PLATFORM=osx-x86_64 + PROTOC=tmp/protobuf/bin/protoc + ;; + *) + echo "Unknown OSTYPE: $OSTYPE" + exit 1 +esac + +# We download a specific version rather than using package managers +# for portability and being able to rely on the version being available +# as soon as it's released on GitHub. +echo "- Downloading protobuf tools..." +cd tmp +curl -sSL \ + https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-$PROTOBUF_PLATFORM.zip \ + --output protobuf.zip +(mkdir protobuf && cd protobuf && unzip -q ../protobuf.zip) +cd .. +chmod +x $PROTOC + +echo "- Downloaded protobuf ${PROTOBUF_VERSION} for ${PROTOBUF_PLATFORM}" + +echo "- Downloading & installing the Go protocol buffers plugin..." +go install google.golang.org/protobuf/cmd/protoc-gen-go@latest +echo "- Protobuf tooling installation complete" + +if [[ -z "${GENERATE_DATA_SOURCE}" ]]; then + echo "- Cloning github.com/googleapis/google-cloudevents into tmp" + # For the moment, just clone google-cloudevents. Later we might make + # it a submodule. We clone quietly, and only with a depth of 1 + # as we don't need history. + dest='tmp/google-cloudevents' + git clone https://github.com/googleapis/google-cloudevents "${dest}" -q --depth 1 +fi + +echo +echo "Configure environment for generate_code.sh:" +echo "- Usage: Configure the path to protobuf tools ('export GENERATE_PROTOC_PATH=$PROTOC')" +if [[ -z "${GENERATE_DATA_SOURCE}" ]]; then + echo "- Usage: Configure the path to proto definitions ('export GENERATE_DATA_SOURCE=${dest}')" +fi