update proto generation and testing pipelines (#358)

This pull request aims to make it possible to generate, format, and lint the protos within this repo.
To accomplish that end, the Dockerfile containing common tools for building the tendermint protos has been moved into this repository and several accompanying changes were made to streamline the proto generation process.

.clang-format

@@ -0,0 +1,11 @@
+---
+Language: Proto
+BasedOnStyle: Google
+IndentWidth: 2
+ColumnLimit: 0
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+SpacesInSquareBrackets: true
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true

.github/workflows/proto-check.yml

@@ -0,0 +1,24 @@
+name: Proto Check
+# Protobuf runs buf (https://buf.build/) lint and check-breakage
+# This workflow is only run when a file in the proto directory
+# has been modified.
+on:
+  workflow_dispatch: # allow running workflow manually
+  pull_request:
+    paths:
+      - "proto/*"
+jobs:
+  proto-lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 4
+    steps:
+      - uses: actions/checkout@v2.4.0
+      - name: lint
+        run: make proto-lint
+  proto-breakage:
+    runs-on: ubuntu-latest
+    timeout-minutes: 4
+    steps:
+      - uses: actions/checkout@v2.4.0
+      - name: check-breakage
+        run: make proto-check-breaking-ci

.github/workflows/proto-dockerfile.yml

@@ -0,0 +1,60 @@
+# This workflow (re)builds and pushes a Docker image containing the
+# protobuf build tools used by the other workflows.
+#
+# When making changes that require updates to the builder image, you
+# should merge the updates first and wait for this workflow to complete,
+# so that the changes will be available for the dependent workflows.
+#
+# TODO(#7272): Update the target location of the builder image.
+
+name: Build & Push TM Proto Builder
+on:
+  pull_request:
+    paths:
+      - "proto/*"
+  push:
+    branches:
+      - master
+    paths:
+      - "proto/*"
+  schedule:
+    # run this job once a month to recieve any go or buf updates
+    - cron: "* * 1 * *"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=tendermintdev/docker-build-proto
+          VERSION=noop
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
+              VERSION=latest
+            fi
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}"
+          echo ::set-output name=tags::${TAGS}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1.6.0
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1.10.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Publish to Docker Hub
+        uses: docker/build-push-action@v2.7.0
+        with:
+          context: ./proto
+          file: ./proto/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}

.gitignore

@@ -6,3 +6,4 @@
 *.gz
 *.dvi
 .idea
+*.pb.go

Makefile

@@ -0,0 +1,31 @@
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto
+HTTPS_GIT := https://github.com/tendermint/spec.git
+
+###############################################################################
+###                                Protobuf                                 ###
+###############################################################################
+
+proto-all: proto-lint proto-check-breaking
+.PHONY: proto-all
+
+proto-gen:
+	@echo "Generating Protobuf files"
+	@$(DOCKER_PROTO_BUILDER) buf generate --template=./buf.gen.yaml --config ./buf.yaml
+.PHONY: proto-gen
+
+proto-lint:
+	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json --config ./buf.yaml
+.PHONY: proto-lint
+
+proto-format:
+	@echo "Formatting Protobuf files"
+	@$(DOCKER_PROTO_BUILDER) find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \;
+.PHONY: proto-format
+
+proto-check-breaking:
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git --config ./buf.yaml
+.PHONY: proto-check-breaking
+
+proto-check-breaking-ci:
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT) --config ./buf.yaml
+.PHONY: proto-check-breaking-ci

buf.gen.yaml

@@ -0,0 +1,14 @@
+# The version of the generation template (required).
+# The only currently-valid value is v1beta1.
+version: v1beta1
+
+# The plugins to run.
+plugins:
+  # The name of the plugin.
+  - name: gogofaster
+    # The directory where the generated proto output will be written.
+    # The directory is relative to where the generation tool was run.
+    out: proto
+    # Set options to assign import paths to the well-known types
+    # and to enable service generation.
+    opt: Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative

buf.yaml

@@ -0,0 +1,16 @@
+version: v1beta1
+
+build:
+  roots:
+    - proto
+    - third_party/proto
+lint:
+  use:
+    - BASIC
+    - FILE_LOWER_SNAKE_CASE
+    - UNARY_RPC
+  ignore:
+    - gogoproto
+breaking:
+  use:
+    - FILE

proto/Dockerfile

@@ -0,0 +1,25 @@
+# This Dockerfile defines an image containing tools for linting, formatting,
+# and compiling the Tendermint protos.
+
+FROM bufbuild/buf:latest as buf
+
+FROM golang:1.17-alpine
+
+# Install a commonly used set of programs for use with our protos.
+# clang-extra-tools is included here because it provides clang-format,
+# used to format the .proto files.
+RUN apk add --update --no-cache build-base clang-extra-tools curl git && \
+  apk add --update --no-cache build-base clang-extra-tools curl git go && \
+  rm -rf /var/cache/apk/*
+
+ENV GOLANG_PROTOBUF_VERSION=1.3.1 \
+  GOGO_PROTOBUF_VERSION=1.3.2
+
+# Retrieve the go protoc programs and copy them into the PATH
+RUN go install github.com/golang/protobuf/protoc-gen-go@v${GOLANG_PROTOBUF_VERSION} && \
+  go install github.com/gogo/protobuf/protoc-gen-gogo@v${GOGO_PROTOBUF_VERSION} && \
+  go install github.com/gogo/protobuf/protoc-gen-gogofaster@v${GOGO_PROTOBUF_VERSION} && \
+  mv "$(go env GOPATH)"/bin/* /usr/local/bin/
+
+# Copy the 'buf' program out of the buildbuf/buf container.
+COPY --from=buf /usr/local/bin/* /usr/local/bin/

proto/tendermint/blocksync/types.proto

@@ -1,16 +1,15 @@
 syntax = "proto3";
 package tendermint.blocksync;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/blocksync";
-
 import "tendermint/types/block.proto";
 
 // BlockRequest requests a block for a specific height
 message BlockRequest {
   int64 height = 1;
 }
 
-// NoBlockResponse informs the node that the peer does not have block at the requested height
+// NoBlockResponse informs the node that the peer does not have block at the
+// requested height
 message NoBlockResponse {
   int64 height = 1;
 }
@@ -21,8 +20,7 @@ message BlockResponse {
 }
 
 // StatusRequest requests the status of a peer.
-message StatusRequest {
-}
+message StatusRequest {}
 
 // StatusResponse is a peer response to inform their status.
 message StatusResponse {

proto/tendermint/consensus/types.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.consensus;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/consensus";
-
 import "gogoproto/gogo.proto";
 import "tendermint/types/types.proto";
 import "tendermint/libs/bits/types.proto";
@@ -17,15 +15,18 @@ message NewRoundStep {
   int32  last_commit_round        = 5;
 }
 
-// NewValidBlock is sent when a validator observes a valid block B in some round r,
-//i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r.
+// NewValidBlock is sent when a validator observes a valid block B in some round
+// r,
+// i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in
+// the round r.
 // In case the block is also committed, then IsCommit flag is set to true.
 message NewValidBlock {
   int64                          height                = 1;
   int32                          round                 = 2;
-  tendermint.types.PartSetHeader block_part_set_header = 3 [(gogoproto.nullable) = false];
-  tendermint.libs.bits.BitArray  block_parts           = 4;
-  bool                           is_commit             = 5;
+  tendermint.types.PartSetHeader block_part_set_header = 3
+      [(gogoproto.nullable) = false];
+  tendermint.libs.bits.BitArray block_parts = 4;
+  bool                          is_commit   = 5;
 }
 
 // Proposal is sent when a new block is proposed.
@@ -37,7 +38,8 @@ message Proposal {
 message ProposalPOL {
   int64                         height             = 1;
   int32                         proposal_pol_round = 2;
-  tendermint.libs.bits.BitArray proposal_pol       = 3 [(gogoproto.nullable) = false];
+  tendermint.libs.bits.BitArray proposal_pol       = 3
+      [(gogoproto.nullable) = false];
 }
 
 // BlockPart is sent when gossipping a piece of the proposed block.
@@ -65,16 +67,19 @@ message VoteSetMaj23 {
   int64                          height   = 1;
   int32                          round    = 2;
   tendermint.types.SignedMsgType type     = 3;
-  tendermint.types.BlockID       block_id = 4 [(gogoproto.customname) = "BlockID", (gogoproto.nullable) = false];
+  tendermint.types.BlockID       block_id = 4
+      [(gogoproto.customname) = "BlockID", (gogoproto.nullable) = false];
 }
 
-// VoteSetBits is sent to communicate the bit-array of votes seen for the BlockID.
+// VoteSetBits is sent to communicate the bit-array of votes seen for the
+// BlockID.
 message VoteSetBits {
   int64                          height   = 1;
   int32                          round    = 2;
   tendermint.types.SignedMsgType type     = 3;
-  tendermint.types.BlockID       block_id = 4 [(gogoproto.customname) = "BlockID", (gogoproto.nullable) = false];
-  tendermint.libs.bits.BitArray  votes    = 5 [(gogoproto.nullable) = false];
+  tendermint.types.BlockID       block_id = 4
+      [(gogoproto.customname) = "BlockID", (gogoproto.nullable) = false];
+  tendermint.libs.bits.BitArray votes = 5 [(gogoproto.nullable) = false];
 }
 
 message Message {

proto/tendermint/crypto/keys.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.crypto;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/crypto";
-
 import "gogoproto/gogo.proto";
 
 // PublicKey defines the keys available for use with Tendermint Validators

proto/tendermint/crypto/proof.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.crypto;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/crypto";
-
 import "gogoproto/gogo.proto";
 
 message Proof {

proto/tendermint/libs/bits/types.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+package tendermint.libs.bits;
+
+message BitArray {
+  int64           bits  = 1;
+  repeated uint64 elems = 2;
+}

proto/tendermint/mempool/types.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.mempool;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/mempool";
-
 message Txs {
   repeated bytes txs = 1;
 }

proto/tendermint/p2p/conn.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.p2p;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/p2p";
-
 import "gogoproto/gogo.proto";
 import "tendermint/crypto/keys.proto";
 

proto/tendermint/p2p/pex.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.p2p;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/p2p";
-
 import "gogoproto/gogo.proto";
 
 message PexAddress {
@@ -18,9 +16,9 @@ message PexResponse {
 }
 
 message PexMessage {
-  reserved 1, 2; // See https://github.com/tendermint/spec/pull/352
+  reserved 1, 2;  // See https://github.com/tendermint/spec/pull/352
   oneof sum {
-    PexRequest    pex_request     = 3;
-    PexResponse   pex_response    = 4;
+    PexRequest  pex_request  = 3;
+    PexResponse pex_response = 4;
   }
 }

proto/tendermint/p2p/types.proto

@@ -1,8 +1,6 @@
 syntax = "proto3";
 package tendermint.p2p;
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/p2p";
-
 import "gogoproto/gogo.proto";
 import "google/protobuf/timestamp.proto";
 
@@ -36,7 +34,9 @@ message PeerInfo {
 
 message PeerAddressInfo {
   string                    address           = 1;
-  google.protobuf.Timestamp last_dial_success = 2 [(gogoproto.stdtime) = true];
-  google.protobuf.Timestamp last_dial_failure = 3 [(gogoproto.stdtime) = true];
-  uint32                    dial_failures     = 4;
+  google.protobuf.Timestamp last_dial_success = 2
+      [(gogoproto.stdtime) = true];
+  google.protobuf.Timestamp last_dial_failure = 3
+      [(gogoproto.stdtime) = true];
+  uint32 dial_failures = 4;
 }

proto/tendermint/statesync/types.proto

@@ -5,8 +5,6 @@ import "gogoproto/gogo.proto";
 import "tendermint/types/types.proto";
 import "tendermint/types/params.proto";
 
-option go_package = "github.com/tendermint/tendermint/proto/tendermint/statesync";
-
 message Message {
   oneof sum {
     SnapshotsRequest   snapshots_request    = 1;
@@ -58,5 +56,6 @@ message ParamsRequest {
 
 message ParamsResponse {
   uint64                           height           = 1;
-  tendermint.types.ConsensusParams consensus_params = 2 [(gogoproto.nullable) = false];
+  tendermint.types.ConsensusParams consensus_params = 2
+      [(gogoproto.nullable) = false];
 }