[proto tooling] add hermetic protoc compilation managed by bazel (#201)

## Summary

Pin the entire codegen toolchain so a single `make proto` produces
byte-identical output on any host.

Previously, `make proto` was dependent on host-installed versions of
protoc + plugins which led to different output when run on different
machine.

## Test Plan
✅ `make fmt && make proto && make build && make test && make e2e-test`
✅ `make proto` <-- no changes to generated source
✅ Modify .proto -> call `make proto` -> see expected updates to all
`protopb/*` files

## Issues
T3-CODE-204

Author: albertywu

MODULE.bazel

@@ -9,6 +9,9 @@ bazel_dep(name = "rules_proto", version = "7.1.0")
 # dependencies) from source on every cold build. Requires
 # --incompatible_enable_proto_toolchain_resolution (set in .bazelrc).
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
+protoc.toolchain(
+    version = "v29.3",
+)
 use_repo(protoc, "toolchains_protoc_hub")
 
 register_toolchains("@toolchains_protoc_hub//:all")
@@ -37,6 +40,11 @@ go_sdk.host()
 
 go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//:go.mod")
+go_deps.module(
+    path = "google.golang.org/grpc/cmd/protoc-gen-go-grpc",
+    sum = "h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=",
+    version = "v1.5.1",
+)
 
 # All *direct* Go dependencies of the module have to be listed explicitly
 use_repo(
@@ -49,6 +57,7 @@ use_repo(
     "com_github_uber_go_tally_v4",
     "in_gopkg_yaml_v3",
     "org_golang_google_grpc",
+    "org_golang_google_grpc_cmd_protoc_gen_go_grpc",
     "org_golang_google_protobuf",
     "org_golang_x_oauth2",
     "org_uber_go_fx",

Makefile

@@ -26,6 +26,11 @@ YAMLFMT_VERSION ?= v0.16.0
 # goimports version for Go formatting + import fixing
 GOIMPORTS_VERSION ?= v0.33.0
 
+# Proto packages: <domain>/<service> dirs whose protopb/ holds the generated
+# stubs. Each is generated by Bazel into bazel-bin/tool/proto/<domain>_<service>/
+# (the out_dir convention in tool/proto/BUILD.bazel) and copied back here.
+PROTO_PACKAGES = submitqueue/gateway submitqueue/orchestrator stovepipe/gateway stovepipe/orchestrator
+
 # Set REPO_ROOT for docker-compose
 export REPO_ROOT := $(shell pwd)
 
@@ -106,10 +111,7 @@ clean: ## Clean generated files and binaries
 
 clean-proto: ## Clean generated proto files
 	@echo "Cleaning generated proto files..."
-	@rm -rf submitqueue/gateway/protopb/*.pb.go
-	@rm -rf submitqueue/orchestrator/protopb/*.pb.go
-	@rm -rf stovepipe/gateway/protopb/*.pb.go
-	@rm -rf stovepipe/orchestrator/protopb/*.pb.go
+	@rm -f $(foreach p,$(PROTO_PACKAGES),$(p)/protopb/*.pb.go $(p)/protopb/*.pb.yarpc.go)
 	@echo "Proto clean complete!"
 
 deps: tidy-go ## Download and tidy Go dependencies
@@ -338,23 +340,15 @@ mocks: ## Generate mock files using mockgen
 	@echo "Mocks generated successfully!"
 
 proto: ## Generate protobuf files from .proto definitions
-	@echo "Generating protobuf files with protoc..."
-	@protoc --go_out=submitqueue/gateway/protopb --go_opt=paths=source_relative \
-	  --go-grpc_out=submitqueue/gateway/protopb --go-grpc_opt=paths=source_relative \
-	  --yarpc-go_out=submitqueue/gateway/protopb --yarpc-go_opt=paths=source_relative \
-	  --proto_path=submitqueue/gateway/proto submitqueue/gateway/proto/gateway.proto
-	@protoc --go_out=submitqueue/orchestrator/protopb --go_opt=paths=source_relative \
-	  --go-grpc_out=submitqueue/orchestrator/protopb --go-grpc_opt=paths=source_relative \
-	  --yarpc-go_out=submitqueue/orchestrator/protopb --yarpc-go_opt=paths=source_relative \
-	  --proto_path=submitqueue/orchestrator/proto submitqueue/orchestrator/proto/orchestrator.proto
-	@protoc --go_out=stovepipe/gateway/protopb --go_opt=paths=source_relative \
-	  --go-grpc_out=stovepipe/gateway/protopb --go-grpc_opt=paths=source_relative \
-	  --yarpc-go_out=stovepipe/gateway/protopb --yarpc-go_opt=paths=source_relative \
-	  --proto_path=stovepipe/gateway/proto stovepipe/gateway/proto/gateway.proto
-	@protoc --go_out=stovepipe/orchestrator/protopb --go_opt=paths=source_relative \
-	  --go-grpc_out=stovepipe/orchestrator/protopb --go-grpc_opt=paths=source_relative \
-	  --yarpc-go_out=stovepipe/orchestrator/protopb --yarpc-go_opt=paths=source_relative \
-	  --proto_path=stovepipe/orchestrator/proto stovepipe/orchestrator/proto/orchestrator.proto
+	@echo "Generating protobuf files with Bazel..."
+	@$(BAZEL) build //tool/proto:generated
+	@set -e; for pkg in $(PROTO_PACKAGES); do \
+	  out=$$(echo $$pkg | tr / _); base=$$(basename $$pkg); \
+	  for f in $$base.pb.go $$base.pb.yarpc.go $${base}_grpc.pb.go; do \
+	    cp -f bazel-bin/tool/proto/$$out/$$f $$pkg/protopb/$$f; \
+	  done; \
+	done
+	@$(BAZEL) run @rules_go//go -- run golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) -w $(addsuffix /protopb,$(PROTO_PACKAGES))
 	@echo "Protobuf files generated successfully!"
 
 # Bazel query helpers

doc/howto/DEVELOPMENT.md

@@ -86,12 +86,7 @@ GoLand works with Go modules automatically. Open the project root and GoLand wil
 
 ```bash
 # macOS
-brew install protobuf grpcurl
-
-# Go protoc plugins (only if modifying .proto files)
-go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
-go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
-go install go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go@latest
+brew install grpcurl
 ```
 
 ## Common Make Targets
@@ -138,8 +133,8 @@ See [TESTING.md](TESTING.md) for the full testing guide, including integration a
 ## Troubleshooting
 
 **Proto generation fails:**
-- Ensure all three protoc plugins are installed (see Optional Tools above)
-- Check that `protoc` is in your PATH: `which protoc`
+- Run `make proto`; Bazel provides the pinned `protoc` and plugin toolchain.
+- If Bazel cannot fetch tools, check network access and the repository cache configuration in `.bazelrc`.
 
 **Build fails after proto changes:**
 - Run `make proto` to regenerate proto files

stovepipe/gateway/proto/BUILD.bazel

@@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library")
 load("@rules_go//proto:def.bzl", "go_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 
+exports_files(
+    ["gateway.proto"],
+    visibility = ["//tool/proto:__pkg__"],
+)
+
 proto_library(
     name = "gatewaypb_proto",
     srcs = ["gateway.proto"],

stovepipe/orchestrator/proto/BUILD.bazel

@@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library")
 load("@rules_go//proto:def.bzl", "go_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 
+exports_files(
+    ["orchestrator.proto"],
+    visibility = ["//tool/proto:__pkg__"],
+)
+
 proto_library(
     name = "orchestratorpb_proto",
     srcs = ["orchestrator.proto"],

submitqueue/gateway/proto/BUILD.bazel

@@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library")
 load("@rules_go//proto:def.bzl", "go_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 
+exports_files(
+    ["gateway.proto"],
+    visibility = ["//tool/proto:__pkg__"],
+)
+
 proto_library(
     name = "gatewaypb_proto",
     srcs = ["gateway.proto"],

submitqueue/orchestrator/proto/BUILD.bazel

@@ -2,6 +2,11 @@ load("@rules_go//go:def.bzl", "go_library")
 load("@rules_go//proto:def.bzl", "go_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 
+exports_files(
+    ["orchestrator.proto"],
+    visibility = ["//tool/proto:__pkg__"],
+)
+
 proto_library(
     name = "orchestratorpb_proto",
     srcs = ["orchestrator.proto"],

tool/proto/BUILD.bazel

@@ -0,0 +1,35 @@
+load("//tool/proto:proto_codegen.bzl", "go_proto_generated_files")
+
+go_proto_generated_files(
+    name = "submitqueue_gateway",
+    src = "//submitqueue/gateway/proto:gateway.proto",
+    out_dir = "submitqueue_gateway",
+)
+
+go_proto_generated_files(
+    name = "submitqueue_orchestrator",
+    src = "//submitqueue/orchestrator/proto:orchestrator.proto",
+    out_dir = "submitqueue_orchestrator",
+)
+
+go_proto_generated_files(
+    name = "stovepipe_gateway",
+    src = "//stovepipe/gateway/proto:gateway.proto",
+    out_dir = "stovepipe_gateway",
+)
+
+go_proto_generated_files(
+    name = "stovepipe_orchestrator",
+    src = "//stovepipe/orchestrator/proto:orchestrator.proto",
+    out_dir = "stovepipe_orchestrator",
+)
+
+filegroup(
+    name = "generated",
+    srcs = [
+        ":stovepipe_gateway",
+        ":stovepipe_orchestrator",
+        ":submitqueue_gateway",
+        ":submitqueue_orchestrator",
+    ],
+)

tool/proto/proto_codegen.bzl

@@ -0,0 +1,84 @@
+"""Hermetic protobuf code generation helpers."""
+
+def _strip_proto_suffix(filename):
+    if not filename.endswith(".proto"):
+        fail("expected a .proto source, got {}".format(filename))
+    return filename[:-len(".proto")]
+
+def _go_proto_generated_files_impl(ctx):
+    src = ctx.file.src
+    base = _strip_proto_suffix(src.basename)
+
+    out_dir = ctx.attr.out_dir
+    outputs = [
+        ctx.actions.declare_file("{}/{}.pb.go".format(out_dir, base)),
+        ctx.actions.declare_file("{}/{}.pb.yarpc.go".format(out_dir, base)),
+        ctx.actions.declare_file("{}/{}_grpc.pb.go".format(out_dir, base)),
+    ]
+
+    proto_toolchain = ctx.toolchains["@rules_proto//proto:toolchain_type"].proto
+    protoc = proto_toolchain.proto_compiler
+
+    args = ctx.actions.args()
+    for opt in proto_toolchain.protoc_opts:
+        args.add(opt)
+    args.add("--plugin=protoc-gen-go={}".format(ctx.executable._protoc_gen_go.path))
+    args.add("--plugin=protoc-gen-go-grpc={}".format(ctx.executable._protoc_gen_go_grpc.path))
+    args.add("--plugin=protoc-gen-yarpc-go={}".format(ctx.executable._protoc_gen_yarpc_go.path))
+    args.add("--go_out={}".format(outputs[0].dirname))
+    args.add("--go_opt=paths=source_relative")
+    args.add("--go-grpc_out={}".format(outputs[0].dirname))
+    args.add("--go-grpc_opt=paths=source_relative")
+    args.add("--yarpc-go_out={}".format(outputs[0].dirname))
+    args.add("--yarpc-go_opt=paths=source_relative")
+    args.add("--proto_path={}".format(src.dirname))
+    args.add(src.path)
+
+    tools = [
+        protoc,
+        ctx.executable._protoc_gen_go,
+        ctx.executable._protoc_gen_go_grpc,
+        ctx.executable._protoc_gen_yarpc_go,
+    ]
+
+    ctx.actions.run_shell(
+        inputs = [src],
+        outputs = outputs,
+        tools = tools,
+        command = "mkdir -p {out_dir} && {protoc} \"$@\"".format(
+            out_dir = outputs[0].dirname,
+            protoc = protoc.executable.path,
+        ),
+        arguments = [args],
+        mnemonic = "HermeticGoProto",
+        progress_message = "Generating Go protobuf files for {}".format(src.short_path),
+    )
+
+    return [DefaultInfo(files = depset(outputs))]
+
+go_proto_generated_files = rule(
+    implementation = _go_proto_generated_files_impl,
+    attrs = {
+        "src": attr.label(
+            allow_single_file = [".proto"],
+            mandatory = True,
+        ),
+        "out_dir": attr.string(mandatory = True),
+        "_protoc_gen_go": attr.label(
+            default = "@org_golang_google_protobuf//cmd/protoc-gen-go",
+            executable = True,
+            cfg = "exec",
+        ),
+        "_protoc_gen_go_grpc": attr.label(
+            default = "@org_golang_google_grpc_cmd_protoc_gen_go_grpc//:protoc-gen-go-grpc",
+            executable = True,
+            cfg = "exec",
+        ),
+        "_protoc_gen_yarpc_go": attr.label(
+            default = "@org_uber_go_yarpc//encoding/protobuf/protoc-gen-yarpc-go",
+            executable = True,
+            cfg = "exec",
+        ),
+    },
+    toolchains = ["@rules_proto//proto:toolchain_type"],
+)