Containerize protobuf generation and remove node build dep

The Go dependency is still required to build the pack binary.

Use "tilt docker" to use Minikube's Docker instance, if available,
removing the local Docker dependency for Minikube users.

The Makefile continues to not require Docker and runs buf locally.

Remove broken Powershell scripts (can't test on Windows). These scripts
should now be substantially easier to write.

Change-Id: Ie80bf68e0e468a747861bea36fa5b353d9de110d

DEVELOP.md

@@ -5,11 +5,12 @@
 The following dependencies are required for local development:
 
 - [Go](https://golang.org/dl/) >= 1.16.6
-- [Docker](https://docs.docker.com/engine/install/) / moby-engine >= 19.03
 - [Tilt](http://tilt.dev/) >= 0.20.8
-- [NodeJS/npm](https://nodejs.org/en/download/) >= 14
 - Any of the local Kubernetes clusters supported by Tilt.
-  We recommend [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) >= v1.21.0 with the kvm2 driver.
+  We strongly recommend [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) >= 
+  v1.21.0 with the kvm2 driver.
+  - Tilt will use Minikube's embedded Docker server. If Minikube is not used, a local instance of
+    [Docker](https://docs.docker.com/engine/install/) / moby-engine >= 19.03 is required.
 
 See the [Tilt docs](https://docs.tilt.dev/install.html) docs on how to set up your local cluster -
 it won't take more than a few minutes to set up! Example minikube invocation, adjust limits as needed:

Dockerfile.proto

@@ -0,0 +1,47 @@
+# syntax=docker.io/docker/dockerfile:experimental@sha256:de85b2f3a3e8a2f7fe48e8e84a65f6fdd5cd5183afa6412fff9caa6871649c44
+FROM docker.io/golang:1.17.0@sha256:06e92e576fc7a7067a268d47727f3083c0a564331bfcbfdde633157fc91fb17d AS go-tools
+
+RUN mkdir /app
+
+ADD tools/build.sh /app/tools/
+ADD tools/go.* /app/tools/
+
+RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/go \
+	cd /app/tools && CGO_ENABLED=0 ./build.sh
+
+# syntax=docker.io/docker/dockerfile:experimental@sha256:de85b2f3a3e8a2f7fe48e8e84a65f6fdd5cd5183afa6412fff9caa6871649c44
+FROM docker.io/golang:1.17.0@sha256:06e92e576fc7a7067a268d47727f3083c0a564331bfcbfdde633157fc91fb17d AS go-build
+
+COPY --from=go-tools /app /app
+
+ADD buf.* /app
+ADD proto /app/proto
+
+RUN --mount=type=cache,target=/root/.cache \
+	cd /app && \
+	tools/bin/buf lint && \
+	tools/bin/buf generate
+
+FROM node:16-alpine@sha256:004dbac84fed48e20f9888a23e32fa7cf83c2995e174a78d41d9a9dd1e051a20 AS node-build
+
+COPY --from=go-tools /app /app
+
+ADD buf.* /app
+ADD proto /app/proto
+
+ADD tools/package.json /app/tools/
+ADD tools/package-lock.json /app/tools/
+
+RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/root/.npm \
+ 	cd /app/tools && npm ci
+
+RUN --mount=type=cache,target=/root/.cache \
+	cd /app && \
+	tools/bin/buf generate --template buf.gen.web.yaml
+
+FROM scratch AS go-export
+COPY --from=go-build /app/node/pkg/proto pkg/proto
+
+FROM scratch AS node-export
+COPY --from=node-build /app/sdk/js/src/proto sdk/js/src/proto
+COPY --from=node-build /app/explorer/src/proto explorer/src/proto

Makefile

@@ -22,7 +22,9 @@ install:
 
 .PHONY: generate
 generate: dirs
-	./generate-protos.sh
+	cd tools && ./build.sh
+	rm -rf node/pkg/proto
+	tools/bin/buf generate
 
 .PHONY: node
 node: $(BIN)/guardiand

Tiltfile

@@ -210,6 +210,12 @@ build_cloud_function(
     builder = "gcr.io/buildpacks/builder:v1",
 )
 
+local_resource(
+    name = "pack-bin",
+    cmd = "go build -mod=readonly -o bin/pack github.com/buildpacks/pack/cmd/pack",
+    dir = "tools",
+)
+
 k8s_yaml_with_ns("devnet/bigtable.yaml")
 
 k8s_resource("bigtable-emulator", port_forwards = [

generate-protos-web.sh

@@ -1,11 +1,5 @@
 #!/usr/bin/env bash
 
-(
-  cd tools/
-  npm ci
-)
+rm -rf explorer/src/proto sdk/js/src/proto
 
-rm -rf explorer/src/proto
-mkdir -p explorer/src/proto
-
-tools/bin/buf generate --template buf.gen.web.yaml
+DOCKER_BUILDKIT=1 tilt docker build -- --target node-export -f Dockerfile.proto -o type=local,dest=. .

generate-protos.sh

@@ -1,15 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-(
-  cd tools/
-  ./build.sh
-)
-
 # TODO(leo): remove after a while
 rm -rf bridge
 
 rm -rf node/pkg/proto
-
-tools/bin/buf lint
-tools/bin/buf generate
+DOCKER_BUILDKIT=1 tilt docker build -- --target go-export -f Dockerfile.proto -o type=local,dest=node .

generate-wasm.sh

@@ -4,5 +4,5 @@ set -euo pipefail
 
 (
   cd solana
-  DOCKER_BUILDKIT=1 docker build -f Dockerfile.wasm -o type=local,dest=.. .
+  DOCKER_BUILDKIT=1 tilt docker build -- -f Dockerfile.wasm -o type=local,dest=.. .
 )

tools/build.sh

@@ -7,4 +7,3 @@ go build -mod=readonly -o bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grp
 go build -mod=readonly -o bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc
 go build -mod=readonly -o bin/buf github.com/bufbuild/buf/cmd/buf
 go build -mod=readonly -o bin/cobra github.com/spf13/cobra/cobra
-go build -mod=readonly -o bin/pack github.com/buildpacks/pack/cmd/pack