Watch task outputs to avoid cache restorations (vercel#1269)

Start a daemon, notify it of output globs when a task completes, watch those outputs, check if we can skip cache restores on subsequent runs. Uses `grpc` and `protobuf` for communication. Before shipping: - [x] Wire up `--no-daemon` flag to `run` - [x] Add command for `daemon status` - [x] Add command for `daemon stop` - [x] Add command for `daemon start` - [x] Add command for `daemon restart` - [x] Proper daemon logfile management - [x] Ensure that the client tries to start the server after maybe killing a previous incarnation - [x] Investigate golang internal lockfile - [ ] Verify ordering guarantees on file-watching - [ ] Sort out proper syntax for `protos` in `Makefile` - [x] Shut down if repo root is deleted - [x] Proper signal handling - [ ] Handle watching too many files - [ ] Consider using [panicwrap](https://github.com/mitchellh/panicwrap)
cgoinglove · Jun 27, 2022 · 5815666 · 5815666
1 parent e655e44
commit 5815666
Show file tree

Hide file tree

Showing 35 changed files with 2,824 additions and 97 deletions.
diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml
@@ -33,6 +33,17 @@ jobs:
           cache: true
           cache-dependency-path: cli/go.sum
 
+      - name: Set Up Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: "3.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Up Go and GRPC protobuf
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
+
       - uses: pnpm/action-setup@v2.2.2
         with:
           version: 7.2.1
@@ -93,6 +104,17 @@ jobs:
           cache: true
           cache-dependency-path: cli/go.sum
 
+      - name: Set Up Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: "3.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Up Go and GRPC protobuf
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
+
       - uses: pnpm/action-setup@v2.2.2
         with:
           version: 7.2.1

diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
@@ -26,11 +26,27 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
+
       - uses: actions/setup-go@v3
         with:
           go-version: 1.17
           cache: true
           cache-dependency-path: cli/go.sum
+
+      - name: Set Up Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: "3.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Up Go and GRPC protobuf
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
+
+      - name: Setup Protos
+        run: cd cli && make compile-protos
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:

diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -1,5 +1,5 @@
 {
-	"recommendations": [
+  "recommendations": [
     "bradlc.vscode-tailwindcss",
     "christian-kohler.npm-intellisense",
     "dbaeumer.vscode-eslint",
@@ -14,6 +14,7 @@
     "ms-vscode-remote.remote-containers",
     "silvenon.mdx",
     "windmilleng.vscode-go-autotest",
-    "yzhang.markdown-all-in-one"
+    "yzhang.markdown-all-in-one",
+    "zxh404.vscode-proto3"
   ]
 }
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -10,12 +10,28 @@
       "cwd": "${workspaceFolder}/cli",
       "program": "${workspaceFolder}/cli/scripts/e2e/e2e.ts",
       "request": "launch",
-      "skipFiles": [
-        "<node_internals>/**"
-      ],
+      "skipFiles": ["<node_internals>/**"],
       "type": "node",
       "preLaunchTask": "prepare e2e"
     },
+    {
+      "name": "Server",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceRoot}/cli/cmd/turbo",
+      "cwd": "${workspaceRoot}/examples/basic",
+      "args": ["daemon"]
+    },
+    {
+      "name": "Client",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceRoot}/cli/cmd/turbo",
+      "cwd": "${workspaceRoot}/examples/basic",
+      "args": ["run", "build", "-vvv"]
+    },
     {
       "name": "turbo --version",
       "type": "go",
@@ -69,6 +85,6 @@
       "program": "${workspaceRoot}/cli/cmd/turbo",
       "cwd": "${workspaceRoot}",
       "args": ["run", "build", "--force"]
-    },
+    }
   ]
 }
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -1,6 +1,15 @@
 {
   "version": "2.0.0",
   "tasks": [
+    {
+      "runOptions": { "runOn": "folderOpen" },
+      "label": "Compile protobufs",
+      "type": "shell",
+      "options": {
+        "cwd": "${workspaceRoot}/cli"
+      },
+      "command": "make compile-protos"
+    },
     {
       "type": "shell",
       "label": "prepare e2e",
@@ -9,7 +18,7 @@
     {
       "type": "shell",
       "command": "cd ${cwd}/cli && make install",
-      "label": "make install",
+      "label": "make install"
     },
     {
       "type": "shell",

diff --git a/cli/.gitignore b/cli/.gitignore
@@ -0,0 +1 @@
+internal/turbodprotocol/*.go
diff --git a/cli/Makefile b/cli/Makefile
@@ -10,10 +10,21 @@ GO_FLAGS += -trimpath
 
 GO_FILES = $(shell find . -name "*.go")
 SRC_FILES = $(shell find . -name "*.go" | grep -v "_test.go")
+GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/turbod_grpc.pb.go
 
-turbo: $(SRC_FILES) go.mod go.sum
+turbo: $(GENERATED_FILES) $(SRC_FILES) go.mod
 	CGO_ENABLED=0 go build $(GO_FLAGS) ./cmd/turbo
 
+protoc: internal/turbodprotocol/turbod.proto
+	protoc --go_out=. --go_opt=paths=source_relative \
+    --go-grpc_out=. --go-grpc_opt=paths=source_relative \
+    internal/turbodprotocol/turbod.proto
+
+$(GENERATED_FILES): internal/turbodprotocol/turbod.proto
+	make protoc
+
+compile-protos: $(GENERATED_FILES)
+
 ewatch: scripts/...
 	nodemon --exec "make e2e" -e .ts,.go
 
@@ -29,10 +40,12 @@ TURBO_RACE ?= -race
 clean-go:
 	go clean -testcache ./...
 
-test-go: $(GO_FILES) go.mod go.sum
+test-go:$(GENERATED_FILES) $(GO_FILES) go.mod go.sum
 	go test $(TURBO_RACE) ./...
 
-lint-go: $(GO_FILES) go.mod go.sum
+# protos need to be compiled before linting, since linting needs to pick up
+# some types from the generated code
+lint-go: $(GENERATED_FILES) $(GO_FILES) go.mod go.sum
 	golangci-lint run --new-from-rev=main
 
 fmt-go: $(GO_FILES) go.mod go.sum
@@ -49,7 +62,7 @@ cmd/turbo/version.go: version.txt
 	node -e 'console.log(`package main\n\nconst turboVersion = "$(TURBO_VERSION)"`)' > cmd/turbo/version.go.txt
 	mv cmd/turbo/version.go.txt cmd/turbo/version.go
 
-platform-all: cmd/turbo/version.go
+platform-all: cmd/turbo/version.go $(GENERATED_FILES)
 	make -j4 \
 	  platform-windows-64 \
 		platform-windows-32 \

diff --git a/cli/cmd/turbo/main.go b/cli/cmd/turbo/main.go
@@ -10,16 +10,16 @@ import (
 	"github.com/vercel/turborepo/cli/internal/cmd/auth"
 	"github.com/vercel/turborepo/cli/internal/cmd/info"
 	"github.com/vercel/turborepo/cli/internal/config"
+	"github.com/vercel/turborepo/cli/internal/daemon"
 	"github.com/vercel/turborepo/cli/internal/login"
-	"github.com/vercel/turborepo/cli/internal/process"
 	prune "github.com/vercel/turborepo/cli/internal/prune"
 	"github.com/vercel/turborepo/cli/internal/run"
+	"github.com/vercel/turborepo/cli/internal/signals"
 	"github.com/vercel/turborepo/cli/internal/ui"
 	uiPkg "github.com/vercel/turborepo/cli/internal/ui"
 	"github.com/vercel/turborepo/cli/internal/util"
 
 	"github.com/fatih/color"
-	hclog "github.com/hashicorp/go-hclog"
 	"github.com/mitchellh/cli"
 )
 
@@ -66,18 +66,11 @@ func main() {
 		os.Exit(1)
 	}
 
-	var logger hclog.Logger
-	if cf != nil {
-		logger = cf.Logger
-	} else {
-		logger = hclog.Default()
-	}
-	processes := process.NewManager(logger.Named("processes"))
-	signalCh := watchSignals(func() { processes.Close() })
+	signalWatcher := signals.NewWatcher()
 	c.HiddenCommands = []string{"graph"}
 	c.Commands = map[string]cli.CommandFactory{
 		"run": func() (cli.Command, error) {
-			return &run.RunCommand{Config: cf, Ui: ui, Processes: processes},
+			return &run.RunCommand{Config: cf, UI: ui, SignalWatcher: signalWatcher},
 				nil
 		},
 		"prune": func() (cli.Command, error) {
@@ -98,6 +91,9 @@ func main() {
 		"bin": func() (cli.Command, error) {
 			return &info.BinCommand{Config: cf, UI: ui}, nil
 		},
+		"daemon": func() (cli.Command, error) {
+			return &daemon.Command{Config: cf, UI: ui, SignalWatcher: signalWatcher}, nil
+		},
 	}
 
 	// Capture the defer statements below so the "done" message comes last
@@ -178,8 +174,10 @@ func main() {
 	// or to receive a signal, in which case the signal handler above does the cleanup
 	select {
 	case <-doneCh:
-		processes.Close()
-	case <-signalCh:
+		// We finished whatever task we were running
+		signalWatcher.Close()
+	case <-signalWatcher.Done():
+		// We caught a signal, which already called the close handlers
 	}
 	os.Exit(exitCode)
 }
diff --git a/cli/cmd/turbo/signals.go b/cli/cmd/turbo/signals.go
diff --git a/cli/compile-protos b/cli/compile-protos
diff --git a/cli/go.mod b/cli/go.mod
@@ -9,9 +9,11 @@ require (
 	github.com/briandowns/spinner v1.18.1
 	github.com/deckarep/golang-set v1.8.0
 	github.com/fatih/color v1.13.0
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/gobwas/glob v0.2.3
 	github.com/google/chrometracing v0.0.0-20210413150014-55fded0163e7
 	github.com/google/uuid v1.3.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/go-gatedio v0.5.0
 	github.com/hashicorp/go-hclog v1.2.1
 	github.com/hashicorp/go-retryablehttp v0.6.8
@@ -21,6 +23,7 @@ require (
 	github.com/mitchellh/cli v1.1.2
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.4.3
+	github.com/nightlyone/lockfile v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/pyr-sh/dag v1.0.0
 	github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f
@@ -29,6 +32,8 @@ require (
 	github.com/stretchr/testify v1.7.2
 	github.com/yosuke-furukawa/json5 v0.1.1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	google.golang.org/grpc v1.46.0
+	google.golang.org/protobuf v1.28.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.2.0
 )
@@ -38,7 +43,9 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -56,8 +63,10 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/posener/complete v1.2.3 // indirect
 	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
+	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
 	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )