add watchdog goroutine for docker (#898)

Author: BolajiOlajide

CHANGELOG.md

@@ -13,6 +13,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Added
 
+- Batch Changes: Watchdog that checks for docker responsiveness while running commands has been added. [#898](https://github.com/sourcegraph/src-cli/pull/898)
+
 ### Changed
 
 ### Fixed

cmd/src/batch_common.go

@@ -32,10 +32,14 @@ import (
 	"github.com/sourcegraph/src-cli/internal/batches/repozip"
 	"github.com/sourcegraph/src-cli/internal/batches/service"
 	"github.com/sourcegraph/src-cli/internal/batches/ui"
+	"github.com/sourcegraph/src-cli/internal/batches/watchdog"
 	"github.com/sourcegraph/src-cli/internal/batches/workspace"
 	"github.com/sourcegraph/src-cli/internal/cmderrors"
 )
 
+// We check for docker responsiveness every minute
+const dockerWatchDuration = 1 * time.Minute
+
 // batchExecutionFlags are common to batch changes that are executed both
 // locally and remotely.
 type batchExecutionFlags struct {
@@ -251,6 +255,15 @@ type executeBatchSpecOpts struct {
 	client api.Client
 }
 
+func createDockerWatchdog(ctx context.Context, execUI ui.ExecUI) *watchdog.WatchDog {
+	return watchdog.New(dockerWatchDuration, func() {
+		_, err := docker.NCPU(ctx)
+		if err != nil {
+			execUI.DockerWatchDogWarning(errors.Wrap(err, "docker watchdog"))
+		}
+	})
+}
+
 // executeBatchSpec performs all the steps required to upload the batch spec to
 // Sourcegraph, including execution as needed and applying the resulting batch
 // spec if specified.
@@ -263,7 +276,11 @@ func executeBatchSpec(ctx context.Context, opts executeBatchSpecOpts) (err error
 		execUI = &ui.TUI{Out: out}
 	}
 
+	w := createDockerWatchdog(ctx, execUI)
+	go w.Start()
+
 	defer func() {
+		w.Stop()
 		if err != nil {
 			execUI.ExecutionError(err)
 		}

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/sourcegraph/go-diff v0.6.1
 	github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
 	github.com/sourcegraph/scip v0.2.3
-	github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c
+	github.com/sourcegraph/sourcegraph/lib v0.0.0-20221207201520-eeeaf1f74c51
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/net v0.2.0
 	golang.org/x/sync v0.1.0

go.sum

@@ -378,8 +378,8 @@ github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9 h1:JjFyvx9hCD5+Jpu
 github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9/go.mod h1:UxiwB6C3xk3xOySJpW1R0MDUyfGuJRFS5Z8C+SA5p2I=
 github.com/sourcegraph/scip v0.2.3 h1:6nRsTlNTELn+1V7JxLrDVbBpb9H1gAbUF48N7x5B8Y4=
 github.com/sourcegraph/scip v0.2.3/go.mod h1:3D/RLCMvrrSam85RtFmC67SE/jhV9++hpCR5MTDzbA0=
-github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c h1:bjTcIdEx6FvUXIbD1Ckjp7SfvV4jTyJ7RozZH/HVj9M=
-github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c/go.mod h1:PmWmo2ysXWCHcbtp08cDXKGD+8NQzrkj2UjdRBu9H0k=
+github.com/sourcegraph/sourcegraph/lib v0.0.0-20221207201520-eeeaf1f74c51 h1:G9XMyQ0SKC53PP9WPSkQqRFvwkZNYRdMHvdm5K3b8Lw=
+github.com/sourcegraph/sourcegraph/lib v0.0.0-20221207201520-eeeaf1f74c51/go.mod h1:pLXoYrwkmERkPVv1dpLcq0Epo21Kh4EgrdZXdnafmyQ=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152 h1:z/MpntplPaW6QW95pzcAR/72Z5TWDyDnSo0EOcyij9o=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=

internal/batches/ui/exec_ui.go

@@ -52,4 +52,6 @@ type ExecUI interface {
 	UploadingWorkspaceFiles()
 	UploadingWorkspaceFilesWarning(error)
 	UploadingWorkspaceFilesSuccess()
+
+	DockerWatchDogWarning(error)
 }

internal/batches/ui/json_lines.go

@@ -177,6 +177,14 @@ func (ui *JSONLines) WriteAfterStepResult(key string, value execution.AfterStepR
 	})
 }
 
+func (ui *JSONLines) DockerWatchDogWarning(err error) {
+	message := fmt.Sprintf(`It seems your Docker engine might be frozen.
+If there's no progress in the next couple minutes, you may want to try restarting Docker and running the command again.
+Error: %s
+`, err.Error())
+	logOperationFailure(batcheslib.LogEventOperationDockerWatchDog, &batcheslib.DockerWatchDogMetadata{Error: message})
+}
+
 type taskExecutionJSONLines struct {
 	linesTasks  map[*executor.Task]batcheslib.JSONLinesTask
 	binaryDiffs bool

internal/batches/ui/tui.go

@@ -221,6 +221,10 @@ func (ui *TUI) CreatingBatchSpecError(err error) error {
 	return prettyPrintBatchUnlicensedError(ui.Out, err)
 }
 
+func (ui *TUI) DockerWatchDogWarning(err error) {
+	dockerWatchDogWarning(ui.Out, err)
+}
+
 func (ui *TUI) PreviewBatchSpec(batchSpecURL string) {
 	ui.Out.Write("")
 	block := ui.Out.Block(output.Line(batchSuccessEmoji, batchSuccessColor, "To preview or apply the batch spec, go to:"))
@@ -449,3 +453,11 @@ func batchCompletePending(p output.Pending, message string) {
 func batchCompleteWarning(p output.Pending, message string) {
 	p.Complete(output.Line(batchWarningEmoji, batchWarningColor, message))
 }
+
+func dockerWatchDogWarning(out *output.Output, err error) {
+	block := out.Block(output.Line("🐳", output.StyleWarning, "It seems your Docker engine might be frozen."))
+	block.WriteLine(output.Line("", output.StyleWarning, "If there's no progress in the next couple minutes, you may want to try restarting Docker and running the command again."))
+	block.WriteLine(output.Linef("", output.StyleWarning, "Error: %s", err.Error()))
+	block.Write("")
+	block.Close()
+}

internal/batches/watchdog/watchdog.go

@@ -0,0 +1,38 @@
+package watchdog
+
+import (
+	"time"
+
+	"github.com/derision-test/glock"
+)
+
+type WatchDog struct {
+	ticker   glock.Ticker
+	callback func()
+	done     chan struct{}
+}
+
+func New(interval time.Duration, callback func()) *WatchDog {
+	ticker := glock.NewRealTicker(interval)
+	return &WatchDog{
+		ticker:   ticker,
+		callback: callback,
+		done:     make(chan struct{}, 1),
+	}
+}
+
+func (w *WatchDog) Stop() {
+	close(w.done)
+}
+
+func (w *WatchDog) Start() {
+	for {
+		select {
+		case <-w.ticker.Chan():
+			go w.callback()
+		case <-w.done:
+			w.ticker.Stop()
+			return
+		}
+	}
+}

internal/batches/watchdog/watchdog_test.go

@@ -0,0 +1,40 @@
+package watchdog
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/derision-test/glock"
+)
+
+func TestWatchDog(t *testing.T) {
+	ticker := glock.NewMockTicker(5 * time.Minute)
+	var count uint32
+	expectedCount := 100
+	var wg sync.WaitGroup
+
+	mockCallback := func() {
+		atomic.AddUint32(&count, 1)
+		wg.Done()
+	}
+
+	w := &WatchDog{
+		ticker:   ticker,
+		callback: mockCallback,
+		done:     make(chan struct{}, 1),
+	}
+
+	go w.Start()
+	for i := 0; i < expectedCount; i++ {
+		wg.Add(1)
+		ticker.BlockingAdvance(5 * time.Minute)
+	}
+	wg.Wait()
+	w.Stop()
+
+	if count != uint32(expectedCount) {
+		t.Errorf("expected mock callback to be called %d times, got %d", expectedCount, count)
+	}
+}