proc: add waitfor option to attach (#3445)

Adds a waitfor option to 'dlv attach' that waits for a process with a
name starting with a given prefix to appear before attaching to it.

Debugserver on macOS does not support follow-fork mode, but has this
feature instead which is not the same thing but still helps with
multiprocess debugging somewhat.

Documentation/backend_test_health.md

@@ -5,6 +5,8 @@ Tests skipped by each supported backend:
 	* 3 not implemented
 * arm64 skipped = 1
 	* 1 broken - global variable symbolication
+* darwin skipped = 1
+	* 1 waitfor implementation is delegated to debugserver
 * darwin/arm64 skipped = 2
 	* 2 broken - cgo stacktraces
 * darwin/lldb skipped = 1

Documentation/usage/dlv_attach.md

@@ -18,8 +18,11 @@ dlv attach pid [executable] [flags]
 ### Options
 
 ```
-      --continue   Continue the debugged process on start.
-  -h, --help       help for attach
+      --continue                 Continue the debugged process on start.
+  -h, --help                     help for attach
+      --waitfor string           Wait for a process with a name beginning with this prefix
+      --waitfor-duration float   Total time to wait for a process
+      --waitfor-interval float   Interval between checks of the process list, in millisecond (default 1)
 ```
 
 ### Options inherited from parent commands

Makefile

@@ -16,7 +16,7 @@ uninstall:
 	@go run _scripts/make.go uninstall
 
 test: vet
-	@go run _scripts/make.go test
+	@go run _scripts/make.go test -v
 
 vet:
 	@go vet $$(go list ./... | grep -v native)

cmd/dlv/cmds/commands.go

@@ -96,6 +96,10 @@ var (
 	loadConfErr error
 
 	rrOnProcessPid int
+
+	attachWaitFor         string
+	attachWaitForInterval float64
+	attachWaitForDuration float64
 )
 
 const dlvCommandLongDesc = `Delve is a source level debugger for Go programs.
@@ -162,14 +166,17 @@ begin a new debug session.  When exiting the debug session you will have the
 option to let the process continue or kill it.
 `,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 {
+			if len(args) == 0 && attachWaitFor == "" {
 				return errors.New("you must provide a PID")
 			}
 			return nil
 		},
 		Run: attachCmd,
 	}
 	attachCommand.Flags().BoolVar(&continueOnStart, "continue", false, "Continue the debugged process on start.")
+	attachCommand.Flags().StringVar(&attachWaitFor, "waitfor", "", "Wait for a process with a name beginning with this prefix")
+	attachCommand.Flags().Float64Var(&attachWaitForInterval, "waitfor-interval", 1, "Interval between checks of the process list, in millisecond")
+	attachCommand.Flags().Float64Var(&attachWaitForDuration, "waitfor-duration", 0, "Total time to wait for a process")
 	rootCommand.AddCommand(attachCommand)
 
 	// 'connect' subcommand.
@@ -305,7 +312,8 @@ to know what functions your process is executing.
 The output of the trace sub command is printed to stderr, so if you would like to
 only see the output of the trace operations you can redirect stdout.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			os.Exit(traceCmd(cmd, args, conf)) }, 
+			os.Exit(traceCmd(cmd, args, conf))
+		},
 	}
 	traceCommand.Flags().IntVarP(&traceAttachPid, "pid", "p", 0, "Pid to attach to.")
 	traceCommand.Flags().StringVarP(&traceExecFile, "exec", "e", "", "Binary file to exec and trace.")
@@ -647,10 +655,10 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 			ProcessArgs: processArgs,
 			APIVersion:  2,
 			Debugger: debugger.Config{
-				AttachPid:      traceAttachPid,
-				WorkingDir:     workingDir,
-				Backend:        backend,
-				CheckGoVersion: checkGoVersion,
+				AttachPid:            traceAttachPid,
+				WorkingDir:           workingDir,
+				Backend:              backend,
+				CheckGoVersion:       checkGoVersion,
 				DebugInfoDirectories: conf.DebugInfoDirectories,
 			},
 		})
@@ -818,12 +826,17 @@ func getPackageDir(pkg []string) string {
 }
 
 func attachCmd(cmd *cobra.Command, args []string) {
-	pid, err := strconv.Atoi(args[0])
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Invalid pid: %s\n", args[0])
-		os.Exit(1)
+	var pid int
+	if len(args) > 0 {
+		var err error
+		pid, err = strconv.Atoi(args[0])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Invalid pid: %s\n", args[0])
+			os.Exit(1)
+		}
+		args = args[1:]
 	}
-	os.Exit(execute(pid, args[1:], conf, "", debugger.ExecutingOther, args, buildFlags))
+	os.Exit(execute(pid, args, conf, "", debugger.ExecutingOther, args, buildFlags))
 }
 
 func coreCmd(cmd *cobra.Command, args []string) {
@@ -1005,22 +1018,25 @@ func execute(attachPid int, processArgs []string, conf *config.Config, coreFile
 			CheckLocalConnUser: checkLocalConnUser,
 			DisconnectChan:     disconnectChan,
 			Debugger: debugger.Config{
-				AttachPid:            attachPid,
-				WorkingDir:           workingDir,
-				Backend:              backend,
-				CoreFile:             coreFile,
-				Foreground:           headless && tty == "",
-				Packages:             dlvArgs,
-				BuildFlags:           buildFlags,
-				ExecuteKind:          kind,
-				DebugInfoDirectories: conf.DebugInfoDirectories,
-				CheckGoVersion:       checkGoVersion,
-				TTY:                  tty,
-				Stdin:                redirects[0],
-				Stdout:               proc.OutputRedirect{Path: redirects[1]},
-				Stderr:               proc.OutputRedirect{Path: redirects[2]},
-				DisableASLR:          disableASLR,
-				RrOnProcessPid:       rrOnProcessPid,
+				AttachPid:             attachPid,
+				WorkingDir:            workingDir,
+				Backend:               backend,
+				CoreFile:              coreFile,
+				Foreground:            headless && tty == "",
+				Packages:              dlvArgs,
+				BuildFlags:            buildFlags,
+				ExecuteKind:           kind,
+				DebugInfoDirectories:  conf.DebugInfoDirectories,
+				CheckGoVersion:        checkGoVersion,
+				TTY:                   tty,
+				Stdin:                 redirects[0],
+				Stdout:                proc.OutputRedirect{Path: redirects[1]},
+				Stderr:                proc.OutputRedirect{Path: redirects[2]},
+				DisableASLR:           disableASLR,
+				RrOnProcessPid:        rrOnProcessPid,
+				AttachWaitFor:         attachWaitFor,
+				AttachWaitForInterval: attachWaitForInterval,
+				AttachWaitForDuration: attachWaitForDuration,
 			},
 		})
 	default:

pkg/proc/gdbserial/gdbserver.go

@@ -588,7 +588,7 @@ func LLDBLaunch(cmd []string, wd string, flags proc.LaunchFlags, debugInfoDirs [
 // Path is path to the target's executable, path only needs to be specified
 // for some stubs that do not provide an automated way of determining it
 // (for example debugserver).
-func LLDBAttach(pid int, path string, debugInfoDirs []string) (*proc.TargetGroup, error) {
+func LLDBAttach(pid int, path string, waitFor *proc.WaitFor, debugInfoDirs []string) (*proc.TargetGroup, error) {
 	if runtime.GOOS == "windows" {
 		return nil, ErrUnsupportedOS
 	}
@@ -609,12 +609,28 @@ func LLDBAttach(pid int, path string, debugInfoDirs []string) (*proc.TargetGroup
 		if err != nil {
 			return nil, err
 		}
-		args := []string{"-R", fmt.Sprintf("127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port), "--attach=" + strconv.Itoa(pid)}
+		args := []string{"-R", fmt.Sprintf("127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port)}
+
+		if waitFor.Valid() {
+			duration := int(waitFor.Duration.Seconds())
+			if duration == 0 && waitFor.Duration != 0 {
+				// If duration is below the (second) resolution of debugserver pass 1
+				// second (0 means infinite).
+				duration = 1
+			}
+			args = append(args, "--waitfor="+waitFor.Name, fmt.Sprintf("--waitfor-interval=%d", waitFor.Interval.Microseconds()), fmt.Sprintf("--waitfor-duration=%d", duration))
+		} else {
+			args = append(args, "--attach="+strconv.Itoa(pid))
+		}
+
 		if canUnmaskSignals(debugserverExecutable) {
 			args = append(args, "--unmask-signals")
 		}
 		process = commandLogger(debugserverExecutable, args...)
 	} else {
+		if waitFor.Valid() {
+			return nil, proc.ErrWaitForNotImplemented
+		}
 		if _, err = exec.LookPath("lldb-server"); err != nil {
 			return nil, &ErrBackendUnavailable{}
 		}

pkg/proc/interface.go

@@ -2,6 +2,7 @@ package proc
 
 import (
 	"sync"
+	"time"
 
 	"github.com/go-delve/delve/pkg/elfwriter"
 	"github.com/go-delve/delve/pkg/proc/internal/ebpf"
@@ -142,3 +143,10 @@ func (cctx *ContinueOnceContext) GetManualStopRequested() bool {
 	defer cctx.StopMu.Unlock()
 	return cctx.manualStopRequested
 }
+
+// WaitFor is passed to native.Attach and gdbserver.LLDBAttach to wait for a
+// process to start before attaching.
+type WaitFor struct {
+	Name               string
+	Interval, Duration time.Duration
+}

pkg/proc/native/nonative_darwin.go

@@ -21,10 +21,14 @@ func Launch(_ []string, _ string, _ proc.LaunchFlags, _ []string, _ string, _ st
 }
 
 // Attach returns ErrNativeBackendDisabled.
-func Attach(_ int, _ []string) (*proc.TargetGroup, error) {
+func Attach(_ int, _ *proc.WaitFor, _ []string) (*proc.TargetGroup, error) {
 	return nil, ErrNativeBackendDisabled
 }
 
+func waitForSearchProcess(string, map[int]struct{}) (int, error) {
+	return 0, proc.ErrWaitForNotImplemented
+}
+
 // waitStatus is a synonym for the platform-specific WaitStatus
 type waitStatus struct{}
 

pkg/proc/native/proc.go

@@ -1,8 +1,10 @@
 package native
 
 import (
+	"errors"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/go-delve/delve/pkg/proc"
 )
@@ -69,6 +71,23 @@ func newChildProcess(dbp *nativeProcess, pid int) *nativeProcess {
 	}
 }
 
+// WaitFor waits for a process as specified by waitFor.
+func WaitFor(waitFor *proc.WaitFor) (int, error) {
+	t0 := time.Now()
+	seen := make(map[int]struct{})
+	for (waitFor.Duration == 0) || (time.Since(t0) < waitFor.Duration) {
+		pid, err := waitForSearchProcess(waitFor.Name, seen)
+		if err != nil {
+			return 0, err
+		}
+		if pid != 0 {
+			return pid, nil
+		}
+		time.Sleep(waitFor.Interval)
+	}
+	return 0, errors.New("waitfor duration expired")
+}
+
 // BinInfo will return the binary info struct associated with this process.
 func (dbp *nativeProcess) BinInfo() *proc.BinaryInfo {
 	return dbp.bi

pkg/proc/native/proc_darwin.go

@@ -136,8 +136,15 @@ func Launch(cmd []string, wd string, flags proc.LaunchFlags, _ []string, _ strin
 	return tgt, err
 }
 
+func waitForSearchProcess(string, map[int]struct{}) (int, error) {
+	return 0, proc.ErrWaitForNotImplemented
+}
+
 // Attach to an existing process with the given PID.
-func Attach(pid int, _ []string) (*proc.TargetGroup, error) {
+func Attach(pid int, waitFor *proc.WaitFor, _ []string) (*proc.TargetGroup, error) {
+	if waitFor.Valid() {
+		return nil, proc.ErrWaitForNotImplemented
+	}
 	if err := macutil.CheckRosetta(); err != nil {
 		return nil, err
 	}

pkg/proc/native/proc_freebsd.go

@@ -3,6 +3,7 @@ package native
 // #cgo LDFLAGS: -lprocstat
 // #include <stdlib.h>
 // #include "proc_freebsd.h"
+// #include <sys/sysctl.h>
 import "C"
 import (
 	"fmt"
@@ -14,6 +15,7 @@ import (
 
 	sys "golang.org/x/sys/unix"
 
+	"github.com/go-delve/delve/pkg/logflags"
 	"github.com/go-delve/delve/pkg/proc"
 	"github.com/go-delve/delve/pkg/proc/internal/ebpf"
 
@@ -121,7 +123,15 @@ func Launch(cmd []string, wd string, flags proc.LaunchFlags, debugInfoDirs []str
 // Attach to an existing process with the given PID. Once attached, if
 // the DWARF information cannot be found in the binary, Delve will look
 // for external debug files in the directories passed in.
-func Attach(pid int, debugInfoDirs []string) (*proc.TargetGroup, error) {
+func Attach(pid int, waitFor *proc.WaitFor, debugInfoDirs []string) (*proc.TargetGroup, error) {
+	if waitFor.Valid() {
+		var err error
+		pid, err = WaitFor(waitFor)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	dbp := newProcess(pid)
 
 	var err error
@@ -142,6 +152,31 @@ func Attach(pid int, debugInfoDirs []string) (*proc.TargetGroup, error) {
 	return tgt, nil
 }
 
+func waitForSearchProcess(pfx string, seen map[int]struct{}) (int, error) {
+	log := logflags.DebuggerLogger()
+	ps := C.procstat_open_sysctl()
+	defer C.procstat_close(ps)
+	var cnt C.uint
+	procs := C.procstat_getprocs(ps, C.KERN_PROC_PROC, 0, &cnt)
+	defer C.procstat_freeprocs(ps, procs)
+	proc := procs
+	for i := 0; i < int(cnt); i++ {
+		if _, isseen := seen[int(proc.ki_pid)]; isseen {
+			continue
+		}
+		seen[int(proc.ki_pid)] = struct{}{}
+
+		argv := strings.Join(getCmdLineInternal(ps, proc), " ")
+		log.Debugf("waitfor: new process %q", argv)
+		if strings.HasPrefix(argv, pfx) {
+			return int(proc.ki_pid), nil
+		}
+
+		proc = (*C.struct_kinfo_proc)(unsafe.Pointer(uintptr(unsafe.Pointer(proc)) + unsafe.Sizeof(*proc)))
+	}
+	return 0, nil
+}
+
 func initialize(dbp *nativeProcess) (string, error) {
 	comm, _ := C.find_command_name(C.int(dbp.pid))
 	defer C.free(unsafe.Pointer(comm))
@@ -230,7 +265,17 @@ func findExecutable(path string, pid int) string {
 func getCmdLine(pid int) string {
 	ps := C.procstat_open_sysctl()
 	kp := C.kinfo_getproc(C.int(pid))
+	goargv := getCmdLineInternal(ps, kp)
+	C.free(unsafe.Pointer(kp))
+	C.procstat_close(ps)
+	return strings.Join(goargv, " ")
+}
+
+func getCmdLineInternal(ps *C.struct_procstat, kp *C.struct_kinfo_proc) []string {
 	argv := C.procstat_getargv(ps, kp, 0)
+	if argv == nil {
+		return nil
+	}
 	goargv := []string{}
 	for {
 		arg := *argv
@@ -240,9 +285,8 @@ func getCmdLine(pid int) string {
 		argv = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(argv)) + unsafe.Sizeof(*argv)))
 		goargv = append(goargv, C.GoString(arg))
 	}
-	C.free(unsafe.Pointer(kp))
-	C.procstat_close(ps)
-	return strings.Join(goargv, " ")
+	C.procstat_freeargv(ps)
+	return goargv
 }
 
 func trapWait(procgrp *processGroup, pid int) (*nativeThread, error) {