Wait for ovs-vswitchd PID before calling ovs-appctl (#2711)

Otherwise the call may fail and crash the process.

Besides, it cleans up the OVS run files on exit to avoid stale PID
from being used.

Signed-off-by: Quan Tian <qtian@vmware.com>

build/images/scripts/start_ovs

@@ -112,6 +112,7 @@ function quit {
     stop_ovs
     # kill background sleep process
     if [ "$SLEEP_PID" != "" ]; then kill $SLEEP_PID > /dev/null 2>&1 || true; fi
+    cleanup_ovs_run_files
     exit 0
 }
 

pkg/ovs/ovsctl/interface.go

@@ -14,6 +14,7 @@
 package ovsctl
 
 import (
+	"fmt"
 	"net"
 	"os/exec"
 )
@@ -85,3 +86,7 @@ func (e *ExecError) GetErrorOutput() string {
 	}
 	return e.errorOutput
 }
+
+func (e *ExecError) Error() string {
+	return fmt.Sprintf("ExecError: %v, output: %s", e.error, e.errorOutput)
+}

pkg/ovs/ovsctl/ovsctl_others.go

@@ -21,7 +21,9 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
 
@@ -47,12 +49,26 @@ func ovsVSwitchdUDS() string {
 	// are possible. Besides, this value is only used when invoking ovs-appctl (as a new
 	// process) at the moment, so the overhead of reading the PID from file should not be a
 	// concern.
-	pid, err := readOVSVSwitchdPID()
-	if err != nil {
-		klog.ErrorS(err, "Failed to read ovs-vswitchd PID")
+	var pid int
+	var readErr error
+	startTime := time.Now()
+	hasFailure := false
+	pollErr := wait.PollImmediate(50*time.Millisecond, 5*time.Second, func() (bool, error) {
+		pid, readErr = readOVSVSwitchdPID()
+		if readErr != nil {
+			hasFailure = true
+			return false, nil
+		}
+		return true, nil
+	})
+	if pollErr != nil {
+		klog.ErrorS(readErr, "Failed to read ovs-vswitchd PID")
 		// that seems like a reasonable value to return if we cannot read the PID
 		return "/var/run/openvswitch/ovs-vswitchd.*.ctl"
 	}
+	if hasFailure {
+		klog.V(2).InfoS("Waited for ovs-vswitchd PID to be ready", "duration", time.Since(startTime))
+	}
 	return fmt.Sprintf("/var/run/openvswitch/ovs-vswitchd.%d.ctl", pid)
 }