Add new flags to integration tests to specify virtstack

test/cri-containerd/container_downlevel_test.go

@@ -16,7 +16,7 @@ func Test_CreateContainer_DownLevel_WCOW_Hypervisor(t *testing.T) {
 
 	pullRequiredImages(t, []string{imageWindowsNanoserver17763})
 
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisor17763RuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisor17763RuntimeHandler, nil)
 
 	request := &runtime.CreateContainerRequest{
 		Config: &runtime.ContainerConfig{

test/cri-containerd/container_network_test.go

@@ -32,7 +32,7 @@ func Test_Container_Network_LCOW(t *testing.T) {
 	}()
 	log := filepath.Join(dir, "ping.txt")
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 
 	client := newTestRuntimeClient(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -137,7 +137,7 @@ func Test_Container_Network_Hostname(t *testing.T) {
 				pullRequiredImages(t, []string{test.sandboxImage, test.containerImage})
 			}
 
-			sandboxRequest := getRunPodSandboxRequest(t, test.runtimeHandler)
+			sandboxRequest := getRunPodSandboxRequest(t, test.runtimeHandler, nil)
 			sandboxRequest.Config.Hostname = "TestHost"
 
 			client := newTestRuntimeClient(t)

test/cri-containerd/container_test.go

@@ -80,7 +80,7 @@ func Test_RotateLogs_LCOW(t *testing.T) {
 	pullRequiredLcowImages(t, []string{imageLcowK8sPause, image})
 	logrus.SetLevel(logrus.DebugLevel)
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 
 	request := &runtime.CreateContainerRequest{
 		Config: &runtime.ContainerConfig{
@@ -142,7 +142,7 @@ func Test_RunContainer_Events_LCOW(t *testing.T) {
 	defer podcancel()
 	targetNamespace := "k8s.io"
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -211,7 +211,7 @@ func Test_RunContainer_ForksThenExits_ShowsAsExited_LCOW(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
-	podRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	podRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	podID := runPodSandbox(t, client, ctx, podRequest)
 	defer removePodSandbox(t, client, ctx, podID)
 	defer stopPodSandbox(t, client, ctx, podID)
@@ -263,11 +263,14 @@ func Test_RunContainer_ZeroVPMEM_LCOW(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.lcow.preferredrootfstype":         "initrd",
-		"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.lcow.preferredrootfstype":         "initrd",
+			"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0",
+		},
+	)
 
 	podID := runPodSandbox(t, client, ctx, sandboxRequest)
 	defer removePodSandbox(t, client, ctx, podID)
@@ -302,11 +305,14 @@ func Test_RunContainer_ZeroVPMEM_Multiple_LCOW(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.lcow.preferredrootfstype":         "initrd",
-		"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.lcow.preferredrootfstype":         "initrd",
+			"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0",
+		},
+	)
 
 	podID := runPodSandbox(t, client, ctx, sandboxRequest)
 	defer removePodSandbox(t, client, ctx, podID)
@@ -350,7 +356,7 @@ func Test_RunContainer_GMSA_WCOW_Process(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, ctx, sandboxRequest)
 	defer removePodSandbox(t, client, ctx, podID)
@@ -414,7 +420,7 @@ func Test_RunContainer_GMSA_WCOW_Hypervisor(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, ctx, sandboxRequest)
 	defer removePodSandbox(t, client, ctx, podID)
@@ -478,7 +484,7 @@ func Test_RunContainer_SandboxDevice_LCOW(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{
 		SecurityContext: &runtime.LinuxSandboxSecurityContext{
 			Privileged: true,
@@ -544,7 +550,7 @@ func Test_RunContainer_NonDefault_User(t *testing.T) {
 
 	pullRequiredLcowImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
 
-	podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	podID := runPodSandbox(t, client, ctx, podReq)
 	defer removePodSandbox(t, client, ctx, podID)
 	defer stopPodSandbox(t, client, ctx, podID)

test/cri-containerd/container_virtual_device_test.go

@@ -114,28 +114,17 @@ func findTestVirtualDeviceID() (string, error) {
 	return strings.TrimSpace(string(out)), nil
 }
 
-func getGPUPodRequestLCOW(t *testing.T) *runtime.RunPodSandboxRequest {
-	return &runtime.RunPodSandboxRequest{
-		Config: &runtime.PodSandboxConfig{
-			Metadata: &runtime.PodSandboxMetadata{
-				Name:      t.Name(),
-				Namespace: testNamespace,
-			},
-			Annotations: map[string]string{
-				"io.microsoft.virtualmachine.lcow.kerneldirectboot":                  "false",
-				"io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false",
-				"io.microsoft.virtualmachine.lcow.preferredrootfstype":               "initrd",
-				"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount":       "0",
-				"io.microsoft.virtualmachine.lcow.vpcienabled":                       "true",
-				// we believe this is a sufficiently large high MMIO space amount for this test.
-				// if a given gpu device needs more, this test will fail to create the container
-				// and may hang.
-				"io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "64000",
-				"io.microsoft.virtualmachine.lcow.bootfilesrootpath":                 testGPUBootFiles,
-			},
-		},
-		RuntimeHandler: lcowRuntimeHandler,
-	}
+var lcowPodGPUAnnotations = map[string]string{
+	"io.microsoft.virtualmachine.lcow.kerneldirectboot":                  "false",
+	"io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false",
+	"io.microsoft.virtualmachine.lcow.preferredrootfstype":               "initrd",
+	"io.microsoft.virtualmachine.devices.virtualpmem.maximumcount":       "0",
+	"io.microsoft.virtualmachine.lcow.vpcienabled":                       "true",
+	// we believe this is a sufficiently large high MMIO space amount for this test.
+	// if a given gpu device needs more, this test will fail to create the container
+	// and may hang.
+	"io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "64000",
+	"io.microsoft.virtualmachine.lcow.bootfilesrootpath":                 testGPUBootFiles,
 }
 
 func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest {
@@ -216,7 +205,11 @@ func Test_RunContainer_VirtualDevice_GPU_LCOW(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getGPUPodRequestLCOW(t)
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		lcowPodGPUAnnotations,
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -257,7 +250,11 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_LCOW(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getGPUPodRequestLCOW(t)
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		lcowPodGPUAnnotations,
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -305,7 +302,11 @@ func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_LCOW(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getGPUPodRequestLCOW(t)
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		lcowPodGPUAnnotations,
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -374,7 +375,11 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_LCOW(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getGPUPodRequestLCOW(t)
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		lcowRuntimeHandler,
+		lcowPodGPUAnnotations,
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -420,7 +425,7 @@ func Test_RunContainer_VirtualDevice_LocationPath_WCOW_Process(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -461,7 +466,7 @@ func Test_RunContainer_VirtualDevice_ClassGUID_WCOW_Process(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowProcessRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -502,10 +507,13 @@ func Test_RunContainer_VirtualDevice_GPU_WCOW_Hypervisor(t *testing.T) {
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		wcowHypervisorRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
+		},
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -547,10 +555,13 @@ func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_WCOW_Hypervisor(t *testing.T)
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		wcowHypervisorRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
+		},
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -609,10 +620,13 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_WCOW_Hypervisor(t *testing.T)
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		wcowHypervisorRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
+		},
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)
@@ -661,10 +675,13 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_WCOW_Hypervisor(t *tes
 	client := newTestRuntimeClient(t)
 
 	podctx := context.Background()
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
-	sandboxRequest.Config.Annotations = map[string]string{
-		"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
-	}
+	sandboxRequest := getRunPodSandboxRequest(
+		t,
+		wcowHypervisorRuntimeHandler,
+		map[string]string{
+			"io.microsoft.virtualmachine.fullyphysicallybacked": "true",
+		},
+	)
 
 	podID := runPodSandbox(t, client, podctx, sandboxRequest)
 	defer removePodSandbox(t, client, podctx, podID)

test/cri-containerd/createcontainer_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func runCreateContainerTest(t *testing.T, runtimeHandler string, request *runtime.CreateContainerRequest) {
-	sandboxRequest := getRunPodSandboxRequest(t, runtimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, runtimeHandler, nil)
 	runCreateContainerTestWithSandbox(t, sandboxRequest, request)
 }
 
@@ -188,7 +188,7 @@ func Test_CreateContainer_LCOW_Privileged(t *testing.T) {
 
 	pullRequiredLcowImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{
 		SecurityContext: &runtime.LinuxSandboxSecurityContext{
 			Privileged: true,
@@ -222,7 +222,7 @@ func Test_CreateContainer_SandboxDevice_LCOW(t *testing.T) {
 
 	pullRequiredLcowImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
 
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{
 		SecurityContext: &runtime.LinuxSandboxSecurityContext{
 			Privileged: true,
@@ -1227,7 +1227,7 @@ func Test_Mount_ReadOnlyDirReuse_WCOW(t *testing.T) {
 	}
 	defer os.RemoveAll(tempDir)
 
-	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, nil)
 
 	podID := runPodSandbox(t, client, ctx, sandboxRequest)
 	defer removePodSandbox(t, client, ctx, podID)

test/cri-containerd/execcontainer_test.go

@@ -37,7 +37,7 @@ func execContainerLCOW(t *testing.T, uid int64, cmd []string) *runtime.ExecSyncR
 	pullRequiredLcowImages(t, []string{imageLcowK8sPause, imageLcowCosmos})
 
 	// run podsandbox request
-	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, nil)
 	sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{
 		SecurityContext: &runtime.LinuxSandboxSecurityContext{
 			Privileged: true,

test/cri-containerd/logging_binary_test.go

@@ -85,7 +85,7 @@ func Test_Run_Container_With_Binary_Logger(t *testing.T) {
 				pullRequiredImages(t, requiredImages)
 			}
 
-			podReq := getRunPodSandboxRequest(t, test.runtimeHandler)
+			podReq := getRunPodSandboxRequest(t, test.runtimeHandler, nil)
 			podID := runPodSandbox(t, client, ctx, podReq)
 			defer removePodSandbox(t, client, ctx, podID)
 
@@ -123,7 +123,7 @@ func Test_Run_Container_With_Binary_Logger(t *testing.T) {
 				pullRequiredImages(t, requiredImages)
 			}
 
-			podReq := getRunPodSandboxRequest(t, test.runtimeHandler)
+			podReq := getRunPodSandboxRequest(t, test.runtimeHandler, nil)
 			podID := runPodSandbox(t, client, ctx, podReq)
 			defer removePodSandbox(t, client, ctx, podID)
 

test/cri-containerd/main.go

@@ -40,6 +40,9 @@ const (
 	testDriversPath        = "C:\\ContainerPlat\\testdrivers"
 	testGPUBootFiles       = "C:\\ContainerPlat\\LinuxBootFiles\\nvidiagpu"
 
+	testVMServiceAddress = "C:\\ContainerPlat\\vmservice.sock"
+	testVMServiceBinary  = "C:\\Containerplat\\vmservice.exe"
+
 	lcowRuntimeHandler   = "runhcs-lcow"
 	imageLcowK8sPause    = "k8s.gcr.io/pause:3.1"
 	imageLcowAlpine      = "docker.io/library/alpine:latest"
@@ -69,8 +72,10 @@ var (
 
 // Flags
 var (
-	flagFeatures    = testutilities.NewStringSetFlag()
-	flagCRIEndpoint = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.")
+	flagFeatures        = testutilities.NewStringSetFlag()
+	flagCRIEndpoint     = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.")
+	flagVirtstack       = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers")
+	flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service")
 )
 
 // Features