Merge pull request moby#41479 from olljanat/ci-win-containerd-support

Windows CI: Add support for testing with containerd

Dockerfile.windows

@@ -166,12 +166,15 @@ FROM microsoft/windowsservercore
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 ARG GO_VERSION=1.16.7
+ARG CONTAINERD_VERSION=1.5.5
 ARG GOTESTSUM_COMMIT=v0.5.3
 
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
+#  - CONTAINERD_VERSION must be consistent with 'hack/dockerfile/install/containerd.installer' used by Linux.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
 ENV GO_VERSION=${GO_VERSION} `
+    CONTAINERD_VERSION=${CONTAINERD_VERSION} `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\gopath `
     GO111MODULE=off `
@@ -211,7 +214,7 @@ RUN `
     } `
   } `
   `
-  setx /M PATH $('C:\git\cmd;C:\git\usr\bin;'+$Env:PATH+';C:\gcc\bin;C:\go\bin'); `
+  setx /M PATH $('C:\git\cmd;C:\git\usr\bin;'+$Env:PATH+';C:\gcc\bin;C:\go\bin;C:\containerd\bin'); `
   `
   Write-Host INFO: Downloading git...; `
   $location='https://www.nuget.org/api/v2/package/GitForWindows/'+$Env:GIT_VERSION; `
@@ -252,6 +255,16 @@ RUN `
   Remove-Item C:\binutils.zip; `
   Remove-Item C:\gitsetup.zip; `
   `
+  Write-Host INFO: Downloading containerd; `
+  Install-Package -Force 7Zip4PowerShell; `
+  $location='https://github.com/containerd/containerd/releases/download/v'+$Env:CONTAINERD_VERSION+'/containerd-'+$Env:CONTAINERD_VERSION+'-windows-amd64.tar.gz'; `
+  Download-File $location C:\containerd.tar.gz; `
+  New-Item -Path C:\containerd -ItemType Directory; `
+  Expand-7Zip C:\containerd.tar.gz C:\; `
+  Expand-7Zip C:\containerd.tar C:\containerd; `
+  Remove-Item C:\containerd.tar.gz; `
+  Remove-Item C:\containerd.tar; `
+  `
   # Ensure all directories exist that we will require below....
   $srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; `
   Write-Host INFO: Ensuring existence of directory $srcDir...; `

Jenkinsfile

@@ -1308,7 +1308,7 @@ pipeline {
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/containerd.out", "bundles/containerd.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true

cmd/dockerd/daemon_windows.go

@@ -94,6 +94,6 @@ func newCgroupParent(config *config.Config) string {
 }
 
 func (cli *DaemonCli) initContainerD(_ context.Context) (func(time.Duration) error, error) {
-	system.InitContainerdRuntime(cli.Config.Experimental, cli.Config.ContainerdAddr)
+	system.InitContainerdRuntime(cli.Config.ContainerdAddr)
 	return nil, nil
 }

daemon/start_windows.go

@@ -8,8 +8,8 @@ import (
 
 func (daemon *Daemon) getLibcontainerdCreateOptions(_ *container.Container) (string, interface{}, error) {
 	if system.ContainerdRuntimeSupported() {
-		// Set the runtime options to debug regardless of current logging level.
-		return "", &options.Options{Debug: true}, nil
+		opts := &options.Options{}
+		return "io.containerd.runhcs.v1", opts, nil
 	}
 	return "", nil, nil
 }

hack/ci/windows.ps1

@@ -179,6 +179,13 @@ Function Nuke-Everything {
             }
         }
 
+        # Kill any spurious containerd.
+        $pids=$(get-process | where-object {$_.ProcessName -like 'containerd'}).id
+        foreach ($p in $pids) {
+            Write-Host "INFO: Killing containerd with PID $p"
+            Stop-Process -Id $p -Force -ErrorAction SilentlyContinue
+        }
+
         Stop-Process -name "cc1" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
         Stop-Process -name "link" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
         Stop-Process -name "compile" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
@@ -521,6 +528,15 @@ Try {
             Throw "ERROR: gotestsum.exe not found...." `
         }
 
+        docker cp "$COMMITHASH`:c`:\containerd\bin\containerd.exe" $env:TEMP\binary\
+        if (-not (Test-Path "$env:TEMP\binary\containerd.exe")) {
+            Throw "ERROR: containerd.exe not found...." `
+        }
+        docker cp "$COMMITHASH`:c`:\containerd\bin\containerd-shim-runhcs-v1.exe" $env:TEMP\binary\
+        if (-not (Test-Path "$env:TEMP\binary\containerd-shim-runhcs-v1.exe")) {
+            Throw "ERROR: containerd-shim-runhcs-v1.exe not found...." `
+        }
+
         $ErrorActionPreference = "Stop"
 
         # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
@@ -566,8 +582,10 @@ Try {
     $env:GOPATH="$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR"
     Write-Host -ForegroundColor Green "INFO: GOPATH=$env:GOPATH"
 
-    # Set the path to have the version of go from the image at the front
-    $env:PATH="$env:TEMP\go\bin;$env:PATH"
+    # Set the path to have:
+    # - the version of go from the image at the front
+    # - the test binaries, not the host ones.
+    $env:PATH="$env:TEMP\go\bin;$env:TEMP\binary;$env:PATH"
 
     # Set the GOROOT to be our copy of go from the image
     $env:GOROOT="$env:TEMP\go"
@@ -594,6 +612,12 @@ Try {
         $dutArgs += "-D"
     }
 
+    # Arguments: Are we starting the daemon under test in containerd mode?
+    if (-not ("$env:DOCKER_WINDOWS_CONTAINERD_RUNTIME" -eq "")) {
+        Write-Host -ForegroundColor Green "INFO: Running the daemon under test in containerd mode"
+        $dutArgs += "--containerd \\.\pipe\containerd-containerd"
+    }
+
     # Arguments: Are we starting the daemon under test with Hyper-V containers as the default isolation?
     if (-not ("$env:DOCKER_DUT_HYPERV" -eq "")) {
         Write-Host -ForegroundColor Green "INFO: Running the daemon under test with Hyper-V containers as the default"
@@ -616,6 +640,15 @@ Try {
     Write-Host -ForegroundColor Green "INFO: Args: $dutArgs"
     New-Item -ItemType Directory $env:TEMP\daemon -ErrorAction SilentlyContinue  | Out-Null
 
+    # Start containerd first
+    if (-not ("$env:DOCKER_WINDOWS_CONTAINERD_RUNTIME" -eq "")) {
+        Start-Process "$env:TEMP\binary\containerd.exe" `
+                    -ArgumentList "--log-level debug" `
+                    -RedirectStandardOutput "$env:TEMP\containerd.out" `
+                    -RedirectStandardError "$env:TEMP\containerd.err"
+        Write-Host -ForegroundColor Green "INFO: Containerd started successfully."
+    }
+
     # Cannot fathom why, but always writes to stderr....
     Start-Process "$env:TEMP\binary\dockerd-$COMMITHASH" `
                   -ArgumentList $dutArgs `
@@ -840,7 +873,6 @@ Try {
                                                     "`$env`:PATH`='c`:\target;'+`$env:PATH`;  `$env:DOCKER_HOST`='tcp`://'+(ipconfig | select -last 1).Substring(39)+'`:2357'; c:\target\runIntegrationCLI.ps1" | Out-Host } )
         } else  {
             $env:DOCKER_HOST=$DASHH_CUT
-            $env:PATH="$env:TEMP\binary;$env:PATH;"  # Force to use the test binaries, not the host ones.
                 $env:GO111MODULE="off"
             Write-Host -ForegroundColor Green "INFO: DOCKER_HOST at $DASHH_CUT"
 
@@ -943,6 +975,15 @@ Finally {
         Copy-Item  "$env:TEMP\dut.out" "bundles\CIDUT.out" -Force -ErrorAction SilentlyContinue
         Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.err) to bundles\CIDUT.err"
         Copy-Item  "$env:TEMP\dut.err" "bundles\CIDUT.err" -Force -ErrorAction SilentlyContinue
+
+        Write-Host -ForegroundColor Green "INFO: Saving containerd logs to bundles"
+        if (Test-Path -Path "$env:TEMP\containerd.out") {
+            Copy-Item "$env:TEMP\containerd.out" "bundles\containerd.out" -Force -ErrorAction SilentlyContinue
+            Copy-Item "$env:TEMP\containerd.err" "bundles\containerd.err" -Force -ErrorAction SilentlyContinue
+        } else {
+            "" | Out-File -FilePath "bundles\containerd.out"
+            "" | Out-File -FilePath "bundles\containerd.err"
+        }
     }
 
     Set-Location "$env:SOURCES_DRIVE\$env:SOURCES_SUBDIR" -ErrorAction SilentlyContinue

integration-cli/docker_api_stats_test.go

@@ -18,11 +18,13 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/testutil/request"
 	"gotest.tools/v3/assert"
+	"gotest.tools/v3/skip"
 )
 
 var expectedNetworkInterfaceStats = strings.Split("rx_bytes rx_dropped rx_errors rx_packets tx_bytes tx_dropped tx_errors tx_packets", " ")
 
 func (s *DockerSuite) TestAPIStatsNoStreamGetCpu(c *testing.T) {
+	skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true;usleep 100; do echo 'Hello'; done")
 
 	id := strings.TrimSpace(out)
@@ -98,6 +100,7 @@ func (s *DockerSuite) TestAPIStatsStoppedContainerInGoroutines(c *testing.T) {
 }
 
 func (s *DockerSuite) TestAPIStatsNetworkStats(c *testing.T) {
+	skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination")
 	testRequires(c, testEnv.IsLocalDaemon)
 
 	out := runSleepingContainer(c)

integration-cli/docker_cli_commit_test.go

@@ -7,9 +7,11 @@ import (
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/integration-cli/cli"
 	"gotest.tools/v3/assert"
+	"gotest.tools/v3/skip"
 )
 
 func (s *DockerSuite) TestCommitAfterContainerIsDone(c *testing.T) {
+	skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination")
 	out := cli.DockerCmd(c, "run", "-i", "-a", "stdin", "busybox", "echo", "foo").Combined()
 
 	cleanedContainerID := strings.TrimSpace(out)

integration-cli/docker_cli_ps_test.go

@@ -227,6 +227,7 @@ func (s *DockerSuite) TestPsListContainersFilterStatus(c *testing.T) {
 }
 
 func (s *DockerSuite) TestPsListContainersFilterHealth(c *testing.T) {
+	skip.If(c, RuntimeIsWindowsContainerd(), "FIXME. Hang on Windows + containerd combination")
 	existingContainers := ExistingContainerIDs(c)
 	// Test legacy no health check
 	out := runSleepingContainer(c, "--name=none_legacy")

integration-cli/docker_cli_run_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/moby/sys/mountinfo"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
+	"gotest.tools/v3/skip"
 )
 
 // "test123" should be printed by docker run
@@ -1983,6 +1984,7 @@ func (s *DockerSuite) TestRunCidFileCheckIDLength(c *testing.T) {
 }
 
 func (s *DockerSuite) TestRunSetMacAddress(c *testing.T) {
+	skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination")
 	mac := "12:34:56:78:9a:bc"
 	var out string
 	if testEnv.OSType == "windows" {

integration-cli/requirements_test.go

@@ -181,6 +181,10 @@ func RegistryHosting() bool {
 	return err == nil
 }
 
+func RuntimeIsWindowsContainerd() bool {
+	return os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME") == "1"
+}
+
 func SwarmInactive() bool {
 	return testEnv.DaemonInfo.Swarm.LocalNodeState == swarm.LocalNodeStateInactive
 }

integration/container/exec_test.go

@@ -17,6 +17,7 @@ import (
 
 // TestExecWithCloseStdin adds case for moby#37870 issue.
 func TestExecWithCloseStdin(t *testing.T) {
+	skip.If(t, testEnv.RuntimeIsWindowsContainerd(), "FIXME. Hang on Windows + containerd combination")
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.39"), "broken in earlier versions")
 	defer setupTest(t)()
 

pkg/system/init_windows.go

@@ -1,29 +1,18 @@
 package system // import "github.com/docker/docker/pkg/system"
 
-import (
-	"os"
-
-	"github.com/sirupsen/logrus"
-)
-
 var (
-	// containerdRuntimeSupported determines if ContainerD should be the runtime.
-	// As of March 2019, this is an experimental feature.
+	// containerdRuntimeSupported determines if containerd should be the runtime.
 	containerdRuntimeSupported = false
 )
 
-// InitContainerdRuntime sets whether to use ContainerD for runtime
-// on Windows. This is an experimental feature still in development, and
-// also requires an environment variable to be set (so as not to turn the
-// feature on from simply experimental which would also mean LCOW.
-func InitContainerdRuntime(experimental bool, cdPath string) {
-	if experimental && len(cdPath) > 0 && len(os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME")) > 0 {
-		logrus.Warnf("Using ContainerD runtime. This feature is experimental")
+// InitContainerdRuntime sets whether to use containerd for runtime on Windows.
+func InitContainerdRuntime(cdPath string) {
+	if len(cdPath) > 0 {
 		containerdRuntimeSupported = true
 	}
 }
 
-// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported.
+// ContainerdRuntimeSupported returns true if the use of containerd runtime is supported.
 func ContainerdRuntimeSupported() bool {
 	return containerdRuntimeSupported
 }

testutil/environment/environment.go

@@ -162,6 +162,11 @@ func (e *Execution) IsUserNamespace() bool {
 	return root != ""
 }
 
+// RuntimeIsWindowsContainerd returns whether containerd runtime is used on Windows
+func (e *Execution) RuntimeIsWindowsContainerd() bool {
+	return os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME") == "1"
+}
+
 // IsRootless returns whether the rootless mode is enabled
 func (e *Execution) IsRootless() bool {
 	return os.Getenv("DOCKER_ROOTLESS") != ""