Test rework, part 4

Signed-off-by: apostasie <spam_blackhole@farcloser.world>
containerd · Oct 5, 2024 · a374cf1 · a374cf1
1 parent 394015b
commit a374cf1
Show file tree

Hide file tree

Showing 84 changed files with 6,713 additions and 3,749 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -99,12 +99,15 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
       - name: "Run integration tests"
+        run: |
+          docker run -t --rm --privileged test-integration
+      - name: "Run integration tests: flaky"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          command: docker run -t --rm --privileged test-integration
+          command: docker run -t --rm --privileged test-integration-flaky
 
   test-integration-ipv6:
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
@@ -149,18 +152,20 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
-      - name: "Run integration tests"
         # The nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config.
         # Therefore it's hard to debug why the IPv6 tests fail in such an isolation layer.
         # On the other side, using the host network is easier at configuration.
         # Besides, each job is running on a different instance, which means using host network here
         # is safe and has no side effects on others.
+      - name: "Run integration tests"
+        run: docker run --network host -t --rm --privileged test-integration-ipv6
+      - name: "Run integration tests: flaky"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          command: docker run --network host -t --rm --privileged test-integration-ipv6
+          command: docker run --network host -t --rm --privileged test-integration-ipv6-flaky
 
   test-integration-rootless:
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
@@ -230,12 +235,14 @@ jobs:
           fi
           echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
       - name: "Test (network driver=slirp4netns, port driver=builtin)"
+        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}
+      - name: "Test (network driver=slirp4netns, port driver=builtin): flaky"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          command: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}
+          command: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}-flaky
 
   cross:
     runs-on: ubuntu-24.04
@@ -285,21 +292,25 @@ jobs:
         run: |
           sudo apt-get install -y expect
       - name: "Ensure that the integration test suite is compatible with Docker"
+        run: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
+      - name: "Ensure that the integration test suite is compatible with Docker: flaky"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
           # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
+          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-flaky
       - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
+        run: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
+      - name: "Ensure that the IPv6 integration test suite is compatible with Docker: flaky"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
           # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
+          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6 -test.only-flaky
 
   test-integration-windows:
     runs-on: windows-2022

diff --git a/Dockerfile b/Dockerfile
@@ -318,9 +318,13 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
   tar xzf nydus-static.tgz && \
   mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
   rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+CMD ["gotestsum", "--format=testname", "--packages=./cmd/nerdctl/...", \
   "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
+FROM test-integration AS test-integration-flaky
+CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-flaky"]
+
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
 # (`sudo` does not work for this purpose,
@@ -343,16 +347,29 @@ VOLUME /home/rootless/.local/share
 COPY ./Dockerfile.d/test-integration-rootless.sh /
 RUN chmod a+rx /test-integration-rootless.sh
 CMD ["/test-integration-rootless.sh", \
-  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "gotestsum", "--format=testname", "--packages=./cmd/nerdctl/...", \
   "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
+FROM test-integration-rootless AS test-integration-rootless-flaky
+CMD ["/test-integration-rootless.sh", \
+  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-flaky"]
+
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
+FROM test-integration-rootless-flaky AS test-integration-rootless-port-slirp4netns-flaky
+COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
+RUN chown -R rootless:rootless /home/rootless/.config
+
 FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+CMD ["gotestsum", "--format=testname", "--packages=./cmd/nerdctl/...", \
   "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
 
+FROM test-integration AS test-integration-ipv6-flaky
+CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6", "-test.only-flaky"]
+
 FROM base AS demo
diff --git a/cmd/nerdctl/builder/builder_build_linux_test.go b/cmd/nerdctl/builder/builder_build_linux_test.go
@@ -18,17 +18,19 @@ package builder
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
 
-	"github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
+	testhelpers "github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 func TestBuildContextWithOCILayout(t *testing.T) {
-	testutil.RequiresBuild(t)
-	testutil.RegisterBuildCacheCleanup(t)
+	nerdtest.Setup()
 
 	var dockerBuilderArgs []string
 	if testutil.IsDocker() {
@@ -38,48 +40,50 @@ func TestBuildContextWithOCILayout(t *testing.T) {
 		dockerBuilderArgs = []string{"buildx", "--builder", builderName}
 	}
 
-	base := testutil.NewBase(t)
-	imageName := testutil.Identifier(t)
-	ociLayout := "parent"
-	parentImageName := fmt.Sprintf("%s-%s", imageName, ociLayout)
-
-	teardown := func() {
-		base.Cmd("rmi", parentImageName, imageName).Run()
-	}
-	t.Cleanup(teardown)
-	teardown()
-
-	dockerfile := fmt.Sprintf(`FROM %s
+	testCase := &test.Case{
+		Require: nerdtest.Build,
+		Cleanup: func(data test.Data, helpers test.Helpers) {
+			helpers.Anyhow("rmi", fmt.Sprintf("%s-parent", data.Identifier()))
+		},
+		Setup: func(data test.Data, helpers test.Helpers) {
+			dockerfile := fmt.Sprintf(`FROM %s
 LABEL layer=oci-layout-parent
 CMD ["echo", "test-nerdctl-build-context-oci-layout-parent"]`, testutil.CommonImage)
-	buildCtx := helpers.CreateBuildContext(t, dockerfile)
-
-	tarPath := fmt.Sprintf("%s/%s.tar", buildCtx, ociLayout)
-
-	// Create OCI archive from parent image.
-	base.Cmd("build", buildCtx, "--tag", parentImageName).AssertOK()
-	base.Cmd("image", "save", "--output", tarPath, parentImageName).AssertOK()
-
-	// Unpack OCI archive into OCI layout directory.
-	ociLayoutDir := t.TempDir()
-	err := helpers.ExtractTarFile(ociLayoutDir, tarPath)
-	assert.NilError(t, err)
 
-	dockerfile = fmt.Sprintf(`FROM %s
-CMD ["echo", "test-nerdctl-build-context-oci-layout"]`, ociLayout)
-	buildCtx = helpers.CreateBuildContext(t, dockerfile)
-
-	var buildArgs = []string{}
-	if testutil.IsDocker() {
-		buildArgs = dockerBuilderArgs
-	}
-
-	buildArgs = append(buildArgs, "build", buildCtx, fmt.Sprintf("--build-context=%s=oci-layout://%s", ociLayout, ociLayoutDir), "--tag", imageName)
-	if testutil.IsDocker() {
-		// Need to load the container image from the builder to be able to run it.
-		buildArgs = append(buildArgs, "--load")
+			// FIXME: replace with a generic file creation helper - search for all occurrences of temp file creation
+			buildCtx := testhelpers.CreateBuildContext(t, dockerfile)
+			tarPath := fmt.Sprintf("%s/parent.tar", buildCtx)
+
+			helpers.Ensure("build", buildCtx, "--tag", fmt.Sprintf("%s-parent", data.Identifier()))
+			helpers.Ensure("image", "save", "--output", tarPath, fmt.Sprintf("%s-parent", data.Identifier()))
+			helpers.Custom("tar", "Cxf", data.TempDir(), tarPath).Run(&test.Expected{})
+		},
+
+		Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+			dockerfile := `FROM parent
+CMD ["echo", "test-nerdctl-build-context-oci-layout"]`
+			buildCtx := testhelpers.CreateBuildContext(t, dockerfile)
+			var cmd test.TestableCommand
+			if testutil.IsDocker() {
+				cmd = helpers.Command(dockerBuilderArgs...)
+			} else {
+				cmd = helpers.Command()
+			}
+			cmd.WithArgs("build", buildCtx, fmt.Sprintf("--build-context=parent=oci-layout://%s", data.TempDir()), "--tag", data.Identifier())
+			if testutil.IsDocker() {
+				// Need to load the container image from the builder to be able to run it.
+				cmd.WithArgs("--load")
+			}
+			return cmd
+		},
+		Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+			return &test.Expected{
+				Output: func(stdout string, info string, t *testing.T) {
+					assert.Assert(t, strings.Contains(helpers.Capture("run", "--rm", data.Identifier()), "test-nerdctl-build-context-oci-layout"), info)
+				},
+			}
+		},
 	}
 
-	base.Cmd(buildArgs...).AssertOK()
-	base.Cmd("run", "--rm", imageName).AssertOutContains("test-nerdctl-build-context-oci-layout")
+	testCase.Run(t)
 }