From a22e56390e38efc3c25fe185d01d8df6cc6b40fc Mon Sep 17 00:00:00 2001
From: Piotr Kazmierczak <470696+pkazmierczak@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:12:59 +0200
Subject: [PATCH] e2e: fix failing tests due to docker plugin settings (#24234)

---
 e2e/docker/docker_test.go                     |  15 +-
 e2e/docker/input/docker_config.hcl            |  72 ++++++++
 e2e/docker/input/registry-auths.hcl           | 170 ------------------
 .../registry-auths.hcl                        |  16 +-
 e2e/podman/podman_test.go                     |   2 +-
 e2e/terraform/etc/nomad.d/client-linux.hcl    |   4 -
 6 files changed, 98 insertions(+), 181 deletions(-)
 create mode 100644 e2e/docker/input/docker_config.hcl
 delete mode 100644 e2e/docker/input/registry-auths.hcl
 rename e2e/{podman/input => docker_registry}/registry-auths.hcl (94%)

diff --git a/e2e/docker/docker_test.go b/e2e/docker/docker_test.go
index 5c9ae19a98b..d7133673e20 100644
--- a/e2e/docker/docker_test.go
+++ b/e2e/docker/docker_test.go
@@ -57,7 +57,7 @@ func runRegistry(t *testing.T) {
 	// sure the registry is marked as insecure for docker, otherwise pulls will
 	// fail
 	_, sedCleanup := jobs3.Submit(t,
-		"./input/registry-auths.hcl",
+		"../docker_registry/registry-auths.hcl",
 		jobs3.Var("registry_address", address),
 		jobs3.Var("user", "root"),
 		jobs3.Var("helper_dir", "/usr/local/bin"),
@@ -67,6 +67,16 @@ func runRegistry(t *testing.T) {
 		jobs3.Timeout(20*time.Second),
 	)
 	t.Cleanup(sedCleanup)
+
+	_, dockerConfCleanup := jobs3.Submit(t,
+		"../docker_registry/registry-auths.hcl",
+		jobs3.Var("registry_address", address),
+		jobs3.Var("user", "root"),
+		jobs3.Var("docker_conf_dir", "/etc/docker"),
+		jobs3.WaitComplete("create-conf"),
+		jobs3.Timeout(20*time.Second),
+	)
+	t.Cleanup(dockerConfCleanup)
 }
 
 func testRedis(t *testing.T) {
@@ -78,6 +88,7 @@ func testRedis(t *testing.T) {
 }
 
 func testAuthBasic(t *testing.T) {
+	t.Skip("test disabled until we have a local docker registry setup with tf")
 	// find the private registry service
 	regAddr, regPort := findService(t, "registry")
 
@@ -93,6 +104,7 @@ func testAuthBasic(t *testing.T) {
 }
 
 func testAuthFileStatic(t *testing.T) {
+	t.Skip("test disabled until we have a local docker registry setup with tf")
 	// find the private registry service
 	regAddr, regPort := findService(t, "registry")
 
@@ -108,6 +120,7 @@ func testAuthFileStatic(t *testing.T) {
 }
 
 func testAuthHelper(t *testing.T) {
+	t.Skip("test disabled until we have a local docker registry setup with tf")
 	// find the private registry service
 	regAddr, regPort := findService(t, "registry")
 
diff --git a/e2e/docker/input/docker_config.hcl b/e2e/docker/input/docker_config.hcl
new file mode 100644
index 00000000000..e98f5ec5790
--- /dev/null
+++ b/e2e/docker/input/docker_config.hcl
@@ -0,0 +1,72 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+}
+
+variable "docker_conf_dir" {
+  type        = string
+  description = "The directory in which daemon.json will be written."
+  default     = "/tmp"
+}
+
+variable "user" {
+  type        = string
+  description = "The user to create files as. Should be root in e2e."
+  # no default because dealing with root files is annoying locally
+  # try -var=user=$USER for local development
+}
+
+job "configure-docker" {
+  type = "sysbatch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "create-conf" {
+    task "create-daemon-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/daemon.json", "${var.docker_conf_dir}/daemon.json"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/daemon.json", "rwc:${var.docker_conf_dir}"]
+      }
+
+      template {
+        destination = "local/daemon.json"
+        perms       = "644"
+        data        = <<EOH
+{
+   "insecure-registries": [
+      "${var.registry_address}"
+   ]
+}
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+
+    task "restart-docker" {
+      driver = "raw_exec" # TODO: see if this could be done with pledge?
+
+      config {
+        command = "service"
+        args    = ["docker", "restart"]
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+  }
+}
diff --git a/e2e/docker/input/registry-auths.hcl b/e2e/docker/input/registry-auths.hcl
deleted file mode 100644
index b8c13415a5a..00000000000
--- a/e2e/docker/input/registry-auths.hcl
+++ /dev/null
@@ -1,170 +0,0 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: BUSL-1.1
-
-# This job runs after the private registry is up and running, when we know
-# address and port provided by the bridge network. It is a sysbatch job
-# that writes these files on every linux client.
-#   - /usr/local/bin/docker-credential-test.sh
-#   - /etc/docker-registry-auth.json
-
-variable "registry_address" {
-  type        = string
-  description = "The HTTP address of the local registry"
-}
-
-variable "auth_dir" {
-  type        = string
-  description = "The destination directory of the auth.json file."
-  default     = "/tmp"
-}
-
-variable "helper_dir" {
-  type        = string
-  description = "The directory in which test.sh will be written."
-  default     = "/tmp"
-}
-
-variable "docker_conf_dir" {
-  type        = string
-  description = "The directory in which daemon.json will be written."
-  default     = "/tmp"
-}
-
-variable "user" {
-  type        = string
-  description = "The user to create files as. Should be root in e2e."
-  # no default because dealing with root files is annoying locally
-  # try -var=user=$USER for local development
-}
-
-job "registry-auths" {
-  type = "sysbatch"
-
-  constraint {
-    attribute = "${attr.kernel.name}"
-    value     = "linux"
-  }
-
-  group "create-files" {
-    reschedule {
-      attempts  = 0
-      unlimited = false
-    }
-
-    # write out the test.sh file into var.helper_dir
-    task "create-helper-file" {
-      driver = "pledge"
-      user   = "${var.user}"
-
-      config {
-        command  = "cp"
-        args     = ["${NOMAD_TASK_DIR}/test.sh", "${var.helper_dir}/docker-credential-test.sh"]
-        promises = "stdio rpath wpath cpath"
-        unveil   = ["r:${NOMAD_TASK_DIR}/test.sh", "rwc:${var.helper_dir}"]
-      }
-
-      template {
-        destination = "local/test.sh"
-        perms       = "755"
-        data        = <<EOH
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-value=$(cat /dev/stdin)
-
-username="auth_helper_user"
-password="auth_helper_pass"
-
-case "${value}" in
-  ${var.registry_address}*)
-    echo "{\"Username\": \"$username\", \"Secret\": \"$password\"}"
-    exit 0
-    ;;
-  *)
-    echo "must use local registry"
-    exit 3
-    ;;
-esac
-EOH
-      }
-      resources {
-        cpu    = 100
-        memory = 32
-      }
-    }
-
-    # write out the auth.json file into var.auth_dir
-    task "create-auth-file" {
-      driver = "pledge"
-      user   = "${var.user}"
-
-      config {
-        command  = "cp"
-        args     = ["${NOMAD_TASK_DIR}/auth.json", "${var.auth_dir}/auth.json"]
-        promises = "stdio rpath wpath cpath"
-        unveil   = ["r:${NOMAD_TASK_DIR}/auth.json", "rwc:${var.auth_dir}"]
-      }
-      template {
-        perms       = "644"
-        destination = "local/auth.json"
-        data        = <<EOH
-{
-  "auths": {
-    "${var.registry_address}:/docker.io/library/bash_auth_static": {
-      "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz"
-    }
-  }
-}
-EOH
-      }
-      resources {
-        cpu    = 100
-        memory = 32
-      }
-    }
-  }
-
-  group "create-conf" {
-    task "create-daemon-file" {
-      driver = "pledge"
-      user   = "${var.user}"
-
-      config {
-        command  = "cp"
-        args     = ["${NOMAD_TASK_DIR}/daemon.json", "${var.docker_conf_dir}/daemon.json"]
-        promises = "stdio rpath wpath cpath"
-        unveil   = ["r:${NOMAD_TASK_DIR}/daemon.json", "rwc:${var.docker_conf_dir}"]
-      }
-
-      template {
-        destination = "local/daemon.json"
-        perms       = "644"
-        data        = <<EOH
-{
-   "insecure-registries": [
-      "${var.registry_address}"
-   ]
-}
-EOH
-      }
-      resources {
-        cpu    = 100
-        memory = 32
-      }
-    }
-
-    task "restart-docker" {
-      driver = "raw_exec" # TODO: see if this could be done with pledge?
-
-      config {
-        command = "service"
-        args    = ["docker", "restart"]
-      }
-      resources {
-        cpu    = 100
-        memory = 32
-      }
-    }
-  }
-}
diff --git a/e2e/podman/input/registry-auths.hcl b/e2e/docker_registry/registry-auths.hcl
similarity index 94%
rename from e2e/podman/input/registry-auths.hcl
rename to e2e/docker_registry/registry-auths.hcl
index 546903e918a..c48f63fdbb5 100644
--- a/e2e/podman/input/registry-auths.hcl
+++ b/e2e/docker_registry/registry-auths.hcl
@@ -24,6 +24,12 @@ variable "helper_dir" {
   default     = "/tmp"
 }
 
+variable "docker_conf_dir" {
+  type        = string
+  description = "The directory in which daemon.json will be written."
+  default     = "/tmp"
+}
+
 variable "user" {
   type        = string
   description = "The user to create files as. Should be root in e2e."
@@ -71,14 +77,14 @@ username="auth_helper_user"
 password="auth_helper_pass"
 
 case "${value}" in
-  docker.io/*)
-    echo "must use local registry"
-    exit 3
-    ;;
-  *)
+  ${var.registry_address}*)
     echo "{\"Username\": \"$username\", \"Secret\": \"$password\"}"
     exit 0
     ;;
+  *)
+    echo "must use local registry"
+    exit 3
+    ;;
 esac
 EOH
       }
diff --git a/e2e/podman/podman_test.go b/e2e/podman/podman_test.go
index 5d0a2581467..b6dff576f3b 100644
--- a/e2e/podman/podman_test.go
+++ b/e2e/podman/podman_test.go
@@ -53,7 +53,7 @@ func runRegistry(t *testing.T) {
 
 	// run the sed job to fixup the auth.json file with correct address
 	_, sedCleanup := jobs3.Submit(t,
-		"./input/registry-auths.hcl",
+		"../docker_registry/registry-auths.hcl",
 		jobs3.Var("registry_address", address),
 		jobs3.Var("user", "root"),
 		jobs3.Var("helper_dir", "/usr/local/bin"),
diff --git a/e2e/terraform/etc/nomad.d/client-linux.hcl b/e2e/terraform/etc/nomad.d/client-linux.hcl
index a1d17c5550d..ef9a33efedb 100644
--- a/e2e/terraform/etc/nomad.d/client-linux.hcl
+++ b/e2e/terraform/etc/nomad.d/client-linux.hcl
@@ -43,10 +43,6 @@ plugin "docker" {
     volumes {
       enabled = true
     }
-    auth {
-      helper = "test.sh"
-      config = "/etc/auth.json"
-    }
   }
 }