WIP: Kubernetes v1.16 update

Gopkg.toml

@@ -49,38 +49,50 @@
 # see https://github.com/kubernetes/kubernetes/issues/72638.
 [[constraint]]
   name = "k8s.io/kubernetes"
-  version = "1.14.0"
+  source = "github.com/avalluri/kubernetes"
+  branch = "automated-cherry-pick-of-#83609-upstream-release-1.16"
 
 [[override]]
   name = "k8s.io/apimachinery"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/api"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/apiserver"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/apiextensions-apiserver"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/cli-runtime"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/client-go"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/cloud-provider"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/cluster-bootstrap"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/csi-api"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
 [[override]]
   name = "k8s.io/kube-aggregator"
-  version = "kubernetes-1.14.0"
+  version = "kubernetes-1.16.0"
+[[override]]
+  name = "k8s.io/component-base"
+  version = "kubernetes-1.16.0"
+[[override]]
+  name = "k8s.io/csi-translation-lib"
+  version = "kubernetes-1.16.0"
+# There is a api breakage in latest(v1.0.0) prometheus release
+# vendor/k8s.io/component-base/metrics/legacyregistry/registry.go:44:9: undefined: prometheus.InstrumentHandler
+[[override]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.9.2"
 
 # Doesn't do proper releases.
 [[override]]
@@ -92,11 +104,21 @@
   name = "k8s.io/utils"
   branch = "master"
 
+# Latest etcd package name has moved to "go.etcd.io/etcd"
+# But apiserver v1.16 is still using github.com/coreos/etcd/
+# hence lock to appropriate v3.3.15.
 [[override]]
-  # kubernetes/pkg/util/parsers/parsers.go needs the new reference.ParseNormalizedNamed
-  # API call, which isn't yet in any stable release.
-  name = "github.com/docker/distribution"
-  branch = "master"
+  name = "github.com/coreos/etcd"
+  version = "=3.3.15"
+
+# In commit ec11136828c9558db61de95daec25475549dbe7e
+# systemd.Manager interface made LegacyManger, but kubelet
+# is depend on this interface.
+# Hence lock to best known revision
+# https://github.com/kubernetes/kubernetes/blob/2bd9643cee5b3b3a5ecbd3af49d09018f0773c77/go.mod#L102
+[[override]]
+  name = "github.com/opencontainers/runc"
+  revision = "6cc515888830787a93d82138821f0309ad970640"
 
 # This prevents dep from pulling an empty repo for gopkg.in/fsnotify.v1,
 # something that it does not recover from.

pkg/pmem-device-manager/pmd-util.go

@@ -8,7 +8,7 @@ import (
 
 	pmemexec "github.com/intel/pmem-csi/pkg/pmem-exec"
 	"k8s.io/klog"
-	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume/util/hostutil"
 )
 
 const (
@@ -40,11 +40,11 @@ func FlushDevice(dev PmemDeviceInfo, blocks uint64) error {
 		klog.Errorf("FlushDevice: %s is not device", dev.Path)
 		return fmt.Errorf("%s is not device", dev.Path)
 	}
-	devOpen, err := mount.New("").DeviceOpened(dev.Path)
+	devOpen, err := hostutil.NewHostUtil().DeviceOpened(dev.Path)
 	if err != nil {
 		return err
 	}
-	if devOpen == true {
+	if devOpen {
 		return fmt.Errorf("%s is in use", dev.Path)
 	}
 	if blocks == 0 {

pkg/registryserver/registryserver_test.go

@@ -175,12 +175,18 @@ var _ = Describe("pmem registry", func() {
 
 		// This covers different scenarios for connections to the registry.
 		cases := []struct {
-			name, ca, cert, key, peerName, errorText string
+			name, ca, cert, key, peerName, errorRE string
 		}{
-			{"registry should detect man-in-the-middle", ca, evilCert, evilKey, "pmem-registry", "authentication handshake failed: remote error: tls: bad certificate"},
+			// The exact error for the server side depends on whether TLS 1.3 is active (https://golang.org/doc/go1.12#tls_1_3).
+			// It looks like error detection is less precise in that case.
+			{"registry should detect man-in-the-middle", ca, evilCert, evilKey, "pmem-registry",
+				"authentication handshake failed: remote error: tls: bad certificate|all SubConns are in TransientFailure",
+			},
 			{"client should detect man-in-the-middle", evilCA, evilCert, evilKey, "pmem-registry", "transport: authentication handshake failed: x509: certificate signed by unknown authority"},
 			{"client should detect wrong peer", ca, cert, key, "unknown-registry", "transport: authentication handshake failed: x509: certificate is valid for pmem-registry, not unknown-registry"},
-			{"server should detect wrong peer", ca, wrongCert, wrongKey, "pmem-registry", "transport: authentication handshake failed: remote error: tls: bad certificate"},
+			{"server should detect wrong peer", ca, wrongCert, wrongKey, "pmem-registry",
+				"transport: authentication handshake failed: remote error: tls: bad certificate|all SubConns are in TransientFailure",
+			},
 		}
 
 		for _, c := range cases {
@@ -199,7 +205,7 @@ var _ = Describe("pmem registry", func() {
 
 				_, err = client.RegisterController(context.Background(), &req)
 				Expect(err).To(HaveOccurred())
-				Expect(err.Error()).To(ContainSubstring(c.errorText))
+				Expect(err.Error()).To(MatchRegexp(c.errorRE))
 			})
 		}
 	})

test/e2e/e2e.go

@@ -29,6 +29,8 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/framework/podlogs"
 )
 
@@ -83,7 +85,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// #41007. To avoid those pods preventing the whole test runs (and just
 	// wasting the whole run), we allow for some not-ready pods (with the
 	// number equal to the number of allowed not-ready nodes).
-	if err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil {
+	if err := e2epod.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil {
 		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
 		framework.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf)
 		framework.Failf("Error waiting for all pods to be running and ready: %v", err)
@@ -144,9 +146,7 @@ var _ = ginkgo.SynchronizedAfterSuite(func() {
 // E2E tests using the Ginkgo runner.
 // This function is called on each Ginkgo node in parallel mode.
 func RunE2ETests(t *testing.T) {
-	// TODO: with "ginkgo ./test/e2e" we shouldn't get verbose output, but somehow we do.
-	// TODO: use ginkgowrapper.Fail again after upstreaming our enhancements in wrapper.go
-	gomega.RegisterFailHandler(FailWrapper)
+	gomega.RegisterFailHandler(ginkgowrapper.Fail)
 	ginkgo.RunSpecs(t, "PMEM E2E suite")
 }
 

test/e2e/e2e_test.go

@@ -21,8 +21,10 @@ import (
 	"os"
 	"testing"
 
+	"k8s.io/component-base/logs"
 	"k8s.io/klog"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 
 	. "github.com/onsi/ginkgo"
@@ -31,22 +33,28 @@ import (
 	_ "github.com/intel/pmem-csi/test/e2e/storage"
 )
 
-func init() {
+func TestMain(m *testing.M) {
 	klog.SetOutput(GinkgoWriter)
-	klog.InitFlags(flag.CommandLine)
 
+	logs.InitLogs()
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
 	// Skip slow or distruptive tests by default.
 	flag.Set("ginkgo.skip", `\[Slow|Disruptive\]`)
+	flag.Parse()
 
 	// Register framework flags, then handle flags.
-	framework.HandleFlags()
 	framework.AfterReadingAllFlags(&framework.TestContext)
 
 	// We need extra files at runtime.
 	repoRoot := os.Getenv("REPO_ROOT")
 	if repoRoot != "" {
 		testfiles.AddFileSource(RootFileSource{Root: repoRoot})
 	}
+
+	// Now run the test suite.
+	os.Exit(m.Run())
 }
 
 func TestE2E(t *testing.T) {

test/e2e/storage/latebinding.go

@@ -30,6 +30,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
 // TestDynamicLateBindingProvisioning is a variant of k8s.io/kubernetes/test/e2e/storage/testsuites/provisioning.go
@@ -104,12 +105,12 @@ func TestDynamicLateBindingProvisioning(client clientset.Interface, claim *v1.Pe
 func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, id string) {
 	By(fmt.Sprintf("%s: checking the created volume is writable", id))
 	command := "echo 'hello world' > /mnt/test/data || (mount | grep 'on /mnt/test'; false)"
-	pod := testsuites.StartInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-writer-"+id, command, testsuites.NodeSelection{})
+	pod := testsuites.StartInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-writer-"+id, command, e2epod.NodeSelection{})
 	defer func() {
 		// pod might be nil now.
 		testsuites.StopPod(client, pod)
 	}()
-	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace))
+	framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace))
 	runningPod, err := client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
 	Expect(err).NotTo(HaveOccurred(), "get pod %s", id)
 	actualNodeName := runningPod.Spec.NodeName
@@ -118,5 +119,5 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
 
 	By(fmt.Sprintf("%s: checking the created volume is readable and retains data on the same node %q", id, actualNodeName))
 	command = "grep 'hello world' /mnt/test/data"
-	testsuites.RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader-"+id, command, testsuites.NodeSelection{Name: actualNodeName})
+	testsuites.RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader-"+id, command, e2epod.NodeSelection{Name: actualNodeName})
 }

test/e2e/storage/sanity.go

@@ -44,6 +44,8 @@ import (
 	clientexec "k8s.io/client-go/util/exec"
 	"k8s.io/kubernetes/test/e2e/framework"
 	testutils "k8s.io/kubernetes/test/utils"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -65,6 +67,7 @@ var _ = Describe("sanity", func() {
 		// and deletes all extra entries that it does not know about.
 		TargetPath:  "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pmem-sanity-target.XXXXXX",
 		StagingPath: "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pmem-sanity-staging.XXXXXX",
+		IDGen: &sanity.DefaultIDGenerator{},
 	}
 
 	f := framework.NewDefaultFramework("pmem")
@@ -81,7 +84,7 @@ var _ = Describe("sanity", func() {
 			framework.Skipf("driver deployed in production mode")
 		}
 
-		hosts, err := framework.NodeSSHHosts(cs)
+		hosts, err := e2essh.NodeSSHHosts(cs)
 		framework.ExpectNoError(err, "failed to find external/internal IPs for every node")
 		if len(hosts) <= 1 {
 			framework.Failf("not enough nodes with external IP")
@@ -970,7 +973,7 @@ func WaitForPodsWithLabelRunningReady(c clientset.Interface, ns string, label la
 	var current int
 	err = wait.Poll(2*time.Second, timeout,
 		func() (bool, error) {
-			pods, err = framework.WaitForPodsWithLabel(c, ns, label)
+			pods, err = e2epod.WaitForPodsWithLabel(c, ns, label)
 			if err != nil {
 				framework.Logf("Failed to list pods: %v", err)
 				if testutils.IsRetryableAPIError(err) {