Upgrade k8s.io/client-go and k8s keystore tests (#18817) (#18879)

(cherry picked from commit 5bfcc9c)

CHANGELOG.next.asciidoc

@@ -314,6 +314,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update RPM packages contained in Beat Docker images. {issue}17035[17035]
 - Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905]
 - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
+- Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
 
 *Auditbeat*
 

NOTICE.txt

@@ -1853,6 +1853,38 @@ This product includes a number of subcomponents with
 separate copyright notices and license terms. Your use of these
 subcomponents is subject to the terms and conditions of the 
 subcomponent's license, as noted in the LICENSE file.
+--------------------------------------------------------------------
+Dependency: github.com/evanphx/json-patch
+Version: v4.2.0
+License type (autodetected): BSD-3-Clause
+./vendor/github.com/evanphx/json-patch/LICENSE:
+--------------------------------------------------------------------
+Copyright (c) 2014, Evan Phoenix
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+* Neither the name of the Evan Phoenix nor the names of its contributors 
+  may be used to endorse or promote products derived from this software 
+  without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 --------------------------------------------------------------------
 Dependency: github.com/fatih/color
 Version: v1.5.0
@@ -3065,7 +3097,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: github.com/google/gofuzz
-Version: v1.0.0
+Version: v1.1.0
 License type (autodetected): Apache-2.0
 ./vendor/github.com/google/gofuzz/LICENSE:
 --------------------------------------------------------------------
@@ -5397,7 +5429,7 @@ SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: github.com/json-iterator/go
-Version: v1.1.7
+Version: v1.1.8
 License type (autodetected): MIT
 ./vendor/github.com/json-iterator/go/LICENSE:
 --------------------------------------------------------------------
@@ -7120,7 +7152,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/spf13/pflag
-Version: v1.0.3
+Version: v1.0.5
 License type (autodetected): BSD-3-Clause
 ./vendor/github.com/spf13/pflag/LICENSE:
 --------------------------------------------------------------------
@@ -8254,7 +8286,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: gopkg.in/inf.v0
-Version: v0.9.0
+Version: v0.9.1
 License type (autodetected): BSD-3-Clause
 ./vendor/gopkg.in/inf.v0/LICENSE:
 --------------------------------------------------------------------
@@ -8885,7 +8917,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: k8s.io/api
-Revision: b90922c02518
+Version: v0.18.3
 License type (autodetected): Apache-2.0
 ./vendor/k8s.io/api/LICENSE:
 --------------------------------------------------------------------
@@ -8894,7 +8926,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: k8s.io/apimachinery
-Revision: bfcf53abc9f8
+Version: v0.18.3
 License type (autodetected): Apache-2.0
 ./vendor/k8s.io/apimachinery/LICENSE:
 --------------------------------------------------------------------
@@ -8903,7 +8935,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: k8s.io/client-go
-Revision: 78d2af792bab
+Version: v0.18.3
 License type (autodetected): Apache-2.0
 ./vendor/k8s.io/client-go/LICENSE:
 --------------------------------------------------------------------
@@ -8912,27 +8944,43 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: k8s.io/klog
-Version: v0.3.4
-Revision: 6a023d6d0e09
+Version: v1.0.0
 License type (autodetected): Apache-2.0
 ./vendor/k8s.io/klog/LICENSE:
 --------------------------------------------------------------------
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: k8s.io/kube-openapi
+Revision: 61e04a5be9a6
+License type (autodetected): Apache-2.0
+./vendor/k8s.io/kube-openapi/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: k8s.io/utils
-Revision: 3dccf664f023
+Revision: a9aa75ae1b89
 License type (autodetected): Apache-2.0
 ./vendor/k8s.io/utils/LICENSE:
 --------------------------------------------------------------------
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: sigs.k8s.io/structured-merge-diff/v3
+Version: v3.0.0
+License type (autodetected): Apache-2.0
+./vendor/sigs.k8s.io/structured-merge-diff/v3/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: sigs.k8s.io/yaml
-Version: v1.1.1
-Revision: 4cd0c284b15f
+Version: v1.2.0
 License type (autodetected): MIT
 ./vendor/sigs.k8s.io/yaml/LICENSE:
 --------------------------------------------------------------------

dev-tools/generate_notice.py

@@ -352,6 +352,18 @@ def create_notice(filename, beat, copyright, vendor_dir, csvfile, overrides=None
    and/or other materials provided with the distribution.
 """)]
 
+# This is a temporary workaround to solve UNKNOWN licence issue
+# for `evanphx/json-patch`: https://github.com/evanphx/json-patch/pull/105
+BSD_LICENSE_CONTENTS_COMMA_MISSING = [
+    re.sub(r"\s+", " ", """Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:"""),
+    re.sub(r"\s+", " ", """Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer."""),
+    re.sub(r"\s+", " ", """Redistributions in binary form must reproduce the above copyright notice
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+""")]
+
 BSD_LICENSE_3_CLAUSE = [
     re.sub(r"\s+", " ", """Neither the name of"""),
     re.sub(r"\s+", " ", """nor the
@@ -400,7 +412,8 @@ def detect_license_summary(content):
         return "Apache-2.0"
     if any(sentence in content[0:1000] for sentence in MIT_LICENSES):
         return "MIT"
-    if all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS):
+    if all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS) or \
+            all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS_COMMA_MISSING):
         if all(sentence in content[0:1000] for sentence in BSD_LICENSE_3_CLAUSE):
             if all(sentence in content[0:1000] for sentence in BSD_LICENSE_4_CLAUSE):
                 return "BSD-4-Clause"

dev-tools/mage/kubernetes/kubectl.go

@@ -77,7 +77,7 @@ func KubectlDeleteInput(env map[string]string, stdout, stderr io.Writer, manifes
 // KubectlWait waits for a condition to occur for a resource in the kubernetes cluster.
 //
 // KUBECONFIG must be in `env` to target a specific cluster.
-func KubectlWait(env map[string]string, stdout, stderr io.Writer, waitFor, resource string) error {
+func KubectlWait(env map[string]string, stdout, stderr io.Writer, waitFor, resource string, labels string) error {
 	_, err := sh.Exec(
 		env,
 		stdout,
@@ -87,6 +87,8 @@ func KubectlWait(env map[string]string, stdout, stderr io.Writer, waitFor, resou
 		"--timeout=300s",
 		fmt.Sprintf("--for=%s", waitFor),
 		resource,
+		"-l",
+		labels,
 	)
 	return err
 }

dev-tools/mage/kubernetes/kuberemote.go

@@ -164,14 +164,17 @@ func (r *KubeRemote) Run(env map[string]string, stdout io.Writer, stderr io.Writ
 
 // deleteSSHKey deletes SSH key from the cluster.
 func (r *KubeRemote) deleteSSHKey() {
-	_ = r.cs.CoreV1().Secrets(r.namespace).Delete(r.secretName, &metav1.DeleteOptions{})
+	_ = r.cs.CoreV1().Secrets(r.namespace).Delete(context.TODO(), r.secretName, metav1.DeleteOptions{})
 }
 
 // syncSSHKey syncs the SSH key to the cluster.
 func (r *KubeRemote) syncSSHKey() error {
 	// delete before create
 	r.deleteSSHKey()
-	_, err := r.cs.CoreV1().Secrets(r.namespace).Create(createSecretManifest(r.secretName, r.publicKey))
+	_, err := r.cs.CoreV1().Secrets(r.namespace).Create(
+		context.TODO(),
+		createSecretManifest(r.secretName, r.publicKey),
+		metav1.CreateOptions{})
 	if err != nil {
 		return err
 	}
@@ -180,24 +183,32 @@ func (r *KubeRemote) syncSSHKey() error {
 
 // deleteServiceAccount syncs required service account.
 func (r *KubeRemote) deleteServiceAccount() {
-	_ = r.cs.RbacV1().ClusterRoleBindings().Delete(r.name, &metav1.DeleteOptions{})
-	_ = r.cs.RbacV1().ClusterRoles().Delete(r.name, &metav1.DeleteOptions{})
-	_ = r.cs.CoreV1().ServiceAccounts(r.namespace).Delete(r.svcAccName, &metav1.DeleteOptions{})
+	ctx := context.TODO()
+	_ = r.cs.RbacV1().ClusterRoleBindings().Delete(ctx, r.name, metav1.DeleteOptions{})
+	_ = r.cs.RbacV1().ClusterRoles().Delete(ctx, r.name, metav1.DeleteOptions{})
+	_ = r.cs.CoreV1().ServiceAccounts(r.namespace).Delete(ctx, r.svcAccName, metav1.DeleteOptions{})
 }
 
 // syncServiceAccount syncs required service account.
 func (r *KubeRemote) syncServiceAccount() error {
+	ctx := context.TODO()
 	// delete before create
 	r.deleteServiceAccount()
-	_, err := r.cs.CoreV1().ServiceAccounts(r.namespace).Create(createServiceAccountManifest(r.svcAccName))
+	_, err := r.cs.CoreV1().ServiceAccounts(r.namespace).Create(
+		ctx,
+		createServiceAccountManifest(r.svcAccName),
+		metav1.CreateOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to create service account")
 	}
-	_, err = r.cs.RbacV1().ClusterRoles().Create(createClusterRoleManifest(r.name))
+	_, err = r.cs.RbacV1().ClusterRoles().Create(ctx, createClusterRoleManifest(r.name), metav1.CreateOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to create cluster role")
 	}
-	_, err = r.cs.RbacV1().ClusterRoleBindings().Create(createClusterRoleBindingManifest(r.name, r.namespace, r.svcAccName))
+	_, err = r.cs.RbacV1().ClusterRoleBindings().Create(
+		ctx,
+		createClusterRoleBindingManifest(r.name, r.namespace, r.svcAccName),
+		metav1.CreateOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to create cluster role binding")
 	}
@@ -212,17 +223,20 @@ func (r *KubeRemote) createPod(env map[string]string, cmd ...string) (*apiv1.Pod
 	}
 	image := fmt.Sprintf("golang:%s", version)
 	r.deletePod() // ensure it doesn't already exist
-	return r.cs.CoreV1().Pods(r.namespace).Create(createPodManifest(r.name, image, env, cmd, r.workDir, r.destDir, r.secretName, r.svcAccName))
+	return r.cs.CoreV1().Pods(r.namespace).Create(
+		context.TODO(),
+		createPodManifest(r.name, image, env, cmd, r.workDir, r.destDir, r.secretName, r.svcAccName),
+		metav1.CreateOptions{})
 }
 
 // deletePod deletes the pod.
 func (r *KubeRemote) deletePod() {
-	_ = r.cs.CoreV1().Pods(r.namespace).Delete(r.name, &metav1.DeleteOptions{})
+	_ = r.cs.CoreV1().Pods(r.namespace).Delete(context.TODO(), r.name, metav1.DeleteOptions{})
 }
 
 // waitForPod waits for the created pod to match the given condition.
 func (r *KubeRemote) waitForPod(wait time.Duration, condition watchtools.ConditionFunc) (*apiv1.Pod, error) {
-	w, err := r.cs.CoreV1().Pods(r.namespace).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: r.name}))
+	w, err := r.cs.CoreV1().Pods(r.namespace).Watch(context.TODO(), metav1.SingleObject(metav1.ObjectMeta{Name: r.name}))
 	if err != nil {
 		return nil, err
 	}
@@ -276,7 +290,7 @@ func (r *KubeRemote) streamLogs(container string, stdout io.Writer) error {
 		Container: container,
 		Follow:    true,
 	})
-	logs, err := req.Stream()
+	logs, err := req.Stream(context.TODO())
 	if err != nil {
 		return err
 	}

dev-tools/mage/kubernetes/kubernetes.go

@@ -19,11 +19,13 @@ package kubernetes
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/magefile/mage/mg"
 	"github.com/pkg/errors"
@@ -114,6 +116,11 @@ func (d *KubernetesIntegrationTester) Test(dir string, mageTarget string, env ma
 		}
 	}()
 
+	err = waitKubeStateMetricsReadiness(env, stdOut, stdErr)
+	if err != nil {
+		return err
+	}
+
 	// Pass all environment variables inside the pod, except for KUBECONFIG as the test
 	// should use the environment set by kubernetes on the pod.
 	insideEnv := map[string]string{}
@@ -142,6 +149,29 @@ func (d *KubernetesIntegrationTester) InsideTest(test func() error) error {
 	return test()
 }
 
+// waitKubeStateMetricsReadiness waits until kube-state-metrics Pod is ready to receive requests
+func waitKubeStateMetricsReadiness(env map[string]string, stdOut, stdErr io.Writer) error {
+	checkKubeStateMetricsReadyAttempts := 10
+	readyAttempts := 1
+	for {
+		err := KubectlWait(env, stdOut, stdErr, "condition=ready", "pod", "app=kube-state-metrics")
+		if err != nil {
+			if mg.Verbose() {
+				fmt.Println("Kube-state-metrics is not ready yet...retrying")
+			}
+		} else {
+			break
+		}
+		if readyAttempts > checkKubeStateMetricsReadyAttempts {
+			return errors.Wrapf(err, "Timeout waiting for kube-state-metrics")
+		}
+		time.Sleep(6 * time.Second)
+		readyAttempts += 1
+	}
+	// kube-state-metrics ready, return with no error
+	return nil
+}
+
 // kubernetesPodName returns the pod name to use with kubernetes.
 func kubernetesPodName() string {
 	commit, err := mage.CommitHash()

go.mod

@@ -133,7 +133,7 @@ require (
 	github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 // indirect
 	github.com/shirou/gopsutil v2.19.11+incompatible
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.5.0
 	github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
 	github.com/tsg/gopacket v0.0.0-20190320122513-dd3d0e41124a
@@ -159,17 +159,14 @@ require (
 	google.golang.org/api v0.15.0
 	google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb
 	google.golang.org/grpc v1.27.1
-	gopkg.in/inf.v0 v0.9.0
+	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/jcmturner/gokrb5.v7 v7.5.0
 	gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528
 	gopkg.in/yaml.v2 v2.3.0
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb
-	k8s.io/api v0.0.0-20190722141453-b90922c02518
-	k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8
-	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
-	k8s.io/klog v0.3.4-0.20190719014911-6a023d6d0e09 // indirect
-	k8s.io/utils v0.0.0-20190712204705-3dccf664f023 // indirect
-	sigs.k8s.io/yaml v1.1.1-0.20190704183835-4cd0c284b15f // indirect
+	k8s.io/api v0.18.3
+	k8s.io/apimachinery v0.18.3
+	k8s.io/client-go v0.18.3
 )
 
 replace (