diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index d52862b8062cb..0ea4dbf1a7605 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -511,7 +511,7 @@ func runTestCloudformation(t *testing.T, clusterName string, srcDir string, vers t.Logf("actual terraform output in %s", actualPath) } - t.Fatalf("cloudformation output differed from expected") + t.Fatalf("cloudformation output differed from expected. Test file: %s", path.Join(srcDir, expectedCfPath)) } expectedExtracted, err := ioutil.ReadFile(path.Join(srcDir, expectedCfPath+".extracted.yaml")) @@ -542,7 +542,7 @@ func runTestCloudformation(t *testing.T, clusterName string, srcDir string, vers diffString := diff.FormatDiff(expectedValue, extractedValueTrimmed) t.Logf("diff for key %s:\n%s\n\n\n\n\n\n", key, diffString) - t.Fatalf("cloudformation output differed from expected") + t.Fatalf("cloudformation output differed from expected. Test file: %s", path.Join(srcDir, expectedCfPath+".extracted.yaml")) } } } diff --git a/hack/.packages b/hack/.packages index 20666c6e01bd9..13af8f817f968 100644 --- a/hack/.packages +++ b/hack/.packages @@ -155,6 +155,7 @@ k8s.io/kops/upup/pkg/fi/utils k8s.io/kops/upup/pkg/kutil k8s.io/kops/upup/tools/generators/fitask k8s.io/kops/upup/tools/generators/pkg/codegen +k8s.io/kops/util/pkg/exec k8s.io/kops/util/pkg/hashing k8s.io/kops/util/pkg/slice k8s.io/kops/util/pkg/tables diff --git a/nodeup/pkg/model/convenience.go b/nodeup/pkg/model/convenience.go index 5c674e6ad88cd..940a4deed3331 100644 --- a/nodeup/pkg/model/convenience.go +++ b/nodeup/pkg/model/convenience.go @@ -21,7 +21,6 @@ import ( "path/filepath" "sort" "strconv" - "strings" "github.com/golang/glog" "k8s.io/api/core/v1" @@ -187,19 +186,3 @@ func addHostPathMapping(pod *v1.Pod, container *v1.Container, name, path string) func convEtcdSettingsToMs(dur *metav1.Duration) string { return strconv.FormatInt(dur.Nanoseconds()/1000000, 10) } - -// execWithTee returns the command to run the command while piping output to both the log file and stdout/stderr -func execWithTee(cmd string, args []string, logfile string) []string { - // exec so we don't have a shell that doesn't pass signals to the real process - execCmd := "exec " + cmd + " " + strings.Join(args, " ") - - // NOTE: tee & mkfifo is in /usr/bin in the kube-proxy image, but /bin in other images - - // Bash supports something like this, but dash and other limited shells don't - //shCmd := "exec &> >(/usr/bin/tee -a " + logfile + "); " + execCmd - // Instead we create the pipe manually and wire up the tee: - shCmd := "mkfifo /tmp/pipe; (tee -a " + logfile + " < /tmp/pipe & ) ; " + execCmd + " > /tmp/pipe 2>&1" - - // Execute shell command - return []string{"/bin/sh", "-c", shCmd} -} diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index ae52e589a0861..e24f0f10c45b3 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -26,6 +26,7 @@ import ( "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/exec" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -247,7 +248,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { container := &v1.Container{ Name: "kube-apiserver", Image: b.Cluster.Spec.KubeAPIServer.Image, - Command: execWithTee( + Command: exec.WithTee( "/usr/local/bin/kube-apiserver", sortedStrings(flags), "/var/log/kube-apiserver.log"), diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index 07cf7b79179d6..49215a4e9f680 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -24,6 +24,7 @@ import ( "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/exec" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -167,7 +168,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { container := &v1.Container{ Name: "kube-controller-manager", Image: b.Cluster.Spec.KubeControllerManager.Image, - Command: execWithTee( + Command: exec.WithTee( "/usr/local/bin/kube-controller-manager", sortedStrings(flags), "/var/log/kube-controller-manager.log"), diff --git a/nodeup/pkg/model/kube_proxy.go b/nodeup/pkg/model/kube_proxy.go index fd77c0a1d0efa..370d13d252e4b 100644 --- a/nodeup/pkg/model/kube_proxy.go +++ b/nodeup/pkg/model/kube_proxy.go @@ -23,6 +23,7 @@ import ( "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/exec" "github.com/golang/glog" "k8s.io/api/core/v1" @@ -143,7 +144,7 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) { container := &v1.Container{ Name: "kube-proxy", Image: image, - Command: execWithTee( + Command: exec.WithTee( "/usr/local/bin/kube-proxy", sortedStrings(flags), "/var/log/kube-proxy.log"), diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index 2c285d66a8e3d..57a78e9a7802f 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -22,6 +22,7 @@ import ( "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/exec" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -124,7 +125,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { container := &v1.Container{ Name: "kube-scheduler", Image: c.Image, - Command: execWithTee( + Command: exec.WithTee( "/usr/local/bin/kube-scheduler", sortedStrings(flags), "/var/log/kube-scheduler.log"), diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index ece65ec039665..65bbd0b8ff3d3 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -111,6 +111,14 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste spec["kubeControllerManager"] = cs.KubeControllerManager spec["kubeScheduler"] = cs.KubeScheduler spec["masterKubelet"] = cs.MasterKubelet + spec["etcdClusters"] = make(map[string]kops.EtcdClusterSpec, 0) + + for _, etcdCluster := range cs.EtcdClusters { + spec["etcdClusters"].(map[string]kops.EtcdClusterSpec)[etcdCluster.Name] = kops.EtcdClusterSpec{ + Image: etcdCluster.Image, + Version: etcdCluster.Version, + } + } } hooks, err := b.getRelevantHooks(cs.Hooks, ig.Spec.Role) diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index eddc74763d611..027a70913da14 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -173,6 +173,18 @@ func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole, fileAssetSpecRoles InstanceGroup: s("ig-1"), }, }, + Version: "3.1.11", + }, + { + Name: "events", + Members: []*kops.EtcdMemberSpec{ + { + Name: "test", + InstanceGroup: s("ig-1"), + }, + }, + Version: "3.1.11", + Image: "gcr.io/etcd-development/etcd:v3.1.11", }, }, NetworkCIDR: "10.79.0.0/24", diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt index fb0ec4d0f58a8..fcaf1cb6e89c8 100644 --- a/pkg/model/tests/data/bootstrapscript_0.txt +++ b/pkg/model/tests/data/bootstrapscript_0.txt @@ -159,6 +159,12 @@ cloudConfig: docker: logLevel: INFO encryptionConfig: null +etcdClusters: + events: + image: gcr.io/etcd-development/etcd:v3.1.11 + version: 3.1.11 + main: + version: 3.1.11 kubeAPIServer: image: CoreOS kubeControllerManager: diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt index 4f5e78576a658..0ce92cf8d1b3a 100644 --- a/pkg/model/tests/data/bootstrapscript_1.txt +++ b/pkg/model/tests/data/bootstrapscript_1.txt @@ -159,6 +159,12 @@ cloudConfig: docker: logLevel: INFO encryptionConfig: null +etcdClusters: + events: + image: gcr.io/etcd-development/etcd:v3.1.11 + version: 3.1.11 + main: + version: 3.1.11 fileAssets: - content: E1oeAbrnQsSldrIP1BpoP2SDykM= (fingerprint) name: iptables-restore diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt index 4f5e78576a658..0ce92cf8d1b3a 100644 --- a/pkg/model/tests/data/bootstrapscript_2.txt +++ b/pkg/model/tests/data/bootstrapscript_2.txt @@ -159,6 +159,12 @@ cloudConfig: docker: logLevel: INFO encryptionConfig: null +etcdClusters: + events: + image: gcr.io/etcd-development/etcd:v3.1.11 + version: 3.1.11 + main: + version: 3.1.11 fileAssets: - content: E1oeAbrnQsSldrIP1BpoP2SDykM= (fingerprint) name: iptables-restore diff --git a/protokube/pkg/protokube/etcd_manifest.go b/protokube/pkg/protokube/etcd_manifest.go index 62fe822c33f4f..84fa9c7bde05d 100644 --- a/protokube/pkg/protokube/etcd_manifest.go +++ b/protokube/pkg/protokube/etcd_manifest.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/util/pkg/exec" ) // BuildEtcdManifest creates the pod spec, based on the etcd cluster @@ -44,12 +45,7 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod { v1.ResourceCPU: c.CPURequest, }, }, - Command: []string{ - "/bin/sh", "-c", - "/bin/mkfifo /tmp/pipe; " + - "(/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); " + - "exec /usr/local/bin/etcd > /tmp/pipe 2>&1", - }, + Command: exec.WithTee("/usr/local/bin/etcd", []string{}, "/var/log/etcd.log"), } // build the environment variables for etcd service container.Env = buildEtcdEnvironmentOptions(c) diff --git a/protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml b/protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml index bd4dd69c63d1e..d3bc07782e12d 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml @@ -37,8 +37,8 @@ spec: - command: - /bin/sh - -c - - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec - /usr/local/bin/etcd > /tmp/pipe 2>&1 + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd > + /tmp/pipe 2>&1 env: - name: ETCD_NAME value: node0 diff --git a/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml b/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml index be82abf91a854..aed2c82dc33d3 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml @@ -35,8 +35,8 @@ spec: - command: - /bin/sh - -c - - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec - /usr/local/bin/etcd > /tmp/pipe 2>&1 + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd > + /tmp/pipe 2>&1 env: - name: ETCD_NAME value: node0 diff --git a/protokube/tests/integration/build_etcd_manifest/main/tls.yaml b/protokube/tests/integration/build_etcd_manifest/main/tls.yaml index e9fb1a1346462..99145e49b2d3f 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/tls.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/tls.yaml @@ -41,8 +41,8 @@ spec: - command: - /bin/sh - -c - - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec - /usr/local/bin/etcd > /tmp/pipe 2>&1 + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd > + /tmp/pipe 2>&1 env: - name: ETCD_NAME value: node0 diff --git a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml index 6f95e3b8bd836..d4815a4f83c9f 100644 --- a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml @@ -157,6 +157,11 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda storage: overlay,aufs version: 1.11.2 encryptionConfig: null + etcdClusters: + events: + version: 2.2.1 + main: + version: 2.2.1 kubeAPIServer: address: 127.0.0.1 admissionControl: diff --git a/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml index c9216f6e8ccbb..6ee2d034d0eca 100644 --- a/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml @@ -148,6 +148,11 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco storage: overlay,aufs version: 1.11.2 encryptionConfig: null + etcdClusters: + events: + version: 2.2.1 + main: + version: 2.2.1 kubeAPIServer: address: 127.0.0.1 admissionControl: diff --git a/util/pkg/exec/exec.go b/util/pkg/exec/exec.go new file mode 100644 index 0000000000000..10000e55aad37 --- /dev/null +++ b/util/pkg/exec/exec.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exec + +import "strings" + +// WithTee returns the command to run the command while piping output to both the log file and stdout/stderr +func WithTee(cmd string, args []string, logfile string) []string { + // exec so we don't have a shell that doesn't pass signals to the real process + execCmd := "exec " + cmd + " " + strings.Join(args, " ") + + // NOTE: tee & mkfifo is in /usr/bin in the kube-proxy image, but /bin in other images + + // Bash supports something like this, but dash and other limited shells don't + //shCmd := "exec &> >(/usr/bin/tee -a " + logfile + "); " + execCmd + // Instead we create the pipe manually and wire up the tee: + shCmd := "mkfifo /tmp/pipe; (tee -a " + logfile + " < /tmp/pipe & ) ; " + execCmd + " > /tmp/pipe 2>&1" + + // Execute shell command + return []string{"/bin/sh", "-c", shCmd} +}