From 242b9977c7263f0f16e4ce86ded7ba5d898541bf Mon Sep 17 00:00:00 2001 From: Andy Zheng Date: Fri, 12 Feb 2016 08:02:26 -0800 Subject: [PATCH] Remove manifest copies from Trusty support This change revises the way to provide kube-system manifests for clusters on Trusty. Originally, we maintained copies of some manifests under cluster/gce/trusty/kube-manifests, which is not scalable and hard to maintain. With this change, clusters on Trusty will use the same source of manifests as ContainerVM. This change also fixes some minor problems such as shell variables and comments to meet the style guidance better. --- build/common.sh | 17 +-- cluster/gce/trusty/configure.sh | 132 +++++++++++------- .../kube-manifests/kube-apiserver.manifest | 98 ------------- .../kube-controller-manager.manifest | 83 ----------- cluster/gce/trusty/node.yaml | 3 +- .../kube-apiserver/kube-apiserver.manifest | 43 ++---- .../kube-controller-manager.manifest | 37 +---- hack/verify-flags/exceptions.txt | 1 + 8 files changed, 105 insertions(+), 309 deletions(-) delete mode 100644 cluster/gce/trusty/kube-manifests/kube-apiserver.manifest delete mode 100644 cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest diff --git a/build/common.sh b/build/common.sh index b3901a92d2325..328f813997ec2 100755 --- a/build/common.sh +++ b/build/common.sh @@ -925,12 +925,9 @@ function kube::release::package_salt_tarball() { } # This will pack kube-system manifests files for distros without using salt -# such as Ubuntu Trusty. -# -# There are two sources of manifests files: (1) some manifests in the directory -# cluster/saltbase/salt and cluster/addons can be used directly or after minor -# revision, so we copy them from there; (2) otherwise, we will maintain separate -# copies in cluster/gce//kube-manifests. +# such as Ubuntu Trusty. For Trusty, we directly copy manifests from cluster/addons +# and cluster/saltbase/salt. The script of cluster initialization will remove +# the salt configuration and evaluate the variables in the manifests. function kube::release::package_kube_manifests_tarball() { kube::log::status "Building tarball: manifests" @@ -938,7 +935,6 @@ function kube::release::package_kube_manifests_tarball() { rm -rf "${release_stage}" mkdir -p "${release_stage}/trusty" - # Source 1: manifests from cluster/saltbase/salt and cluster/addons local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt" cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/" cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/" @@ -946,6 +942,8 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/" cp "${salt_dir}/etcd/etcd.manifest" "${release_stage}/trusty" cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${release_stage}/trusty" + cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${release_stage}/trusty" + cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${release_stage}/trusty" cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty" cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty" cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty" @@ -954,10 +952,7 @@ function kube::release::package_kube_manifests_tarball() { objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo) tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/trusty" - # Source 2: manifests from cluster/gce//kube-manifests. - # TODO(andyzheng0831): Avoid using separate copies for trusty. We should use whatever - # from cluster/saltbase/salt to minimize maintenance cost. - cp "${KUBE_ROOT}/cluster/gce/trusty/kube-manifests/"* "${release_stage}/trusty" + # This is for coreos only. ContainerVM or Trusty does not use it. cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/" kube::release::clean_cruft diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index 00d79c8326d0c..f2feb1e859b3c 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -23,7 +23,7 @@ config_hostname() { # Set the hostname to the short version. short_hostname=$(hostname -s) - hostname $short_hostname + hostname "${short_hostname}" } config_ip_firewall() { @@ -54,7 +54,7 @@ create_dirs() { download_kube_env() { # Fetch kube-env from GCE metadata server. readonly tmp_install_dir="/var/cache/kubernetes-install" - mkdir -p ${tmp_install_dir} + mkdir -p "${tmp_install_dir}" curl --fail --silent --show-error \ -H "X-Google-Metadata-Request: True" \ -o "${tmp_install_dir}/kube_env.yaml" \ @@ -78,12 +78,12 @@ kind: Config users: - name: kubelet user: - client-certificate-data: ${KUBELET_CERT} - client-key-data: ${KUBELET_KEY} + client-certificate-data: "${KUBELET_CERT}" + client-key-data: "${KUBELET_KEY}" clusters: - name: local cluster: - certificate-authority-data: ${KUBELET_CA_CERT} + certificate-authority-data: "${KUBELET_CA_CERT}" contexts: - context: cluster: local @@ -101,11 +101,11 @@ kind: Config users: - name: kube-proxy user: - token: ${KUBE_PROXY_TOKEN} + token: "${KUBE_PROXY_TOKEN}" clusters: - name: local cluster: - certificate-authority-data: ${CA_CERT} + certificate-authority-data: "${CA_CERT}" contexts: - context: cluster: local @@ -171,7 +171,7 @@ install_kube_binary_config() { echo "Downloading k8s tar file ${k8s_tar}" download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}" # Validate hash. - actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true + actual=$(sha1sum "${k8s_tar}" | awk '{ print $1 }') || true if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" else @@ -198,8 +198,8 @@ install_kube_binary_config() { BINARY_PATH="/usr/local/bin/" fi if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then - cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH} - cp /tmp/kubernetes/server/bin/kubectl ${BINARY_PATH} + cp /tmp/kubernetes/server/bin/kubelet "${BINARY_PATH}" + cp /tmp/kubernetes/server/bin/kubectl "${BINARY_PATH}" fi # Clean up. rm -rf "/tmp/kubernetes" @@ -216,7 +216,7 @@ install_kube_binary_config() { echo "Downloading kube-manifest tar file ${manifests_tar}" download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}" # Validate hash. - actual=$(sha1sum ${manifests_tar} | awk '{ print $1 }') || true + actual=$(sha1sum "${manifests_tar}" | awk '{ print $1 }') || true if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} ==" else @@ -306,12 +306,11 @@ health_monitoring() { echo "Docker daemon failed!" pkill docker fi - if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then + if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then echo "Kubelet is unhealthy!" pkill kubelet fi - # TODO(andyzheng0831): Add master side health monitoring. - sleep ${sleep_seconds} + sleep "${sleep_seconds}" done } @@ -327,29 +326,29 @@ mount_master_pd() { readonly mount_point="/mnt/disks/master-pd" # TODO(zmerlynn): GKE is still lagging in master-pd creation - if [ ! -e ${pd_path} ]; then + if [ ! -e "${pd_path}" ]; then return fi # Format and mount the disk, create directories on it for all of the master's # persistent data, and link them to where they're used. - mkdir -p ${mount_point} - /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" ${pd_path} ${mount_point} >/var/log/master-pd-mount.log || \ + mkdir -p "${mount_point}" + /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \ { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; } # Contains all the data stored in etcd mkdir -m 700 -p "${mount_point}/var/etcd" # Contains the dynamically generated apiserver auth certs and keys mkdir -p "${mount_point}/etc/srv/kubernetes" # Directory for kube-apiserver to store SSH key (if necessary) - mkdir -p /"${mount_point}/etc/srv/sshproxy" + mkdir -p "${mount_point}/etc/srv/sshproxy" ln -s -f "${mount_point}/var/etcd" /var/etcd mkdir -p /etc/srv - ln -s -f /"${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes - ln -s -f /"${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy + ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes + ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy if ! id etcd &>/dev/null; then useradd -s /sbin/nologin -d /var/etcd etcd fi - chown -R etcd /"${mount_point}/var/etcd" + chown -R etcd "${mount_point}/var/etcd" chgrp -R etcd "${mount_point}/var/etcd" } @@ -393,20 +392,20 @@ create_master_auth() { # NB: If this list ever changes, this script actually has to # change to detect the existence of this file, kill any deleted # old tokens and add any new tokens (to handle the upgrade case). - add_token_entry "system:scheduler" ${known_tokens_csv} - add_token_entry "system:controller_manager" ${known_tokens_csv} - add_token_entry "system:logging" ${known_tokens_csv} - add_token_entry "system:monitoring" ${known_tokens_csv} - add_token_entry "system:dns" ${known_tokens_csv} + add_token_entry "system:scheduler" "${known_tokens_csv}" + add_token_entry "system:controller_manager" "${known_tokens_csv}" + add_token_entry "system:logging" "${known_tokens_csv}" + add_token_entry "system:monitoring" "${known_tokens_csv}" + add_token_entry "system:dns" "${known_tokens_csv}" fi if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then cat </etc/gce.conf [global] -token-url = ${TOKEN_URL} -token-body = ${TOKEN_BODY} -project-id = ${PROJECT_ID} -network-name = ${NODE_NETWORK} +token-url = "${TOKEN_URL}" +token-body = "${TOKEN_BODY}" +project-id = "${PROJECT_ID}" +network-name = "${NODE_NETWORK}" EOF fi } @@ -477,10 +476,20 @@ compute_master_manifest_variables() { fi DOCKER_REGISTRY="gcr.io/google_containers" if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then - DOCKER_REGISTRY=${KUBE_DOCKER_REGISTRY} + DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}" fi } +# A helper function for removing salt configuration and comments from a file. +# This is mainly for preparing a manifest file. +# $1: Full path of the file to manipulate +remove_salt_config_comments() { + # Remove salt configuration + sed -i "/^[ |\t]*{[#|%]/d" $1 + # Remove comments + sed -i "/^[ |\t]*#/d" $1 +} + # Starts k8s apiserver. # It prepares the log file, loads the docker image, calculates variables, sets them # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. @@ -526,12 +535,21 @@ start_kube_apiserver() { readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag) src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest" - sed -i -e "s@{{params}}@${params}@g" ${src_file} - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" ${src_file} - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" ${src_file} - sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" ${src_file} - sed -i -e "s@{{kube-apiserver_docker_tag}}@${kube_apiserver_docker_tag}@g" ${src_file} - cp ${src_file} /etc/kubernetes/manifests + remove_salt_config_comments "${src_file}" + # Evaluate variables + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" + sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" + sed -i -e "s@{{secure_port}}@443@g" "${src_file}" + sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } # Starts k8s controller manager. @@ -550,6 +568,9 @@ start_kube_controller_manager() { # Calculate variables and assemble the command line. params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key" + if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then + params="${params} --cloud-config=/etc/gce.conf" + fi if [ -n "${INSTANCE_PREFIX:-}" ]; then params="${params} --cluster-name=${INSTANCE_PREFIX}" fi @@ -573,15 +594,20 @@ start_kube_controller_manager() { readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag) src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest" - sed -i -e "s@{{params}}@${params}@g" ${src_file} - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" ${src_file} - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" ${src_file} - sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" ${src_file} - sed -i -e "s@{{kube-controller-manager_docker_tag}}@${kube_rc_docker_tag}@g" ${src_file} - cp ${src_file} /etc/kubernetes/manifests + remove_salt_config_comments "${src_file}" + # Evaluate variables + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } -# Start k8s scheduler. +# Starts k8s scheduler. # It prepares the log file, loads the docker image, calculates variables, sets them # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. # @@ -607,14 +633,14 @@ start_kube_scheduler() { # Remove salt comments and replace variables with values src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest" - sed -i "/^ *{%/d" ${src_file} - sed -i -e "s@{{params}}@${params}@g" ${src_file} - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" ${src_file} - sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" ${src_file} - cp ${src_file} /etc/kubernetes/manifests + remove_salt_config_comments "${src_file}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } -# Start a fluentd static pod for logging. +# Starts a fluentd static pod for logging. start_fluentd() { if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then @@ -652,7 +678,7 @@ setup_addon_manifests() { chmod 644 "${dst_dir}"/* } -# Start k8s addons static pods. +# Prepares the manifests of k8s addons static pods. prepare_kube_addons() { addon_src_dir="/run/kube-manifests/kubernetes/trusty" addon_dst_dir="/etc/kubernetes/addons" @@ -674,7 +700,7 @@ prepare_kube_addons() { else controller_yaml="${controller_yaml}/heapster-controller.yaml" fi - sed -i "/^ *{%/d" "${controller_yaml}" + remove_salt_config_comments "${controller_yaml}" sed -i -e "s@{{ *heapster_memory *}}@${heapster_memory}@g" "${controller_yaml}" fi cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}" @@ -699,7 +725,7 @@ prepare_kube_addons() { mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}" mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}" # Replace the salt configurations with variable values. - sed -i "/^ *{%/d" "${registry_pv_file}" + remove_salt_config_comments "${controller_yaml}" sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}" sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}" sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}" diff --git a/cluster/gce/trusty/kube-manifests/kube-apiserver.manifest b/cluster/gce/trusty/kube-manifests/kube-apiserver.manifest deleted file mode 100644 index b9cd229f725f2..0000000000000 --- a/cluster/gce/trusty/kube-manifests/kube-apiserver.manifest +++ /dev/null @@ -1,98 +0,0 @@ -{ -"apiVersion": "v1", -"kind": "Pod", -"metadata": { - "name":"kube-apiserver", - "namespace": "kube-system" -}, -"spec":{ -"hostNetwork": true, -"containers":[ - { - "name": "kube-apiserver", - "image": "{{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}}", - "resources": { - "requests": { - "cpu": "250m" - } - }, - "command": [ - "/bin/sh", - "-c", - "/usr/local/bin/kube-apiserver {{params}} 1>>/var/log/kube-apiserver.log 2>&1" - ], - "livenessProbe": { - "httpGet": { - "host": "127.0.0.1", - "port": 8080, - "path": "/healthz" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15 - }, - "ports":[ - { "name": "https", - "containerPort": 443, - "hostPort": 443},{ - "name": "local", - "containerPort": 8080, - "hostPort": 8080} - ], - "volumeMounts": [ - {{cloud_config_mount}} - { "name": "srvkube", - "mountPath": "/etc/srv/kubernetes", - "readOnly": true}, - { "name": "logfile", - "mountPath": "/var/log/kube-apiserver.log", - "readOnly": false}, - { "name": "etcssl", - "mountPath": "/etc/ssl", - "readOnly": true}, - { "name": "varssl", - "mountPath": "/var/ssl", - "readOnly": true}, - { "name": "etcopenssl", - "mountPath": "/etc/openssl", - "readOnly": true}, - { "name": "etcpkitls", - "mountPath": "/etc/pki/tls", - "readOnly": true}, - { "name": "srvsshproxy", - "mountPath": "/etc/srv/sshproxy", - "readOnly": false} - ] - } -], -"volumes":[ - {{cloud_config_volume}} - { "name": "srvkube", - "hostPath": { - "path": "/etc/srv/kubernetes"} - }, - { "name": "logfile", - "hostPath": { - "path": "/var/log/kube-apiserver.log"} - }, - { "name": "etcssl", - "hostPath": { - "path": "/etc/ssl"} - }, - { "name": "varssl", - "hostPath": { - "path": "/var/ssl"} - }, - { "name": "etcopenssl", - "hostPath": { - "path": "/etc/openssl"} - }, - { "name": "etcpkitls", - "hostPath": { - "path": "/etc/pki/tls"} - }, - { "name": "srvsshproxy", - "hostPath": { - "path": "/etc/srv/sshproxy"} - } -] -}} diff --git a/cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest b/cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest deleted file mode 100644 index 6ade1c152470e..0000000000000 --- a/cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest +++ /dev/null @@ -1,83 +0,0 @@ -{ -"apiVersion": "v1", -"kind": "Pod", -"metadata": { - "name":"kube-controller-manager", - "namespace": "kube-system" -}, -"spec":{ -"hostNetwork": true, -"containers":[ - { - "name": "kube-controller-manager", - "image": "{{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}}", - "resources": { - "requests": { - "cpu": "200m" - } - }, - "command": [ - "/bin/sh", - "-c", - "/usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1" - ], - "livenessProbe": { - "httpGet": { - "host": "127.0.0.1", - "port": 10252, - "path": "/healthz" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15 - }, - "volumeMounts": [ - {{cloud_config_mount}} - { "name": "srvkube", - "mountPath": "/etc/srv/kubernetes", - "readOnly": true}, - { "name": "logfile", - "mountPath": "/var/log/kube-controller-manager.log", - "readOnly": false}, - { "name": "etcssl", - "mountPath": "/etc/ssl", - "readOnly": true}, - { "name": "varssl", - "mountPath": "/var/ssl", - "readOnly": true}, - { "name": "etcopenssl", - "mountPath": "/etc/openssl", - "readOnly": true}, - { "name": "etcpkitls", - "mountPath": "/etc/pki/tls", - "readOnly": true} - ] - } -], -"volumes":[ - {{cloud_config_volume}} - { "name": "srvkube", - "hostPath": { - "path": "/etc/srv/kubernetes"} - }, - { "name": "logfile", - "hostPath": { - "path": "/var/log/kube-controller-manager.log"} - }, - { "name": "etcssl", - "hostPath": { - "path": "/etc/ssl"} - }, - { "name": "varssl", - "hostPath": { - "path": "/var/ssl"} - }, - { "name": "etcopenssl", - "hostPath": { - "path": "/etc/openssl"} - }, - { "name": "etcpkitls", - "hostPath": { - "path": "/etc/pki/tls"} - } -] -}} diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index e21f42d19d677..a01b935ed601f 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -264,8 +264,7 @@ Content-Disposition: attachment; filename="kube-node-health-monitoring.conf" description "Kubenetes node health monitoring" -# The termination of kube-proxy upstart job indicates that kubelet and docker are ready. -start on stopped kube-proxy +start on stopped kube-docker respawn diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 2ffd6b271a0cd..52b0089a0032f 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -7,6 +7,11 @@ {% set cloud_config = "" -%} {% set cloud_config_mount = "" -%} {% set cloud_config_volume = "" -%} +{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%} +{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%} + +{% set srv_kube_path = "/srv/kubernetes" -%} +{% set srv_sshproxy_path = "/srv/sshproxy" -%} {% if grains.cloud is defined -%} {% if grains.cloud not in ['vagrant', 'vsphere'] -%} @@ -141,8 +146,9 @@ ], "volumeMounts": [ {{cloud_config_mount}} + {{additional_cloud_config_mount}} { "name": "srvkube", - "mountPath": "/srv/kubernetes", + "mountPath": "{{srv_kube_path}}", "readOnly": true}, { "name": "logfile", "mountPath": "/var/log/kube-apiserver.log", @@ -150,21 +156,9 @@ { "name": "etcssl", "mountPath": "/etc/ssl", "readOnly": true}, - { "name": "usrsharessl", - "mountPath": "/usr/share/ssl", - "readOnly": true}, { "name": "varssl", "mountPath": "/var/ssl", "readOnly": true}, - { "name": "usrssl", - "mountPath": "/usr/ssl", - "readOnly": true}, - { "name": "usrlibssl", - "mountPath": "/usr/lib/ssl", - "readOnly": true}, - { "name": "usrlocalopenssl", - "mountPath": "/usr/local/openssl", - "readOnly": true}, { "name": "etcopenssl", "mountPath": "/etc/openssl", "readOnly": true}, @@ -172,16 +166,17 @@ "mountPath": "/etc/pki/tls", "readOnly": true}, { "name": "srvsshproxy", - "mountPath": "/srv/sshproxy", + "mountPath": "{{srv_sshproxy_path}}", "readOnly": false} ] } ], "volumes":[ {{cloud_config_volume}} + {{additional_cloud_config_volume}} { "name": "srvkube", "hostPath": { - "path": "/srv/kubernetes"} + "path": "{{srv_kube_path}}"} }, { "name": "logfile", "hostPath": { @@ -191,26 +186,10 @@ "hostPath": { "path": "/etc/ssl"} }, - { "name": "usrsharessl", - "hostPath": { - "path": "/usr/share/ssl"} - }, { "name": "varssl", "hostPath": { "path": "/var/ssl"} }, - { "name": "usrssl", - "hostPath": { - "path": "/usr/ssl"} - }, - { "name": "usrlibssl", - "hostPath": { - "path": "/usr/lib/ssl"} - }, - { "name": "usrlocalopenssl", - "hostPath": { - "path": "/usr/local/openssl"} - }, { "name": "etcopenssl", "hostPath": { "path": "/etc/openssl"} @@ -221,7 +200,7 @@ }, { "name": "srvsshproxy", "hostPath": { - "path": "/srv/sshproxy"} + "path": "{{srv_sshproxy_path}}"} } ] }} diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index 5777ef0ee302a..3e857ac887a3b 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -27,6 +27,9 @@ {% set cloud_config = "" -%} {% set cloud_config_mount = "" -%} {% set cloud_config_volume = "" -%} +{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%} +{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%} +{% set srv_kube_path = "/srv/kubernetes" -%} {% if grains.cloud is defined -%} {% if grains.cloud not in ['vagrant', 'vsphere'] -%} @@ -94,8 +97,9 @@ }, "volumeMounts": [ {{cloud_config_mount}} + {{additional_cloud_config_mount}} { "name": "srvkube", - "mountPath": "/srv/kubernetes", + "mountPath": "{{srv_kube_path}}", "readOnly": true}, { "name": "logfile", "mountPath": "/var/log/kube-controller-manager.log", @@ -103,21 +107,9 @@ { "name": "etcssl", "mountPath": "/etc/ssl", "readOnly": true}, - { "name": "usrsharessl", - "mountPath": "/usr/share/ssl", - "readOnly": true}, { "name": "varssl", "mountPath": "/var/ssl", "readOnly": true}, - { "name": "usrssl", - "mountPath": "/usr/ssl", - "readOnly": true}, - { "name": "usrlibssl", - "mountPath": "/usr/lib/ssl", - "readOnly": true}, - { "name": "usrlocalopenssl", - "mountPath": "/usr/local/openssl", - "readOnly": true}, { "name": "etcopenssl", "mountPath": "/etc/openssl", "readOnly": true}, @@ -129,9 +121,10 @@ ], "volumes":[ {{cloud_config_volume}} + {{additional_cloud_config_volume}} { "name": "srvkube", "hostPath": { - "path": "/srv/kubernetes"} + "path": "{{srv_kube_path}}"} }, { "name": "logfile", "hostPath": { @@ -141,26 +134,10 @@ "hostPath": { "path": "/etc/ssl"} }, - { "name": "usrsharessl", - "hostPath": { - "path": "/usr/share/ssl"} - }, { "name": "varssl", "hostPath": { "path": "/var/ssl"} }, - { "name": "usrssl", - "hostPath": { - "path": "/usr/ssl"} - }, - { "name": "usrlibssl", - "hostPath": { - "path": "/usr/lib/ssl"} - }, - { "name": "usrlocalopenssl", - "hostPath": { - "path": "/usr/local/openssl"} - }, { "name": "etcopenssl", "hostPath": { "path": "/etc/openssl"} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 1235780aeedc5..60eb28c5f9c16 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -13,6 +13,7 @@ cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed +cluster/gce/trusty/configure.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl: --address=%(bind_address)s \ cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl: --address=%(bind_address)s \