From 2dd57898d4fac3af8793148619bfe63e9ad492b4 Mon Sep 17 00:00:00 2001 From: Rajat Chopra Date: Thu, 7 Aug 2014 13:02:52 -0700 Subject: [PATCH] add ip per pod across vagrant minions --- .gitignore | 1 + Vagrantfile | 2 +- cluster/saltbase/salt/docker/init.sls | 6 +- cluster/saltbase/salt/sdn/init.sls | 16 ++++ cluster/saltbase/salt/top.sls | 2 + cluster/vagrant/pod-ip-test.sh | 44 ++++++++++ cluster/vagrant/provision-minion.sh | 3 + cluster/vagrant/provision-network.sh | 112 ++++++++++++++++++++++++++ 8 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 cluster/saltbase/salt/sdn/init.sls create mode 100644 cluster/vagrant/pod-ip-test.sh create mode 100755 cluster/vagrant/provision-network.sh diff --git a/.gitignore b/.gitignore index 5d28efcdf80f1..d8884a59f5c58 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ Session.vim # Vagrant .vagrant +network_closure.sh # compiled binaries in third_party /third_party/pkg diff --git a/Vagrantfile b/Vagrantfile index 51eae9363e17d..26f147a242948 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -45,7 +45,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| minion_ip = minion_ips[n] minion.vm.box = kube_box[kube_os]["name"] minion.vm.box_url = kube_box[kube_os]["box_url"] - minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip}" + minion.vm.provision "shell", inline: "/vagrant/cluster/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index}" minion.vm.network "private_network", ip: "#{minion_ip}" minion.vm.hostname = "kubernetes-minion-#{minion_index}" end diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls index e154d64c6fd2a..aaefceafebe7a 100644 --- a/cluster/saltbase/salt/docker/init.sls +++ b/cluster/saltbase/salt/docker/init.sls @@ -4,6 +4,9 @@ {% set environment_file = '/etc/default/docker' %} {% endif %} +bridge-utils: + pkg.installed + {% if grains['os_family'] != 'RedHat' %} docker-repo: @@ -25,9 +28,6 @@ net.ipv4.ip_forward: sysctl.present: - value: 1 -bridge-utils: - pkg.installed - cbr0: container_bridge.ensure: - cidr: {{ grains['cbr-cidr'] }} diff --git a/cluster/saltbase/salt/sdn/init.sls b/cluster/saltbase/salt/sdn/init.sls new file mode 100644 index 0000000000000..4b6a0e5274be8 --- /dev/null +++ b/cluster/saltbase/salt/sdn/init.sls @@ -0,0 +1,16 @@ +{% if grains['os_family'] == 'RedHat' %} + +openvswitch: + pkg: + - installed + service.running: + - enable: True + +sdn: + cmd.wait: + - name: /vagrant/network_closure.sh + - watch: + - pkg: docker-io + - pkg: openvswitch + +{% endif %} diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index eff1a26057449..bb0b133b06f09 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -12,6 +12,8 @@ base: - nsinit {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn-client +{% else %} + - sdn {% endif %} 'roles:kubernetes-master': diff --git a/cluster/vagrant/pod-ip-test.sh b/cluster/vagrant/pod-ip-test.sh new file mode 100644 index 0000000000000..cf43bcba819d9 --- /dev/null +++ b/cluster/vagrant/pod-ip-test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +cd $(dirname ${BASH_SOURCE})/../../ + +# start the cluster with 2 minions +export KUBERNETES_NUM_MINIONS=2 +export KUBERNETES_PROVIDER=vagrant +cluster/kube-up.sh + +echo "Pull an image that runs a web server" +vagrant ssh minion-1 -- sudo docker pull dockerfile/nginx +vagrant ssh minion-2 -- sudo docker pull dockerfile/nginx + +echo "Run the servers" +vagrant ssh minion-1 -- sudo docker run -d dockerfile/nginx +vagrant ssh minion-2 -- sudo docker run -d dockerfile/nginx + +echo "Run ping from minion-1 to docker bridges and to the containers on both minions" +vagrant ssh minion-1 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' +echo "Same pinch from minion-2" +vagrant ssh minion-2 -- 'ping -c 10 10.244.1.1 && ping -c 10 10.244.2.1 && ping -c 10 10.244.1.3 && ping -c 10 10.244.2.3' + +echo "tcp check, curl to both the running webservers from both machines" +vagrant ssh minion-1 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' +vagrant ssh minion-2 -- 'curl 10.244.1.3:80 && curl 10.244.2.3:80' + +echo "All good, destroy the cluster" +vagrant destroy -f diff --git a/cluster/vagrant/provision-minion.sh b/cluster/vagrant/provision-minion.sh index e03cd7104af69..26f63215f9887 100755 --- a/cluster/vagrant/provision-minion.sh +++ b/cluster/vagrant/provision-minion.sh @@ -52,3 +52,6 @@ if [ ! $(which salt-minion) ]; then systemctl enable salt-minion systemctl start salt-minion fi + +# run the networking setup +$(dirname $0)/provision-network.sh $@ diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh new file mode 100755 index 0000000000000..178f0b28ae0e5 --- /dev/null +++ b/cluster/vagrant/provision-network.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# exit on any error +set -e +source $(dirname $0)/provision-config.sh + +MINION_IP=$4 +MINION_ID=$5 +DOCKER_BRIDGE=kbr0 +OVS_SWITCH=obr0 +GRE_TUNNEL_BASE=gre +BRIDGE_BASE=10.244 +BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1 +BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24 +BRIDGE_NETMASK=255.255.255.0 +NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ +POST_NETWORK_SCRIPT=/vagrant/network_closure.sh + +# add docker bridge ifcfg file +cat < ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} +# Generated by yours truly +DEVICE=${DOCKER_BRIDGE} +ONBOOT=yes +TYPE=Bridge +BOOTPROTO=static +IPADDR=${BRIDGE_ADDRESS} +NETMASK=${BRIDGE_NETMASK} +STP=yes +EOF + +# add the ovs bridge ifcfg file +cat < ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH} +DEVICE=${OVS_SWITCH} +ONBOOT=yes +DEVICETYPE=ovs +TYPE=OVSBridge +BOOTPROTO=static +HOTPLUG=no +BRIDGE=${DOCKER_BRIDGE} +EOF + +# now loop through all other minions and create persistent gre tunnels +MINION_IPS=$3 +MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`) +GRE_NUM=0 +for remote_ip in "${MINION_IP_ARRAY[@]}" +do + if [ "${remote_ip}" == "${MINION_IP}" ]; then + continue + fi + ((GRE_NUM++)) || echo + GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM} + # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip} + cat < ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL} +DEVICE=${GRE_TUNNEL} +ONBOOT=yes +DEVICETYPE=ovs +TYPE=OVSTunnel +OVS_BRIDGE=${OVS_SWITCH} +OVS_TUNNEL_TYPE=gre +OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}" +EOF +done + +# add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels +cat < /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE} +${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS} +EOF + + +# generate the post-configure script to be called by salt as cmd.wait +cat < ${POST_NETWORK_SCRIPT} +#!/bin/bash + +set -e +# NAT interface fails to revive on network restart, so OR-gate to true +systemctl restart network.service || true + +# set docker bridge up, and set stp on the ovs bridge +ip link set dev ${DOCKER_BRIDGE} up +ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true + +# modify the docker service file such that it uses the kube docker bridge and not its own +sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service +systemctl daemon-reload +systemctl restart docker.service + +# setup iptables masquerade rules so the pods can reach the internet +iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE + +# persist please +iptables-save >& /etc/sysconfig/iptables + +# self-destruct after doing the job +#rm -f ${POST_NETWORK_SCRIPT} +EOF + +chmod +x ${POST_NETWORK_SCRIPT}