Begin Terraform and Windows support

bin/kubespray

@@ -29,7 +29,7 @@ from kubespray.common import clone_kubespray_git_repo
 from kubespray.configure import Config
 from kubespray.inventory import CfgInventory
 from kubespray.deploy import RunPlaybook
-from kubespray.cloud import AWS, GCE, OpenStack
+from kubespray.cloud import AWS, GCE, OpenStack, Terraform
 display = Display()
 
 
@@ -67,6 +67,24 @@ def openstack(options):
     O.write_inventory()
 
 
+def terraform(options):
+    clone_kubespray_git_repo(options)
+    T = Terraform(options)
+    T.prepare()
+    if options.get('terraform_command') == 'plan':
+        T.plan_instances()
+    elif options.get('terraform_command') == 'apply':
+        T.plan_instances()
+        T.create_instances()
+        T.write_inventory()
+    elif options.get('terraform_command') == 'plan-destroy':
+        T.plan_destroy_instances()
+    elif options.get('terraform_command') == 'destroy':
+        T.plan_destroy_instances()
+        T.destroy_instances()
+        T.write_inventory()
+
+
 def deploy(options):
     Run = RunPlaybook(options)
     Run.ssh_prepare()
@@ -331,6 +349,58 @@ if __name__ == '__main__':
     )
     openstack_parser.set_defaults(func=openstack)
 
+    # terraform
+    terraform_parser = subparsers.add_parser(
+        'terraform', parents=[parent_parser, firststep_parser],
+        help='Create VMs using Terraform and generate inventory',
+    )
+    terraform_parser.add_argument(
+        '--tf_version', dest='tf_version',
+        help='Terraform version'
+    )
+    terraform_parser.add_argument(
+        '--tf_plan_folder', dest='tf_plan_folder',
+        help='Terraform plan folder'
+    )
+    terraform_parser.add_argument(
+        '--tf_state_folder', dest='tf_state_folder',
+        help='Terraform state folder'
+    )
+    terraform_parser.add_argument(
+        '--tf_binary_folder', dest='tf_binary_folder',
+        help='Terraform binary path'
+    )
+    terraform_parser.add_argument(
+        '--cluster-name', dest='cluster_name', help='Name of the cluster'
+    )
+    terraform_parser.add_argument(
+        '--tf_vars', dest='tf_vars', help='List of Terraform vars', nargs="+", metavar="TF_VAR"
+    )
+    terraform_subparsers = terraform_parser.add_subparsers(help='terraform commands', dest='terraform_command')
+    terraform_subparsers.add_parser(
+        'plan',
+        help='Run terraform plan',
+        conflict_handler='resolve',
+    )
+    terraform_subparsers.add_parser(
+        'apply',
+        help='Run terraform apply',
+        conflict_handler='resolve',
+    )
+    terraform_subparsers.add_parser(
+        'plan-destroy',
+        help='Run terraform plan -destroy',
+        conflict_handler='resolve',
+    )
+    terraform_subparsers.add_parser(
+        'destroy',
+        help='Run terraform destroy',
+        conflict_handler='resolve',
+    )
+    #terraform_parser.add_argument('--add', dest='add_node', action='store_true',
+    #    help="Add node to an existing cluster")
+    terraform_parser.set_defaults(func=terraform)
+
     # deploy
     deploy_parser = subparsers.add_parser(
         'deploy', parents=[parent_parser],

setup.py

@@ -13,7 +13,11 @@
     'markupsafe>=0.23',
     'pyasn1>=0.1.8',
     'boto>=2.40.0',
-    'apache-libcloud>=0.20.1'
+    'apache-libcloud>=0.20.1',
+]
+dependency_links = [
+#    'git+https://github.com/mantl/terraform.py.git#egg=ati',
+    'git+https://github.com/titilambert/terraform.py.git#egg=ati',
 ]
 
 test_requirements = [
@@ -39,6 +43,7 @@
     package_dir={'': 'src'},
     package_data={'kubespray': ['files/*.yml'], },
     install_requires=requirements,
+    dependency_links=dependency_links,
     license="GPLv3",
     zip_safe=False,
     keywords='kubespray',

src/kubespray/cloud.py

@@ -25,8 +25,16 @@
 
 import sys
 import os
+import re
+import subprocess
 import yaml
 import json
+import zipfile
+
+import requests
+from ati.terraform import tfstates, iter_states, iterresources
+
+
 from kubespray.inventory import CfgInventory
 from kubespray.common import get_logger, query_yes_no, run_command, which, id_generator, get_cluster_name
 from ansible.utils.display import Display
@@ -397,3 +405,194 @@ def gen_openstack_playbook(self):
                 )
         self.write_local_inventory()
         self.write_playbook()
+
+
+class Terraform(Cloud):
+    TERRAFORM_DOWNLOAD_URL = "https://releases.hashicorp.com/terraform/{version}/terraform_{version}_linux_amd64.zip"
+    TERRAFORM_VERSION_RE = re.compile(r"Terraform v(?P<version>\d\.\d\.\d)")
+
+    def __init__(self, options):
+        Cloud.__init__(self, options, "terraform")
+        self.options = options
+        self.path = options.get("tf_binary_folder")
+        self.binary = "terraform"
+        self.proxy = options.get("http_proxy", "")
+        self.conf_path = options.get("tf_plan_folder")
+
+    @property
+    def version(self):
+        terraform = os.path.join(self.path, self.binary)
+        raw_version = subprocess.check_output([terraform, '--version'])
+        m = re.match(self.TERRAFORM_VERSION_RE, raw_version)
+        return m.group('version')
+
+    @property
+    def _config_env(self):
+        """Return the subprocess env-compatible list from config_file"""
+        config_env = {}
+        for key, value in self.options.get('tf_vars', {}).iteritems():
+            terra_key = 'TF_VAR_{}'.format(key)
+            config_env[terra_key] = str(value)
+
+        return config_env
+
+    def _update(self, version):
+        def _download(url):
+            return requests.get(url, proxies=self.proxy).content
+
+        filepath = os.path.join(self.path, self.binary)
+
+        # Is there a need to update?
+        if os.path.isfile(filepath):
+            if self.version == version:
+                self.logger.info('Terraform v{} up to date'.format(version))
+                return None
+
+        self.logger.info('Downloading terraform v{}...'.format(version))
+
+        terraform_url = self.TERRAFORM_DOWNLOAD_URL.format(version=version)
+        zippath = os.path.join(self.path, ''.join((self.binary, '.zip')))
+
+        try:
+            os.mkdir(self.path)
+        except OSError:
+            # Directory already exist.
+            pass
+
+        with open(zippath, 'w+') as file:
+            file.write(_download(terraform_url))
+
+        with zipfile.ZipFile(zippath, 'r') as zfile:
+            zfile.extractall(os.path.join(self.path))
+
+        os.chmod(os.path.join(self.path, self.binary), 0755)
+        os.remove(zippath)
+
+
+    def prepare(self):
+        # TODO Get tf files
+        self._update(self.options.get("tf_version"))
+
+    def _get(self, conf_path):
+        terraform = os.path.join(self.path, self.binary)
+        subprocess.check_call([
+            terraform,
+            'get',
+            conf_path
+        ], cwd=self.path)
+
+    def plan_instances(self):
+        """The conf_path must include:
+
+        ::
+
+            ssh_keys/ansible.pub
+            infra/*.tf
+
+        :param conf_path: The path leading to ssh_keys/ and infra/
+        :return:
+        """
+        terraform = os.path.join(self.path, self.binary)
+        self._get(self.conf_path)
+
+        subprocess.check_call([
+            terraform,
+            'plan',
+            '-state={}'.format(os.path.join(self.options.get("tf_state_folder"), 'terraform.tfstate')),
+            self.conf_path,
+        ], env=self._config_env, cwd=self.path)
+
+    def create_instances(self):
+        '''Run ansible-playbook for instances creation'''
+        self._update(self.options.get("tf_version"))
+        # TODO
+        # TERRAFORM
+        terraform = os.path.join(self.path, self.binary)
+        self._get(self.options.get('tf_plan_folder'))
+
+        # Little protection against Microsoft errors.
+        for _ in xrange(5):
+            retval = subprocess.call([
+                terraform,
+                'apply',
+                '-state={}'.format(os.path.join(self.options.get("tf_state_folder"), 'terraform.tfstate')),
+                os.path.join(self.options.get('tf_plan_folder')),
+            ], env=self._config_env, cwd=self.path)
+
+            if retval == 0:
+                break
+
+        # Compatibility with other functions.
+        if retval != 0:
+            raise subprocess.CalledProcessError(retval, terraform)
+
+
+    def plan_destroy_instances(self):
+        terraform = os.path.join(self.path, self.binary)
+        self._get(self.options.get('tf_plan_folder'))
+
+        # Little protection against retryable errors.
+        for _ in xrange(5):
+            retval = subprocess.call([
+                terraform,
+                'plan',
+                '-destroy',
+                '-state={}'.format(os.path.join(self.options.get("tf_state_folder"), 'terraform.tfstate')),
+                os.path.join(self.options.get('tf_plan_folder')),
+            ],
+                env=self._config_env,
+                stderr=subprocess.STDOUT,
+                cwd=self.path)
+
+            if retval == 0:
+                break
+
+    def destroy_instances(self):
+        terraform = os.path.join(self.path, self.binary)
+        self._get(self.options.get('tf_plan_folder'))
+
+        # Little protection against retryable errors.
+        for _ in xrange(5):
+            retval = subprocess.call([
+                terraform,
+                'destroy',
+                '-state={}'.format(os.path.join(self.options.get("tf_state_folder"), 'terraform.tfstate')),
+                '-force',
+                os.path.join(self.options.get('tf_plan_folder')),
+            ],
+                env=self._config_env,
+                stderr=subprocess.STDOUT,
+                cwd=self.path)
+
+            if retval == 0:
+                break
+
+        # Compatibility with other functions.
+        if retval != 0:
+            raise subprocess.CalledProcessError(retval, terraform)
+
+    def write_inventory(self):
+        '''Generate the inventory according the instances created'''
+        from ati.terraform import tfstates, iter_states, iterresources, iterhosts
+        hosts = iterhosts(iterresources(tfstates(self.options.get("tf_state_folder"))), None)
+
+
+        self.instances['masters']['json'] = []
+        self.instances['nodes']['json'] = []
+        self.instances['etcds']['json'] = []
+        for host in hosts:
+            hostname = host[0]
+            attrs = host[1]
+            tags = host[2]
+            for tag in tags:
+                if tag == 'role=kube-master':
+                    self.instances['masters']['json'].append(attrs)
+                if tag == 'role=kube-node':
+                    self.instances['nodes']['json'].append(attrs)
+        # TODO handle add node
+        self.options['add_node'] = None
+        # TODO handle etcds == masters
+        if self.instances['etcds']['json'] == []:
+            self.instances['etcds']['json'] = self.instances['masters']['json']
+        self.Cfg.write_inventory(self.instances['masters']['json'], self.instances['nodes']['json'], self.instances['etcds']['json'])
+

src/kubespray/common.py

@@ -106,7 +106,7 @@ def get_cluster_name():
 
 
 def clone_kubespray_git_repo(options):
-    if not options['add_node']:
+    if not options.get('add_node', False):
         if (os.path.isdir(options['kubespray_path']) and not options['assume_yes']
                 and not options['noclone']):
             display.warning(

src/kubespray/inventory.py

@@ -113,7 +113,6 @@ def format_inventory(self, masters, nodes, etcds):
                              {'hostname': 'kube-master', 'hostvars': []}
                              ]},
                          }
-
         if self.platform == 'openstack':
             if self.options['floating_ip']:
                 ip_type = 'public_v4'
@@ -150,7 +149,47 @@ def format_inventory(self, masters, nodes, etcds):
             elif etcds and len(etcds) < 3:
                 etcds = [etcds[0]]
 
-        if self.platform in ['aws', 'gce', 'openstack']:
+        if self.platform == 'terraform':
+            if self.options['add_node']:
+                current_inventory = self.read_inventory()
+                cluster_name = '-'.join(
+                    current_inventory['all']['hosts'][0]['hostname'].split('-')[:-1]
+                )
+                new_inventory = current_inventory
+            else:
+                cluster_name = 'k8s-' + get_cluster_name()
+
+            for host in nodes + masters + etcds:
+                tmp_dict = {'hostname': '%s' % host['name'],
+                            'hostvars': [{'name': 'ansible_ssh_host',
+                                          'value': host['ansible_ssh_host']}]
+                           }
+                # TODO handle windows nodes. Add windows specific attributues
+                attrs = ['ansible_ssh_user']
+                for attr in attrs:
+                    if host.get(attr):
+                        tmp_dict['hostvars'].append({'name': 'ansible_ssh_user',
+                                                     'value': host['ansible_ssh_user']})
+                new_inventory['all']['hosts'].append(tmp_dict)
+
+            if not self.options['add_node']:
+                for host in nodes:
+                    new_inventory['kube-node']['hosts'].append(
+                        {'hostname': '%s' % host['name'],
+                         'hostvars': []}
+                    )
+                for host in masters:
+                    new_inventory['kube-master']['hosts'].append(
+                        {'hostname': '%s' % host['name'],
+                         'hostvars': []}
+                    )
+                for host in etcds:
+                    new_inventory['etcd']['hosts'].append(
+                        {'hostname': '%s' % host['name'],
+                         'hostvars': []}
+                    )
+
+        elif self.platform in ['aws', 'gce', 'openstack']:
             if self.options['add_node']:
                 current_inventory = self.read_inventory()
                 cluster_name = '-'.join(