Support for kerberization

README.rst

@@ -56,6 +56,12 @@ To see full usage instructions for the ``start`` action, use ``-h``/``--help``:
       -h, --help            show this help message and exit
 
     MapR arguments:
+      --kerberos            If specified, enable Kerberos for the cluster (default: False)
+      --kerberos-principals If specified, a comma-separated list of Kerberos user principals
+                            to create in KDC (default: None)
+      --kerberos-ticket-lifetime
+                            If specified, the maximum time period in seconds for which
+                            a ticket may be valid in the realm (default: 86400)
       --license-credentials credentials
                             MapR license credentials to use in the format
                             username:password (default: None)

images/primary_node/Dockerfile

@@ -27,6 +27,8 @@ RUN . /tmp/${REPO_URL_INSERT_SCRIPT} ${MAPR_VERSION} ${MEP_VERSION}
 RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key
 
 RUN yum -y install java-1.8.0-openjdk-devel \
+    krb5-libs \
+    krb5-workstation \
     mapr-cldb \
     mapr-core \
     mapr-fileserver \

images/secondary_node/Dockerfile

@@ -28,6 +28,8 @@ RUN . /tmp/${REPO_URL_INSERT_SCRIPT} ${MAPR_VERSION} ${MEP_VERSION}
 RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key
 
 RUN yum -y install java-1.8.0-openjdk-devel \
+    krb5-libs \
+    krb5-workstation \
     mapr-core \
     mapr-fileserver \
     mapr-hbase-regionserver \

kerberos_helper.py

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from clusterdock.utils import wait_for_condition
+
+KDC_IMAGE_NAME = 'clusterdock/topology_nodebase_kerberos:centos6.6'
+
+KERBEROS_CONFIG_CONTAINER_DIR = '/etc/clusterdock/kerberos'
+
+KDC_ACL_FILENAME = '/var/kerberos/krb5kdc/kadm5.acl'
+KDC_CONF_FILEPATH = '/var/kerberos/krb5kdc/kdc.conf'
+KDC_HOSTNAME = 'kdc'
+KDC_GROUPNAME = 'kdc'
+# Following is the location clusterdock places the keytab file of kerberos_principals
+# so that client nodes like SDC docker can fetch it.
+KDC_KEYTAB_FILEPATH = '{}/clusterdock.keytab'.format(KERBEROS_CONFIG_CONTAINER_DIR)
+KRB5_CONF_FILEPATH = '/etc/krb5.conf'
+MAPR_CONF_PATH = '/opt/mapr/conf'
+MAPR_KEYTAB_FILENAME = 'mapr.keytab'
+# Following is the location clusterdock places the mapr.keytab file so that client nodes like SDC docker can fetch it.
+MAPR_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_CONFIG_CONTAINER_DIR, MAPR_KEYTAB_FILENAME)
+# Following is the location expected by MapR for the mapr.keytab file.
+MAPR_CONF_KEYTAB_FILEPATH = '{}/{}'.format(MAPR_CONF_PATH, MAPR_KEYTAB_FILENAME)
+MAPR_PRINCIPAL = 'mapr/my.cluster.com'
+LINUX_USER_ID_START = 1000
+
+logger = logging.getLogger('clusterdock.{}'.format(__name__))
+
+
+class Kerberos_Helper:
+    """Class to deal with kerberization of the cluster.
+
+    Args:
+        network (:obj:`str`): Docker network to use.
+    """
+    def __init__(self, network):
+        self.network = network
+
+    @property
+    def mapr_principal(self):
+        return '{}@{}'.format(MAPR_PRINCIPAL, self.network.upper())
+
+    def configure_kdc(self, kdc_node, nodes, kerberos_principals, kerberos_ticket_lifetime, quiet):
+        logger.info('Updating KDC configurations ...')
+        realm = self.network.upper()
+
+        logger.debug('Updating krb5.conf ...')
+        krb5_conf = kdc_node.get_file(KRB5_CONF_FILEPATH)
+        # Here '\g<1>' represents group matched in regex which is the original default value of ticket_lifetime.
+        ticket_lifetime_replacement = kerberos_ticket_lifetime if kerberos_ticket_lifetime else '\g<1>'
+        krb5_conf_contents = re.sub(r'EXAMPLE.COM', realm,
+                                    re.sub(r'example.com', self.network,
+                                           re.sub(r'ticket_lifetime = ((.)*)',
+                                                  r'ticket_lifetime = {}'.format(ticket_lifetime_replacement),
+                                                  re.sub(r'kerberos.example.com',
+                                                         kdc_node.fqdn,
+                                                         krb5_conf))))
+        kdc_node.put_file(KRB5_CONF_FILEPATH, krb5_conf_contents)
+
+        logger.debug('Updating kdc.conf ...')
+        kdc_conf = kdc_node.get_file(KDC_CONF_FILEPATH)
+        max_time_replacement = kerberos_ticket_lifetime if kerberos_ticket_lifetime else '1d'
+        kdc_node.put_file(KDC_CONF_FILEPATH,
+                          re.sub(r'EXAMPLE.COM', realm,
+                                 re.sub(r'\[kdcdefaults\]',
+                                        r'[kdcdefaults]\n max_renewablelife = 7d\n max_life = {}'.format(max_time_replacement),
+                                        kdc_conf)))
+
+        logger.debug('Updating kadm5.acl ...')
+        kadm5_acl = kdc_node.get_file(KDC_ACL_FILENAME)
+        kdc_node.put_file(KDC_ACL_FILENAME,
+                          re.sub(r'EXAMPLE.COM', realm, kadm5_acl))
+
+        logger.info('Starting KDC ...')
+
+        kdc_commands = [
+            'kdb5_util create -s -r {} -P kdcadmin'.format(realm),
+            'kadmin.local -q "addprinc -pw {} admin/admin@{}"'.format('acladmin', realm),
+            'kadmin.local -q "addprinc -randkey {}"'.format(self.mapr_principal),
+            'kadmin.local -q "ktadd -k {} {}"'.format(MAPR_KEYTAB_FILEPATH, self.mapr_principal)
+        ]
+
+        if kerberos_principals:
+            principals = ['{}@{}'.format(primary, realm)
+                          for primary in kerberos_principals.split(',')]
+            if kerberos_ticket_lifetime:
+                kdc_commands.extend([('kadmin.local -q "addprinc -maxlife {}sec '
+                                      '-maxrenewlife 5day -randkey {}"'.format(kerberos_ticket_lifetime, principal))
+                                     for principal in principals])
+            else:
+                kdc_commands.extend(['kadmin.local -q "addprinc -randkey {}"'.format(principal)
+                                     for principal in principals])
+            kdc_commands.append('kadmin.local -q '
+                                '"xst -norandkey -k {} {}"'.format(KDC_KEYTAB_FILEPATH,
+                                                                   ' '.join(principals)))
+        kdc_commands.extend(['service krb5kdc start',
+                             'service kadmin start',
+                             'authconfig --enablekrb5 --update',
+                             'cp -f {} {}'.format(KRB5_CONF_FILEPATH,
+                                                  KERBEROS_CONFIG_CONTAINER_DIR)])
+        if kerberos_principals:
+            kdc_commands.append('chmod 644 {}'.format(KDC_KEYTAB_FILEPATH))
+
+        kdc_node.execute(' && '.join(kdc_commands), quiet=quiet)
+
+        _validate_service_health(kdc_node, ['krb5kdc', 'kadmin'], quiet=quiet)
+
+        # nodes are the primary and secondary nodes.
+        for node in nodes:
+            # Copy mapr.keytab file to location where MapR expects
+            commands = ['cp -f {} {}/{}'.format(MAPR_KEYTAB_FILEPATH, MAPR_CONF_PATH, MAPR_KEYTAB_FILENAME),
+                        'chown mapr:mapr {}'.format(MAPR_CONF_KEYTAB_FILEPATH)]
+            # Copy krb5.conf to each node's /etc/krb5.conf
+            commands.append('cp -f {}/{} {}'.format(KERBEROS_CONFIG_CONTAINER_DIR, 'krb5.conf', KRB5_CONF_FILEPATH))
+            node.execute(' && '.join(commands), quiet=quiet)
+
+def _validate_service_health(kdc_node, services, quiet=True):
+    logger.info('Validating health of Kerberos services ...')
+
+    def condition(node, services, quiet):
+        services_with_poor_health = [service
+                                     for service in services
+                                     if node.execute(command='service {} status'.format(service),
+                                                     quiet=quiet).exit_code != 0]
+        if services_with_poor_health:
+            logger.debug('Services with poor health: %s',
+                         ', '.join(services_with_poor_health))
+        # Return True if the list of services with poor health is empty.
+        return not bool(services_with_poor_health)
+    wait_for_condition(condition=condition, condition_args=[kdc_node, services, quiet])
+
+def create_kerberos_cluster_users(nodes, kerberos_principals, quiet):
+    commands = ['useradd -u {} -g hadoop {}'.format(uid, primary)
+                for uid, primary in enumerate(kerberos_principals.split(','),
+                                              start=LINUX_USER_ID_START)]
+    for node in nodes:
+        node.execute('; '.join(commands), quiet=quiet)

start.py

@@ -20,19 +20,21 @@
 from clusterdock.models import Cluster, Node
 from clusterdock.utils import wait_for_condition
 
+from . import kerberos_helper
+
 DEFAULT_NAMESPACE = 'clusterdock'
 EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7 = (6, 0, 0)
 MAPR_CONFIG_DIR = '/opt/mapr/conf'
 MAPR_SERVERTICKET_FILE = 'maprserverticket'
 MCS_SERVER_PORT = 8443
 SECURE_CONFIG_CONTAINER_DIR = '/etc/clusterdock/secure'
-SSL_KEYSTORE_FILE = 'ssl_keystore'
-SSL_TRUSTSTORE_FILE = 'ssl_truststore'
+SSL_KEYSTORE_FILES = 'ssl_keystore'
+SSL_TRUSTSTORE_FILES = 'ssl_truststore'
 
 SECURE_FILES = [
     MAPR_SERVERTICKET_FILE,
-    SSL_KEYSTORE_FILE,
-    SSL_TRUSTSTORE_FILE
+    SSL_KEYSTORE_FILES,
+    SSL_TRUSTSTORE_FILES
 ]
 
 
@@ -68,17 +70,29 @@ def main(args):
                         # Secure cluster needs the ticket to execute rest of commands
                         # after cluster start.
                         environment=['MAPR_TICKETFILE_LOCATION=/opt/mapr/conf/mapruserticket']
-                        if args.secure else [])
+                        if args.secure or args.kerberos else [])
 
     secondary_nodes = [Node(hostname=hostname,
                             group='secondary',
                             image=secondary_node_image,
                             devices=node_disks.get(hostname))
                        for hostname in args.secondary_nodes]
 
-    cluster = Cluster(primary_node, *secondary_nodes)
+    nodes = [primary_node] + secondary_nodes
+    if args.kerberos:
+        logger.info('Creating KDC node...')
+        kerberos_helper_instance = kerberos_helper.Kerberos_Helper(args.network)
+        kerberos_config_host_dir = os.path.realpath(os.path.expanduser(args.clusterdock_config_directory))
+        volumes = [{kerberos_config_host_dir: kerberos_helper.KERBEROS_CONFIG_CONTAINER_DIR}]
+        for node in nodes:
+            node.volumes.extend(volumes)
+
+        kdc_node = Node(hostname=kerberos_helper.KDC_HOSTNAME, group=kerberos_helper.KDC_GROUPNAME,
+                        image=kerberos_helper.KDC_IMAGE_NAME, volumes=volumes)
+
+    cluster = Cluster(*nodes + ([kdc_node] if args.kerberos else []))
 
-    if args.secure:
+    if args.secure or args.kerberos:
         secure_config_host_dir = os.path.expanduser(args.secure_config_directory)
         volumes = [{secure_config_host_dir: SECURE_CONFIG_CONTAINER_DIR}]
         for node in cluster.nodes:
@@ -95,10 +109,22 @@ def main(args):
     cluster.primary_node = primary_node
     cluster.start(args.network, pull_images=args.always_pull)
 
+    # Keep track of whether to suppress DEBUG-level output in commands.
+    quiet = not args.verbose
+
+    if args.kerberos:
+        cluster.kdc_node = kdc_node
+        kerberos_helper_instance.configure_kdc(kdc_node, nodes,
+                                               args.kerberos_principals,
+                                               args.kerberos_ticket_lifetime,
+                                               quiet=quiet)
+        if args.kerberos_principals:
+            kerberos_helper.create_kerberos_cluster_users(nodes, args.kerberos_principals, quiet=quiet)
+
     logger.info('Generating new UUIDs ...')
     cluster.execute('/opt/mapr/server/mruuidgen > /opt/mapr/hostid')
 
-    if not args.secure:
+    if not (args.secure or args.kerberos):
         logger.info('Configuring the cluster ...')
         for node in cluster:
             configure_command = ('/opt/mapr/server/configure.sh -C {0} -Z {0} -RM {0} -HS {0} '
@@ -116,7 +142,7 @@ def main(args):
                              ))
         source_files = ['{}/{}'.format(MAPR_CONFIG_DIR, file) for file in SECURE_FILES]
         commands = [configure_command,
-                    'chmod 600 {}/{}'.format(MAPR_CONFIG_DIR, SSL_KEYSTORE_FILE),
+                    'chmod 600 {}/{}'.format(MAPR_CONFIG_DIR, SSL_KEYSTORE_FILES),
                     'cp -f {src} {dest_dir}'.format(src=' '.join(source_files),
                                                     dest_dir=SECURE_CONFIG_CONTAINER_DIR)]
         primary_node.execute(' && '.join(commands))
@@ -149,6 +175,15 @@ def failure(timeout):
                        time_between_checks=3, timeout=180, success=success, failure=failure)
     mcs_server_host_port = primary_node.host_ports.get(MCS_SERVER_PORT)
 
+    _configure_after_mcs_server_start(primary_node, secondary_nodes, args, mapr_version_tuple,
+                                      kerberos_helper_instance.mapr_principal if args.kerberos else None)
+
+    logger.info('MapR Control System server is now accessible at https://%s:%s',
+                getfqdn(), mcs_server_host_port)
+
+
+def _configure_after_mcs_server_start(primary_node, secondary_nodes, args,
+                                      mapr_version_tuple, mapr_principal=None):
     logger.info('Creating /apps/spark directory on %s ...', primary_node.hostname)
     spark_directory_command = ['hadoop fs -mkdir -p /apps/spark',
                                'hadoop fs -chmod 777 /apps/spark']
@@ -159,21 +194,59 @@ def failure(timeout):
                          '-produceperm p -consumeperm p -topicperm p')
 
     if mapr_version_tuple >= EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7 and args.license_url:
-        license_commands = ['curl --user {} {} > /tmp/lic'.format(args.license_credentials,
-                                                                  args.license_url),
-                            '/opt/mapr/bin/maprcli license add -license /tmp/lic -is_file true',
-                            'rm -rf /tmp/lic']
-        logger.info('Applying license ...')
-        primary_node.execute(' && '.join(license_commands))
+        _apply_license(args, primary_node)
 
     if not args.dont_register_gateway:
-        logger.info('Registering gateway with the cluster ...')
-        register_gateway_commands = ["cat /opt/mapr/conf/mapr-clusters.conf | egrep -o '^[^ ]* '"
-                                     ' > /tmp/cluster-name',
-                                     'maprcli cluster gateway set -dstcluster $(cat '
-                                     '/tmp/cluster-name) -gateways {}'.format(primary_node.fqdn),
-                                     'rm /tmp/cluster-name']
-        primary_node.execute(' && '.join(register_gateway_commands))
+        _register_gateway(primary_node)
 
-    logger.info('MapR Control System server is now accessible at https://%s:%s',
-                getfqdn(), mcs_server_host_port)
+    if args.secure or args.kerberos:
+        primary_node.execute('echo mapr | sudo -u mapr maprlogin password')
+
+    if args.kerberos:
+        _kerberize_cluster(primary_node, secondary_nodes, mapr_principal)
+
+
+def _kerberize_cluster(primary_node, secondary_nodes, mapr_principal):
+    commands = ['service mapr-warden stop',
+                'service mapr-zookeeper stop',
+                ('/opt/mapr/server/configure.sh -K -P {0} -C {1} -Z {1} '
+                 .format(mapr_principal, primary_node.fqdn))]
+    primary_node.execute(' && '.join(commands))
+    for node in secondary_nodes:
+        node.execute(commands[2])
+
+    logger.info('After kerberization, waiting for MapR Control System server to come online ...')
+
+    def condition(address, port):
+        return socket().connect_ex((address, port)) == 0
+
+    def success(time):
+        logger.info('MapR Control System server is online after %s seconds.', time)
+
+    def failure(timeout):
+        raise TimeoutError('Timed out after {} seconds waiting '
+                           'for MapR Control System server to come online.'.format(timeout))
+    wait_for_condition(condition=condition,
+                       condition_args=[primary_node.ip_address, MCS_SERVER_PORT],
+                       time_between_checks=3, timeout=180, success=success, failure=failure)
+    primary_node.execute('kinit -kt {} {}'.format(kerberos_helper.MAPR_CONF_KEYTAB_FILEPATH, mapr_principal))
+    primary_node.execute('maprlogin kerberos')
+
+
+def _apply_license(args, primary_node):
+    license_commands = ['curl --user {} {} > /tmp/lic'.format(args.license_credentials,
+                                                              args.license_url),
+                        '/opt/mapr/bin/maprcli license add -license /tmp/lic -is_file true',
+                        'rm -rf /tmp/lic']
+    logger.info('Applying license ...')
+    primary_node.execute(' && '.join(license_commands))
+
+
+def _register_gateway(primary_node):
+    logger.info('Registering gateway with the cluster ...')
+    register_gateway_commands = ["cat /opt/mapr/conf/mapr-clusters.conf | egrep -o '^[^ ]* '"
+                                 ' > /tmp/cluster-name',
+                                 'maprcli cluster gateway set -dstcluster $(cat '
+                                 '/tmp/cluster-name) -gateways {}'.format(primary_node.fqdn),
+                                 'rm /tmp/cluster-name']
+    primary_node.execute(' && '.join(register_gateway_commands))

topology.yaml

@@ -27,6 +27,16 @@ build args:
         metavar: ver
 
 start args:
+    --kerberos, -k:
+        action: store_true
+        help: If specified, enable Kerberos for the cluster
+    --kerberos-principals:
+        help: If specified, a comma-separated list of Kerberos user principals to create in KDC
+        metavar: princ1,princ2,...
+    --kerberos-ticket-lifetime:
+        default: 86400
+        help: If specified, the maximum time period in seconds for which a ticket may be valid in the realm
+        metavar: seconds
     --license-credentials:
         help: MapR license credentials to use in the format username:password
         metavar: credentials