use zookeeper from kafka distribution

download everything directly on machine (prepare for different inventories)

README.md

@@ -45,9 +45,9 @@ The result if everything wents fine should be
 |:--- |:-- |:-- |:-- |
 |192.168.10.2|mon-1|running elk and metricbeat | 4096 MB RAM |
 |192.168.10.3|mon-2|running grafana, prometheus and metricbeat | 2048 MB RAM |
-|192.168.10.4|kafka-1|running a kafka broker and metricbeat | 2048 MB RAM |
-|192.168.10.5|kafka-2|running a kafka broker and metricbeat | 2048 MB RAM |
-|192.168.10.6|kafka-3|running a kafka broker and metricbeat | 2048 MB RAM |
+|192.168.10.4|kafka-1|running a zookeeper, kafka broker and metricbeat | 2048 MB RAM |
+|192.168.10.5|kafka-2|running a zookeeper, kafka broker and metricbeat | 2048 MB RAM |
+|192.168.10.6|kafka-3|running a zookeeper, kafka broker and metricbeat | 2048 MB RAM |
 
 
 ### Connections
@@ -91,7 +91,7 @@ The result if everything wents fine should be
 
 ```bash
 vagrant ssh kafka-1
-zkCli.sh -server kafka-1:2181/
+zookeeper-shell.sh kafka-1:2181/
 Connecting to kafka-1:2181/
 ...
 
@@ -111,10 +111,10 @@ WatchedEvent state:SyncConnected type:None path:null
 ```bash
 vagrant ssh kafka-1
 
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Create --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Create --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
 
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-topics.sh --create --zookeeper kafka-1:2181 --replication-factor 1 --partitions 4 --topic sample
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-topics.sh --create --zookeeper kafka-1:2181 --replication-factor 1 --partitions 4 --topic sample
 
 ```
 
@@ -123,7 +123,7 @@ Created topic "sample".
 ```
 
 ```bash
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-topics.sh --zookeeper kafka-1:2181 --topic sample --describe
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-topics.sh --zookeeper kafka-1:2181 --topic sample --describe
 ```
 
 ```bash
@@ -139,11 +139,11 @@ Topic:sample	PartitionCount:6	ReplicationFactor:2   Configs:
 ### ACL for producers and consumers
 
 ```bash
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --producer --topic sample --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --producer --topic sample --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE
 
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --consumer --topic sample --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE  --group console --resource-pattern-type PREFIXED
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --consumer --topic sample --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE  --group console --resource-pattern-type PREFIXED
 
-KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --list
+KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --list
 
 ```
 

Vagrantfile

@@ -13,9 +13,7 @@ Vagrant.configure("2") do |config|
   config.vm.box = "markush81/centos7-vbox-guestadditions"
   config.vm.box_check_update = true
 
-  config.vm.synced_folder "download", "/vagrant/download", create: true
   config.vm.synced_folder "exchange", "/home/vagrant/exchange", create: true
-  config.vm.synced_folder "ansible", "/home/vagrant/ansible", create: true
 
   config.trigger.after :destroy do |trigger|
     trigger.run = { inline: 'rm -rf exchange/ssl && rm -rf exchange/ssl-client'}
@@ -57,7 +55,7 @@ Vagrant.configure("2") do |config|
         "-vv"
       ]
     end
-    
+
     mon.vm.provision :ansible do |ansible|
       ansible.compatibility_mode = "2.0"
       ansible.limit = "grafana"

ansible/ansible.cfg

@@ -1 +1,3 @@
+[defaults]
 inventory = inventories/vbox
+host_key_checking = False

ansible/cluster.yml

@@ -21,19 +21,12 @@
   roles:
   - grafana
 
-- hosts: zookeeper
-  remote_user: vagrant
-  serial: 33%
-  roles:
-    - java8
-    - metricbeat
-    - secure
-    - zookeeper
-
 - hosts: kafka
   remote_user: vagrant
   serial: 33%
   roles:
     - java8
+    - metricbeat
     - filebeat
+    - secure
     - kafka

ansible/inventories/vbox/group_vars/all

@@ -1,10 +1,11 @@
-download: /vagrant/download
 exchange: /vagrant/exchange
-ssl_ca_dir: "{{ exchange }}/ssl"
-ssl_client_dir: "{{ exchange }}/ssl-client"
 usr_local: /usr/local
 etc_profiles: /etc/profile.d
 system_units: /etc/systemd/system
+
+secure: true
+ssl_ca_dir: "{{ exchange }}/ssl"
+ssl_client_dir: "{{ exchange }}/ssl-client"
 keystore_pwd: keystore-secret
 key_pwd: key-secret
 truststore_pwd: truststore-secret

ansible/inventories/vbox/group_vars/kafka

@@ -1,2 +1,3 @@
 kafka_home: "{{ usr_local }}/{{ kafka }}"
 kafka_log_dir: "{{ kafka_home }}/logs"
+zookeeper_data_dir: "/var/data"

ansible/roles/filebeat/tasks/main.yml

@@ -30,5 +30,6 @@
 
 - name: setup filebeat
   become: yes
-  command: filebeat setup -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["{{ groups['elk'][0] }}:9200"]' -E 'setup.kibana.host={{ groups['elk'][0] }}:5601'
-  run_once: true
+  shell: filebeat setup -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["{{ groups['elk'][0] }}:9200"]' -E 'setup.kibana.host={{ groups['elk'][0] }}:5601' && touch /tmp/filebeat.setup
+  args:
+    creates: /tmp/filebeat.setup

ansible/roles/java8/tasks/main.yml

@@ -5,12 +5,12 @@
     url: "{{ url }}"
     headers:
       Cookie: 'oraclelicense=accept-securebackup-cookie'
-    dest: "{{ download }}/{{ file }}"
+    dest: "/tmp/{{ file }}"
 
-- name: install {{ download }}/{{ file }}
+- name: install /tmp/{{ file }}
   become: yes
   yum:
-    name: "{{ download }}/{{ file }}"
+    name: "/tmp/{{ file }}"
     state: present
 
 - name: set JAVA_HOME=/usr/java/latest

ansible/roles/kafka/handlers/main.yml

@@ -1,5 +1,20 @@
 ---
 
+- name: restart zookeeper service
+  become: yes
+  systemd:
+    enabled: yes
+    state: restarted
+    name: zookeeper
+    daemon_reload: yes
+  listen: "restart zookeeper"
+
+- name: wait until zookeeper has started
+  wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 2181
+  listen: "restart zookeeper"
+
 - name: restart kafka service
   become: yes
   systemd:

ansible/roles/kafka/tasks/kafka.yml

@@ -0,0 +1,90 @@
+---
+
+- name: copy server.properties {{ kafka_home }}/config/server.properties
+  become: yes
+  template:
+    src: templates/server.properties.j2
+    dest: "{{ kafka_home }}/config/server.properties"
+  notify: "restart kafka"
+
+- name: copy kafka.environment {{ kafka_home }}/kafka.environment
+  become: yes
+  template:
+    src: templates/kafka.environment.j2
+    dest: "{{ kafka_home }}/kafka.environment"
+  notify: "restart kafka"
+
+- name: copy zookeeper_client_jaas.conf {{ kafka_home }}/config/zookeeper_client_jaas.conf
+  become: yes
+  template:
+    src: templates/zookeeper_client_jaas.conf.j2
+    dest: "{{ kafka_home }}/config/zookeeper_client_jaas.conf"
+  notify: "restart kafka"
+
+- name: download {{ prometheus_jmx }}
+  become: yes
+  get_url:
+    url: "{{ prometheus_jmx }}"
+    dest: "{{ kafka_home }}/prometheus-agent.jar"
+
+- name: copy prometheus-agent.yml {{ kafka_home }}/config/prometheus_agent.yml
+  become: yes
+  template:
+    src: templates/prometheus_agent.yml.j2
+    dest: "{{ kafka_home }}/config/prometheus_agent.yml"
+  notify: "restart kafka"
+
+- name: create {{ kafka_log_dir }}
+  become: yes
+  file:
+    path: "{{ kafka_log_dir }}"
+    state: directory
+  notify: "restart kafka"
+
+- name: copy filebeat kafka configs to /etc/filebeat/modules.d/kafka.yml
+  become: yes
+  template:
+    src: templates/filebeat_kafka.yml.j2
+    dest: /etc/filebeat/modules.d/kafka.yml
+
+- name: filebeat enable kafka module
+  become: yes
+  command: filebeat modules enable kafka
+
+- name: copy metricbeat kafka configs to /etc/metricbeat/modules.d/kafka.yml
+  become: yes
+  template:
+    src: templates/metricbeat_kafka.yml.j2
+    dest: /etc/metricbeat/modules.d/kafka.yml
+
+- name: metricbeat modules enable kafka
+  become: yes
+  command: metricbeat modules enable kafka
+
+- name: install kafka systemd unit file
+  become: yes
+  template:
+    src: templates/kafka.service.j2
+    dest: "{{ system_units }}/kafka.service"
+  notify: "restart kafka"
+
+- name: create {{ exchange }}/client-ssl.properties
+  become: yes
+  template:
+    src: templates/client-ssl.properties.j2
+    dest: "{{ ssl_client_dir }}/client-ssl.properties"
+  when: secure|bool
+
+- meta: flush_handlers
+
+- name: allow ClusterAction
+  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation ClusterAction --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
+  when: secure|bool and groups.kafka.index(inventory_hostname) == groups.kafka|length - 1
+
+- name: allow Describe cluster
+  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
+  when: secure|bool and groups.kafka.index(inventory_hostname) == groups.kafka|length - 1
+
+- name: allow Describe __consumer_offsets
+  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_client_jaas.conf /usr/local/kafka/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --topic __consumer_offsets --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
+  when: secure|bool and groups.kafka.index(inventory_hostname) == groups.kafka|length - 1

ansible/roles/kafka/tasks/main.yml

@@ -3,18 +3,17 @@
   run_once: true
   get_url:
     url: "{{ url }}"
-    dest: "{{ download }}/{{ file }}"
+    dest: "/tmp/{{ file }}"
 
-- name: extract {{ download }}/{{ file }}
+- name: extract /tmp/{{ file }}
   become: yes
   unarchive:
-    src: "{{ download }}/{{ file }}"
+    src: "/tmp/{{ file }}"
     dest: "{{ usr_local }}"
     copy: no
-    creates: "{{ kafka_home }}"
     owner: root
     group: root
-  notify: "restart kafka"
+    creates: "{{ kafka_home }}"
 
 - name: link to /usr/local/kafka
   become: yes
@@ -32,91 +31,8 @@
     regexp: '^PATH'
     line: 'PATH=$PATH:{{ kafka_home }}/bin'
 
-- name: copy server.properties {{ kafka_home }}/config/server.properties
-  become: yes
-  template:
-    src: templates/server.properties.j2
-    dest: "{{ kafka_home }}/config/server.properties"
-  notify: "restart kafka"
-
-- name: copy kafka.environment {{ kafka_home }}/kafka.environment
-  become: yes
-  template:
-    src: templates/kafka.environment.j2
-    dest: "{{ kafka_home }}/kafka.environment"
-  notify: "restart kafka"
-
-- name: copy zookeeper_jaas.conf {{ kafka_home }}/config/zookeeper_jaas.conf
-  become: yes
-  template:
-    src: templates/zookeeper_jaas.conf.j2
-    dest: "{{ kafka_home }}/config/zookeeper_jaas.conf"
-  notify: "restart kafka"
-
-- name: download {{ prometheus_jmx }}
-  become: yes
-  get_url:
-    url: "{{ prometheus_jmx }}"
-    dest: "{{ kafka_home }}/prometheus-agent.jar"
-
-- name: copy prometheus-agent.yml {{ kafka_home }}/config/prometheus_agent.yml
-  become: yes
-  template:
-    src: templates/prometheus_agent.yml.j2
-    dest: "{{ kafka_home }}/config/prometheus_agent.yml"
-  notify: "restart kafka"
-
-- name: create {{ kafka_log_dir }}
-  become: yes
-  file:
-    path: "{{ kafka_log_dir }}"
-    state: directory
-  notify: "restart kafka"
-
-- name: copy filebeat kafka configs to /etc/filebeat/modules.d/kafka.yml
-  become: yes
-  template:
-    src: templates/filebeat_kafka.yml.j2
-    dest: /etc/filebeat/modules.d/kafka.yml
-
-- name: filebeat enable kafka module
-  become: yes
-  command: filebeat modules enable kafka
+- import_tasks: zookeeper.yml
+  tags: zookeeper
 
-- name: copy metricbeat kafka configs to /etc/metricbeat/modules.d/kafka.yml
-  become: yes
-  template:
-    src: templates/metricbeat_kafka.yml.j2
-    dest: /etc/metricbeat/modules.d/kafka.yml
-
-- name: metricbeat modules enable kafka
-  become: yes
-  command: metricbeat modules enable kafka
-
-- name: install kafka systemd unit file
-  become: yes
-  template:
-    src: templates/kafka.service.j2
-    dest: "{{ system_units }}/kafka.service"
-  notify: "restart kafka"
-
-- name: create {{ exchange }}/client-ssl.properties
-  become: yes
-  template:
-    src: templates/client-ssl.properties.j2
-    dest: "{{ ssl_client_dir }}/client-ssl.properties"
-  run_once: true
-
-- meta: flush_handlers
-
-- name: allow ClusterAction
-  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation ClusterAction --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
-  run_once: true
-
-- name: allow Describe cluster
-  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --cluster --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
-  run_once: true
-
-- name: allow Describe __consumer_offsets
-  shell: "KAFKA_OPTS=-Djava.security.auth.login.config=/usr/local/kafka/config/zookeeper_jaas.conf /usr/local/kafka/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --operation Describe --topic __consumer_offsets --allow-principal User:CN=kafka,OU=org,O=org,L=home,ST=Bavaria,C=DE"
-  run_once: true
+- import_tasks: kafka.yml
+  tags: kafka