Merge pull request #11 from remerge/kafka-exporter

Add Kafka exporter

Author: sixtus

.copier-answers.yml

@@ -1,6 +1,6 @@
 ---
 # Changes here will be overwritten by Copier
-_commit: v0.39.1
+_commit: v0.44.2-14-gd32fad2
 _src_path: gh:remerge/template
 deploy_branch: production
 google_cloud_gke_master_subnet: null
@@ -14,14 +14,10 @@ project_name: Ansible Role for Kafka
 project_owner: sre
 project_type: ansible-role
 use_ansible: false
-use_consul: false
 use_docker: false
 use_golang: false
 use_google_cloud_compute: false
-use_google_cloud_dns: false
 use_google_cloud_gke: false
-use_google_cloud_public_dns: false
-use_nomad: false
 use_python: true
 use_terraform: false
 use_terraform_google: false

.envrc

@@ -16,6 +16,6 @@ dotenv_if_exists
 PATH_add "${PWD}/bin"
 
 # Load poetry virtual env
-export POETRY_ACTIVE=1 && poetry run true
-export VIRTUAL_ENV="$(poetry env info --path)"
+VIRTUAL_ENV="$(poetry env info --path)"
+export VIRTUAL_ENV POETRY_ACTIVE=1
 PATH_add "${VIRTUAL_ENV}/bin"

.pre-commit-config.yaml

@@ -39,14 +39,14 @@ repos:
 
   # https://github.com/igorshubovych/markdownlint-cli/tags
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: "v0.32.2"
+    rev: "v0.33.0"
     hooks:
       - id: markdownlint-fix
         name: markdownlint
 
   # https://github.com/adrienverge/yamllint/tags
   - repo: https://github.com/adrienverge/yamllint
-    rev: "v1.28.0"
+    rev: "v1.29.0"
     hooks:
       - id: yamllint
         entry: yamllint --strict
@@ -71,7 +71,7 @@ repos:
 
   # https://github.com/ansible/ansible-lint/tags
   - repo: https://github.com/ansible/ansible-lint
-    rev: "v6.10.2"
+    rev: "v6.11.0"
     hooks:
       - id: ansible-lint
         name: ansible-lint
@@ -91,14 +91,14 @@ repos:
 
   # https://github.com/bridgecrewio/checkov/tags
   - repo: https://github.com/bridgecrewio/checkov
-    rev: "2.2.229"
+    rev: "2.2.281"
     hooks:
       - id: checkov
         name: checkov
         args: [--config-file=.checkov.yml]
 
   # https://github.com/jorisroovers/gitlint/tags
   - repo: https://github.com/jorisroovers/gitlint
-    rev: "v0.18.0"
+    rev: "v0.19.0dev"
     hooks:
       - id: gitlint

CONTRIBUTING.md

@@ -48,7 +48,6 @@ recommended to install Ansible with [pipx](https://pypa.github.io/pipx/):
 ```shell
 pipx install --include-deps ansible
 pipx inject --include-apps ansible ansible-lint
-pipx inject ansible netaddr
 ```
 
 You can update all your pipx packages, including Ansible and all injected

defaults/exporter.yml

@@ -0,0 +1,5 @@
+---
+kafka_exporter_version: "1.6.0"
+kafka_exporter_archive: "kafka_exporter-{{ kafka_exporter_version }}.linux-amd64"
+kafka_exporter_checksum: "sha512:18a52a92f8c1ef06175f2cb7e12b81aed75b4a87884fb3c3120c6671e2b25e3a3b52dda02eb952ba22f892598c8a47c474b625d481701d83382650c8b4616550"
+kafka_exporter_url: "https://github.com/danielqsj/kafka_exporter/releases/download/v{{ kafka_exporter_version }}/kafka_exporter-{{ kafka_exporter_version }}.linux-amd64.tar.gz"

defaults/main.yml

@@ -1,23 +1,20 @@
 ---
-# Kafka Version
 scala_version: "2.13"
-kafka_version: "3.3.1"
-jmx_exporter_version: "0.17.2"
+kafka_version: "3.3.2"
 kafka_archive: "kafka_{{ scala_version }}-{{ kafka_version }}"
-jmx_exporter_archive: "jmx_prometheus_javaagent-{{ jmx_exporter_version }}"
-kafka_checksum: "sha512:d14d9cc97d1702649b33378f38060c4a6220850a3d7182d1af20e905cb98728e51f2aeec63121f5773a389d6b66891b7508e5d23584dd1a4e424825e3d54e3e0"
-jmx_exporter_checksum: "sha1:614e3c13696f1df9f87640a67c80dc3d5a7f9619"
+kafka_checksum: "sha512:f56657105a67f6195c1d4795b5fb4a00fc5e2f5cce061d7f7a504063778187b325c6fbe51d7c59f57cd7ef323a37d4c2c780c27a5609a2008852cbfa944287b3"
 kafka_url: "https://downloads.apache.org/kafka/{{ kafka_version }}/{{ kafka_archive }}.tgz"
+
+jmx_exporter_version: "0.17.2"
+jmx_exporter_archive: "jmx_prometheus_javaagent-{{ jmx_exporter_version }}"
+jmx_exporter_checksum: "sha512:26ce9bfae04be23bd41c5e7b97fe2e6b98bf3311734523974c6ea9790bf9f7328fd6ec3a031f0ab7cc77f8a62bcb8bc980de0ac2cebd88d495ec375b8c038bc8"
 jmx_exporter_url: "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/{{ jmx_exporter_version }}/{{ jmx_exporter_archive }}.jar"
 
-# Kafka Installation
 kafka_user: kafka
 kafka_group: kafka
 
-# KRaft Settings
 kafka_process_roles:
   - broker
   - controller
 
-# Listener Settings
 kafka_listener_address: "{{ ansible_default_ipv4.address }}"

handlers/main.yml

@@ -1,6 +1,12 @@
 ---
-- name: Restart kafka
+- name: Restart Kafka
   ansible.builtin.systemd:
     name: kafka
     state: restarted
     daemon_reload: true
+
+- name: Restart Kafka exporter
+  ansible.builtin.systemd:
+    name: kafka-exporter
+    state: restarted
+    daemon_reload: true

poetry.lock

@@ -10,11 +10,11 @@ python = "^3.10"
 [tool.poetry.group.dev.dependencies]
 black = "^22.12.0"
 flake8 = "^6.0.0"
-flake8-bugbear = "^22.12.6"
-pylint = "^2.15.8"
-tox = "^4.0.11"
+flake8-bugbear = "^23.1.20"
+pylint = "^2.15.10"
+tox = "^4.3.5"
 molecule = {extras = ["docker"], version = "^4.0.4"}
-ansible-lint = { version = "^6.10.0", markers = "platform_system != 'Windows'" }
+ansible-lint = { version = "^6.11.0", markers = "platform_system != 'Windows'" }
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

pyproject.toml

@@ -0,0 +1,48 @@
+---
+- name: Download Kafka exporter
+  ansible.builtin.get_url:
+    url: "{{ kafka_exporter_url }}"
+    dest: /tmp/{{ kafka_exporter_archive }}.tar.gz
+    checksum: "{{ kafka_exporter_checksum }}"
+    mode: 0644
+
+- name: Unarchive Kafka exporter
+  ansible.builtin.unarchive:
+    src: /tmp/{{ kafka_exporter_archive }}.tar.gz
+    dest: /opt
+    remote_src: true
+    owner: "{{ kafka_user }}"
+    group: "{{ kafka_group }}"
+    creates: /opt/{{ kafka_exporter_archive }}
+  notify:
+    - Restart Kafka exporter
+
+- name: Create Kafka exporter symlink
+  ansible.builtin.file:
+    src: /opt/{{ kafka_exporter_archive }}
+    dest: /opt/kafka_exporter
+    owner: "{{ kafka_user }}"
+    group: "{{ kafka_group }}"
+    state: link
+  notify:
+    - Restart Kafka exporter
+
+- name: Copy Kafka systemd service
+  ansible.builtin.template:
+    src: kafka-exporter.service.j2
+    dest: /usr/lib/systemd/system/kafka-exporter.service
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - Restart Kafka exporter
+
+- name: Flush handlers to prevent restart after start
+  ansible.builtin.meta: flush_handlers
+
+- name: Start Kafka exporter
+  ansible.builtin.systemd:
+    name: kafka-exporter
+    state: started
+    daemon_reload: true
+    enabled: true