cisagov · cbaxley · Jun 18, 2025 · Jun 19, 2025 · Jun 19, 2025 · Jun 19, 2025
diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml
@@ -42,7 +42,7 @@ jobs:
       ENROLLMENT_TOKEN: ""
       ES_PASSWORD: ""
       KIBANA_PASSWORD: ""
-      ELASTIC_AGENT_VERSION: "8.18.0"
+      ELASTIC_AGENT_VERSION: "9.0.2"
       MINIMEGA_PASSWORD: ${{ secrets.MINIMEGA_PASSWORD }}
 
     steps:
@@ -292,10 +292,10 @@ jobs:
           ")
           echo "Retrieved Policy ID: $POLICY_ID"
 
-          # Retrieve enrollment token using the policy ID
+          # Create enrollment token using the policy ID
           ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
             ssh lme-user@${{ env.AZURE_IP }} '
-              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
+              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment_api_keys\" \
                 -H \"kbn-xsrf: true\" \
                 -H \"Content-Type: application/json\" \
                 -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" |

diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml
@@ -84,7 +84,7 @@ jobs:
             ls -la && \
             cd /home/lme-user/LME/testing/v2/installers && \
             IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
-            ./install_v2/install_pre_v2.1.sh lme-user \$IP_ADDRESS \"pipe-${{ env.UNIQUE_ID }}.password.txt\" '${{ env.LATEST_TAG }}'
+            ./install_v2/install.sh lme-user \$IP_ADDRESS \"pipe-${{ env.UNIQUE_ID }}.password.txt\" '${{ env.LATEST_TAG }}'
           "
 
       - name: Retrieve Elastic password 

diff --git a/ansible/UPGRADE_README.md b/ansible/UPGRADE_README.md
@@ -337,7 +337,7 @@ ansible-playbook rollback_lme.yml
 ping docker.io
 
 # Manually pull images
-sudo podman pull docker.elastic.co/elasticsearch/elasticsearch:8.18.0
+sudo podman pull docker.elastic.co/elasticsearch/elasticsearch:9.0.2
 
 # Check container registry access
 sudo podman login docker.elastic.co

diff --git a/ansible/roles/backup_lme/tasks/main.yml b/ansible/roles/backup_lme/tasks/main.yml
@@ -378,6 +378,4 @@
 
 - name: Backup summary
   debug:
-    msg: "{{ dict(range(1, backup_summary_lines|length + 1) | zip(backup_summary_lines)) }}"
-  vars:
-    backup_summary_lines: "{{ (final_status.stdout + '\n\nBackup location: ' + backup_base_dir + '/backups/' + ansible_date_time.iso8601_basic_short).splitlines() }}" 
+    msg: "{{ (final_status.stdout + '\n\nBackup location: ' + backup_base_dir + '/backups/' + ansible_date_time.iso8601_basic_short).splitlines() }}" 
diff --git a/ansible/roles/fleet/tasks/main.yml b/ansible/roles/fleet/tasks/main.yml
@@ -160,32 +160,36 @@
     var: ca_fingerprint.stdout
   when: debug_mode | bool
 
-- name: Set Fleet server hosts
-  uri:
-    url: "{{ local_kbn_url }}/api/fleet/settings"
-    method: PUT
-    user: "{{ elastic_username }}"
-    password: "{{ elastic_password }}"
-    force_basic_auth: yes
-    validate_certs: no
-    headers:
-      kbn-version: "{{ kibana_version }}"
-      kbn-xsrf: "kibana"
-      Content-Type: "application/json"
-    body_format: json
-    body:
-      fleet_server_hosts: ["https://{{ ipvar }}:{{ fleet_port }}"]
-    status_code: [200]
-  register: fleet_server_hosts_result
-  retries: 3
-  delay: 10
-  until: fleet_server_hosts_result.status == 200
-  no_log: "{{ not debug_mode }}"
-
-- name: Debug Fleet server hosts result
-  debug:
-    msg: "Fleet server hosts successfully set to {{ fleet_server_hosts_result.json.item.fleet_server_hosts }}"
-  when: debug_mode | bool
+# The fleet_server_hosts field has been removed from /api/fleet/settings in version 9.0
+# This configuration should now be done through Kibana configuration
+# Removing the deprecated task below:
+#
+# - name: Set Fleet server hosts
+#   uri:
+#     url: "{{ local_kbn_url }}/api/fleet/settings"
+#     method: PUT
+#     user: "{{ elastic_username }}"
+#     password: "{{ elastic_password }}"
+#     force_basic_auth: yes
+#     validate_certs: no
+#     headers:
+#       kbn-version: "{{ kibana_version }}"
+#       kbn-xsrf: "kibana"
+#       Content-Type: "application/json"
+#     body_format: json
+#     body:
+#       fleet_server_hosts: ["https://{{ ipvar }}:{{ fleet_port }}"]
+#     status_code: [200]
+#   register: fleet_server_hosts_result
+#   retries: 3
+#   delay: 10
+#   until: fleet_server_hosts_result.status == 200
+#   no_log: "{{ not debug_mode }}"
+
+# - name: Debug Fleet server hosts result
+#   debug:
+#     msg: "Fleet server hosts successfully set to {{ fleet_server_hosts_result.json.item.fleet_server_hosts }}"
+#   when: debug_mode | bool
 
 - name: Set Fleet default output hosts
   uri:
@@ -391,7 +395,6 @@
   debug:
     var: "{{ item }}"
   loop:
-    - fleet_server_hosts_result
     - fleet_output_hosts_result
     - fleet_output_fingerprint_result
     - fleet_output_ssl_result

diff --git a/ansible/roles/kibana/tasks/main.yml b/ansible/roles/kibana/tasks/main.yml
@@ -89,4 +89,46 @@
 
     - name: Retry connection check
       include_tasks: main.yml
-      when: (current_retry | int) < max_retries 
+      when: (current_retry | int) < max_retries
+
+- name: Debug Kibana readiness response
+  debug:
+    var: kibana_ready_check
+  when: debug_mode | bool
+
+# Add Fleet server hosts configuration to Kibana config
+# This is required for version 9.0+ as the API no longer supports setting fleet_server_hosts
+- name: Add Fleet server hosts to Kibana configuration
+  lineinfile:
+    path: /opt/lme/config/kibana.yml
+    line: "xpack.fleet.agents.fleet_server.hosts: [\"https://{{ ipvar }}:{{ fleet_port }}\"]"
+    regexp: '^xpack\.fleet\.agents\.fleet_server\.hosts:'
+    state: present
+  become: yes
+  when: ipvar is defined and ipvar != ''
+
+- name: Restart Kibana to apply Fleet server hosts configuration
+  systemd:
+    name: lme-kibana.service
+    state: restarted
+  become: yes
+  when: ipvar is defined and ipvar != ''
+  register: kibana_restart
+
+- name: Wait for Kibana to be ready after restart
+  uri:
+    url: "{{ local_kbn_url }}/api/status"
+    method: GET
+    user: "{{ elastic_username }}"
+    password: "{{ elastic_password }}"
+    force_basic_auth: yes
+    validate_certs: no
+    status_code: 200
+  register: kibana_status_after_restart
+  until: >
+    kibana_status_after_restart.status == 200 and 
+    kibana_status_after_restart.json.status.overall.level == "available"
+  retries: 60
+  delay: 20
+  when: kibana_restart is changed
+  no_log: "{{ not debug_mode }}" 
diff --git a/ansible/rollback_lme.yml b/ansible/rollback_lme.yml
@@ -137,7 +137,7 @@
 
     - name: Show backup list with versions
       debug:
-        msg: "{{ dict(range(1, backup_display_list|length + 1) | zip(backup_display_list)) }}"
+        msg: "{{ backup_display_list }}"
       when: backup_display_list is defined and backup_display_list | length > 0
 
     - name: Prompt for backup selection
@@ -922,4 +922,4 @@
 
     - name: Show cleanup instructions
       debug:
-        msg: "{{ dict(range(1, status_file.stdout_lines|length + 1) | zip(status_file.stdout_lines)) }}" 
+        msg: "{{ status_file.stdout_lines }}" 
diff --git a/ansible/site.yml b/ansible/site.yml
@@ -31,7 +31,7 @@
     fleet_port: 8220
 
     # Kibana version
-    kibana_version: "8.18.0"
+    kibana_version: "9.0.2"
 
     # Default timezone settings
     timezone_area: "Etc"  # Change to your area: America, Europe, Asia, etc.

diff --git a/ansible/upgrade_lme.yml b/ansible/upgrade_lme.yml
@@ -7,7 +7,10 @@
     # Common variables
     clone_directory: "{{ clone_dir | default(playbook_dir + '/..') }}"
     install_user: "{{ ansible_user_id }}"
-    new_lme_version: "2.1.0"
+    new_lme_version: "2.1.1"
+
+    # Backup timestamp - set once and used throughout
+    backup_timestamp: "{{ ansible_date_time.year }}-{{ ansible_date_time.month }}-{{ ansible_date_time.day }}_{{ ansible_date_time.hour }}-{{ ansible_date_time.minute }}"
 
     # Directory configuration
     config_dir: "/etc/lme"
@@ -262,7 +265,21 @@
         dest: "/opt/lme/config/containers.txt"
         remote_src: yes
       register: containers_copied
-
+
+    - name: Remove old elasticsearch repo path for 9.0.2 upgrade
+      lineinfile:
+        path: /opt/lme/config/elasticsearch.yml
+        line: "    - /usr/share/elasticsearch"
+        state: absent
+      register: elasticsearch_config_updated
+
+    - name: Display configuration updates
+      debug:
+        msg: |
+          Configuration updates completed:
+          - Containers file: {{ 'Updated' if containers_copied.changed else 'No changes needed' }}
+          - Elasticsearch config: {{ 'Updated (removed old repo path)' if elasticsearch_config_updated.changed else 'Already correct' }}
+
     - name: Update STACK_VERSION in environment file
       lineinfile:
         path: "{{ env_file_path }}"
@@ -354,7 +371,7 @@
 
     - name: Create backup directory for quadlet files
       file:
-        path: "/opt/lme/backup/quadlet-{{ ansible_date_time.epoch }}"
+        path: "/opt/lme/backup/quadlet-{{ backup_timestamp }}"
         state: directory
         owner: root
         group: root
@@ -516,20 +533,19 @@
 
     - name: Upgrade complete message
       debug:
-        msg: |
-          {% set message_lines = [
-            'LME has been successfully upgraded to version ' + new_lme_version,
-            '',
-            'Container versions:',
-            'Elasticsearch: ' + elasticsearch_image,
-            'Elastic Agent: docker.elastic.co/beats/elastic-agent:' + elasticsearch_version,
-            'Kibana: ' + kibana_image,
-            'Wazuh: ' + wazuh_image,
-            'ElastAlert: ' + elastalert_image,
-            '',
-            'Configuration updates:',
-            '- Quadlet files: Updated with latest container configurations',
-            '- Previous quadlet files backed up to: ' + quadlet_backup_dir.path,
-            '- Systemd daemon: Reloaded to recognize changes'
-          ] %}
-          {{ dict(range(1, message_lines|length + 1) | zip(message_lines)) }} 
+        msg:
+          - "LME has been successfully upgraded to version {{ new_lme_version }}"
+          - ""
+          - "Container versions:"
+          - "- Elasticsearch: {{ elasticsearch_image }}"
+          - "- Elastic Agent: docker.elastic.co/beats/elastic-agent:{{ elasticsearch_version }}"
+          - "- Kibana: {{ kibana_image }}"
+          - "- Wazuh: {{ wazuh_image }}"
+          - "- ElastAlert: {{ elastalert_image }}"
+          - ""
+          - "Configuration updates:"
+          - "- Containers file: {{ 'Updated' if containers_copied.changed else 'No changes needed' }}"
+          - "- Elasticsearch config: {{ 'Updated (removed old repo path)' if elasticsearch_config_updated.changed else 'Already correct' }}"
+          - "- Quadlet files: {{ 'Updated' if quadlet_copy_result.changed else 'No changes needed' }}"
+          - "- Previous quadlet files backed up to: {{ quadlet_backup_dir.path }}"
+          - "- Systemd daemon: {{ 'Reloaded' if daemon_reload_result.changed else 'Already current' }}" 
diff --git a/config/containers.txt b/config/containers.txt
@@ -1,5 +1,5 @@
-docker.elastic.co/elasticsearch/elasticsearch:8.18.0
-docker.elastic.co/beats/elastic-agent:8.18.0
-docker.elastic.co/kibana/kibana:8.18.0
+docker.elastic.co/elasticsearch/elasticsearch:9.0.2
+docker.elastic.co/elastic-agent/elastic-agent:9.0.2
+docker.elastic.co/kibana/kibana:9.0.2
 docker.io/wazuh/wazuh-manager:4.9.1
 docker.io/jertel/elastalert2:2.20.0
diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml
@@ -2,6 +2,5 @@ cluster.name: "docker-cluster"
 network.host: 0.0.0.0
 path:
   repo:
-    - /usr/share/elasticsearch
     - /usr/share/elasticsearch/backups
 
diff --git a/config/example.env b/config/example.env
@@ -5,7 +5,7 @@
 IPVAR=127.0.0.1
 
 # LME Version
-LME_VERSION=2.1.0
+LME_VERSION=2.1.1
 
 # ElasticSearch settings
 ########################
@@ -26,7 +26,7 @@ LOCAL_ES_URL=https://127.0.0.1:9200
 #################
 
 # Version of Elastic products
-STACK_VERSION=8.18.0
+STACK_VERSION=9.0.1
 # Testing pre-releases? Use the SNAPSHOT option below:
 # STACK_VERSION=8.11.0-SNAPSHOT
 #

diff --git a/quadlet/lme-kibana.container b/quadlet/lme-kibana.container
@@ -26,6 +26,6 @@ Volume=lme_certs:/usr/share/kibana/config/certs
 Volume=lme_kibanadata:/usr/share/kibana/data
 Volume=/opt/lme/config/kibana.yml:/usr/share/kibana/config/kibana.yml
 Volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro
-HealthCmd=CMD-SHELL curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'
+HealthCmd=CMD-SHELL curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/[12].* 302'
 Notify=healthy
 UserNS=auto:uidmapping=0:168584:3048,gidmapping=0:168584:3048
diff --git a/scripts/check_fleet_api.sh b/scripts/check_fleet_api.sh
@@ -13,7 +13,7 @@ get_script_path() {
 SCRIPT_DIR="$(get_script_path)"
 
 HEADERS=(
-  -H "kbn-version: 8.18.0"
+  -H "kbn-version: 9.0.2"
   -H "kbn-xsrf: kibana"
   -H 'Content-Type: application/json'
 )

diff --git a/scripts/set-fleet.sh b/scripts/set-fleet.sh
@@ -13,7 +13,7 @@ get_script_path() {
 SCRIPT_DIR="$(get_script_path)"
 
 HEADERS=(
-  -H "kbn-version: 8.18.0"
+  -H "kbn-version: 9.0.2"
   -H "kbn-xsrf: kibana"
   -H 'Content-Type: application/json'
 )
@@ -51,10 +51,13 @@ wait_for_fleet() {
 
 set_fleet_values() {
   fingerprint=$(/nix/var/nix/profiles/default/bin/podman exec -w /usr/share/elasticsearch/config/certs/ca lme-elasticsearch cat ca.crt  | openssl x509 -nout -fingerprint -sha256 | cut -d "=" -f 2| tr -d : | head -n1)
-  fleet_api_response=$(printf '{"fleet_server_hosts": ["%s"]}' "https://${IPVAR}:${FLEET_PORT}" | curl -kL -v --user "elastic:${elastic}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @-)
-
-  echo "Fleet API Response:"
-  echo "$fleet_api_response"
+
+  # The fleet_server_hosts field has been removed from /api/fleet/settings in version 9.0
+  # This configuration should now be done through Kibana configuration
+  # Commenting out the deprecated API call:
+  # fleet_api_response=$(printf '{"fleet_server_hosts": ["%s"]}' "https://${IPVAR}:${FLEET_PORT}" | curl -kL -v --user "elastic:${elastic}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @-)
+  # echo "Fleet API Response:"
+  # echo "$fleet_api_response"
 
   printf '{"hosts": ["%s"]}' "https://${IPVAR}:9200" | curl -kL --silent --user "elastic:${elastic}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq
   printf '{"ca_trusted_fingerprint": "%s"}' "${fingerprint}" | curl -kL --silent --user "elastic:${elastic}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq

diff --git a/scripts/upgrade/version_upgrade.md b/scripts/upgrade/version_upgrade.md
@@ -80,8 +80,8 @@ The installer will:
 ### Container Versions
 
 LME uses these container versions:
-- Elasticsearch/Elastic Agent: 8.18.0
-- Kibana: 8.18.0
+- Elasticsearch/Elastic Agent: 9.0.2
+- Kibana: 9.0.2
 - Wazuh Manager: 4.9.1
 - ElastAlert2: 2.20.0