[HOT FIX] 03042019

fix #215 - Logstash plugins offline install (default) - Logstash mutate statements update - ES Memory Calculation fix - Compose files typo
Cyb3rWard0g · Mar 4, 2019 · 1389aae · 1389aae
1 parent cfb9b98
commit 1389aae
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 46 deletions.
diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh
@@ -18,7 +18,7 @@ if [[ -z "$ES_JAVA_OPTS" ]]; then
       ES_MEMORY="4g"
     else
       # Using GB instead of MB -- because plenty of RAM now
-      ES_MEMORY=$(( AVAILABLE_MEMORY / 1024 ))
+      ES_MEMORY=$(( AVAILABLE_MEMORY / 1024 / 2 ))
       if [ $ES_MEMORY -gt 31 ]; then
         ES_MEMORY="31g"
       else

diff --git a/docker/helk-kibana-analysis-basic.yml b/docker/helk-kibana-analysis-basic.yml
@@ -9,7 +9,7 @@ services:
         target: /usr/share/elasticsearch/config/elasticsearch.yml
     volumes:
       - esdata:/usr/share/elasticsearch/data
-      - ./helk-elasticsearch//scripts:/usr/share/elasticsearch/scripts
+      - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
     entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
       - cluster.name=helk-cluster

diff --git a/docker/helk-kibana-notebook-analysis-basic.yml b/docker/helk-kibana-notebook-analysis-basic.yml
@@ -9,7 +9,7 @@ services:
         target: /usr/share/elasticsearch/config/elasticsearch.yml
     volumes:
       - esdata:/usr/share/elasticsearch/data
-      - ./helk-elasticsearch//scripts:/usr/share/elasticsearch/scripts
+      - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
     entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
       - cluster.name=helk-cluster

diff --git a/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf b/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf
@@ -6,36 +6,52 @@
 filter {
     if [event_id] {
         if [Image] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_1" } }
-            mutate { rename => { "Image" => "process_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_1" }
+                rename => { "Image" => "process_path" }
+            }
         }
         if [Application] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_2" } }
-            mutate { rename => { "Application" => "process_path" } }
+            mutate { 
+                add_field => { "z_logstash_pipeline" => "1523_2" }
+                rename => { "Application" => "process_path" }
+            }
         }
         if [NewProcessName] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_3" } }
-            mutate { rename => { "NewProcessName" => "process_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_3" }
+                rename => { "NewProcessName" => "process_path" }
+            }
         }
         if [ProcessName] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_4" } }
-            mutate { rename => { "ProcessName" => "process_path" }}
+            mutate { 
+                add_field => { "z_logstash_pipeline" => "1523_4" }
+                rename => { "ProcessName" => "process_path" }
+            }
         }
         if [ParentProcessName] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_5" } }
-            mutate { rename => { "ParentProcessName" => "process_parent_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_5" }
+                rename => { "ParentProcessName" => "process_parent_path" }
+            }
         }
         if [ParentImage] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_6" } }
-            mutate { rename => { "ParentImage" => "process_parent_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_6" }
+                rename => { "ParentImage" => "process_parent_path" }
+            }
         }
         if [TargetImage] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_7" } }
-            mutate { rename => { "TargetImage" => "process_target_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_7" }
+                rename => { "TargetImage" => "process_target_path" }
+            }
         }
         if [SourceImage] {
-            mutate { add_field => { "z_logstash_pipeline" => "1523_8" } }
-            mutate { rename => { "SourceImage" => "process_path" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1523_8" }
+                rename => { "SourceImage" => "process_path" }
+            }
         }
         if [ProdessName] {
             mutate { rename => { "ProdessName" => "process_path" } }

diff --git a/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf b/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf
@@ -6,48 +6,70 @@
 filter {
     if [event_id] {
         if [ProcessId] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_2" } }
-            mutate { rename => { "ProcessId" => "process_id" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_2" }
+                rename => { "ProcessId" => "process_id" }
+            }
         }
         if [NewProcessId] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_3" } }
-            mutate { rename => { "NewProcessId" => "process_id" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_3" }
+                rename => { "NewProcessId" => "process_id" }
+            }
         }
         if [ParentProcessId] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_5" } }
-            mutate { rename => { "ParentProcessId" => "process_parent_id" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_5" }
+                rename => { "ParentProcessId" => "process_parent_id" }
+            }
         }
         if [ProcessGuid] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_6" } }
-            mutate { rename => { "ProcessGuid" => "process_guid" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_6" }
+                rename => { "ProcessGuid" => "process_guid" }
+            }
         }
         if [ParentProcessGuid] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_7" } }
-            mutate { rename => { "ParentProcessGuid" => "process_parent_guid" } }
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_7" }
+                rename => { "ParentProcessGuid" => "process_parent_guid" }
+            }
         }
         if [SourceProcessGuid] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_8" } }
-            mutate { rename => { "SourceProcessGuid" => "process_guid" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_8" }
+                rename => { "SourceProcessGuid" => "process_guid" }
+            }   
         }
         if [SourceProcessGUID] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_9" } }
-            mutate { rename => { "SourceProcessGUID" => "process_guid" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_9" }
+                rename => { "SourceProcessGUID" => "process_guid" }
+            }   
         }
         if [SourceProcessId] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_11" } }
-            mutate { rename => { "SourceProcessId" => "process_id" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_11" }
+                rename => { "SourceProcessId" => "process_id" }
+            }   
         }
         if [TargetProcessGuid] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_12" } }
-            mutate { rename => { "TargetProcessGuid" => "process_target_guid" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_12" }
+                rename => { "TargetProcessGuid" => "process_target_guid" }
+            }   
         }
         if [TargetProcessGUID] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_13" } }
-            mutate { rename => { "TargetProcessGUID" => "process_target_guid" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_13" }
+                rename => { "TargetProcessGUID" => "process_target_guid" }
+            }   
         }
         if [TargetProcessId] {
-            mutate { add_field => { "z_logstash_pipeline" => "1524_15" } }
-            mutate { rename => { "TargetProcessId" => "process_target_id" } }   
+            mutate {
+                add_field => { "z_logstash_pipeline" => "1524_15" }
+                rename => { "TargetProcessId" => "process_target_id" }
+            }   
         }
     }
 }
diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh
@@ -86,12 +86,11 @@ if ( logstash-plugin list 'prune' ) && ( logstash-plugin list 'i18n' ) && ( logs
     echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Plugins are already installed"
 else
 # logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-input-lumberjack && logstash-plugin install logstash-output-lumberjack && logstash-plugin install logstash-output-zabbix && logstash-plugin install logstash-filter-geoip && logstash-plugin install logstash-codec-cef && logstash-plugin install logstash-output-syslog && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-output-stdout && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-avro && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-snmptrap && logstash-plugin install logstash-input-snmp && logstash-plugin install logstash-input-jdbc && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone
-	echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install Logstash plugins over the Internet.."
-  if (logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-filter-geoip && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone); then
-    echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Logstash plugins installed via the Internet.."
+	if (logstash-plugin install file:///usr/share/logstash/logstash-offline-plugins-6.6.1.zip); then
+    echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Logstash plugins installed via offline package.."
   else
-    echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install logstash plugins from offline package.."
-    logstash-plugin install file:///usr/share/logstash/logstash-offline-plugins-6.6.1.zip
+    echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install logstash plugins over the Internet.."
+    logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-filter-geoip && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone
   fi
 fi