Closed
Description
I just noticed these NPEs (about every 10 seconds) in my Elasticsearch log on 6.6.0 BC1 (on CentOS7 from rpm package install) default distribution.
[2018-12-21T19:58:26,131][INFO ][o.e.x.w.a.l.ExecutableLoggingAction] [GR0TRcN] executed at 2018-12-21T19:58:26.107Z
[2018-12-21T19:58:30,788][ERROR][o.e.x.m.c.c.ClusterStatsCollector] [GR0TRcN] collector [cluster_stats] failed to collect data
java.lang.NullPointerException: null
at org.elasticsearch.xpack.ml.MachineLearningFeatureSet$Retriever.addJobsUsage(MachineLearningFeatureSet.java:224) ~[?:?]
at org.elasticsearch.xpack.ml.MachineLearningFeatureSet$Retriever.lambda$execute$1(MachineLearningFeatureSet.java:197) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction.lambda$gatherStatsForClosedJobs$3(TransportGetJobsStatsAction.java:164) ~[?:?]
at org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction.lambda$gatherDataCountsAndModelSizeStats$5(TransportGetJobsStatsAction.java:180) ~[?:?]
at org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider.lambda$modelSizeStats$25(JobResultsProvider.java:932) ~[?:?]
at org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider.lambda$searchSingleResult$27(JobResultsProvider.java:945) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onResponse(AbstractSearchAsyncAction.java:313) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onResponse(AbstractSearchAsyncAction.java:50) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase$3.run(FetchSearchPhase.java:213) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:160) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:153) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.ExpandSearchPhase.run(ExpandSearchPhase.java:120) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:160) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:153) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase.moveToNextPhase(FetchSearchPhase.java:206) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase.lambda$innerRun$2(FetchSearchPhase.java:104) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase.innerRun(FetchSearchPhase.java:118) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase.access$000(FetchSearchPhase.java:44) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.action.search.FetchSearchPhase$1.doRun(FetchSearchPhase.java:86) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:759) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) ~[elasticsearch-6.6.0.jar:6.6.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.6.0.jar:6.6.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
Larger log file (just has more of those ^ );
elasticsearch.log.gz
Tal suggested I run these queries in case they might help;
GET _xpack/ml/anomaly_detectors/_all/_stats
{
"count" : 10,
"jobs" : [
{
"job_id" : "filebeat-apache2-access-low_request_rate",
"data_counts" : {
"job_id" : "filebeat-apache2-access-low_request_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-apache2-access-low_request_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141937
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-apache2-access-remote_ip_request_rate",
"data_counts" : {
"job_id" : "filebeat-apache2-access-remote_ip_request_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-apache2-access-remote_ip_request_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141937
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-apache2-access-remote_ip_url_count",
"data_counts" : {
"job_id" : "filebeat-apache2-access-remote_ip_url_count",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-apache2-access-remote_ip_url_count",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141939
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-apache2-access-response_code",
"data_counts" : {
"job_id" : "filebeat-apache2-access-response_code",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-apache2-access-response_code",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141936
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-apache2-access-visitor_rate",
"data_counts" : {
"job_id" : "filebeat-apache2-access-visitor_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-apache2-access-visitor_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141936
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-nginx-access-low_request_rate",
"data_counts" : {
"job_id" : "filebeat-nginx-access-low_request_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-nginx-access-low_request_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141936
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-nginx-access-remote_ip_request_rate",
"data_counts" : {
"job_id" : "filebeat-nginx-access-remote_ip_request_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-nginx-access-remote_ip_request_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141936
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-nginx-access-remote_ip_url_count",
"data_counts" : {
"job_id" : "filebeat-nginx-access-remote_ip_url_count",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-nginx-access-remote_ip_url_count",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141936
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-nginx-access-response_code",
"data_counts" : {
"job_id" : "filebeat-nginx-access-response_code",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-nginx-access-response_code",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141937
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
},
{
"job_id" : "filebeat-nginx-access-visitor_rate",
"data_counts" : {
"job_id" : "filebeat-nginx-access-visitor_rate",
"processed_record_count" : 0,
"processed_field_count" : 0,
"input_bytes" : 0,
"input_field_count" : 0,
"invalid_date_count" : 0,
"missing_field_count" : 0,
"out_of_order_timestamp_count" : 0,
"empty_bucket_count" : 0,
"sparse_bucket_count" : 0,
"bucket_count" : 0,
"input_record_count" : 0
},
"model_size_stats" : {
"job_id" : "filebeat-nginx-access-visitor_rate",
"result_type" : "model_size_stats",
"model_bytes" : 0,
"total_by_field_count" : 0,
"total_over_field_count" : 0,
"total_partition_field_count" : 0,
"bucket_allocation_failures_count" : 0,
"memory_status" : "ok",
"log_time" : 1545423141937
},
"forecasts_stats" : {
"total" : 0,
"forecasted_jobs" : 0
},
"state" : "closed"
}
]
}
And
GET _xpack/ml/anomaly_detectors/_all
{
"count" : 10,
"jobs" : [
{
"job_id" : "filebeat-apache2-access-low_request_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"apache2"
],
"description" : "Apache2 Access Logs: Detect low request rate",
"create_time" : 1545409192253,
"analysis_config" : {
"bucket_span" : "15m",
"summary_count_field_name" : "doc_count",
"detectors" : [
{
"detector_description" : "apache2_access_low_request_rate",
"function" : "low_count",
"detector_index" : 0
}
],
"influencers" : [ ]
},
"analysis_limits" : {
"model_memory_limit" : "10mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Apache2-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-apache2-access-remote_ip_request_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"apache2"
],
"description" : "Apache2 Access Logs: Detect unusual remote_ips - high request rates",
"create_time" : 1545409192226,
"analysis_config" : {
"bucket_span" : "1h",
"detectors" : [
{
"detector_description" : "apache2_access_remote_ip_high_count",
"function" : "high_count",
"over_field_name" : "apache2.access.remote_ip",
"detector_index" : 0
}
],
"influencers" : [
"apache2.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "1024mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Count Explorer",
"url_value" : "kibana#/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.remote_ip,negate:!f,type:phrase,value:'$apache2.access.remote_ip$'),query:(match:(apache2.access.remote_ip:(query:'$apache2.access.remote_ip$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Apache2-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.remote_ip,negate:!f,type:phrase,value:'$apache2.access.remote_ip$'),query:(match:(apache2.access.remote_ip:(query:'$apache2.access.remote_ip$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-apache2-access-remote_ip_url_count",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"apache2"
],
"description" : "Apache2 Access Logs: Detect unusual remote_ips - high distinct count of urls",
"create_time" : 1545409192249,
"analysis_config" : {
"bucket_span" : "1h",
"detectors" : [
{
"detector_description" : "apache2_access_remote_ip_high_dc_url",
"function" : "high_distinct_count",
"field_name" : "apache2.access.url",
"over_field_name" : "apache2.access.remote_ip",
"detector_index" : 0
}
],
"influencers" : [
"apache2.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "1024mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "URL Explorer",
"url_value" : "kibana#/dashboard/ML-Apache2-Remote-IP-URL-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.remote_ip,negate:!f,type:phrase,value:'$apache2.access.remote_ip$'),query:(match:(apache2.access.remote_ip:(query:'$apache2.access.remote_ip$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Apache2-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.remote_ip,negate:!f,type:phrase,value:'$apache2.access.remote_ip$'),query:(match:(apache2.access.remote_ip:(query:'$apache2.access.remote_ip$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-apache2-access-response_code",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"apache2"
],
"description" : "Apache2 Access Logs: Detect unusual response_code rates",
"create_time" : 1545409192225,
"analysis_config" : {
"bucket_span" : "15m",
"detectors" : [
{
"detector_description" : "apache2_access_response_code_rate",
"function" : "count",
"partition_field_name" : "apache2.access.response_code",
"detector_index" : 0
}
],
"influencers" : [
"apache2.access.response_code",
"apache2.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "100mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Count Explorer",
"url_value" : "kibana#/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.response_code,negate:!f,type:phrase,value:'$apache2.access.response_code$'),query:(match:(apache2.access.response_code:(query:'$apache2.access.response_code$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Apache2-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:apache2.access.response_code,negate:!f,type:phrase,value:'$apache2.access.response_code$'),query:(match:(apache2.access.response_code:(query:'$apache2.access.response_code$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'_exists_:apache2.access')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-apache2-access-visitor_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"apache2"
],
"description" : "Apache2 Access Logs: Detect unusual visitor rate",
"create_time" : 1545409192225,
"analysis_config" : {
"bucket_span" : "15m",
"summary_count_field_name" : "dc_remote_ips",
"detectors" : [
{
"detector_description" : "apache2_access_visitor_rate",
"function" : "non_zero_count",
"detector_index" : 0
}
],
"influencers" : [ ]
},
"analysis_limits" : {
"model_memory_limit" : "10mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Apache2-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-nginx-access-low_request_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"nginx"
],
"description" : "Nginx Access Logs: Detect low request rate",
"create_time" : 1545409195632,
"analysis_config" : {
"bucket_span" : "15m",
"summary_count_field_name" : "doc_count",
"detectors" : [
{
"detector_description" : "nginx_access_low_request_rate",
"function" : "low_count",
"detector_index" : 0
}
],
"influencers" : [ ]
},
"analysis_limits" : {
"model_memory_limit" : "10mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-nginx-access-remote_ip_request_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"nginx"
],
"description" : "Nginx Access Logs: Detect unusual remote_ips - high request rates",
"create_time" : 1545409195649,
"analysis_config" : {
"bucket_span" : "1h",
"detectors" : [
{
"detector_description" : "nginx_access_remote_ip_high_count",
"function" : "high_count",
"over_field_name" : "nginx.access.remote_ip",
"detector_index" : 0
}
],
"influencers" : [
"nginx.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "1024mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Count Explorer",
"url_value" : "kibana#/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.remote_ip,negate:!f,type:phrase,value:'$nginx.access.remote_ip$'),query:(match:(nginx.access.remote_ip:(query:'$nginx.access.remote_ip$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.remote_ip,negate:!f,type:phrase,value:'$nginx.access.remote_ip$'),query:(match:(nginx.access.remote_ip:(query:'$nginx.access.remote_ip$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-nginx-access-remote_ip_url_count",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"nginx"
],
"description" : "Nginx Access Logs: Detect unusual remote_ips - high distinct count of urls",
"create_time" : 1545409195632,
"analysis_config" : {
"bucket_span" : "1h",
"detectors" : [
{
"detector_description" : "nginx_access_remote_ip_high_dc_url",
"function" : "high_distinct_count",
"field_name" : "nginx.access.url",
"over_field_name" : "nginx.access.remote_ip",
"detector_index" : 0
}
],
"influencers" : [
"nginx.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "1024mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "URL Explorer",
"url_value" : "kibana#/dashboard/ML-Nginx-Remote-IP-URL-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.remote_ip,negate:!f,type:phrase,value:'$nginx.access.remote_ip$'),query:(match:(nginx.access.remote_ip:(query:'$nginx.access.remote_ip$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.remote_ip,negate:!f,type:phrase,value:'$nginx.access.remote_ip$'),query:(match:(nginx.access.remote_ip:(query:'$nginx.access.remote_ip$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-nginx-access-response_code",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"nginx"
],
"description" : "Nginx Access Logs: Detect unusual response_code rates",
"create_time" : 1545409195631,
"analysis_config" : {
"bucket_span" : "15m",
"detectors" : [
{
"detector_description" : "nginx_access_response_code_rate",
"function" : "count",
"partition_field_name" : "nginx.access.response_code",
"detector_index" : 0
}
],
"influencers" : [
"nginx.access.response_code",
"nginx.access.remote_ip"
]
},
"analysis_limits" : {
"model_memory_limit" : "100mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Count Explorer",
"url_value" : "kibana#/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(description:'',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.response_code,negate:!f,type:phrase,value:'$nginx.access.response_code$'),query:(match:(nginx.access.response_code:(query:'$nginx.access.response_code$',type:phrase))))),query:(query_string:(analyze_wildcard:!t,query:'*')))"
},
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'filebeat-*',key:nginx.access.response_code,negate:!f,type:phrase,value:'$nginx.access.response_code$'),query:(match:(nginx.access.response_code:(query:'$nginx.access.response_code$',type:phrase))))),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'_exists_:nginx.access')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
},
{
"job_id" : "filebeat-nginx-access-visitor_rate",
"job_type" : "anomaly_detector",
"job_version" : "6.6.0",
"groups" : [
"nginx"
],
"description" : "Nginx Access Logs: Detect unusual visitor rate",
"create_time" : 1545409195631,
"analysis_config" : {
"bucket_span" : "15m",
"summary_count_field_name" : "dc_remote_ips",
"detectors" : [
{
"detector_description" : "nginx_access_visitor_rate",
"function" : "non_zero_count",
"detector_index" : 0
}
],
"influencers" : [ ]
},
"analysis_limits" : {
"model_memory_limit" : "10mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config" : {
"enabled" : true
},
"model_snapshot_retention_days" : 1,
"custom_settings" : {
"custom_urls" : [
{
"url_name" : "Raw Data",
"url_value" : "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(columns:!(_source),filters:!(),index:'filebeat-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'*')),sort:!('@timestamp',desc))"
}
]
},
"results_index_name" : "shared"
}
]
}