Libbeat: Add support for API keys in Elasticsearch outputs (elastic#1…

…4324) Adds API key support to both the Elasticsearch and Elasticsearch Monitoring outputs.
fearful-symmetry · Nov 1, 2019 · d592096 · d592096
1 parent ab3def8
commit d592096
Show file tree

Hide file tree

Showing 38 changed files with 353 additions and 46 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -315,6 +315,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add shared_credential_file option in aws related config for specifying credential file directory. {issue}14157[14157] {pull}14178[14178]
 - GA the `script` processor. {pull}14325[14325]
 - Add `fingerprint` processor. {issue}11173[11173] {pull}14205[14205]
+- Add support for API keys in Elasticsearch outputs. {pull}14324[14324]
 
 *Auditbeat*
 

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -414,8 +414,11 @@ output.elasticsearch:
   # Configure escaping HTML symbols in strings.
   #escape_html: false
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 
@@ -1252,8 +1255,11 @@ logging.files:
   # Set gzip compression level.
   #compression_level: 0
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "beats_system"
   #password: "changeme"
 

diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
@@ -121,8 +121,11 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1111,8 +1111,11 @@ output.elasticsearch:
   # Configure escaping HTML symbols in strings.
   #escape_html: false
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 
@@ -1949,8 +1952,11 @@ logging.files:
   # Set gzip compression level.
   #compression_level: 0
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "beats_system"
   #password: "changeme"
 

diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -149,8 +149,11 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -558,8 +558,11 @@ output.elasticsearch:
   # Configure escaping HTML symbols in strings.
   #escape_html: false
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 
@@ -1396,8 +1399,11 @@ logging.files:
   # Set gzip compression level.
   #compression_level: 0
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "beats_system"
   #password: "changeme"
 

diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml
@@ -94,8 +94,11 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -352,8 +352,11 @@ output.elasticsearch:
   # Configure escaping HTML symbols in strings.
   #escape_html: false
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 
@@ -1190,8 +1193,11 @@ logging.files:
   # Set gzip compression level.
   #compression_level: 0
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "beats_system"
   #password: "changeme"
 

diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml
@@ -118,8 +118,11 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 

diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl
@@ -295,8 +295,11 @@ output.elasticsearch:
   # Configure escaping HTML symbols in strings.
   #escape_html: false
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 
@@ -1133,8 +1136,11 @@ logging.files:
   # Set gzip compression level.
   #compression_level: 0
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "beats_system"
   #password: "changeme"
 

diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl
@@ -66,8 +66,11 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Optional protocol and basic auth credentials.
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 {{if not .ExcludeLogstash}}

diff --git a/libbeat/docs/monitoring/monitoring-internal-collection.asciidoc b/libbeat/docs/monitoring/monitoring-internal-collection.asciidoc
@@ -28,10 +28,11 @@ To learn about monitoring in general, see
 //TODO: Not sure if these docs need to be updated to be parallel with other
 //stack components since this is the old way of configuring monitoring. 
 
-. Create a user that has appropriate authority to send system-level monitoring
+. Create an API key or user that has appropriate authority to send system-level monitoring
 data to {es}. For example, you can use the built-in +{beat_monitoring_user}+ user or
 assign the built-in +{beat_monitoring_user}+ role to another user. For more
-information, see <<privileges-to-publish-monitoring>>.
+information on the required privileges, see <<privileges-to-publish-monitoring>>.
+For more information on how to use API keys, see <<beats-api-keys>>.
 
 . Add the `monitoring` settings in the {beatname_uc} configuration file. If you
 configured the {es} output and want to send {beatname_uc} monitoring events to
@@ -42,9 +43,11 @@ the same {es} cluster, specify the following minimal configuration:
 monitoring:
   enabled: true
   elasticsearch:
+    api_key:  id:api_key <1>
     username: {beat_monitoring_user}
     password: somepassword
 --------------------
+<1> Specify one of `api_key` or `username`/`password`.
 +
 If you
 ifndef::no-output-logstash[]
@@ -61,6 +64,7 @@ monitoring:
   cluster_uuid: PRODUCTION_ES_CLUSTER_UUID <1>
   elasticsearch:
     hosts: ["https://example.com:9200", "https://example2.com:9200"] <2>
+    api_key:  id:api_key <3>
     username: {beat_monitoring_user}
     password: somepassword
 --------------------
@@ -70,6 +74,7 @@ Stack Monitoring UI. To get a cluster's `cluster_uuid`,
 call the `GET /` API against that cluster.
 <2> This setting identifies the hosts and port numbers of {es} nodes
 that are part of the monitoring cluster.
+<3> Specify one of `api_key` or `username`/`password`.
 +
 If you want to use PKI authentication to send monitoring events to
 {es}, you must specify a different set of configuration options. For

diff --git a/libbeat/docs/monitoring/monitoring-metricbeat.asciidoc b/libbeat/docs/monitoring/monitoring-metricbeat.asciidoc
@@ -220,11 +220,13 @@ output.elasticsearch:
   
   # Optional protocol and basic auth credentials.
   #protocol: "https"
+  #api_key:  "id:api_key" <2>
   #username: "elastic"
   #password: "changeme"
 ----------------------------------
 <1> In this example, the data is stored on a monitoring cluster with nodes 
 `es-mon-1` and `es-mon-2`.
+<2> Specify one of `api_key` or `username`/`password`.
 
 If you configured the monitoring cluster to use encrypted communications, you
 must access it via HTTPS. For example, use a `hosts` setting like

diff --git a/libbeat/docs/monitoring/shared-monitor-config.asciidoc b/libbeat/docs/monitoring/shared-monitor-config.asciidoc
@@ -32,6 +32,11 @@ The default value is `false`.
 The {es} instances that you want to ship your {beatname_uc} metrics to. This
 configuration option contains the following fields:
 
+==== `api_key`
+
+The detail of the API key to be used to send monitoring information to {es}.
+See <<beats-api-keys>> for more information.
+
 ==== `bulk_max_size`
 
 The maximum number of metrics to bulk in a single {es} bulk API index request.

diff --git a/libbeat/docs/security/api-keys.asciidoc b/libbeat/docs/security/api-keys.asciidoc
@@ -0,0 +1,116 @@
+[role="xpack"]
+[[beats-api-keys]]
+=== Grant access using API keys
+
+Instead of using usernames and passwords, you can use API keys to grant
+access to {es} resources. You can set API keys to expire at a certain time,
+and you can explicitly invalidate them. Any user with the `manage_api_key`
+or `manage_own_api_key` cluster privilege can create API keys.
+
+See the {es} API key documentation for more information:
+
+* {ref}/security-api-create-api-key.html[Create API key]
+* {ref}/security-api-get-api-key.html[Get API key information]
+* {ref}/security-api-invalidate-api-key.html[Invalidate API key]
+
+{beatname_uc} instances typically send both collected data and monitoring
+information to {es}. If you are sending both to the same cluster, you can use the same
+API key. For different clusters, you need to use an API key per cluster.
+
+NOTE: For security reasons, we recommend using a unique API key per Beat instance.
+You can create as many API keys per user as necessary.
+
+[[beats-api-key-publish]]
+==== Create an API key for publishing
+To create an API key to use for writing data to {es}, use the
+{ref}/security-api-create-api-key.html[Create API key API], for example:
+
+[source,console,subs="attributes,callouts"]
+------------------------------------------------------------
+POST /_security/api_key
+{
+  "name": "{beat_default_index_prefix}_host001", <1>
+  "role_descriptors": {
+    "{beat_default_index_prefix}_writer": { <2>
+      "cluster": ["monitor", "read_ilm"],
+      "index": [
+        {
+          "names": ["{beat_default_index_prefix}-*"],
+          "privileges": ["view_index_metadata", "create_doc"]
+        }
+      ]
+    }
+  }
+}
+------------------------------------------------------------
+<1> Name of the API key
+<2> Granted privileges, see <<feature-roles>>
+
+The return value will look something like this:
+
+[source,console-result,subs="attributes,callouts"]
+--------------------------------------------------
+{
+  "id":"TiNAGG4BaaMdaH1tRfuU", <1>
+  "name":"{beat_default_index_prefix}_host001",
+  "api_key":"KnR6yE41RrSowb0kQ0HWoA" <2>
+}
+--------------------------------------------------
+<1> Unique id for this API key
+<2> Generated API key
+
+You can now use this API key in your +{beatname_lc}.yml+ configuration file like this:
+["source","yaml"]
+--------------------
+output.elasticsearch:
+  api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA <1>
+--------------------
+<1> Format is `id:api_key` (as returned by {ref}/security-api-create-api-key.html[Create API key])
+
+[[beats-api-key-monitor]]
+==== Create an API key for monitoring
+To create an API key to use for sending monitoring data to {es}, use the
+{ref}/security-api-create-api-key.html[Create API key API], for example:
+
+[source,console,subs="attributes,callouts"]
+------------------------------------------------------------
+POST /_security/api_key
+{
+  "name": "{beat_default_index_prefix}_host001", <1>
+  "role_descriptors": {
+    "{beat_default_index_prefix}_monitoring": { <2>
+      "cluster": ["monitor"],
+      "index": [
+        {
+          "names": [".monitoring-beats-*"],
+          "privileges": ["create_index", "create"]
+        }
+      ]
+    }
+  }
+}
+------------------------------------------------------------
+<1> Name of the API key
+<2> Granted privileges, see <<feature-roles>>
+
+The return value will look something like this:
+
+[source,console-result,subs="attributes,callouts"]
+--------------------------------------------------
+{
+  "id":"TiNAGG4BaaMdaH1tRfuU", <1>
+  "name":"{beat_default_index_prefix}_host001",
+  "api_key":"KnR6yE41RrSowb0kQ0HWoA" <2>
+}
+--------------------------------------------------
+<1> Unique id for this API key
+<2> Generated API key
+
+You can now use this API key in your +{beatname_lc}.yml+ configuration file like this:
+["source","yml",subs="attributes"]
+--------------------
+monitoring.elasticsearch:
+  api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA <1>
+--------------------
+<1> Format is `id:api_key` (as returned by {ref}/security-api-create-api-key.html[Create API key])
+