add support for api_key authentication in xpack management and monito…

…ring. (elastic#11864)

docs/static/settings/configuration-management-settings.asciidoc

@@ -99,3 +99,10 @@ and `xpack.management.elasticsearch.password`. If `cloud_auth` is configured,
 those settings should not be used.
 The credentials you specify here should be for a user with the `logstash_admin` role, which
 provides access to `.logstash-*` indices for managing configurations.
+
+`xpack.management.elasticsearch.api_key`::
+
+Authenticate using an Elasticsearch API key. Note that this option also requires using SSL.
+
+The API key Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch
+https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].

docs/static/settings/monitoring-settings-legacy.asciidoc

@@ -104,3 +104,10 @@ If you're using {es} in {ecloud}, you can set your auth credentials here.
 This setting is an alternative to both `xpack.monitoring.elasticsearch.username`
 and `xpack.monitoring.elasticsearch.password`. If `cloud_auth` is configured,
 those settings should not be used.
+
+`xpack.monitoring.elasticsearch.api_key`::
+
+Authenticate using an Elasticsearch API key. Note that this option also requires using SSL.
+
+The API key Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch
+https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].

x-pack/lib/config_management/elasticsearch_source.rb

@@ -34,22 +34,9 @@ class RemoteConfigError < LogStash::Error; end
 
       def initialize(settings)
         super(settings)
-        if @settings.get("xpack.management.enabled")
-          if @settings.get_setting("xpack.management.elasticsearch.cloud_id").set?
-            if !@settings.get_setting("xpack.management.elasticsearch.cloud_auth").set?
-              raise ArgumentError.new("You must set credentials using \"xpack.management.elasticsearch.cloud_auth\", " +
-                                      "when using \"xpack.management.elasticsearch.cloud_id\" in logstash.yml")
-            end
-          else
-            if !@settings.get_setting("xpack.management.elasticsearch.password").set?
-              raise ArgumentError.new("You must set the password using \"xpack.management.elasticsearch.password\" in logstash.yml")
-            end
-          end
-        end
-
-        @es_options = es_options_from_settings('management', settings)
 
         if enabled?
+          @es_options = es_options_from_settings('management', settings)
           setup_license_checker(FEATURE_INTERNAL)
           license_check(true)
         end

x-pack/lib/config_management/extension.rb

@@ -29,6 +29,7 @@ def additionals_settings(settings)
         settings.register(LogStash::Setting::ArrayCoercible.new("xpack.management.elasticsearch.hosts", String, [ "https://localhost:9200" ] ))
         settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.cloud_id"))
         settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.cloud_auth"))
+        settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.api_key"))
         settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.proxy"))
         settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.ssl.certificate_authority"))
         settings.register(LogStash::Setting::NullableString.new("xpack.management.elasticsearch.ssl.truststore.path"))

x-pack/lib/helpers/elasticsearch_options.rb

@@ -6,17 +6,34 @@ module LogStash module Helpers
   module ElasticsearchOptions
     extend self
 
-    ES_SETTINGS =%w(
-        ssl.certificate_authority
-        ssl.truststore.path
-        ssl.keystore.path
-        hosts
-        username
-        password
-        cloud_id
-        cloud_auth
-        proxy
-      )
+    ES_SETTINGS = %w(
+      ssl.certificate_authority
+      ssl.truststore.path
+      ssl.keystore.path
+      hosts
+      username
+      password
+      cloud_id
+      cloud_auth
+      api_key
+      proxy
+    )
+
+    # xpack setting to ES output setting
+    SETTINGS_MAPPINGS = {
+      "cloud_id" => "cloud_id",
+      "cloud_auth" => "cloud_auth",
+      "username" => "user",
+      "password" => "password",
+      "api_key" => "api_key",
+      "proxy" => "proxy",
+      "sniffing" => "sniffing",
+      "ssl.certificate_authority" => "cacert",
+      "ssl.truststore.path" => "truststore",
+      "ssl.truststore.password" => "truststore_password",
+      "ssl.keystore.path" => "keystore",
+      "ssl.keystore.password" => "keystore_password",
+    }
 
     # Retrieve elasticsearch options from either specific settings, or modules if the setting is not there and the
     # feature supports falling back to modules if the feature is not specified in logstash.yml
@@ -27,53 +44,53 @@ def es_options_from_settings_or_modules(feature, settings)
     # Populate the Elasticsearch options from LogStashSettings file, based on the feature that is being used.
     # @return Hash
     def es_options_from_settings(feature, settings)
-      prefix = if feature == "monitoring" &&
-                    LogStash::MonitoringExtension.use_direct_shipping?(settings)
-                 ""
-               else
-                 "xpack."
-               end
+      prefix = (feature == "monitoring" && LogStash::MonitoringExtension.use_direct_shipping?(settings)) ? "" : "xpack."
       opts = {}
 
-      if cloud_id = settings.get("#{prefix}#{feature}.elasticsearch.cloud_id")
-        opts['cloud_id'] = cloud_id
-        check_cloud_id_configuration!(feature, settings, prefix)
-      else
-        opts['hosts'] = settings.get("#{prefix}#{feature}.elasticsearch.hosts")
-      end
-      if cloud_auth = settings.get("#{prefix}#{feature}.elasticsearch.cloud_auth")
-        opts['cloud_auth'] = cloud_auth
-        check_cloud_auth_configuration!(feature, settings, prefix)
-      else
-        opts['user'] = settings.get("#{prefix}#{feature}.elasticsearch.username")
-        opts['password'] = settings.get("#{prefix}#{feature}.elasticsearch.password")
-      end
-      if proxysetting = settings.get("#{prefix}#{feature}.elasticsearch.proxy")
-        opts['proxy'] = proxysetting
+      validate_authentication!(feature, settings, prefix)
+
+      # transpose all directly mappable settings
+      SETTINGS_MAPPINGS.each do |xpack_setting, es_setting|
+        v = settings.get("#{prefix}#{feature}.elasticsearch.#{xpack_setting}")
+        opts[es_setting] = v unless v.nil?
       end
 
-      opts['sniffing'] = settings.get("#{prefix}#{feature}.elasticsearch.sniffing")
-      opts['ssl_certificate_verification'] = settings.get("#{prefix}#{feature}.elasticsearch.ssl.verification_mode") == 'certificate'
+      # process remaining settings
 
-      if cacert = settings.get("#{prefix}#{feature}.elasticsearch.ssl.certificate_authority")
-        opts['cacert'] = cacert
-        opts['ssl'] = true
+      unless settings.get("#{prefix}#{feature}.elasticsearch.cloud_id")
+        opts['hosts'] = settings.get("#{prefix}#{feature}.elasticsearch.hosts")
       end
+      opts['ssl_certificate_verification'] = settings.get("#{prefix}#{feature}.elasticsearch.ssl.verification_mode") == 'certificate'
 
-      if truststore = settings.get("#{prefix}#{feature}.elasticsearch.ssl.truststore.path")
-        opts['truststore'] = truststore
-        opts['truststore_password'] = settings.get("#{prefix}#{feature}.elasticsearch.ssl.truststore.password")
+      # if all hosts are using https or any of the ssl related settings are set
+      if ssl?(feature, settings, prefix)
         opts['ssl'] = true
       end
 
-      if keystore = settings.get("#{prefix}#{feature}.elasticsearch.ssl.keystore.path")
-        opts['keystore'] = keystore
-        opts['keystore_password']= settings.get("#{prefix}#{feature}.elasticsearch.ssl.keystore.password")
-        opts['ssl'] = true
+      # the username setting has a default value and should not be included when using another authentication
+      # it is safe to silently remove here since all authentication verifications have been validated at this point.
+      if settings.set?("#{prefix}#{feature}.elasticsearch.cloud_auth") || settings.set?("#{prefix}#{feature}.elasticsearch.api_key")
+        opts.delete('user')
       end
+
       opts
     end
 
+    def ssl?(feature, settings, prefix)
+      return true if verify_https_scheme(feature, settings, prefix)
+      return true if settings.set?("#{prefix}#{feature}.elasticsearch.cloud_id") # cloud_id always resolves to https hosts
+      return true if settings.set?("#{prefix}#{feature}.elasticsearch.ssl.certificate_authority")
+      return true if settings.set?("#{prefix}#{feature}.elasticsearch.ssl.truststore.path") && settings.set?("#{prefix}#{feature}.elasticsearch.ssl.truststore.password")
+      return true if settings.set?("#{prefix}#{feature}.elasticsearch.ssl.keystore.path") && settings.set?("#{prefix}#{feature}.elasticsearch.ssl.keystore.password")
+
+      return false
+    end
+
+    HTTPS_SCHEME = /^https:\/\/.+/
+    def verify_https_scheme(feature, settings, prefix)
+      hosts = Array(settings.get("#{prefix}#{feature}.elasticsearch.hosts"))
+      hosts.all? {|host| host.match?(HTTPS_SCHEME)}
+    end
 
     # Elasticsearch settings can be extracted from the modules settings inside the configuration.
     # Few options will be supported, however - the modules security configuration is
@@ -113,9 +130,6 @@ def only_modules_configured?(feature, settings)
     end
 
     # If no settings are configured, then assume that the feature has not been configured.
-    # The assumption is that with security setup, at least one setting (password or certificates)
-    # should be configured. If security is not setup, and defaults 'just work' for monitoring, then
-    # this will need to be reconsidered.
     def feature_configured?(feature, settings)
       ES_SETTINGS.each do |option|
         return true if settings.set?("xpack.#{feature}.elasticsearch.#{option}")
@@ -146,20 +160,62 @@ def extract_module_settings(settings)
 
     private
 
-    def check_cloud_id_configuration!(feature, settings, prefix)
-      return if !settings.set?("#{prefix}#{feature}.elasticsearch.hosts")
+    def validate_authentication!(feature, settings, prefix)
+      provided_cloud_id = settings.set?("#{prefix}#{feature}.elasticsearch.cloud_id")
+      provided_hosts = settings.set?("#{prefix}#{feature}.elasticsearch.hosts")
+      provided_cloud_auth = settings.set?("#{prefix}#{feature}.elasticsearch.cloud_auth")
+      provided_api_key = settings.set?("#{prefix}#{feature}.elasticsearch.api_key")
+      provided_username = settings.set?("#{prefix}#{feature}.elasticsearch.username")
+      provided_password = settings.set?("#{prefix}#{feature}.elasticsearch.password")
+
+      # note that the username setting has a default value and in the verifications below
+      # we can test on the password option being set as a proxy to using basic auth because
+      # if the username is not explicitly set it will use its default value.
+
+      if provided_cloud_auth && (provided_username || provided_password)
+        raise ArgumentError.new(
+          "Both #{prefix}#{feature}.elasticsearch.cloud_auth and " +
+          "#{prefix}#{feature}.elasticsearch.username/password " +
+          "specified, please only use one of those"
+        )
+      end
+
+      if provided_username && !provided_password
+        raise(ArgumentError,
+          "When using #{prefix}#{feature}.elasticsearch.username, " +
+          "#{prefix}#{feature}.elasticsearch.password must also be set"
+        )
+      end
 
-      raise ArgumentError.new("Both \"#{prefix}#{feature}.elasticsearch.cloud_id\" and " +
-                              "\"#{prefix}#{feature}.elasticsearch.hosts\" specified, please only use one of those.")
-    end
+      if provided_cloud_id
+        if provided_hosts
+          raise(ArgumentError,
+            "Both #{prefix}#{feature}.elasticsearch.cloud_id and " +
+            "#{prefix}#{feature}.elasticsearch.hosts specified, please only use one of those"
+          )
+        end
+      end
 
-    def check_cloud_auth_configuration!(feature, settings, prefix)
-      return if !settings.set?("#{prefix}#{feature}.elasticsearch.username") &&
-                !settings.set?("#{prefix}#{feature}.elasticsearch.password")
+      authentication_count = 0
+      authentication_count += 1 if provided_cloud_auth
+      authentication_count += 1 if provided_password
+      authentication_count += 1 if provided_api_key
+
+      if authentication_count == 0
+        # when no explicit authentication is set it is relying on default username
+        # but without and explicit password set
+        raise(ArgumentError,
+          "With the default #{prefix}#{feature}.elasticsearch.username, " +
+          "#{prefix}#{feature}.elasticsearch.password must be set"
+        )
+      end
 
-      raise ArgumentError.new("Both \"#{prefix}#{feature}.elasticsearch.cloud_auth\" and " +
-                              "\"#{prefix}#{feature}.elasticsearch.username\"/\"#{prefix}#{feature}.elasticsearch.password\" " +
-                              "specified, please only use one of those.")
-    end
+      if authentication_count > 1
+        raise(ArgumentError, "Multiple authentication options are specified, please only use one of #{prefix}#{feature}.elasticsearch.username/password, #{prefix}#{feature}.elasticsearch.cloud_auth or #{prefix}#{feature}.elasticsearch.api_key")
+      end
 
+      if provided_api_key && !ssl?(feature, settings, prefix)
+        raise(ArgumentError, "Using api_key authentication requires SSL/TLS secured communication")
+      end
+    end
   end end end

x-pack/lib/monitoring/internal_pipeline_source.rb

@@ -13,10 +13,10 @@ class InternalPipelineSource < LogStash::Config::Source::Base
     include LogStash::Util::Loggable
     FEATURE = 'monitoring'
 
-    def initialize(pipeline_config, agent)
+    def initialize(pipeline_config, agent, settings)
       super(pipeline_config.settings)
       @pipeline_config = pipeline_config
-      @settings = LogStash::SETTINGS.clone
+      @settings = settings
       @agent = agent
       @es_options = es_options_from_settings_or_modules(FEATURE, @settings)
       setup_license_checker(FEATURE)

x-pack/lib/monitoring/monitoring.rb

@@ -31,7 +31,9 @@ def initialize(node_uuid,
         @password = es_settings['password']
         @cloud_id = es_settings['cloud_id']
         @cloud_auth = es_settings['cloud_auth']
+        @api_key = es_settings['api_key']
         @proxy = es_settings['proxy']
+        @ssl = es_settings['ssl']
         @ca_path = es_settings['cacert']
         @truststore_path = es_settings['truststore']
         @truststore_password = es_settings['truststore_password']
@@ -41,8 +43,8 @@ def initialize(node_uuid,
         @ssl_certificate_verification = (es_settings['verification_mode'] == 'certificate')
       end
 
-      attr_accessor :system_api_version, :es_hosts, :user, :password, :node_uuid, :cloud_id, :cloud_auth, :proxy
-      attr_accessor :ca_path, :truststore_path, :truststore_password
+      attr_accessor :system_api_version, :es_hosts, :user, :password, :node_uuid, :cloud_id, :cloud_auth, :api_key
+      attr_accessor :proxy, :ssl, :ca_path, :truststore_path, :truststore_password
       attr_accessor :keystore_path, :keystore_password, :sniffing, :ssl_certificate_verification
 
       def collection_interval
@@ -69,8 +71,12 @@ def auth?
         user && password
       end
 
+      def api_key?
+        api_key
+      end
+
       def ssl?
-        ca_path || (truststore_path && truststore_password) || (keystore_path && keystore_password)
+        ssl || ca_path || (truststore_path && truststore_password) || (keystore_path && keystore_password)
       end
 
       def truststore?
@@ -132,7 +138,7 @@ def after_agent(runner)
 
         logger.trace("registering the metrics pipeline")
         LogStash::SETTINGS.set("node.uuid", runner.agent.id)
-        internal_pipeline_source = LogStash::Monitoring::InternalPipelineSource.new(setup_metrics_pipeline, runner.agent)
+        internal_pipeline_source = LogStash::Monitoring::InternalPipelineSource.new(setup_metrics_pipeline, runner.agent, LogStash::SETTINGS.clone)
         runner.source_loader.add_source(internal_pipeline_source)
       rescue => e
         logger.error("Failed to set up the metrics pipeline", :message => e.message, :backtrace => e.backtrace)
@@ -255,6 +261,7 @@ def register_monitoring_settings(settings, prefix = "")
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.proxy"))
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.cloud_id"))
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.cloud_auth"))
+      settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.api_key"))
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.ssl.certificate_authority"))
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.ssl.truststore.path"))
       settings.register(LogStash::Setting::NullableString.new("#{prefix}monitoring.elasticsearch.ssl.truststore.password"))

x-pack/lib/template.cfg.erb

@@ -12,13 +12,20 @@ input {
 }
 output {
   elasticsearch_monitoring {
+  <% if auth? %>
+    user => "<%= user %>"
+    password => "<%= password %>"
+  <% end %>
+  <% if api_key? %>
+    api_key => "<%= api_key %>"
+  <% end %>
   <% if cloud_id? %>
     cloud_id => "<%= cloud_id %>"
-    <% if cloud_auth %>
-    cloud_auth => "<%= cloud_auth %>"
-    <% end %>
   <% else %>
     hosts => <%= es_hosts %>
+  <% end %>
+  <% if cloud_auth %>
+    cloud_auth => "<%= cloud_auth %>"
   <% end %>
     bulk_path => "<%= monitoring_endpoint %>"
     manage_template => false
@@ -28,10 +35,6 @@ output {
   <% if proxy? %>
     proxy => "<%= proxy %>"
   <% end %>
-  <% if auth? && !cloud_auth? %>
-    user => "<%= user %>"
-    password => "<%= password %>"
-  <% end %>
   <% if ssl? %>
     ssl => true
     <% if ca_path %>