Introduce a new flag to explicitly permit legacy monitoring

.buildkite/scripts/benchmark/config/logstash.yml

@@ -3,6 +3,7 @@ pipeline.workers: ${WORKER}
 pipeline.batch.size: ${BATCH_SIZE}
 queue.type: ${QTYPE}
 
+allow.legacy:.monitoring: true
 xpack.monitoring.enabled: true
 xpack.monitoring.elasticsearch.username: ${MONITOR_ES_USER}
 xpack.monitoring.elasticsearch.password: ${MONITOR_ES_PW}

config/logstash.yml

@@ -334,6 +334,10 @@
 # Default to direct, optionally can be switched to heap to select Java heap space.
 # pipeline.buffer.type: direct
 #
+#
+# Flag to allow the legacy internal monitoring (default: false)
+# allow.legacy.monitoring: false
+#
 # ------------ X-Pack Settings (not applicable for OSS build)--------------
 #
 # X-Pack Monitoring

docker/data/logstash/env2yaml/env2yaml.go

@@ -17,7 +17,6 @@ package main
 import (
 	"errors"
 	"fmt"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"log"
 	"os"
@@ -86,6 +85,7 @@ var validSettings = []string{
 	"api.auth.basic.password_policy.include.symbol",
 	"allow_superuser",
 	"monitoring.cluster_uuid",
+	"allow.legacy.monitoring",
 	"xpack.monitoring.enabled",
 	"xpack.monitoring.collection.interval",
 	"xpack.monitoring.elasticsearch.hosts",

docs/static/monitoring/monitoring-internal-legacy.asciidoc

@@ -7,6 +7,8 @@
 
 deprecated[7.9.0]
 
+NOTE: Starting from version 9.0, legacy monitoring is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.
+
 ==== Components for legacy collection
 
 Monitoring {ls} with legacy collection uses these components:

docs/static/settings-file.asciidoc

@@ -365,4 +365,8 @@ separating each log lines per pipeline could be helpful in case you need to trou
 | Determine where to allocate memory buffers, for plugins that leverage them.
 Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future.
 | `direct` Check out <<reducing-off-heap-usage>> for more info.
+
+| `allow.legacy.monitoring`
+| Set to `true` to allow <<monitoring-internal-collection-legacy,internal legacy monitoring>>.
+| `false`
 |=======================================================================

logstash-core/lib/logstash/api/commands/default_metadata.rb

@@ -71,6 +71,7 @@ def http_address
 
         private
         def enabled_xpack_monitoring?
+          LogStash::SETTINGS.get_value("allow.legacy.monitoring") &&
           LogStash::SETTINGS.registered?("xpack.monitoring.enabled") &&
           LogStash::SETTINGS.get("xpack.monitoring.enabled")
         end

logstash-core/lib/logstash/environment.rb

@@ -110,7 +110,8 @@ module Environment
             Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
             Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on
     Setting::NullableString.new("monitoring.cluster_uuid"),
-            Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"])
+            Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]),
+           Setting::Boolean.new("allow.legacy.monitoring", false)
   # post_process
   ].each {|setting| SETTINGS.register(setting) }
 

logstash-core/spec/logstash/api/commands/default_metadata_spec.rb

@@ -54,13 +54,22 @@ def registerIfNot(setting)
         )
     end
 
-    it "check monitoring exist when monitoring is enabled" do
+    it "check monitoring section exist when legacy monitoring is enabled and allowed" do
+      LogStash::SETTINGS.set_value("allow.legacy.monitoring", true)
       LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true)
       expect(report.keys).to include(
         :monitoring
         )
     end
 
+    it "check monitoring section does not appear when legacy monitoring is not allowed but enabled" do
+      LogStash::SETTINGS.set_value("allow.legacy.monitoring", false)
+      LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true)
+      expect(report.keys).not_to include(
+                               :monitoring
+                             )
+    end
+
     it "check monitoring does not appear when not enabled and nor cluster_uuid is defined" do
       LogStash::SETTINGS.set_value("xpack.monitoring.enabled", false)
       LogStash::SETTINGS.get_setting("monitoring.cluster_uuid").reset

x-pack/lib/monitoring/monitoring.rb

@@ -155,8 +155,10 @@ def after_agent(runner)
       # For versions prior to 6.3 the default value of "xpack.monitoring.enabled" was true
       # For versions 6.3+ the default of "xpack.monitoring.enabled" is false.
       # To help keep passivity, assume that if "xpack.monitoring.elasticsearch.hosts" has been set that monitoring should be enabled.
-      # return true if xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured
+      # return true if allow.legacy.monitoring=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured
       def monitoring_enabled?(settings)
+        log_warn_if_legacy_is_enabled_and_not_allowed(settings)
+        return false unless settings.get_value("allow.legacy.monitoring")
         return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled")
         return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled")
 
@@ -170,6 +172,15 @@ def monitoring_enabled?(settings)
         end
       end
 
+      def log_warn_if_legacy_is_enabled_and_not_allowed(settings)
+        allowed = settings.get_value("allow.legacy.monitoring")
+        legacy_monitoring_enabled = (settings.get_value("xpack.monitoring.enabled") || settings.get_value("monitoring.enabled"))
+        if !allowed && legacy_monitoring_enabled
+          logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.")
+        end
+      end
+      private :log_warn_if_legacy_is_enabled_and_not_allowed
+
       def setup_metrics_pipeline
         settings = LogStash::SETTINGS.clone
 

x-pack/qa/integration/management/multiple_pipelines_spec.rb

@@ -33,6 +33,7 @@
         "xpack.management.elasticsearch.hosts" => ["http://localhost:9200"],
         "xpack.management.elasticsearch.username" => "elastic",
         "xpack.management.elasticsearch.password" => elastic_password,
+        "allow.legacy.monitoring" => true,
         "xpack.monitoring.elasticsearch.username" => "elastic",
         "xpack.monitoring.elasticsearch.password" => elastic_password
 

x-pack/qa/integration/monitoring/direct_shipping_spec.rb

@@ -15,6 +15,7 @@
 
     @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1  --pipeline.id #{SecureRandom.hex(8)}", {
       :settings => {
+        "allow.legacy.monitoring" => true,
         "monitoring.enabled" => true,
         "monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"],
         "monitoring.collection.interval" => "1s",

x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb

@@ -85,6 +85,7 @@ def start_monitoring_logstash(config, prefix = "")
   end
   logstash_with_empty_default("bin/logstash -e '#{config}' -w 1", {
     :settings => {
+      "allow.legacy.monitoring" => true,
       "#{mon_prefix}monitoring.enabled" => true,
       "#{mon_prefix}monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"],
       "#{mon_prefix}monitoring.collection.interval" => "1s",

x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb

@@ -14,6 +14,7 @@
 
     @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", {
       :settings => {
+        "allow.legacy.monitoring" => true,
         "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"],
         "xpack.monitoring.collection.interval" => "1s",
         "xpack.monitoring.elasticsearch.username" => "elastic",

x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb

@@ -16,6 +16,7 @@
 
     @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", {
       :settings => {
+        "allow.legacy.monitoring" => true,
         "xpack.monitoring.collection.interval" => "1s",
         "xpack.monitoring.elasticsearch.username" => "elastic",
         "xpack.monitoring.elasticsearch.password" => elastic_password

x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb

@@ -14,6 +14,7 @@
 
     @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", {
       :settings => {
+        "allow.legacy.monitoring" => true,
         "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"],
         "xpack.monitoring.collection.interval" => "1s",
         "queue.type" => "persisted",

x-pack/spec/monitoring/pipeline_register_hook_spec.rb

@@ -24,6 +24,18 @@
   }
 
   context 'validate monitoring settings' do
+    it "it's not enabled with just xpack.monitoring.enabled set to true" do
+      expect(subject.monitoring_enabled?(settings)).to be_falsey
+    end
+
+    context 'when legacy monitoring is allowed and xpack monitoring is enabled' do
+      let(:settings) { super().merge({"allow.legacy.monitoring" => true}) }
+
+      it "then internal monitoring should be effectively enabled" do
+        expect(subject.monitoring_enabled?(settings)).to be_truthy
+      end
+    end
+
     it "work without any monitoring settings" do
       settings.set_value("xpack.monitoring.enabled", true)
       expect(subject.generate_pipeline_config(settings)).to be_truthy