Add xpack.apm.tracing.names.include setting for filtering (#80871)

This commit adds a dynamic cluster setting called `xpack.apm.tracing.names.include`
which allows the user to filter on the names of the transactions for which
tracing is enabled.

Author: dimitris-athanasiou

x-pack/plugin/apm-integration/src/internalClusterTest/java/org/elasticsearch/xpack/apm/ApmIT.java

@@ -10,6 +10,7 @@
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.sdk.trace.data.SpanData;
 
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -42,11 +43,13 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.cluster.service.MasterService.STATE_UPDATE_ACTION_NAME;
 import static org.elasticsearch.indices.recovery.PeerRecoverySourceService.Actions.START_RECOVERY;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
@@ -294,4 +297,101 @@ public void testDoesNotRecordSpansWhenDisabled() {
                 .actionGet();
         }
     }
+
+    public void testFilterByNameGivenSingleCompleteMatch() {
+
+        client().admin()
+            .cluster()
+            .updateSettings(
+                new ClusterUpdateSettingsRequest().persistentSettings(
+                    Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), "cluster:monitor/tasks/lists").build()
+                )
+            )
+            .actionGet();
+
+        APMTracer.CAPTURING_SPAN_EXPORTER.clear();// removing start related events
+
+        try {
+            client().admin().cluster().prepareListTasks().get();
+
+            var parentTasks = APMTracer.CAPTURING_SPAN_EXPORTER.findSpanByName("cluster:monitor/tasks/lists").collect(toList());
+            assertThat(parentTasks, hasSize(1));
+
+            var childrenTasks = APMTracer.CAPTURING_SPAN_EXPORTER.findSpanByName("cluster:monitor/tasks/lists[n]").collect(toList());
+            assertThat(childrenTasks, empty());
+        } finally {
+            client().admin()
+                .cluster()
+                .updateSettings(
+                    new ClusterUpdateSettingsRequest().persistentSettings(
+                        Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), (String) null).build()
+                    )
+                )
+                .actionGet();
+        }
+    }
+
+    public void testFilterByNameGivenSinglePattern() {
+
+        client().admin()
+            .cluster()
+            .updateSettings(
+                new ClusterUpdateSettingsRequest().persistentSettings(
+                    Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), "*/tasks/lists*").build()
+                )
+            )
+            .actionGet();
+
+        APMTracer.CAPTURING_SPAN_EXPORTER.clear();// removing start related events
+
+        try {
+            client().admin().cluster().prepareListTasks().get();
+
+            var parentTasks = APMTracer.CAPTURING_SPAN_EXPORTER.findSpanByName("cluster:monitor/tasks/lists").collect(toList());
+            assertThat(parentTasks, hasSize(1));
+
+            var childrenTasks = APMTracer.CAPTURING_SPAN_EXPORTER.findSpanByName("cluster:monitor/tasks/lists[n]").collect(toList());
+            assertThat(childrenTasks, hasSize(internalCluster().size()));
+        } finally {
+            client().admin()
+                .cluster()
+                .updateSettings(
+                    new ClusterUpdateSettingsRequest().persistentSettings(
+                        Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), (String) null).build()
+                    )
+                )
+                .actionGet();
+        }
+    }
+
+    public void testFilterByNameGivenTwoPatterns() {
+
+        client().admin()
+            .cluster()
+            .updateSettings(
+                new ClusterUpdateSettingsRequest().persistentSettings(
+                    Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), "*/tasks/lists,*/nodes/stats").build()
+                )
+            )
+            .actionGet();
+
+        APMTracer.CAPTURING_SPAN_EXPORTER.clear();// removing start related events
+
+        try {
+            client().admin().cluster().prepareListTasks().get();
+            client().admin().cluster().nodesStats(new NodesStatsRequest()).actionGet();
+
+            var spans = APMTracer.CAPTURING_SPAN_EXPORTER.getCapturedSpans().stream().map(SpanData::getName).collect(Collectors.toSet());
+            assertThat(spans, contains("cluster:monitor/nodes/stats", "cluster:monitor/tasks/lists"));
+        } finally {
+            client().admin()
+                .cluster()
+                .updateSettings(
+                    new ClusterUpdateSettingsRequest().persistentSettings(
+                        Settings.builder().put(APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING.getKey(), (String) null).build()
+                    )
+                )
+                .actionGet();
+        }
+    }
 }

x-pack/plugin/apm-integration/src/main/java/org/elasticsearch/xpack/apm/APM.java

@@ -69,7 +69,12 @@ public Collection<Object> createComponents(
 
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(APMTracer.APM_ENABLED_SETTING, APMTracer.APM_ENDPOINT_SETTING, APMTracer.APM_TOKEN_SETTING);
+        return List.of(
+            APMTracer.APM_ENABLED_SETTING,
+            APMTracer.APM_ENDPOINT_SETTING,
+            APMTracer.APM_TOKEN_SETTING,
+            APMTracer.APM_TRACING_NAMES_INCLUDE_SETTING
+        );
     }
 
     public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) {

x-pack/plugin/apm-integration/src/main/java/org/elasticsearch/xpack/apm/APMTracer.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
@@ -46,13 +47,15 @@
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Queue;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -67,6 +70,13 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
     static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting("xpack.apm.tracing.enabled", false, Dynamic, NodeScope);
     static final Setting<SecureString> APM_ENDPOINT_SETTING = SecureSetting.secureString("xpack.apm.endpoint", null);
     static final Setting<SecureString> APM_TOKEN_SETTING = SecureSetting.secureString("xpack.apm.token", null);
+    static final Setting<List<String>> APM_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting(
+        "xpack.apm.tracing.names.include",
+        Collections.emptyList(),
+        Function.identity(),
+        Dynamic,
+        NodeScope
+    );
 
     private final Semaphore shutdownPermits = new Semaphore(Integer.MAX_VALUE);
     private final Map<String, Span> spans = ConcurrentCollections.newConcurrentMap();
@@ -78,6 +88,8 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
     private volatile boolean enabled;
     private volatile APMServices services;
 
+    private List<String> includeNames;
+
     /** This class is required to make all open telemetry services visible at once */
     private static class APMServices {
         private final SdkTracerProvider provider;
@@ -97,7 +109,9 @@ public APMTracer(Settings settings, ThreadPool threadPool, ClusterService cluste
         this.endpoint = APM_ENDPOINT_SETTING.get(settings);
         this.token = APM_TOKEN_SETTING.get(settings);
         this.enabled = APM_ENABLED_SETTING.get(settings);
+        this.includeNames = APM_TRACING_NAMES_INCLUDE_SETTING.get(settings);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_ENABLED_SETTING, this::setEnabled);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_TRACING_NAMES_INCLUDE_SETTING, this::setIncludeNames);
     }
 
     public boolean isEnabled() {
@@ -113,6 +127,10 @@ private void setEnabled(boolean enabled) {
         }
     }
 
+    private void setIncludeNames(List<String> includeNames) {
+        this.includeNames = includeNames;
+    }
+
     @Override
     protected void doStart() {
         if (enabled) {
@@ -195,6 +213,11 @@ public void onTraceStarted(Traceable traceable) {
         if (services == null) {
             return;
         }
+
+        if (isSpanNameIncluded(traceable.getSpanName()) == false) {
+            return;
+        }
+
         spans.computeIfAbsent(traceable.getSpanId(), spanId -> {
             // services might be in shutdown state by this point, but this is handled by the open telemetry internally
             final SpanBuilder spanBuilder = services.tracer.spanBuilder(traceable.getSpanName());
@@ -245,6 +268,12 @@ public void onTraceStarted(Traceable traceable) {
         });
     }
 
+    private boolean isSpanNameIncluded(String name) {
+        // Alternatively we could use automata here but it is much more complex
+        // and it needs wrapping like done for use in the security plugin.
+        return includeNames.isEmpty() || Regex.simpleMatch(includeNames, name);
+    }
+
     private Context getParentSpanContext(OpenTelemetry openTelemetry) {
         // If we already have a non-root span context that should be the parent
         if (Context.current() != Context.root()) {