Merge branch 'main' into add-alibabacloud-inference

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -167,7 +168,7 @@ public void setUp() throws Exception {
             .build();
 
         Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build();
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
 
         ClusterService clusterService = new ClusterService(
             Settings.EMPTY,

docs/changelog/111917.yaml

@@ -0,0 +1,7 @@
+pr: 111917
+summary: "[ES|QL] Cast mixed numeric types to a common numeric type for Coalesce and\
+  \ In at Analyzer"
+area: ES|QL
+type: enhancement
+issues:
+ - 111486

docs/changelog/112133.yaml

@@ -0,0 +1,5 @@
+pr: 112133
+summary: Add telemetry for repository usage
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/112217.yaml

@@ -0,0 +1,5 @@
+pr: 112217
+summary: Fix template alias parsing livelock
+area: Indices APIs
+type: bug
+issues: []

docs/changelog/112230.yaml

@@ -0,0 +1,5 @@
+pr: 112230
+summary: Fix connection timeout for `OpenIdConnectAuthenticator` get Userinfo
+area: Security
+type: bug
+issues: []

docs/reference/cluster/stats.asciidoc

@@ -1282,6 +1282,31 @@ They are included here for expert users, but should otherwise be ignored.
 
 =====
 
+====
+
+`repositories`::
+(object) Contains statistics about the <<snapshot-restore,snapshot>> repositories defined in the cluster, broken down
+by repository type.
++
+.Properties of `repositories`
+[%collapsible%open]
+=====
+
+`count`:::
+(integer) The number of repositories of this type in the cluster.
+
+`read_only`:::
+(integer) The number of repositories of this type in the cluster which are registered read-only.
+
+`read_write`:::
+(integer) The number of repositories of this type in the cluster which are not registered as read-only.
+
+Each repository type may also include other statistics about the repositories of that type here.
+
+=====
+
+====
+
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
 
@@ -1579,6 +1604,9 @@ The API returns the following response:
    },
    "snapshots": {
      ...
+   },
+   "repositories": {
+     ...
    }
 }
 --------------------------------------------------
@@ -1589,6 +1617,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/"snapshots": \{[^\}]*\}/"snapshots": $body.$_path/]
+// TESTRESPONSE[s/"repositories": \{[^\}]*\}/"repositories": $body.$_path/]
 // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
 // TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
 // TESTRESPONSE[s/"search": \{[^\}]*\}/"search": $body.$_path/]
@@ -1600,7 +1629,7 @@ The API returns the following response:
 //    the plugins that will be in it. And because we figure folks don't need to
 //    see an exhaustive list anyway.
 // 2. Similarly, ignore the contents of `network_types`, `discovery_types`,
-//    `packaging_types` and `snapshots`.
+//    `packaging_types`, `snapshots` and `repositories`.
 // 3. Ignore the contents of the (nodes) count object, as what's shown here
 //    depends on the license. Voting-only nodes are e.g. only shown when this
 //    test runs with a basic license.

libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java

@@ -10,14 +10,15 @@
 import org.elasticsearch.logstashbridge.StableBridgeAPI;
 import org.elasticsearch.logstashbridge.common.SettingsBridge;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
 
 public class ThreadPoolBridge extends StableBridgeAPI.Proxy<ThreadPool> {
 
     public ThreadPoolBridge(final SettingsBridge settingsBridge) {
-        this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP));
+        this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders()));
     }
 
     public ThreadPoolBridge(final ThreadPool delegate) {

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 import org.hamcrest.Matchers;
@@ -86,7 +87,11 @@ public void setup() throws IOException {
             "e4a3411cdd7b21eaf18675da5a7f9f360d33c6882363b2c19c38715834c9e836  GeoIP2-City_20240709.tar.gz".getBytes(StandardCharsets.UTF_8)
         );
         clusterService = mock(ClusterService.class);
-        threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(Settings.EMPTY, Set.of(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING))
         );

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -92,7 +93,11 @@ public void setup() throws IOException {
         httpClient = mock(HttpClient.class);
         when(httpClient.getBytes(anyString())).thenReturn("[]".getBytes(StandardCharsets.UTF_8));
         clusterService = mock(ClusterService.class);
-        threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(
                 Settings.EMPTY,

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -26,6 +26,7 @@
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.core.Strings.format;
@@ -175,4 +176,9 @@ protected ByteSizeValue chunkSize() {
     public boolean isReadOnly() {
         return readonly;
     }
+
+    @Override
+    protected Set<String> getExtraUsageFeatures() {
+        return storageService.getExtraUsageFeatures(Repository.CLIENT_NAME.get(getMetadata().settings()));
+    }
 }

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -24,6 +24,7 @@
 import java.net.Proxy;
 import java.net.URL;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
@@ -165,4 +166,15 @@ public void refreshSettings(Map<String, AzureStorageSettings> clientsSettings) {
         this.storageSettings = Map.copyOf(clientsSettings);
         // clients are built lazily by {@link client(String, LocationMode)}
     }
+
+    /**
+     * For Azure repositories, we report the different kinds of credentials in use in the telemetry.
+     */
+    public Set<String> getExtraUsageFeatures(String clientName) {
+        try {
+            return getClientSettings(clientName).credentialsUsageFeatures();
+        } catch (Exception e) {
+            return Set.of();
+        }
+    }
 }

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -29,6 +29,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 final class AzureStorageSettings {
 
@@ -130,6 +131,7 @@ final class AzureStorageSettings {
     private final int maxRetries;
     private final Proxy proxy;
     private final boolean hasCredentials;
+    private final Set<String> credentialsUsageFeatures;
 
     private AzureStorageSettings(
         String account,
@@ -150,6 +152,12 @@ private AzureStorageSettings(
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
+        this.credentialsUsageFeatures = Strings.hasText(key) ? Set.of("uses_key_credentials")
+            : Strings.hasText(sasToken) ? Set.of("uses_sas_token")
+            : SocketAccess.doPrivilegedException(() -> System.getenv("AZURE_FEDERATED_TOKEN_FILE")) == null
+                ? Set.of("uses_default_credentials", "uses_managed_identity")
+            : Set.of("uses_default_credentials", "uses_workload_identity");
+
         // Register the proxy if we have any
         // Validate proxy settings
         if (proxyType.equals(Proxy.Type.DIRECT) && ((proxyPort != 0) || Strings.hasText(proxyHost))) {
@@ -366,4 +374,8 @@ private String deriveURIFromSettings(boolean isPrimary) {
             throw new IllegalArgumentException(e);
         }
     }
+
+    public Set<String> credentialsUsageFeatures() {
+        return credentialsUsageFeatures;
+    }
 }

modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml

@@ -235,6 +235,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.azure.count: 1 }
+  - gte: { repositories.azure.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml

@@ -232,6 +232,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.gcs.count: 1 }
+  - gte: { repositories.gcs.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml

@@ -345,6 +345,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml

@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml

@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml

@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml

@@ -257,6 +257,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 

modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportSettings;
 import org.junit.After;
@@ -52,7 +53,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase {
 
     @Before
     public void startThreadPool() {
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         NetworkService networkService = new NetworkService(Collections.emptyList());
         PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY);
         nettyTransport = new Netty4Transport(