Merge remote-tracking branch 'upstream/main' into final-policies

CHANGELOG.md

@@ -112,6 +112,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Tiered caching] Make IndicesRequestCache implementation configurable [EXPERIMENTAL] ([#12533](https://github.com/opensearch-project/OpenSearch/pull/12533))
 - Add kuromoji_completion analyzer and filter ([#4835](https://github.com/opensearch-project/OpenSearch/issues/4835))
 - The org.opensearch.bootstrap.Security should support codebase for JAR files with classifiers ([#12586](https://github.com/opensearch-project/OpenSearch/issues/12586))
+- [Metrics Framework] Adds support for asynchronous gauge metric type. ([#12642](https://github.com/opensearch-project/OpenSearch/issues/12642))
+- Make search query counters dynamic to support all query types ([#12601](https://github.com/opensearch-project/OpenSearch/pull/12601))
 - [Tiered caching] Add policies controlling which values can enter pluggable caches [EXPERIMENTAL] ([#12542](https://github.com/opensearch-project/OpenSearch/pull/12542))
 
 ### Dependencies
@@ -132,7 +134,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `codecov/codecov-action` from 3 to 4 ([#12585](https://github.com/opensearch-project/OpenSearch/pull/12585))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.1 to 3.9.2 ([#12580](https://github.com/opensearch-project/OpenSearch/pull/12580))
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#12579](https://github.com/opensearch-project/OpenSearch/pull/12579))
-- Bump Jackson version from 2.16.1 to 2.16.2 ([#12611](https://github.com/opensearch-project/OpenSearch/pull/12611))
+- Bump Jackson version from 2.16.1 to 2.17.0 ([#12611](https://github.com/opensearch-project/OpenSearch/pull/12611), [#12662](https://github.com/opensearch-project/OpenSearch/pull/12662))
 - Bump `aws-sdk-java` from 2.20.55 to 2.20.86 ([#12251](https://github.com/opensearch-project/OpenSearch/pull/12251))
 - Bump `reactor-netty` from 1.1.15 to 1.1.17 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
 - Bump `reactor` from 3.5.14 to 3.5.15 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
@@ -152,10 +154,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298))
 - Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481))
 - Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919))
+- Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
 - Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193))
 - Fix `terms` query on `float` field when `doc_values` are turned off by reverting back to `FloatPoint` from `FloatField` ([#12499](https://github.com/opensearch-project/OpenSearch/pull/12499))
 - Fix get task API does not refresh resource stats ([#11531](https://github.com/opensearch-project/OpenSearch/pull/11531))
 - onShardResult and onShardFailure are executed on one shard causes opensearch jvm crashed ([#12158](https://github.com/opensearch-project/OpenSearch/pull/12158))
+- Avoid overflow when sorting missing last on `epoch_millis` datetime field ([#12676](https://github.com/opensearch-project/OpenSearch/pull/12676))
 
 ### Security
 

buildSrc/version.properties

@@ -7,8 +7,8 @@ bundled_jdk = 21.0.2+13
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.16.2
-jackson_databind  = 2.16.2
+jackson           = 2.17.0
+jackson_databind  = 2.17.0
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0

client/sniffer/licenses/jackson-core-2.17.0.jar.sha1

@@ -0,0 +1 @@
+a6e5058ef9720623c517252d17162f845306ff3a

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.0.jar.sha1

@@ -0,0 +1 @@
+880a742337010da4c851f843d8cac150e22dff9f

distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.0.jar.sha1

@@ -0,0 +1 @@
+7173e9e1d4bc6d7ca03bc4eeedcd548b8b580b34

libs/core/licenses/jackson-core-2.17.0.jar.sha1

@@ -0,0 +1 @@
+a6e5058ef9720623c517252d17162f845306ff3a

libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistry.java

@@ -8,7 +8,11 @@
 
 package org.opensearch.telemetry.metrics;
 
+import org.opensearch.telemetry.metrics.tags.Tags;
+
+import java.io.Closeable;
 import java.io.IOException;
+import java.util.function.Supplier;
 
 /**
  * Default implementation for {@link MetricsRegistry}
@@ -39,6 +43,11 @@ public Histogram createHistogram(String name, String description, String unit) {
         return metricsTelemetry.createHistogram(name, description, unit);
     }
 
+    @Override
+    public Closeable createGauge(String name, String description, String unit, Supplier<Double> valueProvider, Tags tags) {
+        return metricsTelemetry.createGauge(name, description, unit, valueProvider, tags);
+    }
+
     @Override
     public void close() throws IOException {
         metricsTelemetry.close();

libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistry.java

@@ -9,8 +9,10 @@
 package org.opensearch.telemetry.metrics;
 
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.telemetry.metrics.tags.Tags;
 
 import java.io.Closeable;
+import java.util.function.Supplier;
 
 /**
  * MetricsRegistry helps in creating the metric instruments.
@@ -47,4 +49,18 @@ public interface MetricsRegistry extends Closeable {
      * @return histogram.
      */
     Histogram createHistogram(String name, String description, String unit);
+
+    /**
+     * Creates the Observable Gauge type of Metric. Where the value provider will be called at a certain frequency
+     * to capture the value.
+     *
+     * @param name          name of the observable gauge.
+     * @param description   any description about the metric.
+     * @param unit          unit of the metric.
+     * @param valueProvider value provider.
+     * @param tags          attributes/dimensions of the metric.
+     * @return closeable to dispose/close the Gauge metric.
+     */
+    Closeable createGauge(String name, String description, String unit, Supplier<Double> valueProvider, Tags tags);
+
 }

libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopMetricsRegistry.java

@@ -12,8 +12,11 @@
 import org.opensearch.telemetry.metrics.Counter;
 import org.opensearch.telemetry.metrics.Histogram;
 import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.tags.Tags;
 
+import java.io.Closeable;
 import java.io.IOException;
+import java.util.function.Supplier;
 
 /**
  *No-op {@link MetricsRegistry}
@@ -44,6 +47,11 @@ public Histogram createHistogram(String name, String description, String unit) {
         return NoopHistogram.INSTANCE;
     }
 
+    @Override
+    public Closeable createGauge(String name, String description, String unit, Supplier<Double> valueProvider, Tags tags) {
+        return () -> {};
+    }
+
     @Override
     public void close() throws IOException {
 

libs/telemetry/src/test/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistryTests.java

@@ -8,8 +8,12 @@
 
 package org.opensearch.telemetry.metrics;
 
+import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.io.Closeable;
+import java.util.function.Supplier;
+
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -59,4 +63,20 @@ public void testHistogram() {
         assertSame(mockHistogram, histogram);
     }
 
+    @SuppressWarnings("unchecked")
+    public void testGauge() {
+        Closeable mockCloseable = mock(Closeable.class);
+        when(
+            defaultMeterRegistry.createGauge(any(String.class), any(String.class), any(String.class), any(Supplier.class), any(Tags.class))
+        ).thenReturn(mockCloseable);
+        Closeable closeable = defaultMeterRegistry.createGauge(
+            "org.opensearch.telemetry.metrics.DefaultMeterRegistryTests.testObservableGauge",
+            "test observable gauge",
+            "ms",
+            () -> 1.0,
+            Tags.EMPTY
+        );
+        assertSame(mockCloseable, closeable);
+    }
+
 }

libs/x-content/licenses/jackson-core-2.17.0.jar.sha1

@@ -0,0 +1 @@
+a6e5058ef9720623c517252d17162f845306ff3a

libs/x-content/licenses/jackson-dataformat-cbor-2.17.0.jar.sha1

@@ -0,0 +1 @@
+6833c8573452d583e4af650a7424d547606b2501

libs/x-content/licenses/jackson-dataformat-smile-2.17.0.jar.sha1

@@ -0,0 +1 @@
+f10183857607fde789490d33ea46372a2d2b0c72

libs/x-content/licenses/jackson-dataformat-yaml-2.17.0.jar.sha1

@@ -0,0 +1 @@
+57a963c6258c49febc11390082d8503f71bb15a9

modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java

@@ -73,7 +73,6 @@ public void onRemoval(RemovalNotification<K, V> notification) {
                             diskCache.put(notification.getKey(), notification.getValue());
                         }
                     }
-                    removalListener.onRemoval(notification);
                 }
             })
                 .setKeyType(builder.cacheConfig.getKeyType())

modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java

@@ -11,6 +11,9 @@
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 
@@ -23,9 +26,12 @@ public class MockDiskCache<K, V> implements ICache<K, V> {
     int maxSize;
     long delay;
 
-    public MockDiskCache(int maxSize, long delay) {
+    private final RemovalListener<K, V> removalListener;
+
+    public MockDiskCache(int maxSize, long delay, RemovalListener<K, V> removalListener) {
         this.maxSize = maxSize;
         this.delay = delay;
+        this.removalListener = removalListener;
         this.cache = new ConcurrentHashMap<K, V>();
     }
 
@@ -38,7 +44,7 @@ public V get(K key) {
     @Override
     public void put(K key, V value) {
         if (this.cache.size() >= maxSize) { // For simplification
-            return;
+            this.removalListener.onRemoval(new RemovalNotification<>(key, value, RemovalReason.EVICTED));
         }
         try {
             Thread.sleep(delay);
@@ -101,7 +107,10 @@ public MockDiskCacheFactory(long delay, int maxSize) {
 
         @Override
         public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType, Map<String, Factory> cacheFactories) {
-            return new Builder<K, V>().setMaxSize(maxSize).setDeliberateDelay(delay).build();
+            return new Builder<K, V>().setMaxSize(maxSize)
+                .setDeliberateDelay(delay)
+                .setRemovalListener(config.getRemovalListener())
+                .build();
         }
 
         @Override
@@ -117,7 +126,7 @@ public static class Builder<K, V> extends ICacheBuilder<K, V> {
 
         @Override
         public ICache<K, V> build() {
-            return new MockDiskCache<K, V>(this.maxSize, this.delay);
+            return new MockDiskCache<K, V>(this.maxSize, this.delay, this.getRemovalListener());
         }
 
         public Builder<K, V> setMaxSize(int maxSize) {

modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java

@@ -47,7 +47,7 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
-            onHeapCacheSize,
+            keyValueSize,
             randomIntBetween(1, 4),
             removalListener,
             Settings.builder()
@@ -153,10 +153,6 @@ public CachedQueryResult.PolicyValues apply(String s) {
             LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
-        long actualDiskCacheSize = tieredSpilloverCache.getDiskCache().count();
-        assertEquals(actualDiskCacheSize, removalListener.evictionsMetric.count()); // Evictions from onHeap equal to
-        // disk cache size.
-
         tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
         tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
 
@@ -301,9 +297,6 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
-        long actualDiskCacheSize = tieredSpilloverCache.getDiskCache().count();
-        assertEquals(actualDiskCacheSize, removalListener.evictionsMetric.count()); // Evictions from onHeap equal to
-        // disk cache size.
 
         tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
         tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
@@ -339,15 +332,15 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
         }
     }
 
-    public void testComputeIfAbsentWithEvictionsFromBothTier() throws Exception {
+    public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
         int totalSize = onHeapCacheSize + diskCacheSize;
         int keyValueSize = 50;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
-            onHeapCacheSize,
+            keyValueSize,
             diskCacheSize,
             removalListener,
             Settings.builder()
@@ -360,13 +353,13 @@ public void testComputeIfAbsentWithEvictionsFromBothTier() throws Exception {
                 .build(),
             0
         );
-
         int numOfItems = randomIntBetween(totalSize + 1, totalSize * 3);
         for (int iter = 0; iter < numOfItems; iter++) {
             LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), tieredCacheLoader);
         }
-        assertTrue(removalListener.evictionsMetric.count() > 0);
+        int evictions = numOfItems - (totalSize);
+        assertEquals(evictions, removalListener.evictionsMetric.count());
     }
 
     public void testGetAndCount() throws Exception {
@@ -377,7 +370,7 @@ public void testGetAndCount() throws Exception {
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
-            onHeapCacheSize,
+            keyValueSize,
             diskCacheSize,
             removalListener,
             Settings.builder()
@@ -429,7 +422,7 @@ public void testPut() {
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
-            onHeapCacheSize,
+            keyValueSize,
             diskCacheSize,
             removalListener,
             Settings.builder()
@@ -530,7 +523,7 @@ public void testInvalidate() {
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
-            onHeapCacheSize,
+            keyValueSize,
             diskCacheSize,
             removalListener,
             Settings.builder()
@@ -755,7 +748,7 @@ public String load(String key) {
         assertEquals(1, numberOfTimesKeyLoaded); // It should be loaded only once.
     }
 
-    public void testConcurrencyForEvictionFlow() throws Exception {
+    public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exception {
         int diskCacheSize = randomIntBetween(450, 800);
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
@@ -839,7 +832,6 @@ public String load(String key) {
         countDownLatch.await();
         assertNotNull(actualValue.get());
         countDownLatch1.await();
-        assertEquals(1, removalListener.evictionsMetric.count());
         assertEquals(1, tieredSpilloverCache.getOnHeapCache().count());
         assertEquals(1, onDiskCache.count());
         assertNotNull(onDiskCache.get(keyToBeEvicted));
@@ -1117,7 +1109,6 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
                     .build()
             )
             .build();
-
         ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize);
 
         TieredSpilloverCache.Builder<String, String> builder = new TieredSpilloverCache.Builder<String, String>().setCacheType(

modules/ingest-geoip/licenses/jackson-annotations-2.17.0.jar.sha1

@@ -0,0 +1 @@
+880a742337010da4c851f843d8cac150e22dff9f

modules/ingest-geoip/licenses/jackson-databind-2.17.0.jar.sha1

@@ -0,0 +1 @@
+7173e9e1d4bc6d7ca03bc4eeedcd548b8b580b34

plugins/crypto-kms/licenses/jackson-annotations-2.17.0.jar.sha1

@@ -0,0 +1 @@
+880a742337010da4c851f843d8cac150e22dff9f