Merge branch 'main' into fix-remove-skip-tags-for-version-7.x

Signed-off-by: Sriram Ganesh <srignsh22@gmail.com>

Author: rgsriram

CHANGELOG.md

@@ -16,25 +16,40 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add FilterFieldType for developers who want to wrap MappedFieldType ([#17627](https://github.com/opensearch-project/OpenSearch/pull/17627))
 - [Rule Based Auto-tagging] Add in-memory rule processing service ([#17365](https://github.com/opensearch-project/OpenSearch/pull/17365))
 - [Security Manager Replacement] Create initial Java Agent to intercept Socket::connect calls ([#17724](https://github.com/opensearch-project/OpenSearch/pull/17724))
+- Faster `terms_query` with already sorted terms ([#17714](https://github.com/opensearch-project/OpenSearch/pull/17714))
 - Add ingestion management APIs for pause, resume and get ingestion state ([#17631](https://github.com/opensearch-project/OpenSearch/pull/17631))
 - [Security Manager Replacement] Enhance Java Agent to intercept System::exit ([#17746](https://github.com/opensearch-project/OpenSearch/pull/17746))
 - [Security Manager Replacement] Add a policy parser for Java agent security policies ([#17753](https://github.com/opensearch-project/OpenSearch/pull/17753))
 - [Security Manager Replacement] Implement File Interceptor and add integration tests ([#17760](https://github.com/opensearch-project/OpenSearch/pull/17760))
 - [Security Manager Replacement] Enhance Java Agent to intercept Runtime::halt ([#17757](https://github.com/opensearch-project/OpenSearch/pull/17757))
+- [Security Manager Replacement] Phase off SecurityManager usage in favor of Java Agent ([#17861](https://github.com/opensearch-project/OpenSearch/pull/17861))
 - Support AutoExpand for SearchReplica ([#17741](https://github.com/opensearch-project/OpenSearch/pull/17741))
+- Add TLS enabled SecureNetty4GrpcServerTransport ([#17796](https://github.com/opensearch-project/OpenSearch/pull/17796))
 - Implement fixed interval refresh task scheduling ([#17777](https://github.com/opensearch-project/OpenSearch/pull/17777))
+- [Tiered caching] Create a single cache manager for all the disk caches. ([#17513](https://github.com/opensearch-project/OpenSearch/pull/17513))
 - Add GRPC DocumentService and Bulk endpoint ([#17727](https://github.com/opensearch-project/OpenSearch/pull/17727))
 - Added scale to zero (`search_only` mode) support for OpenSearch reader writer separation ([#17299](https://github.com/opensearch-project/OpenSearch/pull/17299)
 - [Star Tree] [Search] Resolving numeric range aggregation with metric aggregation using star-tree ([#17273](https://github.com/opensearch-project/OpenSearch/pull/17273))
 - Added Search Only strict routing setting ([#17803](https://github.com/opensearch-project/OpenSearch/pull/17803))
 - Disable the index API for ingestion engine ([#17768](https://github.com/opensearch-project/OpenSearch/pull/17768))
+- Add SearchService and Search GRPC endpoint ([#17830](https://github.com/opensearch-project/OpenSearch/pull/17830))
+- Add update and delete support in pull-based ingestion ([#17822](https://github.com/opensearch-project/OpenSearch/pull/17822))
+- Allow maxPollSize and pollTimeout in IngestionSource to be configurable ([#17863](https://github.com/opensearch-project/OpenSearch/pull/17863))
+- [Star Tree] [Search] Add query changes to support unsigned-long in star tree ([#17275](https://github.com/opensearch-project/OpenSearch/pull/17275))
+- Add `ApproximateMatchAllQuery` that targets match_all queries and approximates sorts ([#17772](https://github.com/opensearch-project/OpenSearch/pull/17772))
+- Add TermsQuery support to Search GRPC endpoint ([#17888](https://github.com/opensearch-project/OpenSearch/pull/17888))
+- Support sub agg in filter rewrite optimization ([#17447](https://github.com/opensearch-project/OpenSearch/pull/17447)
+- Disable scoring of keyword term search by default, fallback logic with new use_similarity:true parameter ([#17889](https://github.com/opensearch-project/OpenSearch/pull/17889))
 
 ### Changed
 - Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912))
 - Increase the floor segment size to 16MB ([#17699](https://github.com/opensearch-project/OpenSearch/pull/17699))
 - Unwrap singleton DocValues in global ordinal value source of composite histogram aggregation ([#17740](https://github.com/opensearch-project/OpenSearch/pull/17740))
 - Unwrap singleton DocValues in date histogram aggregation. ([#17643](https://github.com/opensearch-project/OpenSearch/pull/17643))
 - Introduce 512 byte limit to search and ingest pipeline IDs ([#17786](https://github.com/opensearch-project/OpenSearch/pull/17786))
+- Avoid skewed segment replication lag metric ([#17831](https://github.com/opensearch-project/OpenSearch/pull/17831))
+- Increase the default segment counter step size when replica promoting ([#17568](https://github.com/opensearch-project/OpenSearch/pull/17568))
+- [WLM] Rename QueryGroup to WorkloadGroup ([#17901](https://github.com/opensearch-project/OpenSearch/pull/17901))
 
 ### Dependencies
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 10.0.2 ([#17607](https://github.com/opensearch-project/OpenSearch/pull/17607), [#17669](https://github.com/opensearch-project/OpenSearch/pull/17669))
@@ -55,17 +70,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `reactor_netty` from 1.1.26 to 1.2.3 ([#17322](https://github.com/opensearch-project/OpenSearch/pull/17322), [#17377](https://github.com/opensearch-project/OpenSearch/pull/17377))
 - Bump `com.google.api.grpc:proto-google-iam-v1` from 1.33.0 to 1.49.1 ([#17811](https://github.com/opensearch-project/OpenSearch/pull/17811))
 - Bump `com.azure:azure-core` from 1.54.1 to 1.55.3 ([#17810](https://github.com/opensearch-project/OpenSearch/pull/17810))
+- Bump `org.apache.poi` version from 5.2.5 to 5.4.1 in /plugins/ingest-attachment ([#17887](https://github.com/opensearch-project/OpenSearch/pull/17887))
+- Bump `org.opensearch:protobufs` from 0.2.0 to 0.3.0 ([#17888](https://github.com/opensearch-project/OpenSearch/pull/17888))
 
 ### Changed
 
 ### Deprecated
 
 ### Removed
 - Remove deprecated `batch_size` parameter from `_bulk` ([#14283](https://github.com/opensearch-project/OpenSearch/issues/14283))
+- Remove `FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY_SETTING` since range query approximation is no longer experimental ([#17769](https://github.com/opensearch-project/OpenSearch/pull/17769))
 
 ### Fixed
 - Fix bytes parameter on `_cat/recovery` ([#17598](https://github.com/opensearch-project/OpenSearch/pull/17598))
 - Fix slow performance of FeatureFlag checks ([#17611](https://github.com/opensearch-project/OpenSearch/pull/17611))
+- Fix shard recovery in pull-based ingestion to avoid skipping messages ([#17868](https://github.com/opensearch-project/OpenSearch/pull/17868)))
 
 ### Security
 

build.gradle

@@ -433,11 +433,12 @@ gradle.projectsEvaluated {
 
     project.tasks.withType(Test) { task ->
       if (task != null) {
-        if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
-          task.jvmArgs += ["-Djava.security.manager=allow"]
-        }
-        if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) {
-          task.jvmArgs += ["--add-modules=jdk.incubator.vector"]
+        task.jvmArgs += ["--add-modules=jdk.incubator.vector"]
+
+        // Add Java Agent for security sandboxing
+        if (!(project.path in [':build-tools', ":libs:agent-sm:bootstrap", ":libs:agent-sm:agent"])) {
+          dependsOn(project(':libs:agent-sm:agent').prepareAgent)
+          jvmArgs += ["-javaagent:" + project(':libs:agent-sm:agent').jar.archiveFile.get()]
         }
       }
     }

buildSrc/build.gradle

@@ -110,12 +110,12 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
   api 'org.apache.rat:apache-rat:0.15'
   api "commons-io:commons-io:${props.getProperty('commonsio')}"
-  api "net.java.dev.jna:jna:5.14.0"
+  api "net.java.dev.jna:jna:5.16.0"
   api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.8'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'
   api 'com.networknt:json-schema-validator:1.2.0'

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -115,9 +115,6 @@ public void execute(Task t) {
                             test.jvmArgs("--illegal-access=warn");
                         }
                     }
-                    if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) {
-                        test.jvmArgs("-Djava.security.manager=allow");
-                    }
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

buildSrc/src/main/java/org/opensearch/gradle/agent/JavaAgent.java

@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.agent;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+import java.util.Objects;
+
+/**
+ * Gradle plugin to automatically configure the OpenSearch Java agent
+ * for test tasks in OpenSearch plugin projects.
+ */
+public class JavaAgent implements Plugin<Project> {
+
+    /**
+     * Plugin implementation that sets up java agent configuration and applies it to test tasks.
+     */
+    @Override
+    public void apply(Project project) {
+        Configuration agentConfiguration = project.getConfigurations().findByName("agent");
+        if (agentConfiguration == null) {
+            agentConfiguration = project.getConfigurations().create("agent");
+        }
+
+        project.afterEvaluate(p -> {
+            String opensearchVersion = getOpensearchVersion(p);
+            p.getDependencies().add("agent", "org.opensearch:opensearch-agent-bootstrap:" + opensearchVersion);
+            p.getDependencies().add("agent", "org.opensearch:opensearch-agent:" + opensearchVersion);
+        });
+
+        Configuration finalAgentConfiguration = agentConfiguration;
+        TaskProvider<Copy> prepareJavaAgent = project.getTasks().register("prepareJavaAgent", Copy.class, task -> {
+            task.from(finalAgentConfiguration);
+            task.into(new File(project.getBuildDir(), "agent"));
+        });
+
+        project.getTasks().withType(Test.class).configureEach(testTask -> {
+            testTask.dependsOn(prepareJavaAgent);
+
+            final String opensearchVersion = getOpensearchVersion(project);
+
+            testTask.doFirst(task -> {
+                File agentJar = new File(project.getBuildDir(), "agent/opensearch-agent-" + opensearchVersion + ".jar");
+
+                testTask.jvmArgs("-javaagent:" + agentJar.getAbsolutePath());
+            });
+        });
+    }
+
+    /**
+     * Gets the OpenSearch version from project properties, with a fallback default.
+     *
+     * @param project The Gradle project
+     * @return The OpenSearch version to use
+     */
+    private String getOpensearchVersion(Project project) {
+        return Objects.requireNonNull(project.property("opensearch_version")).toString();
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.java-agent.properties

@@ -0,0 +1,9 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+implementation-class=org.opensearch.gradle.agent.JavaAgent

buildSrc/src/test/java/org/opensearch/gradle/agent/JavaAgentTests.java

@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.agent;
+
+import org.opensearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.Copy;
+import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+public class JavaAgentTests extends GradleUnitTestCase {
+    private TemporaryFolder projectDir;
+    private final String PREPARE_JAVA_AGENT_TASK = "prepareJavaAgent";
+
+    @Before
+    public void setUp() throws IOException {
+        projectDir = new TemporaryFolder();
+        projectDir.create();
+    }
+
+    @After
+    public void tearDown() {
+        projectDir.delete();
+    }
+
+    /**
+     * This test is used to verify that adding the 'opensearch.java-agent' to the project
+     * creates the necessary agent configuration and tasks. This is basically
+     * a behavioral test of the {@link JavaAgent#apply(Project)} method.
+     */
+    @Test
+    public void applyJavaAgentPlugin() {
+        // Create an empty project and apply the JavaAgent plugin
+        Project project = ProjectBuilder.builder().build();
+        project.getPluginManager().apply(JavaAgent.class);
+
+        // Verify the agent configuration was created
+        Configuration agentConfig = project.getConfigurations().findByName("agent");
+        assertNotNull("Agent configuration should be created", agentConfig);
+
+        // Verify the prepareJavaAgent task was created and is of the right type
+        assertNotNull("prepareJavaAgent task should be created", project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK));
+        assertTrue("prepareJavaAgent task should be of type Copy", project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK) instanceof Copy);
+
+        // Verify the destination directory of the Copy task
+        Copy prepareTask = (Copy) project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK);
+        assertEquals(
+            "Destination directory should be build/agent",
+            new File(project.getBuildDir(), "agent"),
+            prepareTask.getDestinationDir()
+        );
+    }
+}

client/rest-high-level/src/test/resources/org/opensearch/bootstrap/test.policy

@@ -8,4 +8,5 @@
 
 grant {
   permission java.net.SocketPermission "*", "connect,resolve";
+  permission java.net.NetPermission "accessUnixDomainSocket";
 };

distribution/archives/build.gradle

@@ -38,6 +38,9 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
       into('lib') {
         with libFiles()
       }
+      into('agent') {
+        with agentFiles()
+      }
       into('config') {
         dirPermissions {
           unix 0750
@@ -226,3 +229,9 @@ subprojects {
 
   group = "org.opensearch.distribution"
 }
+
+tasks.each {
+  if (it.name.startsWith("build")) {
+    it.dependsOn project(':libs:agent-sm:agent').assemble
+  }
+}

distribution/build.gradle

@@ -357,6 +357,18 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
+    agentFiles = {
+      copySpec {
+        from(project(':libs:agent-sm:agent').prepareAgent) {
+          include '**/*.jar'
+          exclude '**/*-javadoc.jar'
+          exclude '**/*-sources.jar'
+          // strip the version since jvm.options is using agent without version
+          rename("opensearch-agent-${project.version}.jar", "opensearch-agent.jar")
+        }
+      }
+    }
+
     modulesFiles = { platform ->
       copySpec {
         eachFile {