Merge branch 'main' into patch-1

.github/workflows/gradle-check.yml

@@ -7,6 +7,10 @@ on:
       - 'dependabot/**'
   pull_request_target:
     types: [opened, synchronize, reopened]
+  workflow_run:
+    workflows: ["Gradle Precommit"]
+    types:
+      - completed
 
 permissions:
   contents: read # to fetch code (actions/checkout)

.github/workflows/precommit.yml

@@ -1,9 +1,12 @@
 name: Gradle Precommit
 on: [pull_request]
-
+ 
 jobs:
   precommit:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest] 
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 11
@@ -13,4 +16,4 @@ jobs:
           distribution: adopt
       - name: Run Gradle
         run: |
-          ./gradlew precommit --parallel
+          ./gradlew javadoc precommit --parallel

CHANGELOG.md

@@ -1,8 +1,11 @@
 # CHANGELOG
+
 Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
+
 ### Added
+
 - GitHub Workflows token permission hardening ([#4587](https://github.com/opensearch-project/OpenSearch/pull/4587))
 - Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001))
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
@@ -21,8 +24,16 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513))
 - [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993))
 - Added missing javadocs for `:example-plugins` modules ([#4540](https://github.com/opensearch-project/OpenSearch/pull/4540))
+- Add missing Javadoc tag descriptions in unit tests ([#4629](https://github.com/opensearch-project/OpenSearch/pull/4629))
+- Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
+- Added precommit support for windows ([#4676](https://github.com/opensearch-project/OpenSearch/pull/4676))
+- Added precommit support for MacOS ([#4682](https://github.com/opensearch-project/OpenSearch/pull/4682))
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
+- Bumps `reactor-netty-http` from 1.0.18 to 1.0.23
+- Bumps `jettison` from 1.5.0 to 1.5.1
+- Bumps `azure-storage-common` from 12.18.0 to 12.18.1
+- Bumps `forbiddenapis` from 3.3 to 3.4
 
 
 ### Dependencies
@@ -36,8 +47,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
+- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4 ([#4644](https://github.com/opensearch-project/OpenSearch/pull/4644))
 
 ### Changed
+
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
@@ -46,16 +59,25 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402))
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
 - [Remote Store] Change behaviour in replica recovery for remote translog enabled indices ([#4318](https://github.com/opensearch-project/OpenSearch/pull/4318))
+- PUT api for weighted shard routing ([#4272](https://github.com/opensearch-project/OpenSearch/pull/4272))
 - Unmute test RelocationIT.testRelocationWhileIndexingRandom ([#4580](https://github.com/opensearch-project/OpenSearch/pull/4580))
 - Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084))
-
+- Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360))
+- Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
+- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582))
+- Add APIs (GET/PUT) to decommission awareness attribute ([#4261](https://github.com/opensearch-project/OpenSearch/pull/4261))
+- Improve Gradle pre-commit checks to pre-empt Jenkins build ([#4660](https://github.com/opensearch-project/OpenSearch/pull/4660))
+- Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661))
+- Backport Apache Lucene version change for 2.4.0 ([#4677](https://github.com/opensearch-project/OpenSearch/pull/4677))
 
 ### Deprecated
 
 ### Removed
 - Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
+- Unused object and import within TransportClusterAllocationExplainAction ([#4639](https://github.com/opensearch-project/OpenSearch/pull/4639))
 
 ### Fixed
+
 - `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289))
 - PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296))
 - `opensearch.bat` and `opensearch-service.bat install` failing to run, missing logs directory ([#4305](https://github.com/opensearch-project/OpenSearch/pull/4305))
@@ -73,19 +95,34 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Update flaky testOnNewCheckpointFromNewPrimaryCancelOngoingReplication unit test ([#4414](https://github.com/opensearch-project/OpenSearch/pull/4414))
 - Fixed the `_cat/shards/10_basic.yml` test cases fix.
 - [Segment Replication] Fix timeout issue by calculating time needed to process getSegmentFiles ([#4426](https://github.com/opensearch-project/OpenSearch/pull/4426))
-- [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/
+- [Bug]: gradle check failing with java heap OutOfMemoryError ([#4328](https://github.com/opensearch-project/OpenSearch/))
 - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
 - Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
 - Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
 - Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
 - [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544))
 - Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
+- Fixing PIT flaky tests ([4632](https://github.com/opensearch-project/OpenSearch/pull/4632))
+- Fixed day of year defaulting for round up parser ([#4627](https://github.com/opensearch-project/OpenSearch/pull/4627))
+- Fixed the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570))
+- [Segment Replication] Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4630](https://github.com/opensearch-project/OpenSearch/pull/4630))
+- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))
+- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499))
+- Fixed the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570))
+- [Segment Replication] Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4630](https://github.com/opensearch-project/OpenSearch/pull/4630))
+- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))
+- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499))
+- Attempt to fix Github workflow for Gradle Check job ([#4679](https://github.com/opensearch-project/OpenSearch/pull/4679))
+- Fix flaky DecommissionControllerTests.testTimesOut ([4683](https://github.com/opensearch-project/OpenSearch/pull/4683))
 
 ### Security
+
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
 
 ## [2.x]
+
 ### Added
+
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
 - Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217))
@@ -101,11 +138,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Removed
 
 ### Fixed
+
 - PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296))
 - Commit workflow for dependabot changelog helper ([#4331](https://github.com/opensearch-project/OpenSearch/pull/4331))
 
 ### Security
 
-
 [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD
 [2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x

buildSrc/src/main/java/org/opensearch/gradle/Jdk.java

@@ -128,7 +128,7 @@ public void setArchitecture(final String architecture) {
                 "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
             );
         }
-        this.architecture.set(architecture);
+        this.architecture.set(jdkArchitecture);
     }
 
     public String getBaseVersion() {

buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java

@@ -9,83 +9,72 @@
 
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.logging.Logger;
-import org.gradle.api.logging.Logging;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
-import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
 
 import java.nio.file.Path;
 import org.gradle.api.Task;
+import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
 
 public class Publish implements Plugin<Project> {
 
-    private static final Logger LOGGER = Logging.getLogger(Publish.class);
-
-    public final static String EXTENSION_NAME = "zipmavensettings";
+    // public final static String PLUGIN_ZIP_PUBLISH_POM_TASK = "generatePomFileForPluginZipPublication";
     public final static String PUBLICATION_NAME = "pluginZip";
     public final static String STAGING_REPO = "zipStaging";
-    public final static String PLUGIN_ZIP_PUBLISH_POM_TASK = "generatePomFileForPluginZipPublication";
-    public final static String LOCALMAVEN = "publishToMavenLocal";
     public final static String LOCAL_STAGING_REPO_PATH = "/build/local-staging-repo";
-    public String zipDistributionLocation = "/build/distributions/";
+    // TODO: Does the path ^^ need to use platform dependant file separators ?
+
+    private boolean isZipPublicationPresent(Project project) {
+        PublishingExtension pe = project.getExtensions().findByType(PublishingExtension.class);
+        if (pe == null) {
+            return false;
+        }
+        return pe.getPublications().findByName(PUBLICATION_NAME) != null;
+    }
 
-    public static void configMaven(Project project) {
+    private void addLocalMavenRepo(Project project) {
         final Path buildDirectory = project.getRootDir().toPath();
-        project.getPluginManager().apply(MavenPublishPlugin.class);
         project.getExtensions().configure(PublishingExtension.class, publishing -> {
             publishing.repositories(repositories -> {
                 repositories.maven(maven -> {
                     maven.setName(STAGING_REPO);
                     maven.setUrl(buildDirectory.toString() + LOCAL_STAGING_REPO_PATH);
                 });
             });
+        });
+    }
+
+    private void addZipArtifact(Project project) {
+        project.getExtensions().configure(PublishingExtension.class, publishing -> {
             publishing.publications(publications -> {
                 MavenPublication mavenZip = (MavenPublication) publications.findByName(PUBLICATION_NAME);
-
-                if (mavenZip == null) {
-                    mavenZip = publications.create(PUBLICATION_NAME, MavenPublication.class);
+                if (mavenZip != null) {
+                    mavenZip.artifact(project.getTasks().named("bundlePlugin"));
                 }
-
-                String groupId = mavenZip.getGroupId();
-                if (groupId == null) {
-                    // The groupId is not customized thus we get the value from "project.group".
-                    // See https://docs.gradle.org/current/userguide/publishing_maven.html#sec:identity_values_in_the_generated_pom
-                    groupId = getProperty("group", project);
-                }
-
-                String artifactId = project.getName();
-                String pluginVersion = getProperty("version", project);
-                mavenZip.artifact(project.getTasks().named("bundlePlugin"));
-                mavenZip.setGroupId(groupId);
-                mavenZip.setArtifactId(artifactId);
-                mavenZip.setVersion(pluginVersion);
             });
         });
     }
 
-    static String getProperty(String name, Project project) {
-        if (project.hasProperty(name)) {
-            Object property = project.property(name);
-            if (property != null) {
-                return property.toString();
-            }
-        }
-        return null;
-    }
-
     @Override
     public void apply(Project project) {
+        project.getPluginManager().apply("nebula.maven-base-publish");
+        project.getPluginManager().apply(MavenPublishPlugin.class);
         project.afterEvaluate(evaluatedProject -> {
-            configMaven(project);
-            Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
-            if (validatePluginZipPom != null) {
-                project.getTasks().getByName("validatePluginZipPom").dependsOn("generatePomFileForNebulaPublication");
-            }
-            Task publishPluginZipPublicationToZipStagingRepository = project.getTasks()
-                .findByName("publishPluginZipPublicationToZipStagingRepository");
-            if (publishPluginZipPublicationToZipStagingRepository != null) {
-                publishPluginZipPublicationToZipStagingRepository.dependsOn("generatePomFileForNebulaPublication");
+            if (isZipPublicationPresent(project)) {
+                addLocalMavenRepo(project);
+                addZipArtifact(project);
+                Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
+                if (validatePluginZipPom != null) {
+                    validatePluginZipPom.dependsOn("generatePomFileForNebulaPublication");
+                }
+                Task publishPluginZipPublicationToZipStagingRepository = project.getTasks()
+                    .findByName("publishPluginZipPublicationToZipStagingRepository");
+                if (publishPluginZipPublicationToZipStagingRepository != null) {
+                    publishPluginZipPublicationToZipStagingRepository.dependsOn("generatePomFileForNebulaPublication");
+                }
+            } else {
+                project.getLogger()
+                    .warn(String.format("Plugin 'opensearch.pluginzip' is applied but no '%s' publication is defined.", PUBLICATION_NAME));
             }
         });
     }