[Re-opened for testing] Bump io.grpc:grpc-context from 1.56.1 to 1.57.1 in /plugins/repository-gcs

CHANGELOG-3.0.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 - Added pull-based Ingestion (APIs, for ingestion source, a Kafka plugin, and IngestionEngine that pulls data from the ingestion source) ([#16958](https://github.com/opensearch-project/OpenSearch/pull/16958))
 - Added ConfigurationUtils to core for the ease of configuration parsing [#17223](https://github.com/opensearch-project/OpenSearch/pull/17223)
+- Support for FIPS-140-3 compliance through environment variable ([#3420](https://github.com/opensearch-project/OpenSearch/pull/14912))
 
 ### Dependencies
 - Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))

build.gradle

@@ -65,7 +65,6 @@ apply from: 'gradle/ide.gradle'
 apply from: 'gradle/forbidden-dependencies.gradle'
 apply from: 'gradle/formatting.gradle'
 apply from: 'gradle/local-distribution.gradle'
-apply from: 'gradle/fips.gradle'
 apply from: 'gradle/run.gradle'
 apply from: 'gradle/missing-javadoc.gradle'
 apply from: 'gradle/code-coverage.gradle'
@@ -472,8 +471,8 @@ gradle.projectsEvaluated {
   }
 }
 
-// test retry configuration
 subprojects {
+  // test retry configuration
   tasks.withType(Test).configureEach {
     develocity.testRetry {
       if (BuildParams.isCi()) {
@@ -559,6 +558,22 @@ subprojects {
       }
     }
   }
+
+  // test with FIPS-140-3 enabled
+  plugins.withType(JavaPlugin).configureEach {
+    tasks.withType(Test).configureEach { testTask ->
+      if (BuildParams.inFipsJvm) {
+        testTask.jvmArgs += "-Dorg.bouncycastle.fips.approved_only=true"
+      }
+    }
+  }
+  plugins.withId('opensearch.testclusters') {
+    testClusters.configureEach {
+      if (BuildParams.inFipsJvm) {
+        keystorePassword 'notarealpasswordphrase'
+      }
+    }
+  }
 }
 
 // eclipse configuration

buildSrc/build.gradle

@@ -229,12 +229,8 @@ if (project != rootProject) {
 
   forbiddenPatterns {
     exclude '**/*.wav'
-    exclude '**/*.p12'
-    exclude '**/*.jks'
-    exclude '**/*.crt'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
-    exclude '**/*.bcfks'
   }
 
   testingConventions {

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -164,6 +164,12 @@
                 test.systemProperty("tests.seed", BuildParams.getTestSeed());
             }
 
+            var securityFile = BuildParams.isInFipsJvm() ? "fips_java.security" : "java.security";
+            test.systemProperty(
+                "java.security.properties",
+                project.getRootProject().getLayout().getProjectDirectory() + "/distribution/src/config/" + securityFile
+            );
+
             // don't track these as inputs since they contain absolute paths and break cache relocatability
             File gradleHome = project.getGradle().getGradleUserHomeDir();
             String gradleVersion = project.getGradle().getGradleVersion();

buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.gradle.http;
 
+import de.thetaphi.forbiddenapis.SuppressForbidden;
+
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 
@@ -216,15 +218,15 @@ KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
     }
 
     private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
-        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        var keyStore = getKeyStoreInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
         try (InputStream input = new FileInputStream(trustStoreFile)) {
             keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
         }
         return keyStore;
     }
 
     private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
-        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        var store = getKeyStoreInstance(KeyStore.getDefaultType());
         store.load(null, null);
         final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
         int counter = 0;
@@ -239,6 +241,11 @@ private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOExce
         return store;
     }
 
+    @SuppressForbidden
+    private KeyStore getKeyStoreInstance(String type) throws KeyStoreException {
+        return KeyStore.getInstance(type);
+    }
+
     private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
         checkForTrustEntry(trustStore);
         TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());

buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.info;
+
+import java.util.function.Function;
+
+public class FipsBuildParams {
+
+    public static final String FIPS_BUILD_PARAM = "crypto.standard";
+
+    public static final String FIPS_ENV_VAR = "OPENSEARCH_CRYPTO_STANDARD";
+
+    private static String fipsMode;
+
+    public static void init(Function<String, Object> fipsValue) {
+        fipsMode = (String) fipsValue.apply(FIPS_BUILD_PARAM);
+    }
+
+    private FipsBuildParams() {}
+
+    public static boolean isInFipsMode() {
+        return "FIPS-140-3".equals(fipsMode);
+    }
+
+    public static String getFipsMode() {
+        return fipsMode;
+    }
+
+}

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -109,6 +109,8 @@
         File rootDir = project.getRootDir();
         GitInfo gitInfo = gitInfo(rootDir);
 
+        FipsBuildParams.init(project::findProperty);
+
         BuildParams.init(params -> {
             // Initialize global build parameters
             boolean isInternal = GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null;
@@ -129,7 +131,7 @@
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(isInternal);
             params.setDefaultParallel(findDefaultParallel(project));
-            params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
+            params.setInFipsJvm(FipsBuildParams.isInFipsMode());
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
             if (isInternal) {
                 params.setBwcVersions(resolveBwcVersions(rootDir));
@@ -179,7 +181,11 @@
             LOGGER.quiet("  JAVA_HOME             : " + gradleJvm.getJavaHome());
         }
         LOGGER.quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
-        LOGGER.quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
+        if (FipsBuildParams.isInFipsMode()) {
+            LOGGER.quiet("  Crypto Standard       : " + FipsBuildParams.getFipsMode());
+        } else {
+            LOGGER.quiet("  Crypto Standard       : any-supported");
+        }
         LOGGER.quiet("=======================================");
     }
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -83,8 +83,13 @@ public class ForbiddenPatternsTask extends DefaultTask {
         .exclude("**/*.ico")
         .exclude("**/*.jar")
         .exclude("**/*.zip")
+        .exclude("**/*.p12")
         .exclude("**/*.jks")
         .exclude("**/*.crt")
+        .exclude("**/*.der")
+        .exclude("**/*.pem")
+        .exclude("**/*.key")
+        .exclude("**/*.bcfks")
         .exclude("**/*.keystore")
         .exclude("**/*.png");
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionsTasks.java

@@ -40,6 +40,7 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.jvm.JvmTestSuite;
@@ -87,6 +88,7 @@
 
     private final NamedDomainObjectContainer<TestingConventionRule> naming;
     private final Project project;
+    private final ConfigurableFileCollection extraClassPath = getProject().files();
 
     @Inject
     public TestingConventionsTasks(Project project) {
@@ -398,7 +400,16 @@
 
     @Classpath
     public FileCollection getTestsClassPath() {
-        return Util.getJavaTestSourceSet(project).get().getRuntimeClasspath();
+        return Util.getJavaTestSourceSet(project).get().getRuntimeClasspath().plus(extraClassPath);
+    }
+
+    @Classpath
+    public ConfigurableFileCollection getExtraClassPath() {
+        return extraClassPath;
+    }
+
+    public void addExtraClassPath(Object... paths) {
+        extraClassPath.from(paths);
     }
 
     private Map<String, File> walkPathAndLoadClasses(File testRoot) {

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -46,6 +46,7 @@
 import org.opensearch.gradle.Version;
 import org.opensearch.gradle.VersionProperties;
 import org.opensearch.gradle.info.BuildParams;
+import org.opensearch.gradle.info.FipsBuildParams;
 import org.gradle.api.Action;
 import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
@@ -546,17 +547,21 @@
             logToProcessStdout("installed plugins");
         }
 
+        if (FipsBuildParams.isInFipsMode() && keystorePassword.isEmpty()) {
+            throw new TestClustersException("Can not start " + this + " in FIPS JVM, missing keystore password");
+        }
+
         logToProcessStdout("Creating opensearch keystore with password set to [" + keystorePassword + "]");
         if (keystorePassword.length() > 0) {
-            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword, "opensearch-keystore", "create", "-p");
+            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword + "\n", "opensearch-keystore", "create", "-p");
         } else {
             runOpenSearchBinScript("opensearch-keystore", "-v", "create");
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
 
-            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", "-x", key));
+            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", key));
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
                 File file = entry.getValue();
@@ -738,7 +743,12 @@
     }
 
     private void runKeystoreCommandWithPassword(String keystorePassword, String input, CharSequence... args) {
-        final String actualInput = keystorePassword.length() > 0 ? keystorePassword + "\n" + input : input;
+        final String actualInput;
+        if (keystorePassword.length() > 0) {
+            actualInput = keystorePassword + "\n" + input + "\n" + input;
+        } else {
+            actualInput = input + "\n" + input;
+        }
         runOpenSearchBinScriptWithInput(actualInput, "opensearch-keystore", args);
     }
 
@@ -786,6 +796,9 @@
         // Override the system hostname variables for testing
         defaultEnv.put("HOSTNAME", HOSTNAME_OVERRIDE);
         defaultEnv.put("COMPUTERNAME", COMPUTERNAME_OVERRIDE);
+        if (FipsBuildParams.isInFipsMode()) {
+            defaultEnv.put(FipsBuildParams.FIPS_ENV_VAR, FipsBuildParams.getFipsMode());
+        }
 
         Set<String> commonKeys = new HashSet<>(environment.keySet());
         commonKeys.retainAll(defaultEnv.keySet());