Allow Integ Tests to run in a FIPS-140 JVM (#31989)

* Complete changes for running IT in a fips JVM

- Mute :x-pack:qa:sql:security:ssl:integTest as it
  cannot run in FIPS 140 JVM until the SQL CLI supports key/cert.
- Set default JVM keystore/truststore password in top level build
  script for all integTest tasks in a FIPS 140 JVM
- Changed top level x-pack build script to use keys and certificates
  for trust/key material when spinning up clusters for IT

Author: jkakavas

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -131,6 +131,9 @@ class BuildPlugin implements Plugin<Project> {
                 runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
 
+            String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
+            boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript))
+
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
@@ -202,6 +205,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.buildChecksDone = true
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
+            project.rootProject.ext.inFipsJvm = inFipsJvm
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -213,6 +217,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
+        project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
     }
 
     private static String findCompilerJavaHome() {
@@ -770,7 +775,11 @@ class BuildPlugin implements Plugin<Project> {
                     systemProperty property.getKey(), property.getValue()
                 }
             }
-
+            // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
+            if (project.inFipsJvm) {
+                systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+            }
             boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
             enableSystemAssertions assertionsEnabled
             enableAssertions assertionsEnabled

plugins/discovery-gce/build.gradle

@@ -22,36 +22,6 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
-
-// needed to be consistent with ssl host checking
-String host = InetAddress.getLoopbackAddress().getHostAddress();
-
-// location of keystore and files to generate it
-File keystore = new File(project.buildDir, 'keystore/test-node.jks')
-
-// generate the keystore
-task createKey(type: LoggedExec) {
-  doFirst {
-    project.delete(keystore.parentFile)
-    keystore.parentFile.mkdirs()
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', keystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=' + host,
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
-}
-
-// add keystore to test classpath: it expects it there
-sourceSets.test.resources.srcDir(keystore.parentFile)
-processTestResources.dependsOn(createKey)
-
 dependencyLicenses {
   mapping from: /google-.*/, to: 'google'
 }

server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java

@@ -205,7 +205,14 @@ public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
                             assertThat(nodesMap.size(), equalTo(cluster().size()));
                             for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
                                 assertThat(nodeResponse.reloadException(), notNullValue());
-                                assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                // Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
+                                // password returns a SecurityException if the DataInputStream can't be fully consumed
+                                if (inFipsJvm()) {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
+                                } else {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                }
+
                             }
                         } catch (final AssertionError e) {
                             reloadSettingsError.set(e);

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -176,6 +176,7 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -2364,4 +2365,7 @@ protected void assertSeqNos() throws Exception {
         });
     }
 
+    public static boolean inFipsJvm() {
+        return Security.getProviders()[0].getName().toLowerCase(Locale.ROOT).contains("fips");
+    }
 }

x-pack/plugin/build.gradle

@@ -104,39 +104,28 @@ integTestRunner {
   systemProperty 'tests.rest.blacklist', blacklist.join(',')
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
-// Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-task createNodeKeyStore(type: LoggedExec) {
-  doFirst {
-    if (nodeKeystore.parentFile.exists() == false) {
-      nodeKeystore.parentFile.mkdirs()
-    }
-    if (nodeKeystore.exists()) {
-      delete nodeKeystore
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
+
+// Add key and certs to test classpath: it expects them there
+// User cert and key PEM files instead of a JKS Keystore for the cluster's trust material so that
+// it can run in a FIPS 140 JVM
+// TODO: Remove all existing uses of cross project file references when the new approach for referencing static files is available
+// https://github.com/elastic/elasticsearch/pull/32201
+task copyKeyCerts(type: Copy) {
+  from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
+    include 'testnode.crt', 'testnode.pem'
     }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
+  into keystoreDir
 }
-
 // Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-  dependsOn createNodeKeyStore
+  dependsOn copyKeyCerts
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
   setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
@@ -145,17 +134,19 @@ integTestCluster {
   setting 'xpack.monitoring.exporters._local.enabled', 'false'
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.transport.ssl.enabled', 'true'
-  setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+  setting 'xpack.security.transport.ssl.key', nodeKey.name
+  setting 'xpack.security.transport.ssl.certificate', nodeCert.name
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
   setting 'xpack.security.audit.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-  keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+  keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
   distribution = 'zip' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-  extraConfigFile nodeKeystore.name, nodeKeystore
+  extraConfigFile nodeKey.name, nodeKey
+  extraConfigFile nodeCert.name, nodeCert
 
   waitCondition = { NodeInfo node, AntBuilder ant ->
       File tmpFile = new File(node.cwd, 'wait.success')

x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml

@@ -4,7 +4,6 @@
       xpack.ssl.certificates: {}
 
   - length: { $body: 1 }
-  - match: { $body.0.path: "test-node.jks" }
-  - match: { $body.0.format: "jks" }
-  - match: { $body.0.alias: "test-node" }
+  - match: { $body.0.path: "testnode.crt" }
+  - match: { $body.0.format: "PEM" }
   - match: { $body.0.has_private_key: true }

x-pack/qa/full-cluster-restart/build.gradle

@@ -125,8 +125,8 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
-            .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+    from project(':x-pack:plugin:core')
+      .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }
 

x-pack/qa/ml-native-multi-node-tests/build.gradle

@@ -18,59 +18,45 @@ integTestRunner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
-// Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-task createNodeKeyStore(type: LoggedExec) {
-    doFirst {
-        if (nodeKeystore.parentFile.exists() == false) {
-            nodeKeystore.parentFile.mkdirs()
-        }
-        if (nodeKeystore.exists()) {
-            delete nodeKeystore
-        }
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
+// Add key and certs to test classpath: it expects it there
+task copyKeyCerts(type: Copy) {
+    from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
+        include 'testnode.crt', 'testnode.pem'
     }
-    executable = new File(project.runtimeJavaHome, 'bin/keytool')
-    standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-    args '-genkey',
-            '-alias', 'test-node',
-            '-keystore', nodeKeystore,
-            '-keyalg', 'RSA',
-            '-keysize', '2048',
-            '-validity', '712',
-            '-dname', 'CN=smoke-test-plugins-ssl',
-            '-keypass', 'keypass',
-            '-storepass', 'keypass'
+    into keystoreDir
 }
-
-// Add keystores to test classpath: it expects it there
+// Add keys and cets to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-    dependsOn createNodeKeyStore
+    dependsOn copyKeyCerts
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.ml.enabled', 'true'
     setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.authc.token.enabled', 'true'
     setting 'xpack.security.transport.ssl.enabled', 'true'
-    setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+    setting 'xpack.security.transport.ssl.key', nodeKey.name
+    setting 'xpack.security.transport.ssl.certificate', nodeCert.name
     setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
     setting 'xpack.security.audit.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
 
     keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-    keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+    keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
     numNodes = 3
 
     setupCommand 'setupDummyUser',
             'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-    extraConfigFile nodeKeystore.name, nodeKeystore
+    extraConfigFile nodeKey.name, nodeKey
+    extraConfigFile nodeCert.name, nodeCert
 
     waitCondition = { node, ant ->
         File tmpFile = new File(node.cwd, 'wait.success')

x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java

@@ -124,9 +124,11 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
 
     @Override
     protected Settings externalClusterClientSettings() {
-        Path keyStore;
+        Path key;
+        Path certificate;
         try {
-            keyStore = PathUtils.get(getClass().getResource("/test-node.jks").toURI());
+            key = PathUtils.get(getClass().getResource("/testnode.pem").toURI());
+            certificate = PathUtils.get(getClass().getResource("/testnode.crt").toURI());
         } catch (URISyntaxException e) {
             throw new IllegalStateException("error trying to get keystore path", e);
         }
@@ -135,8 +137,9 @@ protected Settings externalClusterClientSettings() {
         builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true);
         builder.put("xpack.security.transport.ssl.enabled", true);
-        builder.put("xpack.security.transport.ssl.keystore.path", keyStore.toAbsolutePath().toString());
-        builder.put("xpack.security.transport.ssl.keystore.password", "keypass");
+        builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.key_passphrase", "testnode");
         builder.put("xpack.security.transport.ssl.verification_mode", "certificate");
         return builder.build();
     }

x-pack/qa/rolling-upgrade/build.gradle

@@ -107,7 +107,7 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
+    from project(':x-pack:plugin:core')
             .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }