Merge branch 'main' of github.com:opensearch-project/security into ap…

…i-token

build.gradle

@@ -65,7 +65,7 @@ plugins {
     id 'com.diffplug.spotless' version '6.25.0'
     id 'checkstyle'
     id 'com.netflix.nebula.ospackage' version "11.10.0"
-    id "org.gradle.test-retry" version "1.5.10"
+    id "org.gradle.test-retry" version "1.6.0"
     id 'eclipse'
     id "com.github.spotbugs" version "5.2.5"
     id "com.google.osdetector" version "1.7.3"
@@ -471,7 +471,7 @@ configurations {
         resolutionStrategy {
             force 'commons-codec:commons-codec:1.17.1'
             force 'org.slf4j:slf4j-api:1.7.36'
-            force 'org.scala-lang:scala-library:2.13.14'
+            force 'org.scala-lang:scala-library:2.13.15'
             force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
             force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
             force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
@@ -482,7 +482,7 @@ configurations {
             force "io.netty:netty-transport:${versions.netty}"
             force "io.netty:netty-transport-native-unix-common:${versions.netty}"
             force "com.github.luben:zstd-jni:${versions.zstd}"
-            force "org.xerial.snappy:snappy-java:1.1.10.6"
+            force "org.xerial.snappy:snappy-java:1.1.10.7"
             force "com.google.guava:guava:${guava_version}"
 
             // for spotbugs dependency conflict
@@ -495,7 +495,7 @@ configurations {
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
-            force "com.google.errorprone:error_prone_annotations:2.31.0"
+            force "com.google.errorprone:error_prone_annotations:2.32.0"
             force "org.checkerframework:checker-qual:3.47.0"
             force "ch.qos.logback:logback-classic:1.5.8"
         }
@@ -580,7 +580,7 @@ dependencies {
     implementation 'commons-cli:commons-cli:1.9.0'
     implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
     implementation 'org.ldaptive:ldaptive:1.2.3'
-    implementation 'com.nimbusds:nimbus-jose-jwt:9.40'
+    implementation 'com.nimbusds:nimbus-jose-jwt:9.41.1'
     implementation 'com.rfksystems:blake2b:2.0.0'
     implementation 'com.password4j:password4j:1.8.2'
     //JWT
@@ -602,7 +602,7 @@ dependencies {
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
     runtimeOnly 'commons-codec:commons-codec:1.17.1'
     runtimeOnly 'org.cryptacular:cryptacular:1.2.7'
-    compileOnly 'com.google.errorprone:error_prone_annotations:2.31.0'
+    compileOnly 'com.google.errorprone:error_prone_annotations:2.32.0'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
     runtimeOnly 'org.ow2.asm:asm:9.7'
@@ -639,7 +639,7 @@ dependencies {
     runtimeOnly 'org.lz4:lz4-java:1.8.0'
     runtimeOnly 'org.slf4j:slf4j-api:1.7.36'
     runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.6'
+    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.7'
     runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2'
     runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}"
     runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.7.0'
@@ -696,7 +696,7 @@ dependencies {
     testRuntimeOnly ("org.springframework:spring-core:${spring_version}") {
         exclude(group:'org.springframework', module: 'spring-jcl' )
     }
-    testRuntimeOnly 'org.scala-lang:scala-library:2.13.14'
+    testRuntimeOnly 'org.scala-lang:scala-library:2.13.15'
     testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
     testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') {
         exclude(group:'ch.qos.logback', module: 'logback-classic' )
@@ -718,7 +718,7 @@ dependencies {
     integrationTestImplementation 'junit:junit:4.13.2'
     integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
     integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.16.1'
+    integrationTestImplementation 'commons-io:commons-io:2.17.0'
     integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'

src/main/java/org/opensearch/security/configuration/ClusterInfoHolder.java

@@ -29,6 +29,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import org.opensearch.Version;
 import org.opensearch.cluster.ClusterChangedEvent;
 import org.opensearch.cluster.ClusterStateListener;
 import org.opensearch.cluster.node.DiscoveryNode;
@@ -67,6 +68,17 @@ public boolean isInitialized() {
         return initialized;
     }
 
+    public Version getMinNodeVersion() {
+        if (nodes == null) {
+            if (log.isDebugEnabled()) {
+                log.debug("Cluster Info Holder not initialized yet for 'nodes'");
+            }
+            return null;
+        }
+
+        return nodes.getMinNodeVersion();
+    }
+
     public Boolean hasNode(DiscoveryNode node) {
         if (nodes == null) {
             if (log.isDebugEnabled()) {

src/main/java/org/opensearch/security/filter/SecurityFilter.java

@@ -185,7 +185,7 @@ private <Request extends ActionRequest, Response extends ActionResponse> void ap
             }
 
             if (threadContext.getTransient(ConfigConstants.USE_JDK_SERIALIZATION) == null) {
-                threadContext.putTransient(ConfigConstants.USE_JDK_SERIALIZATION, false);
+                threadContext.putTransient(ConfigConstants.USE_JDK_SERIALIZATION, true);
             }
 
             final ComplianceConfig complianceConfig = auditLog.getComplianceConfig();

src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java

@@ -18,6 +18,7 @@
 package org.opensearch.security.ssl;
 
 import java.io.File;
+import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -1223,9 +1224,10 @@ private List<String> getOtherName(List<?> altName) {
             final ASN1Sequence sequence = ASN1Sequence.getInstance(asn1Primitive);
             final ASN1ObjectIdentifier asn1ObjectIdentifier = ASN1ObjectIdentifier.getInstance(sequence.getObjectAt(0));
             final ASN1TaggedObject asn1TaggedObject = ASN1TaggedObject.getInstance(sequence.getObjectAt(1));
-            ASN1Object maybeTaggedAsn1Primitive = asn1TaggedObject.getBaseObject();
+            Method getObjectMethod = getObjectMethod();
+            ASN1Object maybeTaggedAsn1Primitive = (ASN1Primitive) getObjectMethod.invoke(asn1TaggedObject);
             if (maybeTaggedAsn1Primitive instanceof ASN1TaggedObject) {
-                maybeTaggedAsn1Primitive = ASN1TaggedObject.getInstance(maybeTaggedAsn1Primitive).getBaseObject();
+                maybeTaggedAsn1Primitive = (ASN1Primitive) getObjectMethod.invoke(maybeTaggedAsn1Primitive);
             }
             if (maybeTaggedAsn1Primitive instanceof ASN1String) {
                 return ImmutableList.of(asn1ObjectIdentifier.getId(), maybeTaggedAsn1Primitive.toString());
@@ -1237,4 +1239,13 @@ private List<String> getOtherName(List<?> altName) {
             throw new RuntimeException("Couldn't parse subject alternative names", ioe);
         }
     }
+
+    static Method getObjectMethod() throws ClassNotFoundException, NoSuchMethodException {
+        Class<?> asn1TaggedObjectClass = Class.forName("org.bouncycastle.asn1.ASN1TaggedObject");
+        try {
+            return asn1TaggedObjectClass.getMethod("getBaseObject");
+        } catch (NoSuchMethodException ex) {
+            return asn1TaggedObjectClass.getMethod("getObject");
+        }
+    }
 }

src/main/java/org/opensearch/security/support/Base64Helper.java

@@ -35,11 +35,11 @@ public static String serializeObject(final Serializable object, final boolean us
     }
 
     public static String serializeObject(final Serializable object) {
-        return serializeObject(object, false);
+        return serializeObject(object, true);
     }
 
     public static Serializable deserializeObject(final String string) {
-        return deserializeObject(string, false);
+        return deserializeObject(string, true);
     }
 
     public static Serializable deserializeObject(final String string, final boolean useJDKDeserialization) {
@@ -69,4 +69,28 @@ public static String ensureJDKSerialized(final String string) {
         // If we see an exception now, we want the caller to see it -
         return Base64Helper.serializeObject(serializable, true);
     }
+
+    /**
+     * Ensures that the returned string is custom serialized.
+     *
+     * If the supplied string is a JDK serialized representation, will deserialize it and further serialize using
+     * custom, otherwise returns the string as is.
+     *
+     * @param string original string, can be JDK or custom serialized
+     * @return custom serialized string
+     */
+    public static String ensureCustomSerialized(final String string) {
+        Serializable serializable;
+        try {
+            serializable = Base64Helper.deserializeObject(string, true);
+        } catch (Exception e) {
+            // We received an exception when de-serializing the given string. It is probably custom serialized.
+            // Try to deserialize using custom
+            Base64Helper.deserializeObject(string, false);
+            // Since we could deserialize the object using custom, the string is already custom serialized, return as is
+            return string;
+        }
+        // If we see an exception now, we want the caller to see it -
+        return Base64Helper.serializeObject(serializable, false);
+    }
 }

src/main/java/org/opensearch/security/transport/SecurityInterceptor.java

@@ -39,6 +39,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsAction;
 import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.opensearch.action.get.GetRequest;
@@ -231,13 +232,22 @@ && getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_ROL
             }
 
             try {
-                if (serializationFormat == SerializationFormat.JDK) {
-                    Map<String, String> jdkSerializedHeaders = new HashMap<>();
-                    HeaderHelper.getAllSerializedHeaderNames()
-                        .stream()
-                        .filter(k -> headerMap.get(k) != null)
-                        .forEach(k -> jdkSerializedHeaders.put(k, Base64Helper.ensureJDKSerialized(headerMap.get(k))));
-                    headerMap.putAll(jdkSerializedHeaders);
+                if (clusterInfoHolder.getMinNodeVersion() == null || clusterInfoHolder.getMinNodeVersion().before(Version.V_2_14_0)) {
+                    if (serializationFormat == SerializationFormat.JDK) {
+                        Map<String, String> jdkSerializedHeaders = new HashMap<>();
+                        HeaderHelper.getAllSerializedHeaderNames()
+                            .stream()
+                            .filter(k -> headerMap.get(k) != null)
+                            .forEach(k -> jdkSerializedHeaders.put(k, Base64Helper.ensureJDKSerialized(headerMap.get(k))));
+                        headerMap.putAll(jdkSerializedHeaders);
+                    } else if (serializationFormat == SerializationFormat.CustomSerializer_2_11) {
+                        Map<String, String> customSerializedHeaders = new HashMap<>();
+                        HeaderHelper.getAllSerializedHeaderNames()
+                            .stream()
+                            .filter(k -> headerMap.get(k) != null)
+                            .forEach(k -> customSerializedHeaders.put(k, Base64Helper.ensureCustomSerialized(headerMap.get(k))));
+                        headerMap.putAll(customSerializedHeaders);
+                    }
                 }
                 getThreadContext().putHeader(headerMap);
             } catch (IllegalArgumentException iae) {

src/test/java/org/opensearch/security/ssl/SSLTest.java

@@ -17,6 +17,7 @@
 
 package org.opensearch.security.ssl;
 
+import java.lang.reflect.Method;
 import java.net.SocketException;
 import java.nio.file.Paths;
 import java.security.Security;
@@ -1291,4 +1292,18 @@ public void testHttpsAndNodeSSLPemExtendedUsageEnabled() throws Exception {
         Assert.assertTrue(res.contains("CN=node-0.example.com,OU=SSL,O=Test,L=Test,C=DE"));
         Assert.assertTrue(rh.executeSimpleRequest("_nodes/settings?pretty").contains(clusterInfo.clustername));
     }
+
+    @Test
+    public void testGetObjectMethod() {
+        try {
+            Method method = DefaultSecurityKeyStore.getObjectMethod();
+            Assert.assertNotNull("Method should not be null", method);
+            Assert.assertTrue(
+                "One of the expected methods should be available",
+                method.getName().equals("getBaseObject") || method.getName().equals("getObject")
+            );
+        } catch (ClassNotFoundException | NoSuchMethodException e) {
+            Assert.fail("Exception should not be thrown: " + e.getMessage());
+        }
+    }
 }

src/test/java/org/opensearch/security/support/Base64HelperTest.java

@@ -53,6 +53,15 @@ public void testEnsureJDKSerialized() {
         assertThat(Base64Helper.ensureJDKSerialized(customSerialized), is(jdkSerialized));
     }
 
+    @Test
+    public void testEnsureCustomSerialized() {
+        String test = "string";
+        String jdkSerialized = Base64Helper.serializeObject(test, true);
+        String customSerialized = Base64Helper.serializeObject(test, false);
+        assertThat(Base64Helper.ensureCustomSerialized(jdkSerialized), is(customSerialized));
+        assertThat(Base64Helper.ensureCustomSerialized(customSerialized), is(customSerialized));
+    }
+
     @Test
     public void testDuplicatedItemSizes() {
         var largeObject = new HashMap<String, Object>();