[netty#5401] Support -Djdk.tls.ephemeralDHKeySize=num when using Open…

…SslContext

Motivation:

Java8+ adds support set a DH key size via a System property (jdk.tls.ephemeralDHKeySize). We should respect this when using OpenSSL.

Modifications:

Respect system property.

Result:

More consistent SSL implementation.

handler/src/main/java/io/netty/handler/ssl/OpenSslContext.java

@@ -33,7 +33,9 @@
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.X509TrustManager;
+import java.security.AccessController;
 import java.security.PrivateKey;
+import java.security.PrivilegedAction;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateExpiredException;
 import java.security.cert.CertificateNotYetValidException;
@@ -62,6 +64,7 @@ public abstract class OpenSslContext extends SslContext {
     private static final boolean JDK_REJECT_CLIENT_INITIATED_RENEGOTIATION =
             SystemPropertyUtil.getBoolean("jdk.tls.rejectClientInitiatedRenegotiation", false);
     private static final List<String> DEFAULT_CIPHERS;
+    private static final Integer DH_KEY_LENGTH;
 
     // TODO: Maybe make configurable ?
     protected static final int VERIFY_DEPTH = 10;
@@ -121,6 +124,28 @@ public SelectedListenerFailureBehavior selectedListenerFailureBehavior() {
         if (logger.isDebugEnabled()) {
             logger.debug("Default cipher suite (OpenSSL): " + ciphers);
         }
+
+        Integer dhLen = null;
+
+        try {
+            String dhKeySize = AccessController.doPrivileged(new PrivilegedAction<String>() {
+                @Override
+                public String run() {
+                    return SystemPropertyUtil.get("jdk.tls.ephemeralDHKeySize");
+                }
+            });
+            if (dhKeySize != null) {
+                try {
+                    dhLen = Integer.parseInt(dhKeySize);
+                } catch (NumberFormatException e) {
+                    logger.debug("OpenSslContext only support -Djdk.tls.ephemeralDHKeySize={int}, but got: "
+                            + dhKeySize);
+                }
+            }
+        } catch (Throwable ignore) {
+            // ignore
+        }
+        DH_KEY_LENGTH = dhLen;
     }
 
     OpenSslContext(Iterable<String> ciphers, CipherSuiteFilter cipherFilter, ApplicationProtocolConfig apnCfg,
@@ -202,6 +227,10 @@ public SelectedListenerFailureBehavior selectedListenerFailureBehavior() {
                 // See https://github.com/netty/netty-tcnative/issues/100
                 SSLContext.setMode(ctx, SSLContext.getMode(ctx) | SSL.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
 
+                if (DH_KEY_LENGTH != null) {
+                    SSLContext.setTmpDHLength(ctx, DH_KEY_LENGTH);
+                }
+
                 /* List the ciphers that are permitted to negotiate. */
                 try {
                     SSLContext.setCipherSuite(ctx, CipherSuiteConverter.toOpenSsl(unmodifiableCiphers));