[fix] match BCJSSE key types when choosing the TLS credential

src/main/java/org/jruby/ext/openssl/SSLContext.java

@@ -1239,18 +1239,53 @@ public String chooseServerAlias(String keyType, java.security.Principal[] issuer
             return chooseAlias(keyType);
         }
 
-        // BCJSSE asks per negotiated key type (RSA / EC / ...)
+        // BCJSSE asks per negotiated key type: "RSA", "EC/secp256r1" or "Ed25519",
+        // or (TLS 1.2) a legacy auth type such as "ECDHE_ECDSA" or "ECDHE_RSA"
         private String chooseAlias(final String keyType) {
             if (keyType == null) return null;
             for (int i = 0; i < internalContext.credentials.size(); i++) {
                 final Credential credential = internalContext.credentials.get(i);
-                if (credential.privateKey != null && keyType.equalsIgnoreCase(credential.keyType)) {
+                if (credential.privateKey != null && keyTypeMatches(keyType, credential)) {
                     return credential.alias; // pick the first credential that matches
                 }
             }
             return null;
         }
 
+        private static boolean keyTypeMatches(final String keyType, final Credential credential) {
+            final int slash = keyType.indexOf('/');
+            final String keyAlgorithm = slash < 0 ? keyType : keyType.substring(0, slash);
+            switch (keyAlgorithm.toUpperCase(java.util.Locale.ROOT)) {
+                case "ECDHE_ECDSA":
+                    return "EC".equalsIgnoreCase(credential.keyType);
+                case "ECDHE_RSA": case "DHE_RSA":
+                    return "RSA".equalsIgnoreCase(credential.keyType);
+                case "DHE_DSS":
+                    return "DSA".equalsIgnoreCase(credential.keyType);
+                case "ED25519": case "ED448": // credential key type is "EdDSA"
+                    if (!"EdDSA".equalsIgnoreCase(credential.keyType)) return false;
+                    final String algorithm = credential.privateKey.getAlgorithm();
+                    return keyAlgorithm.equalsIgnoreCase(algorithm) || "EdDSA".equalsIgnoreCase(algorithm);
+                default:
+                    if (!keyAlgorithm.equalsIgnoreCase(credential.keyType)) return false;
+                    return slash < 0 || isOnCurve(credential.privateKey, keyType.substring(slash + 1));
+            }
+        }
+
+        private static boolean isOnCurve(final java.security.PrivateKey key, final String namedGroup) {
+            if (!(key instanceof java.security.interfaces.ECKey)) return false;
+            try {
+                final java.security.AlgorithmParameters params = java.security.AlgorithmParameters.getInstance("EC");
+                params.init(new java.security.spec.ECGenParameterSpec(namedGroup));
+                final java.security.spec.ECParameterSpec group = params.getParameterSpec(java.security.spec.ECParameterSpec.class);
+                final java.security.spec.ECParameterSpec own = ((java.security.interfaces.ECKey) key).getParams();
+                return group.getCurve().equals(own.getCurve()) && group.getOrder().equals(own.getOrder());
+            }
+            catch (java.security.GeneralSecurityException e) {
+                return false;
+            }
+        }
+
         private Credential credentialByAlias(String alias) {
             if (alias == null) return null;
             for (int i = 0; i < internalContext.credentials.size(); i++) {

test/ssl/test_ssl.rb

@@ -709,6 +709,34 @@ def test_add_certificate_multiple_certs
     }
   end
 
+  # An explicit TLS version selects BCJSSE, which asks the key manager for
+  # "ECDHE_ECDSA" / "EC/<group>" / "Ed25519" rather than "EC" or "EdDSA"
+  def test_server_key_types_with_explicit_tls_version
+    keys = {
+      "P-256" => OpenSSL::PKey::EC.generate("prime256v1"),
+      "P-384" => OpenSSL::PKey::EC.generate("secp384r1"),
+    }
+    # BCJSSE in FIPS mode does not offer the ed25519 signature scheme
+    keys["Ed25519"] = OpenSSL::PKey.generate_key("ED25519") unless fips?
+    keys.each do |name, key|
+      dn = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{name}")
+      cert = issue_cert(dn, key, 10, [["keyUsage", "digitalSignature", true]], @ca_cert, @ca_key)
+      [:TLSv1_2, :TLSv1_3].each do |version|
+        ctx_proc = -> ctx {
+          ctx.ssl_version = version
+          ctx.cert = cert
+          ctx.key = key
+        }
+        start_server(OpenSSL::SSL::VERIFY_NONE, true, ctx_proc: ctx_proc) { |server, port|
+          server_connect(port, OpenSSL::SSL::SSLContext.new(version)) { |ssl|
+            assert_equal dn.to_s, ssl.peer_cert.subject.to_s, "#{name} #{version}"
+            ssl.puts "abc"; assert_equal "abc\n", ssl.gets
+          }
+        }
+      end
+    end
+  end
+
   def test_add_certificate_validates_private_key_and_chain
     ctx = OpenSSL::SSL::SSLContext.new
     public_key = OpenSSL::PKey.read(@svr_key.public_to_der)