feat(remote): add support for custom certificate validation callbacks (#686)

* feat(remote): add support for custom certificate validation callbacks from Akka.NET v1.5.55

- Updated Akka.NET from v1.5.53 to v1.5.55 to access new SSL/TLS features
- Added CustomValidator property to SslOptions to support CertificateValidationCallback
- Updated RemoteOptions.Build to use appropriate DotNettySslSetup constructor based on settings:
  - 5-parameter constructor when CustomValidator is provided (v1.5.55+)
  - 4-parameter constructor when RequireMutualAuthentication or ValidateCertificateHostname is set
  - 2-parameter constructor for legacy scenarios (backward compatibility)
- Added comprehensive test coverage for CustomValidator configuration
- Updated API approval tests to reflect new public API surface

This enhancement allows users to implement custom certificate validation logic such as:
- Certificate pinning
- Subject/issuer matching
- Business-specific validation rules
- Advanced mTLS scenarios

The implementation maintains full backward compatibility while providing access to the powerful
new CertificateValidation features introduced in Akka.NET v1.5.55.

* Fix SSL configuration precedence issue

Fixed bug where both HOCON SSL configuration and DotNettySslSetup were being emitted simultaneously. DotNettySslSetup ALWAYS takes precedence when present, making HOCON SSL settings ineffective and potentially confusing.

Changes:
- Modified RemoteOptions.Build to only emit HOCON SSL config when X509Certificate is null
- Added comprehensive comments explaining SSL configuration strategy with link to Akka.NET issue #7914
- Updated tests to use CertificateOptions instead of X509Certificate when testing HOCON SSL configuration
- Both WithRemotingNewSslSettingsHoconTest and WithRemotingConfiguratorNewSslSettingsTest now properly test HOCON config

This ensures users understand that:
1. X509Certificate object → DotNettySslSetup (programmatic, takes precedence)
2. X509Certificate null + SSL settings → HOCON configuration only

Related to: akkadotnet/akka.net#7914

* Add comprehensive SSL/TLS documentation with CustomValidator examples

Added detailed SSL/TLS configuration documentation to Akka.Remote.Hosting README including:
- Basic SSL configuration with certificate file (HOCON-based)
- SSL configuration with X509Certificate2 object (programmatic)
- Advanced custom certificate validation example (certificate pinning)
- Examples using all CertificateValidation helper methods from Akka.NET v1.5.55:
  * ValidateChain() - Standard chain validation
  * ValidateHostname() - Hostname validation
  * PinnedCertificate() - Certificate pinning
  * ValidateSubject() - Subject pattern matching
  * ValidateIssuer() - Issuer validation
  * Combine() - Combining multiple validators
- Complete SSL configuration options reference
- Important note about DotNettySslSetup vs HOCON precedence

This helps users understand how to use the new CustomValidator feature for enhanced security scenarios.

* Simplify SSL/TLS documentation to single example

Author: Aaronontheweb

Directory.Build.props

@@ -33,7 +33,7 @@
     <TestSdkVersion>17.11.1</TestSdkVersion>
     <CoverletVersion>6.0.3</CoverletVersion>
     <XunitRunneVisualstudio>3.1.5</XunitRunneVisualstudio>
-    <AkkaVersion>1.5.53</AkkaVersion>
+    <AkkaVersion>1.5.55</AkkaVersion>
     <MicrosoftExtensionsVersion>[6.0.0,)</MicrosoftExtensionsVersion>
     <SystemTextJsonVersion>[6.0.10,)</SystemTextJsonVersion>
   </PropertyGroup>

src/Akka.Hosting.API.Tests/verify/CoreApiSpec.ApproveRemoting.verified.txt

@@ -54,6 +54,7 @@
     {
         public SslOptions() { }
         public Akka.Remote.Hosting.SslCertificateOptions CertificateOptions { get; set; }
+        public Akka.Remote.Transport.DotNetty.CertificateValidationCallback? CustomValidator { get; set; }
         public bool? RequireMutualAuthentication { get; set; }
         public bool? SuppressValidation { get; set; }
         public bool? ValidateCertificateHostname { get; set; }

src/Akka.Remote.Hosting.Tests/RemoteConfigurationSpecs.cs

@@ -449,7 +449,6 @@ public void WithRemotingOptionsSslDisabledCertificateTest()
     public void WithRemotingNewSslSettingsHoconTest()
     {
         // arrange
-        var certificate = new X509Certificate2("./Resources/akka-validcert.pfx", "password");
         var builder = new AkkaConfigurationBuilder(new ServiceCollection(), "test");
         builder.WithRemoting(new RemoteOptions
         {
@@ -459,7 +458,13 @@ public void WithRemotingNewSslSettingsHoconTest()
                 SuppressValidation = false,
                 RequireMutualAuthentication = false, // Explicitly set to false for testing
                 ValidateCertificateHostname = true,  // Explicitly set to true for testing
-                X509Certificate = certificate
+                // NOTE: Not providing X509Certificate so HOCON configuration will be generated
+                // When X509Certificate is provided, DotNettySslSetup is used instead of HOCON
+                CertificateOptions = new SslCertificateOptions
+                {
+                    Path = "./Resources/akka-validcert.pfx",
+                    Password = "password"
+                }
             }
         });
 
@@ -471,6 +476,10 @@ public void WithRemotingNewSslSettingsHoconTest()
         sslConfig.GetBoolean("suppress-validation").Should().BeFalse();
         sslConfig.GetBoolean("require-mutual-authentication").Should().BeFalse();
         sslConfig.GetBoolean("validate-certificate-hostname").Should().BeTrue();
+
+        var certConfig = sslConfig.GetConfig("certificate");
+        certConfig.GetString("path").Should().Be("./Resources/akka-validcert.pfx");
+        certConfig.GetString("password").Should().Be("password");
     }
 
     [Fact(DisplayName = "RemoteOptions with new SSL/TLS settings should properly configure DotNettySslSetup")]
@@ -503,6 +512,45 @@ public void WithRemotingNewSslSettingsDotNettySslSetupTest()
         setup.ValidateCertificateHostname.Should().BeTrue();
     }
 
+    [Fact(DisplayName = "RemoteOptions with CustomValidator should properly configure DotNettySslSetup with custom validation")]
+    public void WithRemotingCustomValidatorDotNettySslSetupTest()
+    {
+        // arrange
+        var certificate = new X509Certificate2("./Resources/akka-validcert.pfx", "password");
+        var builder = new AkkaConfigurationBuilder(new ServiceCollection(), "test");
+
+        // Create a simple custom validator for testing
+        Transport.DotNetty.CertificateValidationCallback customValidator = (cert, chain, peer, errors, log) =>
+        {
+            // This is just a test validator - in real usage, this would contain actual validation logic
+            return cert != null && cert.Thumbprint == certificate.Thumbprint;
+        };
+
+        builder.WithRemoting(new RemoteOptions
+        {
+            EnableSsl = true,
+            Ssl = new SslOptions
+            {
+                SuppressValidation = false,
+                RequireMutualAuthentication = true,
+                ValidateCertificateHostname = false,
+                X509Certificate = certificate,
+                CustomValidator = customValidator
+            }
+        });
+
+        // act
+        var setup = (DotNettySslSetup)builder.Setups.First(s => s is DotNettySslSetup);
+
+        // assert
+        setup.Certificate.Should().Be(certificate);
+        setup.SuppressValidation.Should().BeFalse();
+        setup.RequireMutualAuthentication.Should().BeTrue();
+        setup.ValidateCertificateHostname.Should().BeFalse();
+        setup.CustomValidator.Should().NotBeNull();
+        setup.CustomValidator.Should().BeSameAs(customValidator);
+    }
+
     [Fact(DisplayName = "RemoteOptions without new SSL/TLS settings should use default values")]
     public void WithRemotingDefaultSslSettingsTest()
     {
@@ -533,14 +581,16 @@ public void WithRemotingDefaultSslSettingsTest()
     public void WithRemotingConfiguratorNewSslSettingsTest()
     {
         // arrange
-        var certificate = new X509Certificate2("./Resources/akka-validcert.pfx", "password");
         var builder = new AkkaConfigurationBuilder(new ServiceCollection(), "test");
         builder.WithRemoting(opt =>
         {
             opt.EnableSsl = true;
             opt.Ssl.RequireMutualAuthentication = true;
             opt.Ssl.ValidateCertificateHostname = false;
-            opt.Ssl.X509Certificate = certificate;
+            // Use CertificateOptions instead of X509Certificate to test HOCON configuration
+            // When X509Certificate is provided, DotNettySslSetup takes precedence and HOCON is not emitted
+            opt.Ssl.CertificateOptions.Path = "./Resources/akka-validcert.pfx";
+            opt.Ssl.CertificateOptions.Password = "password";
         });
 
         // act

src/Akka.Remote.Hosting/README.md

@@ -52,3 +52,39 @@ using var host = new HostBuilder()
 
 await host.RunAsync();
 ```
+
+## SSL/TLS Configuration
+
+Akka.Remote supports SSL/TLS encryption for secure communication between actor systems. Starting with Akka.NET v1.5.55, you can provide custom certificate validation callbacks using the `CertificateValidation` helper class.
+
+```csharp
+using System.Security.Cryptography.X509Certificates;
+using Akka.Remote.Transport.DotNetty;
+
+var certificate = new X509Certificate2("/path/to/certificate.pfx", "certificate-password");
+
+using var host = new HostBuilder()
+    .ConfigureServices((context, services) =>
+    {
+        services.AddAkka("secureSystem", (builder, provider) =>
+        {
+            builder.WithRemoting(options =>
+            {
+                options.HostName = "127.0.0.1";
+                options.Port = 4053;
+                options.EnableSsl = true;
+                options.Ssl.X509Certificate = certificate;
+
+                // Use built-in validators for common scenarios
+                options.Ssl.CustomValidator = CertificateValidation.Combine(
+                    CertificateValidation.ValidateChain(),
+                    CertificateValidation.ValidateSubject("CN=*.mycompany.com")
+                );
+            });
+        });
+    }).Build();
+
+await host.RunAsync();
+```
+
+Available `CertificateValidation` methods: `ValidateChain()`, `ValidateHostname()`, `PinnedCertificate()`, `ValidateSubject()`, `ValidateIssuer()`, and `Combine()`.

src/Akka.Remote.Hosting/RemoteOptions.cs

@@ -7,9 +7,11 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
+using System.Net.Security;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using Akka.Configuration;
+using Akka.Event;
 using Akka.Hosting;
 using Akka.Remote.Transport.DotNetty;
 
@@ -105,21 +107,35 @@ internal void Build(AkkaConfigurationBuilder builder)
             if (sb.Length > 0)
                 builder.AddHocon(sb.ToString(), HoconAddMode.Prepend);
 
+            // SSL configuration strategy:
+            // 1. If X509Certificate object is provided -> Use DotNettySslSetup (takes precedence over HOCON)
+            // 2. If X509Certificate is null but SSL settings configured -> Use HOCON configuration only
+            //
+            // Important: DotNettySslSetup ALWAYS takes precedence when present, causing HOCON SSL settings
+            // to be completely ignored. We must not emit both to avoid confusion.
+            // See: https://github.com/akkadotnet/akka.net/blob/dev/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs#L163-L164
+
             if (EnableSsl is false || Ssl.X509Certificate == null)
                 return;
 
             var suppressValidation = Ssl.SuppressValidation ?? false;
             var requireMutualAuth = Ssl.RequireMutualAuthentication ?? true; // Default to true as per v1.5.52
             var validateHostname = Ssl.ValidateCertificateHostname ?? false; // Default to false as per v1.5.53
 
-            // Use the 4-parameter constructor if any of the new settings are provided, otherwise use the legacy constructor for backward compatibility
-            if (Ssl.RequireMutualAuthentication.HasValue || Ssl.ValidateCertificateHostname.HasValue)
+            // Choose the appropriate constructor based on which settings are provided
+            if (Ssl.CustomValidator != null)
+            {
+                // Use the 5-parameter constructor with custom validator (v1.5.55+)
+                builder.AddSetup(new DotNettySslSetup(Ssl.X509Certificate, suppressValidation, requireMutualAuth, validateHostname, Ssl.CustomValidator));
+            }
+            else if (Ssl.RequireMutualAuthentication.HasValue || Ssl.ValidateCertificateHostname.HasValue)
             {
+                // Use the 4-parameter constructor (v1.5.52/v1.5.53+)
                 builder.AddSetup(new DotNettySslSetup(Ssl.X509Certificate, suppressValidation, requireMutualAuth, validateHostname));
             }
             else
             {
-                // Use legacy constructor for backward compatibility when new settings are not specified
+                // Use legacy 2-parameter constructor for backward compatibility when new settings are not specified
                 builder.AddSetup(new DotNettySslSetup(Ssl.X509Certificate, suppressValidation));
             }
         }
@@ -180,8 +196,15 @@ private void Build(StringBuilder builder)
                 {
                     if(Ssl is null)
                         throw new ConfigurationException("Ssl property need to be populated when EnableSsl is set to true.");
-                
-                    Ssl.Build(tcpSb);
+
+                    // Only emit HOCON SSL configuration if we're NOT going to create a DotNettySslSetup
+                    // When DotNettySslSetup is present, it takes precedence and HOCON SSL settings are ignored
+                    // See: https://github.com/akkadotnet/akka.net/issues/7914 and the warning at
+                    // https://github.com/akkadotnet/akka.net/blob/dev/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs#L163-L164
+                    if (Ssl.X509Certificate == null)
+                    {
+                        Ssl.Build(tcpSb);
+                    }
                 }
             }
 
@@ -240,6 +263,30 @@ public sealed class SslOptions
         /// </summary>
         public bool? ValidateCertificateHostname { get; set; }
 
+        /// <summary>
+        /// <para>
+        /// Custom certificate validation callback for advanced validation scenarios.
+        /// When provided, this callback takes precedence over config-based validation.
+        /// </para>
+        /// <para>
+        /// Use this to implement custom validation logic such as certificate pinning,
+        /// subject/issuer matching, or other business-specific validation rules.
+        /// </para>
+        /// <para>
+        /// The callback parameters are:
+        /// - X509Certificate2?: The peer certificate to validate
+        /// - X509Chain?: The X509 chain for validation
+        /// - string: The remote peer identifier
+        /// - SslPolicyErrors: SSL policy errors from standard validation
+        /// - ILoggingAdapter: Logger for diagnostics
+        /// </para>
+        /// <para>
+        /// Returns true to accept the certificate, false to reject it.
+        /// </para>
+        /// <b>Available since:</b> Akka.NET v1.5.55
+        /// </summary>
+        public Transport.DotNetty.CertificateValidationCallback? CustomValidator { get; set; }
+
         internal void Build(StringBuilder builder)
         {
             var sb = new StringBuilder();