Handle RFC 6761 special-use domain names in Dns resolution

src/libraries/System.Net.NameResolution/src/System/Net/Dns.cs

@@ -416,6 +416,68 @@ private static bool ValidateAddressFamily(ref AddressFamily addressFamily, strin
             return true;
         }
 
+        /// <summary>
+        /// Checks if the given host name matches a reserved name or is a subdomain of it.
+        /// For example, IsReservedName("foo.localhost", "localhost") returns true.
+        /// Also handles trailing dots: IsReservedName("foo.localhost.", "localhost") returns true.
+        /// Returns false for malformed hostnames (starting with dot or containing consecutive dots).
+        /// </summary>
+        private static bool IsReservedName(string hostName, string reservedName)
+        {
+            // Reject malformed hostnames - let OS resolver handle them (and reject them)
+            if (hostName.StartsWith('.') || hostName.Contains("..", StringComparison.Ordinal))
+            {
+                return false;
+            }
+
+            // Strip trailing dot if present (DNS root notation)
+            ReadOnlySpan<char> hostSpan = hostName.AsSpan();
+            if (hostSpan.Length > 0 && hostSpan[hostSpan.Length - 1] == '.')
+            {
+                hostSpan = hostSpan.Slice(0, hostSpan.Length - 1);
+            }
+
+            // Matches "reservedName" exactly, or "*.reservedName" (subdomain)
+            return hostSpan.EndsWith(reservedName, StringComparison.OrdinalIgnoreCase) &&
+                   (hostSpan.Length == reservedName.Length ||
+                    hostSpan[hostSpan.Length - reservedName.Length - 1] == '.');
+        }
+
+        /// <summary>
+        /// Checks if the given host name is a subdomain of localhost (e.g., "foo.localhost").
+        /// Plain "localhost" or "localhost." returns false.
+        /// </summary>
+        private static bool IsLocalhostSubdomain(string hostName)
+        {
+            // Strip trailing dot for length comparison
+            int length = hostName.Length;
+            if (length > 0 && hostName[length - 1] == '.')
+            {
+                length--;
+            }
+
+            // Must be longer than "localhost" (not just equal with trailing dot)
+            return length > "localhost".Length && IsReservedName(hostName, "localhost");
+        }
+
+        /// <summary>
+        /// Tries to handle RFC 6761 "invalid" domain names.
+        /// Returns true if the host name is an invalid domain (exception will be set).
+        /// </summary>
+        private static bool TryHandleRfc6761InvalidDomain(string hostName, out SocketException? exception)
+        {
+            // RFC 6761 Section 6.4: "invalid" and "*.invalid" must always return NXDOMAIN.
+            if (IsReservedName(hostName, "invalid"))
+            {
+                if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Returning NXDOMAIN for 'invalid' domain");
+                exception = CreateException(SocketError.HostNotFound, 0);
+                return true;
+            }
+
+            exception = null;
+            return false;
+        }
+
         private static object GetHostEntryOrAddressesCore(string hostName, bool justAddresses, AddressFamily addressFamily, NameResolutionActivity? activityOrDefault = default)
         {
             ValidateHostName(hostName);
@@ -429,17 +491,43 @@ private static object GetHostEntryOrAddressesCore(string hostName, bool justAddr
             // NameResolutionActivity may have already been set if we're being called from RunAsync.
             NameResolutionActivity activity = activityOrDefault ?? NameResolutionTelemetry.Log.BeforeResolution(hostName);
 
+            // RFC 6761 Section 6.4: "invalid" domains must return NXDOMAIN.
+            if (TryHandleRfc6761InvalidDomain(hostName, out SocketException? invalidDomainException))
+            {
+                NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: invalidDomainException);
+                throw invalidDomainException!;
+            }
+
+            // Track if this is a localhost subdomain for fallback handling.
+            bool isLocalhostSubdomain = IsLocalhostSubdomain(hostName);
+
             object result;
             try
             {
                 SocketError errorCode = NameResolutionPal.TryGetAddrInfo(hostName, justAddresses, addressFamily, out string? newHostName, out string[] aliases, out IPAddress[] addresses, out int nativeErrorCode);
 
                 if (errorCode != SocketError.Success)
                 {
+                    // RFC 6761 Section 6.3: If localhost subdomain fails, fall back to resolving plain "localhost".
+                    if (isLocalhostSubdomain)
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain resolution failed, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: CreateException(errorCode, nativeErrorCode));
+                        return GetHostEntryOrAddressesCore("localhost", justAddresses, addressFamily);
+                    }
+
                     if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(hostName, $"{hostName} DNS lookup failed with {errorCode}");
                     throw CreateException(errorCode, nativeErrorCode);
                 }
 
+                // RFC 6761 Section 6.3: If localhost subdomain returns empty addresses, fall back to plain "localhost".
+                if (isLocalhostSubdomain && addresses.Length == 0)
+                {
+                    if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: justAddresses ? addresses : (object)new IPHostEntry { AddressList = addresses, HostName = newHostName!, Aliases = aliases }, exception: null);
+                    return GetHostEntryOrAddressesCore("localhost", justAddresses, addressFamily);
+                }
+
                 result = justAddresses ? (object)
                     addresses :
                     new IPHostEntry
@@ -559,13 +647,6 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
                     Task.FromCanceled<IPHostEntry>(cancellationToken);
             }
 
-            if (!ValidateAddressFamily(ref family, hostName, justAddresses, out object? resultOnFailure))
-            {
-                return justAddresses ? (Task)
-                    Task.FromResult((IPAddress[])resultOnFailure) :
-                    Task.FromResult((IPHostEntry)resultOnFailure);
-            }
-
             object asyncState;
 
             // See if it's an IP Address.
@@ -588,6 +669,31 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
             }
             else
             {
+                // Validate hostname before any processing
+                ValidateHostName(hostName);
+
+                // Check address family support after validation
+                if (!ValidateAddressFamily(ref family, hostName, justAddresses, out object? resultOnFailure))
+                {
+                    return justAddresses ? (Task)
+                        Task.FromResult((IPAddress[])resultOnFailure) :
+                        Task.FromResult((IPHostEntry)resultOnFailure);
+                }
+
+                // RFC 6761 Section 6.4: "invalid" domains must return NXDOMAIN.
+                if (TryHandleRfc6761InvalidDomain(hostName, out SocketException? invalidDomainException))
+                {
+                    NameResolutionActivity activity = NameResolutionTelemetry.Log.BeforeResolution(hostName);
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: invalidDomainException);
+                    return justAddresses ? (Task)
+                        Task.FromException<IPAddress[]>(invalidDomainException!) :
+                        Task.FromException<IPHostEntry>(invalidDomainException!);
+                }
+
+                // For localhost subdomains (RFC 6761 Section 6.3), we try the OS resolver first.
+                // If it fails or returns empty, we fall back to resolving plain "localhost".
+                // This fallback logic is handled in GetHostEntryOrAddressesCore and GetAddrInfoWithTelemetryAsync.
+
                 if (NameResolutionPal.SupportsGetAddrInfoAsync)
                 {
 #pragma warning disable CS0162 // Unreachable code detected -- SupportsGetAddrInfoAsync is a constant on *nix.
@@ -596,10 +702,11 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
                     // instead of calling the synchronous version in the ThreadPool.
                     // If it fails, we will fall back to ThreadPool as well.
 
-                    ValidateHostName(hostName);
-
+                    // Always use the telemetry-enabled path for localhost subdomains to ensure fallback handling.
+                    // For other hostnames, use the non-telemetry path if diagnostics are disabled.
+                    bool isLocalhostSubdomain = IsLocalhostSubdomain(hostName);
                     Task? t;
-                    if (NameResolutionTelemetry.AnyDiagnosticsEnabled())
+                    if (NameResolutionTelemetry.AnyDiagnosticsEnabled() || isLocalhostSubdomain)
                     {
                         t = justAddresses
                             ? GetAddrInfoWithTelemetryAsync<IPAddress[]>(hostName, justAddresses, family, cancellationToken)
@@ -653,31 +760,71 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
 
             if (task != null)
             {
-                return CompleteAsync(task, hostName, startingTimestamp);
+                bool isLocalhostSubdomain = IsLocalhostSubdomain(hostName);
+                return CompleteAsync(task, hostName, justAddresses, addressFamily, isLocalhostSubdomain, startingTimestamp, cancellationToken);
             }
 
             // If resolution even did not start don't bother with telemetry.
             // We will retry on thread-pool.
             return null;
 
-            static async Task<T> CompleteAsync(Task task, string hostName, long startingTimeStamp)
+            static async Task<T> CompleteAsync(Task task, string hostName, bool justAddresses, AddressFamily addressFamily, bool isLocalhostSubdomain, long startingTimeStamp, CancellationToken cancellationToken)
             {
                 NameResolutionActivity activity = NameResolutionTelemetry.Log.BeforeResolution(hostName, startingTimeStamp);
                 Exception? exception = null;
                 T? result = null;
+                bool fallbackOccurred = false;
                 try
                 {
                     result = await ((Task<T>)task).ConfigureAwait(false);
+
+                    // RFC 6761 Section 6.3: If localhost subdomain returns empty addresses, fall back to plain "localhost".
+                    if (isLocalhostSubdomain && result is IPAddress[] addresses && addresses.Length == 0)
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: null);
+                        fallbackOccurred = true;
+
+                        // Resolve plain "localhost" instead
+                        return await ((Task<T>)(justAddresses
+                            ? (Task)Dns.GetHostAddressesAsync("localhost", addressFamily, cancellationToken)
+                            : Dns.GetHostEntryAsync("localhost", addressFamily, cancellationToken))).ConfigureAwait(false);
+                    }
+
+                    if (isLocalhostSubdomain && result is IPHostEntry entry && entry.AddressList.Length == 0)
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: null);
+                        fallbackOccurred = true;
+
+                        // Resolve plain "localhost" instead
+                        return await ((Task<T>)(Task)Dns.GetHostEntryAsync("localhost", addressFamily, cancellationToken)).ConfigureAwait(false);
+                    }
+
                     return result;
                 }
+                catch (SocketException ex) when (isLocalhostSubdomain)
+                {
+                    // RFC 6761 Section 6.3: If localhost subdomain fails, fall back to resolving plain "localhost".
+                    if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain resolution failed, falling back to 'localhost'");
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: ex);
+                    fallbackOccurred = true;
+
+                    return await ((Task<T>)(justAddresses
+                        ? (Task)Dns.GetHostAddressesAsync("localhost", addressFamily, cancellationToken)
+                        : Dns.GetHostEntryAsync("localhost", addressFamily, cancellationToken))).ConfigureAwait(false);
+                }
                 catch (Exception ex)
                 {
                     exception = ex;
                     throw;
                 }
                 finally
                 {
-                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: exception);
+                    if (!fallbackOccurred)
+                    {
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: exception);
+                    }
                 }
             }
         }

src/libraries/System.Net.NameResolution/tests/FunctionalTests/GetHostAddressesTest.cs

@@ -204,6 +204,110 @@ static async Task RunTest(string useAsync)
                 Assert.Empty(addresses);
             }
         }
+
+        // RFC 6761 Section 6.4: "invalid" and "*.invalid" must always return NXDOMAIN (HostNotFound).
+        [Theory]
+        [InlineData("invalid")]
+        [InlineData("invalid.")]
+        [InlineData("test.invalid")]
+        [InlineData("test.invalid.")]
+        [InlineData("foo.bar.invalid")]
+        [InlineData("INVALID")]
+        [InlineData("Test.INVALID")]
+        public async Task DnsGetHostAddresses_InvalidDomain_ThrowsHostNotFound(string hostName)
+        {
+            SocketException ex = Assert.ThrowsAny<SocketException>(() => Dns.GetHostAddresses(hostName));
+            Assert.Equal(SocketError.HostNotFound, ex.SocketErrorCode);
+
+            ex = await Assert.ThrowsAnyAsync<SocketException>(() => Dns.GetHostAddressesAsync(hostName));
+            Assert.Equal(SocketError.HostNotFound, ex.SocketErrorCode);
+        }
+
+        // RFC 6761 Section 6.3: "*.localhost" subdomains - OS resolver is tried first,
+        // falling back to plain "localhost" resolution if OS resolver fails or returns empty.
+        [Theory]
+        [InlineData("foo.localhost")]
+        [InlineData("bar.foo.localhost")]
+        [InlineData("test.localhost")]
+        public async Task DnsGetHostAddresses_LocalhostSubdomain_ReturnsLoopback(string hostName)
+        {
+            // The subdomain goes to OS resolver first. If it fails (likely on most systems),
+            // it falls back to resolving plain "localhost", which should return loopback addresses.
+            IPAddress[] addresses = Dns.GetHostAddresses(hostName);
+            Assert.True(addresses.Length >= 1, "Expected at least one loopback address");
+            Assert.All(addresses, addr => Assert.True(IPAddress.IsLoopback(addr), $"Expected loopback address but got: {addr}"));
+
+            addresses = await Dns.GetHostAddressesAsync(hostName);
+            Assert.True(addresses.Length >= 1, "Expected at least one loopback address");
+            Assert.All(addresses, addr => Assert.True(IPAddress.IsLoopback(addr), $"Expected loopback address but got: {addr}"));
+        }
+
+        // RFC 6761: "*.localhost" subdomains should respect AddressFamily parameter.
+        // OS resolver is tried first, falling back to plain "localhost" resolution.
+        [Theory]
+        [InlineData(AddressFamily.InterNetwork)]
+        [InlineData(AddressFamily.InterNetworkV6)]
+        public async Task DnsGetHostAddresses_LocalhostSubdomain_RespectsAddressFamily(AddressFamily addressFamily)
+        {
+            // Skip IPv6 test if OS doesn't support it.
+            if (addressFamily == AddressFamily.InterNetworkV6 && !Socket.OSSupportsIPv6)
+            {
+                return;
+            }
+
+            string hostName = "test.localhost";
+
+            // The subdomain goes to OS resolver first. If it fails, it falls back to
+            // resolving plain "localhost" with the same address family filter.
+            IPAddress[] addresses = Dns.GetHostAddresses(hostName, addressFamily);
+            Assert.True(addresses.Length >= 1, "Expected at least one address");
+            Assert.All(addresses, addr => Assert.Equal(addressFamily, addr.AddressFamily));
+
+            addresses = await Dns.GetHostAddressesAsync(hostName, addressFamily);
+            Assert.True(addresses.Length >= 1, "Expected at least one address");
+            Assert.All(addresses, addr => Assert.Equal(addressFamily, addr.AddressFamily));
+        }
+
+        // RFC 6761: Verify that localhost subdomains return the same addresses as plain "localhost"
+        // since the fallback delegates to localhost resolution.
+        [Fact]
+        public async Task DnsGetHostAddresses_LocalhostSubdomain_ReturnsSameAsLocalhost()
+        {
+            IPAddress[] localhostAddresses = Dns.GetHostAddresses("localhost");
+            IPAddress[] subdomainAddresses = Dns.GetHostAddresses("foo.localhost");
+
+            // Both should return loopback addresses (the subdomain falls back to localhost resolution)
+            Assert.True(localhostAddresses.Length >= 1);
+            Assert.True(subdomainAddresses.Length >= 1);
+
+            // The addresses should be equivalent (same loopback addresses)
+            Assert.Equal(
+                localhostAddresses.OrderBy(a => a.ToString()).ToArray(),
+                subdomainAddresses.OrderBy(a => a.ToString()).ToArray());
+
+            // Async version
+            localhostAddresses = await Dns.GetHostAddressesAsync("localhost");
+            subdomainAddresses = await Dns.GetHostAddressesAsync("bar.localhost");
+
+            Assert.Equal(
+                localhostAddresses.OrderBy(a => a.ToString()).ToArray(),
+                subdomainAddresses.OrderBy(a => a.ToString()).ToArray());
+        }
+
+        // RFC 6761: Localhost subdomains with trailing dot should work (e.g., "foo.localhost.")
+        [Theory]
+        [InlineData("foo.localhost.")]
+        [InlineData("bar.test.localhost.")]
+        public async Task DnsGetHostAddresses_LocalhostSubdomainWithTrailingDot_ReturnsLoopback(string hostName)
+        {
+            IPAddress[] addresses = Dns.GetHostAddresses(hostName);
+            Assert.True(addresses.Length >= 1, "Expected at least one loopback address");
+            Assert.All(addresses, addr => Assert.True(IPAddress.IsLoopback(addr), $"Expected loopback address but got: {addr}"));
+
+            addresses = await Dns.GetHostAddressesAsync(hostName);
+            Assert.True(addresses.Length >= 1, "Expected at least one loopback address");
+            Assert.All(addresses, addr => Assert.True(IPAddress.IsLoopback(addr), $"Expected loopback address but got: {addr}"));
+        }
     }
 
     // Cancellation tests are sequential to reduce the chance of timing issues.