fix: retry with another upstream if the previous failed

Do not return response to the client if we got SERVFAIL or REFUSED,
until we run out of upstreams.

Fixes #9143

Signed-off-by: Dmitriy Matrenichev <dmitry.matrenichev@siderolabs.com>
(cherry picked from commit a5bd770)

internal/pkg/dns/dns.go

@@ -144,6 +144,10 @@ func (h *Handler) ServeDNS(ctx context.Context, wrt dns.ResponseWriter, msg *dns
 			break
 		}
 
+		if resp != nil && (resp.Rcode == dns.RcodeServerFailure || resp.Rcode == dns.RcodeRefused) {
+			continue
+		}
+
 		if ctx.Err() != nil || err == nil {
 			break
 		}

internal/pkg/dns/dns_test.go

@@ -64,12 +64,20 @@ func TestDNS(t *testing.T) {
 			expectedCode: dnssrv.RcodeNameError,
 			errCheck:     check.NoError(),
 		},
+		{
+			// The first one will return SERVFAIL and the second will return REFUSED. We should try both.
+			name:         `should return "refused"`,
+			hostname:     "dnssec-failed.org",
+			nameservers:  []string{"1.1.1.1", "ns-1098.awsdns-09.org."},
+			expectedCode: dnssrv.RcodeRefused,
+			errCheck:     check.NoError(),
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			stop := newServer(t, test.nameservers...)
-			defer stop()
+			t.Cleanup(stop)
 
 			time.Sleep(10 * time.Millisecond)
 
@@ -81,8 +89,6 @@ func TestDNS(t *testing.T) {
 			}
 
 			t.Logf("r: %s", r)
-
-			stop()
 		})
 	}
 }