DNS resolving with timeout.

Author: and1truong

dialoptions.go

@@ -77,6 +77,7 @@ type dialOptions struct {
 	defaultServiceConfig        *ServiceConfig // defaultServiceConfig is parsed from defaultServiceConfigRawJSON.
 	defaultServiceConfigRawJSON *string
 	resolvers                   []resolver.Builder
+	resolveTimeout              time.Duration
 	idleTimeout                 time.Duration
 	recvBufferPool              SharedBufferPool
 }
@@ -694,6 +695,13 @@ func WithIdleTimeout(d time.Duration) DialOption {
 	})
 }
 
+// WithResolveTimeout returns a DialOption that configures a DNS resolving timeout.
+func WithResolveTimeout(d time.Duration) DialOption {
+	return newFuncDialOption(func(o *dialOptions) {
+		o.resolveTimeout = d
+	})
+}
+
 // WithRecvBufferPool returns a DialOption that configures the ClientConn
 // to use the provided shared buffer pool for parsing incoming messages. Depending
 // on the application's workload, this could result in reduced memory allocation.

internal/resolver/dns/dns_resolver.go

@@ -100,7 +100,9 @@ type dnsBuilder struct{}
 
 // Build creates and starts a DNS resolver that watches the name resolution of
 // the target.
-func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
+	resolver.Resolver, error,
+) {
 	host, port, err := parseTarget(target.Endpoint(), defaultPort)
 	if err != nil {
 		return nil, err
@@ -118,13 +120,18 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts
 	d := &dnsResolver{
 		host:                 host,
 		port:                 port,
+		timeout:              opts.Timeout,
 		ctx:                  ctx,
 		cancel:               cancel,
 		cc:                   cc,
 		rn:                   make(chan struct{}, 1),
 		disableServiceConfig: opts.DisableServiceConfig,
 	}
 
+	if d.timeout == 0 {
+		d.timeout = 1 * time.Minute
+	}
+
 	d.resolver, err = internal.NewNetResolver(target.URL.Host)
 	if err != nil {
 		return nil, err
@@ -152,6 +159,7 @@ type dnsResolver struct {
 	host     string
 	port     string
 	resolver internal.NetResolver
+	timeout  time.Duration
 	ctx      context.Context
 	cancel   context.CancelFunc
 	cc       resolver.ClientConn
@@ -221,18 +229,18 @@ func (d *dnsResolver) watcher() {
 	}
 }
 
-func (d *dnsResolver) lookupSRV() ([]resolver.Address, error) {
+func (d *dnsResolver) lookupSRV(ctx context.Context) ([]resolver.Address, error) {
 	if !EnableSRVLookups {
 		return nil, nil
 	}
 	var newAddrs []resolver.Address
-	_, srvs, err := d.resolver.LookupSRV(d.ctx, "grpclb", "tcp", d.host)
+	_, srvs, err := d.resolver.LookupSRV(ctx, "grpclb", "tcp", d.host)
 	if err != nil {
 		err = handleDNSError(err, "SRV") // may become nil
 		return nil, err
 	}
 	for _, s := range srvs {
-		lbAddrs, err := d.resolver.LookupHost(d.ctx, s.Target)
+		lbAddrs, err := d.resolver.LookupHost(ctx, s.Target)
 		if err != nil {
 			err = handleDNSError(err, "A") // may become nil
 			if err == nil {
@@ -269,8 +277,8 @@ func handleDNSError(err error, lookupType string) error {
 	return err
 }
 
-func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult {
-	ss, err := d.resolver.LookupTXT(d.ctx, txtPrefix+d.host)
+func (d *dnsResolver) lookupTXT(ctx context.Context) *serviceconfig.ParseResult {
+	ss, err := d.resolver.LookupTXT(ctx, txtPrefix+d.host)
 	if err != nil {
 		if envconfig.TXTErrIgnore {
 			return nil
@@ -297,8 +305,8 @@ func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult {
 	return d.cc.ParseServiceConfig(sc)
 }
 
-func (d *dnsResolver) lookupHost() ([]resolver.Address, error) {
-	addrs, err := d.resolver.LookupHost(d.ctx, d.host)
+func (d *dnsResolver) lookupHost(ctx context.Context) ([]resolver.Address, error) {
+	addrs, err := d.resolver.LookupHost(ctx, d.host)
 	if err != nil {
 		err = handleDNSError(err, "A")
 		return nil, err
@@ -316,8 +324,12 @@ func (d *dnsResolver) lookupHost() ([]resolver.Address, error) {
 }
 
 func (d *dnsResolver) lookup() (*resolver.State, error) {
-	srv, srvErr := d.lookupSRV()
-	addrs, hostErr := d.lookupHost()
+	ctxSRV, cancelSRV := context.WithTimeout(d.ctx, d.timeout)
+	defer cancelSRV()
+	srv, srvErr := d.lookupSRV(ctxSRV)
+	ctxHost, cancelHost := context.WithTimeout(d.ctx, d.timeout)
+	defer cancelHost()
+	addrs, hostErr := d.lookupHost(ctxHost)
 	if hostErr != nil && (srvErr != nil || len(srv) == 0) {
 		return nil, hostErr
 	}
@@ -327,7 +339,9 @@ func (d *dnsResolver) lookup() (*resolver.State, error) {
 		state = grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: srv})
 	}
 	if !d.disableServiceConfig {
-		state.ServiceConfig = d.lookupTXT()
+		ctxTXT, cancelTXT := context.WithTimeout(d.ctx, d.timeout)
+		defer cancelTXT()
+		state.ServiceConfig = d.lookupTXT(ctxTXT)
 	}
 	return &state, nil
 }

internal/resolver/dns/dns_resolver_test.go

@@ -135,7 +135,8 @@ func buildResolverWithTestClientConn(t *testing.T, target string) (resolver.Reso
 	}
 
 	tcc := &testutils.ResolverClientConn{Logger: t, UpdateStateF: updateStateF, ReportErrorF: reportErrorF}
-	r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", target))}, tcc, resolver.BuildOptions{})
+	rOpts := resolver.BuildOptions{Timeout: 100 * time.Millisecond}
+	r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", target))}, tcc, rOpts)
 	if err != nil {
 		t.Fatalf("Failed to build DNS resolver for target %q: %v\n", target, err)
 	}
@@ -144,6 +145,29 @@ func buildResolverWithTestClientConn(t *testing.T, target string) (resolver.Reso
 	return r, stateCh, errCh
 }
 
+// Test verifies that when the DNS resolver gets timeout error when net.Resolver
+// takes too long to resolve a target.
+func (s) TestResolveTimeout(t *testing.T) {
+	const target = "timeoutaddress"
+	tr := &testNetResolver{}
+	tr.UpdateHostLookupTable(map[string][]string{target: {"1.2.3.4"}})
+	overrideNetResolver(t, tr)
+
+	r, _, errCh := buildResolverWithTestClientConn(t, target)
+	r.ResolveNow(resolver.ResolveNowOptions{})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for an error from the resolver")
+	case err := <-errCh:
+		if err == nil || !strings.Contains(err.Error(), "timed out") {
+			t.Fatalf(`we expect to see timed out error`)
+		}
+	}
+}
+
 // Waits for a state update from the DNS resolver and verifies the following:
 // - wantAddrs matches the list of addresses in the update
 // - wantBalancerAddrs matches the list of grpclb addresses in the update
@@ -401,7 +425,9 @@ func (s) TestDNSResolver_Basic(t *testing.T) {
 			txtLookupTable: map[string][]string{
 				"_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood),
 			},
-			wantAddrs:         []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}},
+			wantAddrs: []resolver.Address{
+				{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort},
+			},
 			wantBalancerAddrs: nil,
 			wantSC:            scJSON,
 		},
@@ -468,16 +494,23 @@ func (s) TestDNSResolver_Basic(t *testing.T) {
 			txtLookupTable: map[string][]string{
 				"_grpc_config.srv.ipv6.single.fake": txtRecordServiceConfig(txtRecordNonMatching),
 			},
-			wantAddrs:         nil,
-			wantBalancerAddrs: []resolver.Address{{Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.single.fake"}},
-			wantSC:            "{}",
+			wantAddrs: nil,
+			wantBalancerAddrs: []resolver.Address{
+				{
+					Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.single.fake",
+				},
+			},
+			wantSC: "{}",
 		},
 		{
 			name:   "ipv6_with_SRV_and_multiple_grpclb_address",
 			target: "srv.ipv6.multi.fake",
 			hostLookupTable: map[string][]string{
 				"srv.ipv6.multi.fake": nil,
-				"ipv6.multi.fake":     {"2607:f8b0:400a:801::1001", "2607:f8b0:400a:801::1002", "2607:f8b0:400a:801::1003"},
+				"ipv6.multi.fake": {
+					"2607:f8b0:400a:801::1001", "2607:f8b0:400a:801::1002",
+					"2607:f8b0:400a:801::1003",
+				},
 			},
 			srvLookupTable: map[string][]*net.SRV{
 				"_grpclb._tcp.srv.ipv6.multi.fake": {&net.SRV{Target: "ipv6.multi.fake", Port: 1234}},
@@ -880,8 +913,10 @@ func (s) TestDisableServiceConfig(t *testing.T) {
 				"_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood),
 			},
 			disableServiceConfig: false,
-			wantAddrs:            []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}},
-			wantSC:               scJSON,
+			wantAddrs: []resolver.Address{
+				{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort},
+			},
+			wantSC: scJSON,
 		},
 		{
 			name:            "true",
@@ -891,8 +926,10 @@ func (s) TestDisableServiceConfig(t *testing.T) {
 				"_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood),
 			},
 			disableServiceConfig: true,
-			wantAddrs:            []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}},
-			wantSC:               "{}",
+			wantAddrs: []resolver.Address{
+				{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort},
+			},
+			wantSC: "{}",
 		},
 	}
 
@@ -1043,7 +1080,9 @@ func (s) TestCustomAuthority(t *testing.T) {
 			// Override the address dialer to verify the authority being passed.
 			origAddressDialer := dnsinternal.AddressDialer
 			errChan := make(chan error, 1)
-			dnsinternal.AddressDialer = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) {
+			dnsinternal.AddressDialer = func(authority string) func(ctx context.Context, network, address string) (
+				net.Conn, error,
+			) {
 				if authority != test.wantAuthority {
 					errChan <- fmt.Errorf("wrong custom authority passed to resolver. target: %s got authority: %s want authority: %s", test.authority, authority, test.wantAuthority)
 				} else {

internal/resolver/dns/fake_net_resolver_test.go

@@ -20,7 +20,9 @@ package dns_test
 
 import (
 	"context"
+	"fmt"
 	"net"
+	"strings"
 	"sync"
 
 	"google.golang.org/grpc/internal/testutils"
@@ -47,6 +49,10 @@ func (tr *testNetResolver) LookupHost(ctx context.Context, host string) ([]strin
 	tr.mu.Lock()
 	defer tr.mu.Unlock()
 
+	if strings.Contains(host, "timeoutaddress") {
+		return nil, fmt.Errorf("timed out")
+	}
+
 	if addrs, ok := tr.hostLookupTable[host]; ok {
 		return addrs, nil
 	}

resolver/resolver.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/url"
 	"strings"
+	"time"
 
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials"
@@ -168,6 +169,9 @@ type BuildOptions struct {
 	// field. In most cases though, it is not appropriate, and this field may
 	// be ignored.
 	Dialer func(context.Context, string) (net.Conn, error)
+	// Timeout specifies time limit for DNS resolver to resolve a target,
+	// which can be SRV , Host, TXT records.
+	Timeout time.Duration
 }
 
 // An Endpoint is one network endpoint, or server, which may have multiple

resolver_wrapper.go

@@ -75,6 +75,7 @@ func (ccr *ccResolverWrapper) start() error {
 			DialCreds:            ccr.cc.dopts.copts.TransportCredentials,
 			CredsBundle:          ccr.cc.dopts.copts.CredsBundle,
 			Dialer:               ccr.cc.dopts.copts.Dialer,
+			Timeout:              ccr.cc.dopts.resolveTimeout,
 		}
 		var err error
 		ccr.resolver, err = ccr.cc.resolverBuilder.Build(ccr.cc.parsedTarget, ccr, opts)