Skip to content

Commit

Permalink
Tests: improved resolving tests stability.
Browse files Browse the repository at this point in the history 
Various fixes include:

 - resolver polling instead of simple timeout
 - generation tracking to ensure reload was complete
 - ipv6 in resolver is disabled to results handling across targets
  • Loading branch information
@vlhomutov
vlhomutov committed Feb 24, 2023
1 parent b70cfb1 commit c8dbd35
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 74 deletions.
47 changes: 46 additions & 1 deletion tests/lib/Test/Nginx.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,11 +443,38 @@ sub waitforsocket($) {
return undef;
}

sub wait_for_resolver {
my ($self, $server, $port, $name, $expected) = @_;

my $dig = `which dig`;

if (not $dig) {
print("# warn: dig not found, falling back to timeout\n");
select undef, undef, undef, 5;
return 2;
}

my $reply;

for (1 .. 50) {
$reply = `dig \@$server -p $port +short +timeout=1 $name`;
return 1 if index($reply, $expected) != -1;
select undef, undef, undef, 0.1;
}
return undef;
}

sub reload() {
my ($self) = @_;
my ($self, $api_gen_url) = @_;

return $self unless $self->{_started};

my $generation;

if ($api_gen_url) {
$generation = http_get_value($api_gen_url);
}

my $pid = $self->read_file('nginx.pid');

if ($^O eq 'MSWin32') {
Expand All @@ -463,6 +490,16 @@ sub reload() {
kill 'HUP', $pid;
}

if ($api_gen_url) {
my $new_generation;
for (1 .. 50) {
$new_generation = http_get_value($api_gen_url);
return if $new_generation == $generation + 1;

select undef, undef, undef, 0.1;
}
}

return $self;
}

Expand Down Expand Up @@ -904,6 +941,14 @@ sub http_gzip_like {
}
}

sub http_get_value {
my ($uri) = @_;
my $response = http_get($uri);
my ($headers, $body) = split /\n\r/, $response, 2;
$body =~ s/^\s+|\s+$//g;
return $body;
}

###############################################################################

1;
Expand Down
70 changes: 24 additions & 46 deletions tests/upstream_resolve.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ my $t = Test::Nginx->new()
# see https://trac.nginx.org/nginx/ticket/1831
plan(skip_all => "perl >= 5.32 required") if ($t->has_module('perl') && $] < 5.032000);

$t->plan(14)->write_file_expand('nginx.conf', <<'EOF');
$t->plan(13)->write_file_expand('nginx.conf', <<'EOF');
%%TEST_GLOBALS%%
Expand All @@ -56,7 +56,7 @@ http {
server backup.example.com:%%PORT_8081%% resolve backup;
server nonexist.example.com:%%PORT_8081%% resolve backup;
resolver 127.0.0.1:5353 valid=1s;
resolver 127.0.0.1:5353 valid=1s ipv6=off;
}
upstream u2 {
Expand Down Expand Up @@ -182,8 +182,7 @@ EOF
my $dconf = $t->testdir()."/dns.conf";

$t->run_daemon('dnsmasq', '-C', $tdir."/dns.conf", '-k', "--log-facility=$tdir/dns.log", '-q');

# let the dnsmasq execute;
$t->wait_for_resolver('127.0.0.1', 5353, 'foo.example.com', '127.0.0.1');

$t->run();

Expand All @@ -193,44 +192,31 @@ my @ports = my ($port1, $port2) = (port(8081), port(8082));

my ($j, $v);

# give 1 request for each backend (note: due to IPv6 there may be 4 peers)
http_get('/'); http_get('/');
# wait for nginx resolver to complete query
for (1 .. 50) {
last if http_get('/') =~ qr /200 OK/;
select undef, undef, undef, 0.1;
}

# expect that upstream contains addresses from 'test_hosts' file

$j = get_json("/api/status/http/upstreams/u/peers/");

my $n4 = 0;

foreach my $addr ( keys %$j ) {

#print(Dumper($j->{$addr}));

if ($addr eq "127.0.0.1:$port1") {
$n4 = $n4 + 1;
is($j->{$addr}->{'server'}, 'foo.example.com:'.$port1, 'foo.example.com is resolved');
}

if ($addr eq "127.0.0.2:$port2") {
$n4 = $n4 + 1;
is($j->{$addr}->{'server'}, 'bar.example.com:'.$port2, 'bar.example.com resolved');
}
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.1:$port1");
is($j->{'server'}, 'foo.example.com:'.$port1, 'foo.example.com is resolved');

if ($addr eq "127.0.0.3:$port1") {
is($j->{$addr}->{'server'}, 'backup.example.com:'.$port1, 'backup.example.com addr 1 resolved');
}
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.2:$port2");
is($j->{'server'}, 'bar.example.com:'.$port2, 'bar.example.com resolved');

if ($addr eq "127.0.0.4:$port1") {
is($j->{$addr}->{'server'}, 'backup.example.com:'.$port1, 'backup.example.com addr 2 resolved');
}
}
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.3:$port1");
is($j->{'server'}, 'backup.example.com:'.$port1, 'backup.example.com addr 1 resolved');

is($n4, 2, 'foo and bar.example.com resolved into 2 ipv4 addresses');
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.4:$port1");
is($j->{'server'}, 'backup.example.com:'.$port1, 'backup.example.com addr 2 resolved');


# perform reload to trigger the codepath for pre-resolve
$t->reload();
small_delay();
$t->reload('/api/status/angie/generation');

# no need to wait for resolver, we expect cached result

$j = get_json("/api/status/http/upstreams/u/peers/");
is($j->{"127.0.0.1:$port1"}->{'server'}, "foo.example.com:$port1", 'foo.example.com is found after reload');
Expand All @@ -241,8 +227,7 @@ is($j->{"127.0.0.2:$port2"}->{'server'}, "bar.example.com:$port2", 'bar.example.
# and verify upstreams are still accessible

$t->stop_daemons();
$t->reload();
small_delay();
$t->reload('/api/status/angie/generation');

$j = get_json("/api/status/http/upstreams/u/peers/");
is($j->{"127.0.0.1:$port1"}->{'server'}, "foo.example.com:$port1", 'foo.example.com is saved');
Expand All @@ -251,8 +236,10 @@ is($j->{"127.0.0.2:$port2"}->{'server'}, "bar.example.com:$port2", 'bar.example.

# start DNS server with new config, addresses changed
$t->run_daemon('dnsmasq', '-C', $tdir."/dns2.conf", '-k', "--log-facility=$tdir/dns.log", '-q');
$t->wait_for_resolver('127.0.0.1', 5353, 'foo.example.com', '127.0.0.3');

# reload nginx to force resolve
$t->reload();
$t->reload('/api/status/angie/generation');
# wait 3 seconds to ensure re-resolve (valid=1s)
select undef, undef, undef, 3;

Expand Down Expand Up @@ -291,7 +278,7 @@ Host: localhost
EOF

small_delay();
select undef, undef, undef, 1;

$j = get_json("/api/status/http/upstreams/u2/peers/127.0.0.5:$port1");
is($j->{'refs'}, 2, "2 long requests to backend started, ref");
Expand All @@ -318,12 +305,3 @@ sub get_json {

return $json;
}

sub wait_for_dns {
# TODO: waitport() for UDP sockets; avoid delay
select undef, undef, undef, 2;
}

sub small_delay {
select undef, undef, undef, 1;
}
40 changes: 13 additions & 27 deletions tests/upstream_srv.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ plan(skip_all => "JSON::PP not installed") if $@;
plan(skip_all => 'OS is not linux') if $^O ne 'linux';

my $t = Test::Nginx->new()->has(qw/http http_api proxy upstream_zone/)
->has_daemon("dnsmasq")->plan(3)
->has_daemon("dnsmasq")->plan(2)
->write_file_expand('nginx.conf', <<'EOF');
%%TEST_GLOBALS%%
Expand All @@ -48,7 +48,7 @@ http {
zone z 1m;
server backends.example.com service=http resolve;
resolver 127.0.0.1:5454 valid=1s;
resolver 127.0.0.1:5454 valid=1s ipv6=off;
}
server {
Expand Down Expand Up @@ -111,43 +111,29 @@ EOF
my $dconf = $t->testdir()."/dns.conf";

$t->run_daemon('dnsmasq', '-C', $tdir."/dns.conf", '-k', "--log-facility=$tdir/dns.log", '-q');
$t->run();
$t->wait_for_resolver('127.0.0.1', 5464, 'b1.example.com', '127.0.0.1');

# let the dnsmasq execute;
# TODO: waitport() for UDP sockets; avoid delay
select undef, undef, undef, 3;
$t->run();

###############################################################################

my @ports = my ($port1, $port2) = (port(8081), port(8082));

my ($j, $v);

# give 1 request for each backend
http_get('/'); http_get('/');
# TODO: no idea what 1st backend is going to be; maybe IPv6, which is not listened
# wait for nginx resolver to complete query
for (1 .. 50) {
last if http_get('/') =~ qr /200 OK/;
select undef, undef, undef, 0.1;
}

# expect that upstream contains addresses from 'test_hosts' file

$j = get_json("/api/status/http/upstreams/u/peers/");

my $n4 = 0;

foreach my $addr ( keys %$j ) {

#print(Dumper($j->{$addr}));

if ($addr eq "127.0.0.1:$port1") {
$n4 = $n4 + 1;
is($j->{$addr}->{'server'}, 'backends.example.com', 'b1 address resolved');
}
if ($addr eq "127.0.0.2:$port2") {
$n4 = $n4 + 1;
is($j->{$addr}->{'server'}, 'backends.example.com', 'b2 address resolved');
}
}
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.1:$port1");
is($j->{'server'}, 'backends.example.com', 'b1 address resolved from srv');

is($n4, 2, 'backends.example.com resolved into 2 ipv4 addresses');
$j = get_json("/api/status/http/upstreams/u/peers/127.0.0.2:$port2");
is($j->{'server'}, 'backends.example.com', 'b2 address resolved from srv');


###############################################################################
Expand Down

0 comments on commit c8dbd35

Please sign in to comment.