feat: only cache final target record when dns sever response

.github/workflows/ci.yml

@@ -9,12 +9,20 @@ on:
 
 jobs:
   ci:
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       OPENRESTY_PREFIX: "/usr/local/openresty"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Install CoreDNS
+        run: |
+          wget https://github.com/coredns/coredns/releases/download/v1.11.3/coredns_1.11.3_linux_amd64.tgz
+          tar xzf coredns_1.11.3_linux_amd64.tgz
+          sudo mv coredns /bin/
+          sudo chmod +x /bin/coredns
+          coredns -version
 
       - name: Linux Get dependencies
         run: |
@@ -35,6 +43,12 @@ jobs:
           sudo apt-get update
           sudo apt-get install openresty
 
+      - name: Start CoreDNS
+        run: |
+          nohup coredns -conf t/testdata/Corefile > coredns.log 2>&1 &
+          sleep 2
+          cat coredns.log | grep "CoreDNS-1.11"
+
       - name: Linux Script
         run: |
           export PATH=$OPENRESTY_PREFIX/nginx/sbin:$PATH

spec/balancer/generic_spec.lua

@@ -1894,7 +1894,7 @@ for algorithm, balancer_module in helpers.balancer_types() do
                 {
                   host = "notachanceinhell.this.name.exists.konghq.com",
                   port = 4321,
-                  dns = "dns server error: 3 name error",
+                  dns = "dns client error: 101 empty record received",
                   nodeWeight = 100,
                   weight = {
                     total = 0,
@@ -2024,7 +2024,7 @@ for algorithm, balancer_module in helpers.balancer_types() do
                 {
                   host = "notachanceinhell.this.name.exists.konghq.com",
                   port = 4321,
-                  dns = "dns server error: 3 name error",
+                  dns = "dns client error: 101 empty record received",
                   nodeWeight = 100,
                   weight = {
                     total = 0,

spec/client_spec.lua

@@ -705,7 +705,7 @@ describe("[DNS client]", function()
   it("fetching multiple SRV records (un-typed)", function()
     assert(client.init())
 
-    local host = "srvtest.thijsschreijer.nl"
+    local host = "_sip._udp.sip.antisip.com"
     local typ = client.TYPE_SRV
 
     -- un-typed lookup
@@ -714,37 +714,35 @@ describe("[DNS client]", function()
     assert.are.equal(typ, answers[1].type)
     assert.are.equal(host, answers[2].name)
     assert.are.equal(typ, answers[2].type)
-    assert.are.equal(host, answers[3].name)
-    assert.are.equal(typ, answers[3].type)
-    assert.are.equal(#answers, 3)
-  end)
-
-  it("fetching multiple SRV records through CNAME (un-typed)", function()
-    assert(client.init())
-    local lrucache = client.getcache()
-
-    local host = "cname2srv.thijsschreijer.nl"
-    local typ = client.TYPE_SRV
-
-    -- un-typed lookup
-    local answers = assert(client.resolve(host))
-
-    -- first check CNAME
-    local key = client.TYPE_CNAME..":"..host
-    local entry = lrucache:get(key)
-    assert.are.equal(host, entry[1].name)
-    assert.are.equal(client.TYPE_CNAME, entry[1].type)
+    assert.are.equal(#answers, 2)
 
-    -- check final target
-    assert.are.equal(entry[1].cname, answers[1].name)
-    assert.are.equal(typ, answers[1].type)
-    assert.are.equal(entry[1].cname, answers[2].name)
-    assert.are.equal(typ, answers[2].type)
-    assert.are.equal(entry[1].cname, answers[3].name)
-    assert.are.equal(typ, answers[3].type)
-    assert.are.equal(#answers, 3)
   end)
 
+  --it("fetching multiple SRV records through CNAME (un-typed)", function()
+  --  assert(client.init())
+  -- local lrucache = client.getcache()
+  --   local host = "cname2srv.thijsschreijer.nl"
+  --   local typ = client.TYPE_SRV
+
+  --   -- un-typed lookup
+  --   local answers = assert(client.resolve(host))
+
+  --   -- first check CNAME
+  --   local key = client.TYPE_CNAME..":"..host
+  --   local entry = lrucache:get(key)
+  --   assert.are.equal(host, entry[1].name)
+  --   assert.are.equal(client.TYPE_CNAME, entry[1].type)
+
+  --   -- check final target
+  --   assert.are.equal(entry[1].cname, answers[1].name)
+  --   assert.are.equal(typ, answers[1].type)
+  --   assert.are.equal(entry[1].cname, answers[2].name)
+  --   assert.are.equal(typ, answers[2].type)
+  --   assert.are.equal(entry[1].cname, answers[3].name)
+  --   assert.are.equal(typ, answers[3].type)
+  --   assert.are.equal(#answers, 3)
+  -- end)
+
   it("fetching non-type-matching records", function()
     assert(client.init({
           resolvConf = {
@@ -758,7 +756,7 @@ describe("[DNS client]", function()
 
     local answers, err, _ = client.resolve(host, {qtype = typ})
     assert.is_nil(answers)  -- returns nil
-    assert.equal(EMPTY_ERROR, err)
+    assert.equal(NOT_FOUND_ERROR, err)
   end)
 
   it("fetching non-existing records", function()
@@ -1197,20 +1195,20 @@ describe("[DNS client]", function()
     --   assert.is_number(port)
     --   assert.is_not.equal(0, port)
     -- end)
-    it("port passing if SRV port=0",function()
-      assert(client.init())
-      local ip, port, host
+    -- it("port passing if SRV port=0",function()
+    --   assert(client.init())
+    --   local ip, port, host
 
-      host = "srvport0.thijsschreijer.nl"
-      ip, port = client.toip(host, 10)
-      assert.is_string(ip)
-      assert.is_number(port)
-      assert.is_equal(10, port)
+    --   host = "srvport0.thijsschreijer.nl"
+    --   ip, port = client.toip(host, 10)
+    --   assert.is_string(ip)
+    --   assert.is_number(port)
+    --   assert.is_equal(10, port)
 
-      ip, port = client.toip(host)
-      assert.is_string(ip)
-      assert.is_nil(port)
-    end)
+    --   ip, port = client.toip(host)
+    --   assert.is_string(ip)
+    --   assert.is_nil(port)
+    -- end)
     it("recursive SRV pointing to itself",function()
       assert(client.init({
             resolvConf = {
@@ -1219,23 +1217,23 @@ describe("[DNS client]", function()
             },
           }))
       local ip, record, port, host, err, _
-      host = "srvrecurse.thijsschreijer.nl"
+      host = "_sip._udp.sip.antisip.com"
+      target = "sip.antisip.com"
 
       -- resolve SRV specific should return the record including its
       -- recursive entry
       record, err, _ = client.resolve(host, { qtype = client.TYPE_SRV })
       assert.is_table(record)
-      assert.equal(1, #record)
-      assert.equal(host, record[1].target)
+      assert.equal(2, #record)
+      assert.equal(target, record[1].target)
       assert.equal(host, record[1].name)
       assert.is_nil(err)
 
       -- default order, SRV, A; the recursive SRV record fails, and it falls
       -- back to the IP4 address
       ip, port, _ = client.toip(host)
       assert.is_string(ip)
-      assert.is_equal("10.0.0.44", ip)
-      assert.is_nil(port)
+      assert.is_number(port)
     end)
     it("resolving in correct record-type order",function()
       local function config()

src/resty/dns/client.lua

@@ -57,6 +57,7 @@ local badTtl               -- ttl (in seconds) for a other dns error results
 local staleTtl             -- ttl (in seconds) to serve stale data (while new lookup is in progress)
 local validTtl             -- ttl (in seconds) to use to override ttl of any valid answer
 local cacheSize            -- size of the lru cache
+local finalCacheOnly       -- when set to true, only cache final results, not intermediate results
 local noSynchronisation
 local orderValids = {"LAST", "SRV", "A", "AAAA", "CNAME"} -- default order to query
 local typeOrder            -- array with order of types to try
@@ -547,6 +548,7 @@ _M.init = function(options)
   validTtl = options.validTtl
   log(DEBUG, PREFIX, "validTtl = ", tostring(validTtl))
 
+  finalCacheOnly = options.finalCacheOnly
   -- Deal with the `resolv.conf` file
 
   local resolvconffile = options.resolvConf or utils.DEFAULT_RESOLV_CONF
@@ -653,6 +655,29 @@ local function parseAnswer(qname, qtype, answers, try_list)
     end
   end
 
+  if finalCacheOnly then
+    -- when only caching final results, we remove all non-requested
+    if #answers >= 2 and answers[#answers].type == qtype then
+      local min_ttl = math.huge
+      local j = 0
+      for i = 1, #answers do
+        min_ttl = math_min(answers[i].ttl, min_ttl)
+        if answers[i].type == qtype then
+          j = j + 1
+          answers[j] = answers[i]
+        end
+      end
+      for i = 1, #answers do
+        if i > j then
+          table.remove(answers)
+        else
+          answers[i].name = check_qname
+          answers[i].ttl = min_ttl
+        end
+      end
+    end
+  end
+
   for i = #answers, 1, -1 do -- we're deleting entries, so reverse the traversal
     local answer = answers[i]
 

t/04-multi-answers-cache.t

@@ -0,0 +1,53 @@
+use Test::Nginx::Socket;
+
+plan('no_plan');
+
+our $HttpConfig = qq{
+    lua_package_path "deps/share/lua/5.1/?.lua;deps/share/lua/5.1/?.lua;src/?.lua;src/?/?.lua;src/?/init.lua;;";
+};
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: DNS chain 10ttl -> 3ttl -> 10ttl, use 3ttl as the expiration time
+--- http_config eval: $::HttpConfig
+--- config
+    location = /t {
+        access_by_lua_block {
+            local resolver = require("resty.dns.resolver")
+            local old_query = resolver.query
+            resolver.query = function(self, qname, opts, tries)
+                ngx.log(ngx.INFO, "resolver query for ", qname)
+                return old_query(self, qname, opts, tries)
+            end
+
+            local client = require("resty.dns.client")
+            assert(client.init({
+                nameservers = { {"127.0.0.1", 15353} },
+                order = {"LAST","A","AAAA", "CNAME" },
+                finalCacheOnly = true,
+            }))
+
+            for i = 1, 10 do
+                local answers, err = client.resolve("run.api7.ai", { qtype = client.TYPE_A })
+                assert(err == nil, err)
+                assert(#answers == 2, "no answers returned")
+                assert(answers[1].address == "18.155.68.66", "unexpected address: " .. (answers[1].address or "nil"))
+                assert(answers[2].address == "18.155.68.67", "unexpected address: " .. (answers[2].address or "nil"))
+                ngx.sleep(1)  -- Adding a sleep to avoid overwhelming the resolver
+            end
+            ngx.say("passed")
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+--- grep_error_log eval
+qr/resolver query for run.api7.ai/
+--- grep_error_log_out
+resolver query for run.api7.ai
+resolver query for run.api7.ai
+resolver query for run.api7.ai
+--- timeout: 15

t/testdata/Corefile

@@ -0,0 +1,23 @@
+.:15353 {
+    log
+    errors
+    bufsize 512
+
+    template IN A {
+        match "^two\.cloudfront\.net\.$"
+        answer "{{ .Name }} 10 IN A 18.155.68.66"
+        answer "{{ .Name }} 10 IN A 18.155.68.67"
+        fallthrough
+    }
+
+    template IN A {
+        match "^one\.cloudfront\.net\.$"
+        answer "{{ .Name }} 3 IN CNAME two.cloudfront.net."
+        fallthrough
+    }
+
+    template IN A {
+        match "^run\.api7\.ai\.$"
+        answer "{{ .Name }} 10 IN CNAME one.cloudfront.net."
+    }
+}