feat(page): add page_size parameter to allow overriding page size f…

…rom (#58)

* feat(page): add `page_size` parameter to allow overriding page size from
caller side

This is useful for DB-less reads as DB-less defines it's own page size
which might differ from what this library provides

KAG-5342

README.md

@@ -227,20 +227,31 @@ from the `txn` table when `commit()` returned an error is undefined.
 
 #### page
 
-**syntax:** *res, err = prefix.page(start, prefix, db?)*
+**syntax:** *res, err_or_more = prefix.page(start, prefix, db?, page_size?)*
 
 **context:** *any context*
 
 Return all keys `>= start` and starts with `prefix`. If `db` is omitted,
 it defaults to `"_default"`.
 
+If `page_size` is specified, up to `page_size` results will be returned. However,
+`page_size` can not be set to less than `2` due to internal implementation limitations.
+
 The return value of this function is a table `res` where `res[1].key` and `res[1].value`
 corresponds to the first key and value, `res[2].key` and `res[2].value` corresponds to the
 second and etc. If no keys matched the provided criteria, then an empty table will be
 returned.
 
+In case of success, the second return value will be a boolean indicating if more keys are
+possibly present. However, even when this value is `true`, it is possible subsequent `page`
+might return an empty list. If this value is `false`, then it is guaranteed no more keys
+matching the `prefix` is available.
+
 In case of errors, `nil` and an string describing the reason of the failure will be returned.
 
+This is a low level function, most of the use case should use the higher level
+[lmdb.prefix](#prefix) iterator instead.
+
 [Back to TOC](#table-of-contents)
 
 ## Directives

lib/resty/lmdb/prefix.lua

@@ -26,9 +26,15 @@ local get_string_buf_size = base.get_string_buf_size
 local assert = assert
 
 
-function _M.page(start, prefix, db)
+function _M.page(start, prefix, db, page_size)
+    if not page_size then
+        page_size = DEFAULT_OPS_SIZE
+    end
+
+    assert(page_size >= 2, "'page_size' can not be less than 2")
+
     local value_buf_size = get_string_buf_size()
-    local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", DEFAULT_OPS_SIZE)
+    local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", page_size)
 
     ops[0].opcode = C.NGX_LMDB_OP_PREFIX
     ops[0].key.data = start
@@ -50,7 +56,7 @@ function _M.page(start, prefix, db)
 
 ::again::
     local buf = get_string_buf(value_buf_size, false)
-    local ret = C.ngx_lua_resty_lmdb_ffi_prefix(ops, DEFAULT_OPS_SIZE,
+    local ret = C.ngx_lua_resty_lmdb_ffi_prefix(ops, page_size,
                     buf, value_buf_size, err_ptr)
     if ret == NGX_ERROR then
         return nil, ffi_string(err_ptr[0])
@@ -83,8 +89,8 @@ function _M.page(start, prefix, db)
         res[i] = pair
     end
 
-    -- if ret == DEFAULT_OPS_SIZE, then it is possible there are more keys
-    return res, ret == DEFAULT_OPS_SIZE
+    -- if ret == page_size, then it is possible there are more keys
+    return res, ret == page_size
 end
 
 

t/10-prefix.t

@@ -214,3 +214,142 @@ done
 [error]
 [warn]
 [crit]
+
+
+
+=== TEST 6: prefix.page() operation
+--- http_config eval: $::HttpConfig
+--- main_config eval: $::MainConfig
+--- config
+    location = /t {
+        content_by_lua_block {
+            local l = require("resty.lmdb")
+
+            ngx.say(l.db_drop(true))
+            ngx.say(l.set("test", "value"))
+            ngx.say(l.set("test1", "value1"))
+            ngx.say(l.set("test2", "value2"))
+            ngx.say(l.set("test3", "value3"))
+            ngx.say(l.set("u", "value4"))
+            ngx.say(l.set("u1", "value5"))
+
+            local p = require("resty.lmdb.prefix")
+
+            local res, err = p.page("test", "test")
+            if not res then
+                ngx.say("page errored: ", err)
+            end
+
+            for _, pair in ipairs(res) do
+                ngx.say("key: ", pair.key, " value: ", pair.value)
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+true
+true
+true
+true
+true
+true
+true
+key: test value: value
+key: test1 value: value1
+key: test2 value: value2
+key: test3 value: value3
+--- no_error_log
+[error]
+[warn]
+[crit]
+
+
+
+=== TEST 7: prefix.page() operation with custom page size
+--- http_config eval: $::HttpConfig
+--- main_config eval: $::MainConfig
+--- config
+    location = /t {
+        content_by_lua_block {
+            local l = require("resty.lmdb")
+
+            ngx.say(l.db_drop(true))
+            ngx.say(l.set("test", "value"))
+            ngx.say(l.set("test1", "value1"))
+            ngx.say(l.set("test2", "value2"))
+            ngx.say(l.set("test3", "value3"))
+            ngx.say(l.set("u", "value4"))
+            ngx.say(l.set("u1", "value5"))
+
+            local p = require("resty.lmdb.prefix")
+
+            local res, err = p.page("test", "test", nil, 2)
+            if not res then
+                ngx.say("page errored: ", err)
+            end
+
+            ngx.say("FIRST PAGE")
+            for _, pair in ipairs(res) do
+                ngx.say("key: ", pair.key, " value: ", pair.value)
+            end
+
+            res, err = p.page("test1\x00", "test", nil, 2)
+            if not res then
+                ngx.say("page errored: ", err)
+            end
+
+            ngx.say("SECOND PAGE")
+            for _, pair in ipairs(res) do
+                ngx.say("key: ", pair.key, " value: ", pair.value)
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+true
+true
+true
+true
+true
+true
+true
+FIRST PAGE
+key: test value: value
+key: test1 value: value1
+SECOND PAGE
+key: test2 value: value2
+key: test3 value: value3
+--- no_error_log
+[error]
+[warn]
+[crit]
+
+
+
+=== TEST 8: prefix.page() operation with invalid page size
+--- http_config eval: $::HttpConfig
+--- main_config eval: $::MainConfig
+--- config
+    location = /t {
+        content_by_lua_block {
+            local l = require("resty.lmdb")
+
+            ngx.say(l.db_drop(true))
+            local p = require("resty.lmdb.prefix")
+
+            ngx.say(l.set("test", "value"))
+            ngx.say(pcall(p.page, "test", "test", nil, 1))
+        }
+    }
+--- request
+GET /t
+--- response_body
+true
+true
+false.../lua-resty-lmdb/lua-resty-lmdb/lib/resty/lmdb/prefix.lua:34: 'page_size' can not be less than 2
+--- no_error_log
+[error]
+[warn]
+[crit]