From 890b3caf45bd052e319e48349ef393ec93e08ac4 Mon Sep 17 00:00:00 2001 From: Datong Sun Date: Fri, 11 Oct 2024 21:09:06 +1100 Subject: [PATCH] feat(page): add `page_size` parameter to allow overriding page size from (#58) * feat(page): add `page_size` parameter to allow overriding page size from caller side This is useful for DB-less reads as DB-less defines it's own page size which might differ from what this library provides KAG-5342 --- README.md | 13 +++- lib/resty/lmdb/prefix.lua | 16 +++-- t/10-prefix.t | 139 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c54717f..cf844fd 100644 --- a/README.md +++ b/README.md @@ -227,20 +227,31 @@ from the `txn` table when `commit()` returned an error is undefined. #### page -**syntax:** *res, err = prefix.page(start, prefix, db?)* +**syntax:** *res, err_or_more = prefix.page(start, prefix, db?, page_size?)* **context:** *any context* Return all keys `>= start` and starts with `prefix`. If `db` is omitted, it defaults to `"_default"`. +If `page_size` is specified, up to `page_size` results will be returned. However, +`page_size` can not be set to less than `2` due to internal implementation limitations. + The return value of this function is a table `res` where `res[1].key` and `res[1].value` corresponds to the first key and value, `res[2].key` and `res[2].value` corresponds to the second and etc. If no keys matched the provided criteria, then an empty table will be returned. +In case of success, the second return value will be a boolean indicating if more keys are +possibly present. However, even when this value is `true`, it is possible subsequent `page` +might return an empty list. If this value is `false`, then it is guaranteed no more keys +matching the `prefix` is available. + In case of errors, `nil` and an string describing the reason of the failure will be returned. +This is a low level function, most of the use case should use the higher level +[lmdb.prefix](#prefix) iterator instead. + [Back to TOC](#table-of-contents) ## Directives diff --git a/lib/resty/lmdb/prefix.lua b/lib/resty/lmdb/prefix.lua index 57d6630..989530b 100644 --- a/lib/resty/lmdb/prefix.lua +++ b/lib/resty/lmdb/prefix.lua @@ -26,9 +26,15 @@ local get_string_buf_size = base.get_string_buf_size local assert = assert -function _M.page(start, prefix, db) +function _M.page(start, prefix, db, page_size) + if not page_size then + page_size = DEFAULT_OPS_SIZE + end + + assert(page_size >= 2, "'page_size' can not be less than 2") + local value_buf_size = get_string_buf_size() - local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", DEFAULT_OPS_SIZE) + local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", page_size) ops[0].opcode = C.NGX_LMDB_OP_PREFIX ops[0].key.data = start @@ -50,7 +56,7 @@ function _M.page(start, prefix, db) ::again:: local buf = get_string_buf(value_buf_size, false) - local ret = C.ngx_lua_resty_lmdb_ffi_prefix(ops, DEFAULT_OPS_SIZE, + local ret = C.ngx_lua_resty_lmdb_ffi_prefix(ops, page_size, buf, value_buf_size, err_ptr) if ret == NGX_ERROR then return nil, ffi_string(err_ptr[0]) @@ -83,8 +89,8 @@ function _M.page(start, prefix, db) res[i] = pair end - -- if ret == DEFAULT_OPS_SIZE, then it is possible there are more keys - return res, ret == DEFAULT_OPS_SIZE + -- if ret == page_size, then it is possible there are more keys + return res, ret == page_size end diff --git a/t/10-prefix.t b/t/10-prefix.t index fa04276..8ca2352 100644 --- a/t/10-prefix.t +++ b/t/10-prefix.t @@ -214,3 +214,142 @@ done [error] [warn] [crit] + + + +=== TEST 6: prefix.page() operation +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.db_drop(true)) + ngx.say(l.set("test", "value")) + ngx.say(l.set("test1", "value1")) + ngx.say(l.set("test2", "value2")) + ngx.say(l.set("test3", "value3")) + ngx.say(l.set("u", "value4")) + ngx.say(l.set("u1", "value5")) + + local p = require("resty.lmdb.prefix") + + local res, err = p.page("test", "test") + if not res then + ngx.say("page errored: ", err) + end + + for _, pair in ipairs(res) do + ngx.say("key: ", pair.key, " value: ", pair.value) + end + } + } +--- request +GET /t +--- response_body +true +true +true +true +true +true +true +key: test value: value +key: test1 value: value1 +key: test2 value: value2 +key: test3 value: value3 +--- no_error_log +[error] +[warn] +[crit] + + + +=== TEST 7: prefix.page() operation with custom page size +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.db_drop(true)) + ngx.say(l.set("test", "value")) + ngx.say(l.set("test1", "value1")) + ngx.say(l.set("test2", "value2")) + ngx.say(l.set("test3", "value3")) + ngx.say(l.set("u", "value4")) + ngx.say(l.set("u1", "value5")) + + local p = require("resty.lmdb.prefix") + + local res, err = p.page("test", "test", nil, 2) + if not res then + ngx.say("page errored: ", err) + end + + ngx.say("FIRST PAGE") + for _, pair in ipairs(res) do + ngx.say("key: ", pair.key, " value: ", pair.value) + end + + res, err = p.page("test1\x00", "test", nil, 2) + if not res then + ngx.say("page errored: ", err) + end + + ngx.say("SECOND PAGE") + for _, pair in ipairs(res) do + ngx.say("key: ", pair.key, " value: ", pair.value) + end + } + } +--- request +GET /t +--- response_body +true +true +true +true +true +true +true +FIRST PAGE +key: test value: value +key: test1 value: value1 +SECOND PAGE +key: test2 value: value2 +key: test3 value: value3 +--- no_error_log +[error] +[warn] +[crit] + + + +=== TEST 8: prefix.page() operation with invalid page size +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.db_drop(true)) + local p = require("resty.lmdb.prefix") + + ngx.say(l.set("test", "value")) + ngx.say(pcall(p.page, "test", "test", nil, 1)) + } + } +--- request +GET /t +--- response_body +true +true +false.../lua-resty-lmdb/lua-resty-lmdb/lib/resty/lmdb/prefix.lua:34: 'page_size' can not be less than 2 +--- no_error_log +[error] +[warn] +[crit]