diff --git a/core/src/Kivi.zig b/core/src/Kivi.zig index b0d934b..a520707 100644 --- a/core/src/Kivi.zig +++ b/core/src/Kivi.zig @@ -38,6 +38,9 @@ fn upper_power_of_two(n_arg: u64) u64 { fn key_to_possible_index(self: *const Kivi, key: []const u8) usize { return std.hash.Wyhash.hash(0, key) & (self.entries.len - 1); } +fn stringcpy(dest: []u8, src: []const u8) void { + @memcpy(dest.ptr[0..src.len], src.ptr[0..src.len]); +} pub fn init_default_allocator(self: *Kivi, config: *const Config) !usize { var gpa = GPA{}; @@ -69,27 +72,34 @@ pub fn init(allocator: std.mem.Allocator, config: *const Config) !Kivi { return Kivi{ .allocator = allocator, .entries = entries, .table_size = maximum_elements, .keys_mmap = keys_mmap, .values_mmap = values_mmap }; } -pub fn set(self: *Kivi, key: []const u8, value: []const u8) !usize { +pub fn undo_key_reserve(self: *Kivi, slice: []u8) void { + self.keys_mmap.cursor -= slice.len; +} +pub fn reserve_key(self: *Kivi, size: usize) ![]u8 { const key_cursor = self.keys_mmap.cursor; errdefer self.keys_mmap.cursor = key_cursor; + return try self.keys_mmap.reserve(size); +} +pub fn reserve(self: *Kivi, key_slice: []u8, size: usize) ![]u8 { var retrying = false; var rehashing = false; - var index = self.key_to_possible_index(key); + var index = self.key_to_possible_index(key_slice); while (true) { if (self.entries[index].key == null) { + const value_slice = try self.values_mmap.reserve(size); self.entries[index] = Entry{ - .key = try self.keys_mmap.push(key), - .value = try self.values_mmap.push(value), + .key = key_slice, + .value = value_slice, }; - return value.len; + return value_slice; } index += 1; if (index >= self.entries.len) { if (!rehashing) { - index = self.key_to_possible_index(key); + index = self.key_to_possible_index(key_slice); rehashing = true; } else { if (!retrying) { @@ -104,23 +114,27 @@ pub fn set(self: *Kivi, key: []const u8, value: []const u8) !usize { return error.NoFreeSlot; } +pub fn set(self: *Kivi, key: []const u8, value: []const u8) !usize { + const key_slice = try self.reserve_key(key.len); + errdefer self.undo_key_reserve(key_slice); + + const value_slice = try self.reserve(key_slice, value.len); + + stringcpy(key_slice, key); + stringcpy(value_slice, value); + + return value_slice.len; +} -pub fn get(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { +pub fn get_slice(self: *const Kivi, key: []const u8) ![]u8 { var retrying = false; var rehashing = false; var index = self.key_to_possible_index(key); while (true) { - const entry = self.entries[index]; - if (entry.key != null) { - if (std.mem.eql(u8, key, entry.key.?)) { - if (value != null) { - @memcpy(value.?[0..entry.value.len], entry.value.ptr[0..entry.value.len]); - } - - return entry.value.len; + if (self.entries[index].key != null) { + if (std.mem.eql(u8, key, self.entries[index].key.?)) { + return self.entries[index].value; } - } else { - return error.NotFound; } index += 1; @@ -141,25 +155,27 @@ pub fn get(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { return error.NotFound; } +pub fn get(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { + const value_slice = try self.get_slice(key); -pub fn del(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { + if (value != null) { + stringcpy(value.?, value_slice); + } + + return value_slice.len; +} + +pub fn del_slice(self: *const Kivi, key: []const u8) ![]u8 { var retrying = false; var rehashing = false; var index = self.key_to_possible_index(key); while (true) { - var entry = self.entries[index]; - if (entry.key != null) { - if (std.mem.eql(u8, key, entry.key.?)) { - if (value != null) { - @memcpy(value.?[0..entry.value.len], entry.value.ptr[0..entry.value.len]); - } - + if (self.entries[index].key != null) { + if (std.mem.eql(u8, key, self.entries[index].key.?)) { self.entries[index].key = null; - return entry.value.len; + return self.entries[index].value; } - } else { - return error.NotFound; } index += 1; @@ -180,6 +196,15 @@ pub fn del(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { return error.NotFound; } +pub fn del(self: *const Kivi, key: []const u8, value: ?[]u8) !usize { + const value_slice = try self.del_slice(key); + + if (value != null) { + stringcpy(value.?, value_slice); + } + + return value_slice.len; +} pub fn deinit(self: *Kivi) void { self.keys_mmap.deinit(); diff --git a/core/src/Mmap.zig b/core/src/Mmap.zig index 8cf0c89..f7b6d56 100644 --- a/core/src/Mmap.zig +++ b/core/src/Mmap.zig @@ -64,6 +64,19 @@ pub fn push(self: *Mmap, data: []const u8) ![]u8 { return slice; } +pub fn reserve(self: *Mmap, size: usize) ![]u8 { + const starting_pos = self.cursor; + const ending_pos = starting_pos + size; + + if (ending_pos > self.protected_mem_cursor) { + try self.mprotect(); + } + + self.cursor = ending_pos; + + return self.mem[starting_pos..ending_pos]; +} + pub fn deinit(self: *Mmap) void { if (!is_windows) { std.os.munmap(self.mem); diff --git a/core/src/codegen/generate.zig b/core/src/codegen/generate.zig index 8e43738..e5a4b52 100644 --- a/core/src/codegen/generate.zig +++ b/core/src/codegen/generate.zig @@ -41,7 +41,6 @@ fn generate_C_headers(comptime Type: type, writer: anytype) !void { \\ \\ , .{ @alignOf(Type), @sizeOf(Type) }); - // std.debug.panic("Thing: {any}", .{.{ Type, @alignOf(Type), @sizeOf(Type) }}); } inline for (@typeInfo(Type).Struct.decls) |decl| { if (std.mem.startsWith(u8, decl.name, "kivi_") or std.mem.eql(u8, decl.name, "setup_debug_handlers") or std.mem.eql(u8, decl.name, "dump_stack_trace") or std.mem.eql(u8, decl.name, "Config")) { diff --git a/core/src/include/kivi.h b/core/src/include/kivi.h index 545198d..8701eca 100644 --- a/core/src/include/kivi.h +++ b/core/src/include/kivi.h @@ -2,7 +2,7 @@ #include struct __attribute__((aligned(8))) Kivi { - char __opaque[128]; + char __opaque[120]; }; struct Config { diff --git a/drivers/js/index.d.ts b/drivers/js/index.d.ts index 0941845..9455fde 100644 --- a/drivers/js/index.d.ts +++ b/drivers/js/index.d.ts @@ -1,5 +1,5 @@ interface KiviConfig { - forceUseRuntimeFFI: ?boolean, + forceUseRuntimeFFI: boolean | undefined, } export class Kivi { diff --git a/drivers/js/index.js b/drivers/js/index.js index 537d968..ce38d76 100644 --- a/drivers/js/index.js +++ b/drivers/js/index.js @@ -54,10 +54,6 @@ export class Kivi { * @returns {(string|null)} */ get(key) { - if (key.length > 4096) { - throw new Error("Key is too long!"); - } - return this.#InnerKivi.get(key); } /** @@ -67,13 +63,6 @@ export class Kivi { * @returns {void} */ set(key, value) { - if (key.length > 4096) { - throw new Error("Key is too long!"); - } - if (value.length > 4096) { - throw new Error("Value is too long!"); - } - if (!this.#InnerKivi.set(key, value)) { throw new Error("Failed to insert!"); } diff --git a/drivers/js/nodejs/build.zig b/drivers/js/nodejs/build.zig index 28170ac..072692f 100644 --- a/drivers/js/nodejs/build.zig +++ b/drivers/js/nodejs/build.zig @@ -31,8 +31,6 @@ fn formatTarget(target: std.Target, allocator: std.mem.Allocator, suffix: []cons } pub fn build(b: *std.build.Builder) !void { - const x = b.addModule("core-build", .{ .source_file = std.Build.LazyPath.relative("../../../core/build.zig") }); - const target = b.standardTargetOptions(.{}); optimize = b.standardOptimizeOption(.{}); const target_info = try std.zig.system.NativeTargetInfo.detect(target); @@ -43,13 +41,10 @@ pub fn build(b: *std.build.Builder) !void { }); const shared = b.addSharedLibrary(.{ .name = "kivi-node-addon", .root_source_file = std.Build.LazyPath.relative("src/main.zig"), .target = target, .optimize = optimize }); shared.force_pic = true; - shared.bundle_compiler_rt = true; shared.linker_allow_shlib_undefined = true; shared.addModule("Kivi", kivi_mod); shared.addIncludePath(std.build.LazyPath.relative("src/napi-headers")); - shared.addModule("core-build", x); - if (optimize == .ReleaseFast) { shared.strip = true; shared.single_threaded = true; diff --git a/drivers/js/nodejs/src/main.zig b/drivers/js/nodejs/src/main.zig index 4f53d3d..420f330 100644 --- a/drivers/js/nodejs/src/main.zig +++ b/drivers/js/nodejs/src/main.zig @@ -5,8 +5,7 @@ const ntypes = @cImport({ @cInclude("node_api.h"); }); -const KEYS_DEFAULT_BUF_SIZE: comptime_int = 256 * 1024; -const VALUES_DEFAULT_BUF_SIZE: comptime_int = 2 * 1024 * 1024; +const KEYS_DEFAULT_BUF_SIZE: comptime_int = 500 * 1024; fn get_args(env: ntypes.napi_env, info: ntypes.napi_callback_info, arg_count: [*c]usize, args: [*c]ntypes.napi_value) usize { var cb_status: ntypes.napi_status = symbols.napi_get_cb_info(env, info, arg_count, args, null, null); @@ -25,16 +24,30 @@ fn arg_to_kivi(env: ntypes.napi_env, arraybuffer: ntypes.napi_value) ?*Kivi { } return null; } -fn string_to_buffer(env: ntypes.napi_env, arraybuffer: ntypes.napi_value, buf: []u8, bufsize: usize) usize { +fn get_string_length(env: ntypes.napi_env, arraybuffer: ntypes.napi_value) usize { var len: usize = undefined; - if (symbols.napi_get_value_string_utf8(env, arraybuffer, buf.ptr, bufsize, &len) == ntypes.napi_ok) { + if (symbols.napi_get_value_string_utf8(env, arraybuffer, null, 0, &len) == ntypes.napi_ok) { return len; } return 0; } -fn buffer_to_string(env: ntypes.napi_env, buf: []u8, bufsize: usize) ntypes.napi_value { +fn stack_string_to_buffer(env: ntypes.napi_env, arraybuffer: ntypes.napi_value, buf: []u8) usize { + var len: usize = undefined; + if (symbols.napi_get_value_string_utf8(env, arraybuffer, buf.ptr, buf.len, &len) == ntypes.napi_ok) { + return len; + } + return 0; +} +fn string_to_buffer(env: ntypes.napi_env, arraybuffer: ntypes.napi_value, buf: []u8) usize { + var len: usize = undefined; + if (symbols.napi_get_value_string_utf8(env, arraybuffer, buf.ptr, buf.len + 1, &len) == ntypes.napi_ok) { + return len + 1; + } + return 0; +} +fn buffer_to_string(env: ntypes.napi_env, buf: []u8) ntypes.napi_value { var string: ntypes.napi_value = undefined; - if (symbols.napi_create_string_utf8(env, buf.ptr, bufsize, &string) == ntypes.napi_ok) { + if (symbols.napi_create_string_utf8(env, buf.ptr, buf.len, &string) == ntypes.napi_ok) { return string; } return null; @@ -60,6 +73,28 @@ fn new_undefined(env: ntypes.napi_env) ntypes.napi_value { } return null; } +fn allocate_temp_key(self: *Kivi, env: ntypes.napi_env, napi_buffer: ntypes.napi_value, should_be_freed: *bool) ![]u8 { + var temp_buf: [KEYS_DEFAULT_BUF_SIZE]u8 = undefined; + var length = get_string_length(env, napi_buffer); + + if (length > KEYS_DEFAULT_BUF_SIZE) { + var key_buf = self.allocator.alloc(u8, length) catch return error.Failed; + should_be_freed.* = true; + + const written_len = string_to_buffer(env, napi_buffer, key_buf); + if (written_len == 0) { + return error.Failed; + } + + return key_buf; + } else if (length == 0) { + return error.Failed; + } + + _ = string_to_buffer(env, napi_buffer, &temp_buf); + + return temp_buf[0..length]; +} pub export fn kivi_init_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) ntypes.napi_value { var args_count: usize = 1; @@ -89,19 +124,22 @@ pub export fn kivi_get_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) var argc: usize = get_args(env, info, &args_count, &args); if (argc == 0) return new_undefined(env); - var key_buf: [KEYS_DEFAULT_BUF_SIZE]u8 = undefined; - var key_len: usize = string_to_buffer(env, args[1], &key_buf, KEYS_DEFAULT_BUF_SIZE); - if (key_len == 0) { - return new_null(env); + var self = arg_to_kivi(env, args[0]).?; + + var should_be_freed = false; + const key = allocate_temp_key(self, env, args[1], &should_be_freed) catch return new_null(env); + defer { + if (should_be_freed) { + self.allocator.free(key); + } } - var value_buf: [VALUES_DEFAULT_BUF_SIZE]u8 = undefined; - var value_len: usize = arg_to_kivi(env, args[0]).?.get(key_buf[0..key_len], &value_buf) catch 0; - if (value_len == 0) { + var value = self.get_slice(key) catch return new_null(env); + if (value.len == 0) { return new_null(env); } - return buffer_to_string(env, &value_buf, value_len); + return buffer_to_string(env, value); } pub export fn kivi_set_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) ntypes.napi_value { var args_count: usize = 3; @@ -109,18 +147,32 @@ pub export fn kivi_set_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) var argc: usize = get_args(env, info, &args_count, &args); if (argc == 0) return new_undefined(env); - var key_buf: [KEYS_DEFAULT_BUF_SIZE]u8 = undefined; - var key_len: usize = string_to_buffer(env, args[1], &key_buf, KEYS_DEFAULT_BUF_SIZE); + var self = arg_to_kivi(env, args[0]).?; + + var key_len: usize = get_string_length(env, args[1]); if (key_len == 0) { return new_unint(env, 0); } + var key_buf = self.reserve_key(key_len) catch return new_unint(env, 0); + const written_key_len = string_to_buffer(env, args[1], key_buf); + if (written_key_len == 0) { + return new_unint(env, 0); + } - var value_buf: [VALUES_DEFAULT_BUF_SIZE]u8 = undefined; - var value_len: usize = string_to_buffer(env, args[2], &value_buf, VALUES_DEFAULT_BUF_SIZE); - - var kivi_result: usize = arg_to_kivi(env, args[0]).?.set(key_buf[0..key_len], value_buf[0..value_len]) catch 0; + var value_len: usize = get_string_length(env, args[2]); + if (value_len == 0) { + return new_unint(env, 0); + } + var value_buf = self.reserve(key_buf, value_len) catch { + self.undo_key_reserve(key_buf); + return new_unint(env, 0); + }; + const written_value_len = string_to_buffer(env, args[2], value_buf); + if (written_value_len == 0) { + return new_unint(env, 0); + } - return new_unint(env, @intCast(kivi_result)); + return new_unint(env, @intCast(value_len)); } pub export fn kivi_del_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) ntypes.napi_value { var args_count: usize = 2; @@ -128,19 +180,22 @@ pub export fn kivi_del_js(env: ntypes.napi_env, info: ntypes.napi_callback_info) var argc: usize = get_args(env, info, &args_count, &args); if (argc == 0) return new_undefined(env); - var key_buf: [KEYS_DEFAULT_BUF_SIZE]u8 = undefined; - var key_len: usize = string_to_buffer(env, args[1], &key_buf, KEYS_DEFAULT_BUF_SIZE); - if (key_len == 0) { - return new_null(env); + var self = arg_to_kivi(env, args[0]).?; + + var should_be_freed = false; + const key = allocate_temp_key(self, env, args[1], &should_be_freed) catch return new_null(env); + defer { + if (should_be_freed) { + self.allocator.free(key); + } } - var value_buf: [VALUES_DEFAULT_BUF_SIZE]u8 = undefined; - var value_len: usize = arg_to_kivi(env, args[0]).?.del(key_buf[0..key_len], &value_buf) catch 0; - if (value_len == 0) { + var value = self.del_slice(key) catch return new_null(env); + if (value.len == 0) { return new_null(env); } - return buffer_to_string(env, &value_buf, value_len); + return buffer_to_string(env, value); } pub export fn node_api_module_get_api_version_v1() i32 { diff --git a/drivers/js/nodejs/src/symbols.zig b/drivers/js/nodejs/src/symbols.zig index 690a31b..a60601d 100644 --- a/drivers/js/nodejs/src/symbols.zig +++ b/drivers/js/nodejs/src/symbols.zig @@ -1,6 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const c = @cImport({ +pub const c = @cImport({ @cInclude("node_api.h"); });