Merge pull request #40 from HarperDB/key-ranges

Add support for key ranges

Author: cb1kenobi

.gitignore

@@ -1,7 +1,7 @@
 build
-coverage
+/coverage
 deps
 dist
 node_modules
 .DS_Store
-.env
+.env

README.md

@@ -17,10 +17,11 @@ Creates a new database instance.
 - `path: string` The path to write the database files to. This path does not
   need to exist, but the parent directories do.
 - `options: object` [optional]
-  - `noBlockCache: boolean` When `true`, disables the block cache. Block caching is enabled by default and the cache is shared across all database instances.
   - `name:string` The column family name. Defaults to `"default"`.
+  - `noBlockCache: boolean` When `true`, disables the block cache. Block caching is enabled by default and the cache is shared across all database instances.
   - `parallelismThreads: number` The number of background threads to use for flush and compaction. Defaults to `1`.
   - `pessimistic: boolean` When `true`, throws conflict errors when they occur instead of waiting until commit. Defaults to `false`.
+  - `store: Store` A custom store that handles all interaction between the `RocksDatabase` or `Transaction` instances and the native database interface. See [Custom Store](#custom-store) for more information.
 
 ### `db.config(options)`
 
@@ -82,6 +83,72 @@ Note that all errors are returned as rejected promises.
 
 Synchronous version of `get()`.
 
+### `db.getEntry(key): MaybePromise`
+
+Retrieves a value for the given key as an "entry" object.
+
+```typescript
+const { value } = await db.getEntry('foo');
+```
+
+### `db.getKeys(options?: IteratorOptions): RangeIterable`
+
+Retrieves all keys within a range.
+
+```typescript
+for (const { key, value } of db.getKeys()) {
+  console.log({ key, value });
+}
+```
+
+### `db.getKeysCount(options?: RangeOptions): number`
+
+Retrieves the number of keys within a range.
+
+```typescript
+const total = db.getKeysCount();
+const range = db.getKeysCount({ start: 'a', end: 'z' });
+```
+
+### `db.getRange(options?: IteratorOptions): RangeIterable`
+
+Retrieves a range of keys and their values. Supports both synchronous and asynchronous iteration.
+
+```typescript
+// sync
+for (const { key, value } of db.getRange()) {
+  console.log({ key, value });
+}
+
+// async
+for await (const { key, value } of db.getRange()) {
+  console.log({ key, value });
+}
+
+// key range
+for (const { key, value } of db.getRange({ start: 'a', end: 'z' })) {
+  console.log({ key, value });
+}
+```
+
+### `db.getValues(key: Key, options?: IteratorOptions): RangeIterable`
+
+Retrieves all values for the given key.
+
+```typescript
+for (const { value } of db.getValues('a')) {
+  console.log({ value });
+}
+```
+
+### `db.getValuesCount(key: Key, options?: RangeOptions): number`
+
+Retrieves the number of values for the given key.
+
+```typescript
+const count = db.getValuesCount('a');
+```
+
 ### `db.put(key, value, options?): Promise`
 
 Stores a value for a given key.
@@ -154,6 +221,219 @@ db.transactionSync((txn: Transaction) => {
 });
 ```
 
+### `class RangeIterable`
+
+An iterable that provides a rich set of methods for working with ranges of items.
+
+#### `.asArray: any[] | Promise<any[]>`
+
+Collects the iterator results in an array and returns it.
+
+```typescript
+const array = db.getRange().asArray;
+```
+
+If the iterable is asynchronous, then it will return a promise.
+
+```typescript
+const array = await db.getRange().asArray;
+```
+
+#### `.at(index: number): any`
+
+Returns the item at the given index.
+
+```typescript
+const item = db.getRange().at(0);
+```
+
+#### `.concat(iterable: Iterable): RangeIterable`
+
+Concatenates the iterable with another iterable.
+
+```typescript
+const concatenated = db.getRange().concat(db.getRange());
+```
+
+#### `.drop(limit: number): RangeIterable`
+
+Returns a new iterable with the first `limit` items removed.
+
+```typescript
+for (const { key, value } of db.getRange().drop(10)) {
+  console.log({ key, value });
+}
+```
+
+#### `.every(callback: (value, index) => boolean): boolean`
+
+Returns `true` if the callback returns `true` for every item of the iterable.
+
+```typescript
+const isAllValid = db.getRange().every(item => item.value.length > 0);
+```
+
+#### `.filter(callback: (value, index) => boolean): RangeIterable`
+
+Returns a new iterable containing only the values for which the callback returns `true`.
+
+```typescript
+const filtered = db.getRange().filter(item => item.value.length > 0);
+```
+
+#### `.find(callback: (value, index) => boolean): any`
+
+Returns the first item of the iterable for which the callback returns `true`.
+
+```typescript
+const found = db.getRange().find(item => item.value.length > 0);
+```
+
+#### `.flatMap(callback: (value, index) => any): RangeIterable`
+
+Returns a new iterable with the results of a callback function, then flattens the results.
+
+```typescript
+const flattened = db.getRange().flatMap(item => [item, item]);
+```
+
+#### `.forEach(callback: (value, index) => void): void`
+
+Calls a function for each item of the iterable.
+
+```typescript
+db.getRange().forEach(item => console.log(item));
+```
+
+#### `.map(callback: (value, index) => any): RangeIterable`
+
+Returns a new iterable with the results of calling a callback function.
+
+```typescript
+const mapped = db.getRange().map(item => item.value.length);
+```
+
+#### `.mapError(catchCallback: (error) => Error): RangeIterable`
+
+Catch errors thrown during iteration and allow iteration to continue.
+
+```typescript
+const mapped = db.getRange().mapError(error => new Error('Error: ' + error.message));
+```
+
+#### `.reduce(callback: (prev, current, index) => any): any`
+
+Reduces the iterable to a single value.
+
+```typescript
+const sum = db.getRange().reduce((acc, item) => acc + item.value.length, 0);
+```
+
+#### `.slice(start: number, end?: number): RangeIterable`
+
+Returns a new iterable with the items between the start and end indices.
+
+```typescript
+const sliced = db.getRange().slice(10, 20);
+```
+
+#### `.some(callback: (value, index) => boolean): boolean`
+
+Returns `true` if the callback returns `true` for any item of the iterable.
+
+```typescript
+const hasEven = db.getRange().some(item => item.value.length % 2 === 0);
+```
+
+#### `.take(limit: number): RangeIterable`
+
+Returns a new iterable with the first `limit` items.
+
+```typescript
+for (const { key, value } of db.getRange().take(10)) {
+  console.log({ key, value });
+}
+```
+
+## Custom Store
+
+The store is a class that sits between the `RocksDatabase` or `Transaction`
+instance and the native RocksDB interface. It owns the native RocksDB instance
+along with various settings including encoding and the db name. It handles all interactions with the native RocksDB instance.
+
+The default `Store` contains the following methods which can be overridden:
+
+- `constructor(path, options?)`
+- `close()`
+- `decodeKey(key)`
+- `decodeValue(value)`
+- `encodeKey(key)`
+- `encodeValue(value)`
+- `get(context, key, resolve, reject, txnId?)`
+- `getCount(context, options?, txnId?)`
+- `getRange(context, options?)`
+- `getSync(context, key, options?)`
+- `getValuesCount(context, key, options?)`
+- `isOpen()`
+- `open()`
+- `putSync(context, key, value, options?)`
+- `removeSync(context, key, options?)`
+
+To use it, extend the default `Store` and pass in an instance of your store
+into the `RocksDatabase` constructor.
+
+```typescript
+import { RocksDatabase, Store } from '@harperdb/rocksdb-js';
+
+class MyStore extends Store {
+  get(context, key, resolve, reject, txnId) {
+    console.log('Getting:' key);
+    return super.get(context, key, resolve, reject, txnId);
+  }
+
+  putSync(context, key, value, options) {
+    console.log('Putting:', key);
+    return super.putSync(context, key, value, options);
+  }
+}
+
+const myStore = new MyStore('path/to/db');
+const db = await RocksDatabase.open(myStore);
+await db.put('foo', 'bar');
+console.log(await db.get('foo'));
+```
+
+## Interfaces
+
+### `RocksDBOptions`
+
+- `options: object`
+  - `adaptiveReadahead: boolean` When `true`, RocksDB will do some enhancements for prefetching the data. Defaults to `true`. Note that RocksDB defaults this to `false`.
+  - `asyncIO: boolean` When `true`, RocksDB will prefetch some data async and apply it if reads are sequential and its internal automatic prefetching. Defaults to `true`. Note that RocksDB defaults this to `false`.
+  - `autoReadaheadSize: boolean` When `true`, RocksDB will auto-tune the readahead size during scans internally based on the block cache data when block caching is enabled, an end key (e.g. upper bound) is set, and prefix is the same as the start key. Defaults to `true`.
+  - `backgroundPurgeOnIteratorCleanup: boolean` When `true`, after the iterator is closed, a background job is scheduled to flush the job queue and delete obsolete files. Defaults to `true`. Note that RocksDB defaults this to `false`.
+  - `fillCache: boolean` When `true`, the iterator will fill the block cache. Filling the block cache is not desirable for bulk scans and could impact eviction order. Defaults to `false`. Note that RocksDB defaults this to `true`.
+  - `readaheadSize: number` The RocksDB readahead size. RocksDB does auto-readahead for iterators when there is more than two reads for a table file. The readahead starts at 8KB and doubles on every additional read up to 256KB. This option can help if most of the range scans are large and if a larger readahead than that enabled by auto-readahead is needed. Using a large readahead size (> 2MB) can typically improve the performance of forward iteration on spinning disks. Defaults to `0`.
+  - `tailing: boolean` When `true`, creates a "tailing iterator" which is a special iterator that has a view of the complete database including newly added data and
+  is optimized for sequential reads. This will return records that were inserted into the database after the creation of the iterator. Defaults to `false`.
+
+### `RangeOptions`
+
+Extends `RocksDBOptions`.
+
+- `options: object`
+  - `end: Key | Uint8Array` The range end key, otherwise known as the "upper bound". Defaults to the last key in the database.
+  - `exclusiveStart: boolean` When `true`, the iterator will exclude the first key if it matches the start key. Defaults to `false`.
+  - `inclusiveEnd: boolean` When `true`, the iterator will include the last key if it matches the end key. Defaults to `false`.
+  - `start: Key | Uint8Array` The range start key, otherwise known as the "lower bound". Defaults to the first key in the database.
+
+### `IteratorOptions`
+
+Extends `RangeOptions`.
+
+- `options: object`
+  - `reverse: boolean` When `true`, the iterator will iterate in reverse order. Defaults to `false`.
+
 ## Development
 
 This package requires Node.js 18 or higher, pnpm, and a C++ compiler.
@@ -172,7 +452,7 @@ To compile everything including the native binding and the TypeScript source, ru
 pnpm build
 ```
 
-To compile only the native binding, run:
+To configure and compile only the native binding, run:
 
 ```bash
 pnpm rebuild

binding.gyp

@@ -11,6 +11,8 @@
 				'src/binding/database.cpp',
 				'src/binding/db_descriptor.cpp',
 				'src/binding/db_handle.cpp',
+				'src/binding/db_iterator.cpp',
+				'src/binding/db_iterator_handle.cpp',
 				'src/binding/db_registry.cpp',
 				'src/binding/db_settings.cpp',
 				'src/binding/transaction_handle.cpp',

package.json

@@ -2,8 +2,15 @@
   "name": "@harperdb/rocksdb-js",
   "version": "0.0.0",
   "description": "",
-  "exports": "./dist/index.js",
-  "types": "./dist/index.d.js",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.js",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
   "type": "module",
   "gypfile": true,
   "scripts": {
@@ -22,7 +29,7 @@
     "prebuild": "echo 'TODO'",
     "rebuild": "node-gyp rebuild",
     "rebuild:debug": "node-gyp rebuild --coverage --debug --verbose",
-    "test": "node --expose-gc ./node_modules/vitest/vitest.mjs --allowOnly --reporter=verbose",
+    "test": "node --expose-gc ./node_modules/vitest/vitest.mjs --reporter=verbose",
     "type-check": "tsc --noEmit"
   },
   "files": [
@@ -33,6 +40,7 @@
     "vendor"
   ],
   "dependencies": {
+    "@harperdb/extended-iterable": "1.0.1",
     "msgpackr": "1.11.2",
     "ordered-binary": "1.5.3"
   },