From 44eadf691cb07e0381d21ccae4feeaadad6df689 Mon Sep 17 00:00:00 2001 From: Tom Aylott Date: Tue, 13 May 2025 12:30:34 -0400 Subject: [PATCH 1/6] monorepo warning --- docs/docs/installation.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/docs/installation.md b/docs/docs/installation.md index 160823ee..443d6c7b 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -24,6 +24,12 @@ This package only runs on `iOS`, `Android` and `macOS`, same as `expo-sqlite`, t SQLite is very customizable on compilation level. op-sqlite also allows you add extensions or even change the base implementation. You can do this by adding the following to your `package.json`: +> [!IMPORTANT] +> When using a monorepo, be sure to add the "op-sqlite" config to the monorepo root `package.json` file. +> This is necessary because the `node_modules/@op-engineering/op-sqlite/op-sqlite.podspec` will search for first the `package.json` file in a parent directory. +> Alternative, you may be able to solve this by blocking this package from being hoisted to the root `node_modules`. +> Check your ios/Podfile.lock to see where it's being installed. + ```json { // ... the rest of your package.json From dd7cbe87404b75ca1a8e14248c6c32b693740689 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 15 May 2025 09:19:48 +0200 Subject: [PATCH 2/6] Update template --- .github/ISSUE_TEMPLATE/REQUEST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/REQUEST.yml b/.github/ISSUE_TEMPLATE/REQUEST.yml index db896662..764b69a3 100644 --- a/.github/ISSUE_TEMPLATE/REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/REQUEST.yml @@ -5,6 +5,6 @@ body: - type: textarea attributes: label: What do you need? - placeholder: Please be specific, also bear in mind depending on your requirement a paid sponsorship might be necessary. + placeholder: - Depending on your requirement a paid sponsorship might be necessary.\n- Be specific\n- Make sure you already updated to the latest version and checked the documentation validations: required: true From 31439787b6c0344dd03e5cc8c863648ddf326df9 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 15 May 2025 09:22:53 +0200 Subject: [PATCH 3/6] Update template --- .github/ISSUE_TEMPLATE/REQUEST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/REQUEST.yml b/.github/ISSUE_TEMPLATE/REQUEST.yml index 764b69a3..6502ccb4 100644 --- a/.github/ISSUE_TEMPLATE/REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/REQUEST.yml @@ -5,6 +5,6 @@ body: - type: textarea attributes: label: What do you need? - placeholder: - Depending on your requirement a paid sponsorship might be necessary.\n- Be specific\n- Make sure you already updated to the latest version and checked the documentation + placeholder: Depending on your requirement a paid sponsorship might be necessary. Be specific. Make sure you already updated to the latest version and checked the documentation validations: required: true From d47598037fda2a081022dcf23f14436de83503f8 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 15 May 2025 12:11:39 +0200 Subject: [PATCH 4/6] Update docs --- docs/docs/key_value_storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/key_value_storage.md b/docs/docs/key_value_storage.md index 3d3ee1b8..9d217e4c 100644 --- a/docs/docs/key_value_storage.md +++ b/docs/docs/key_value_storage.md @@ -4,7 +4,7 @@ sidebar_position: 8 # Key-Value Storage -OP-SQLite provides a simple key-value storage API compatible with react-native-async-storage. It should be much faster than async-storage (whilst slower than MMKV) but comes with the convenience of not having to add one more dependency to your app. For convenience it also has sync versions of the methods. If you use SQLCipher the data inside will also be encrypted. +OP-SQLite provides a simple key-value storage API compatible with react-native-async-storage. Mostly as a convenience, use at your own caution. It uses sqlite as a simple text storage, so it might be slower than using other key-value packages or writing data to disk. If you use SQLCipher the data inside will also be encrypted. ```ts import { Storage } from '@op-engineering/op-sqlite'; From cfc57fd78a7426d01f0e4eb615127f8e1411947c Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 15 May 2025 15:46:22 +0200 Subject: [PATCH 5/6] Sanitizes the passed params (looking for array buffers) recursively --- example/src/tests/queries.spec.ts | 41 ++++++++++++++++- src/index.ts | 74 ++++++++++++++++--------------- 2 files changed, 78 insertions(+), 37 deletions(-) diff --git a/example/src/tests/queries.spec.ts b/example/src/tests/queries.spec.ts index 92307a21..8112c625 100644 --- a/example/src/tests/queries.spec.ts +++ b/example/src/tests/queries.spec.ts @@ -600,7 +600,7 @@ export function queriesTests() { await db.executeBatch(commands); const res = await db.execute('SELECT * FROM User'); - // console.log(res); + expect(res.rows).to.eql([ {id: id1, name: name1, age: age1, networth: networth1, nickname: null}, { @@ -613,6 +613,45 @@ export function queriesTests() { ]); }); + it('Batch execute with BLOB', async () => { + let db = open({ + name: 'queries.sqlite', + encryptionKey: 'test', + }); + + await db.execute('DROP TABLE IF EXISTS User;'); + await db.execute( + 'CREATE TABLE IF NOT EXISTS User (id TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL, age INT, networth BLOB, nickname TEXT) STRICT;', + ); + const id1 = '1'; + const name1 = 'name1'; + const age1 = 12; + const networth1 = new Uint8Array([1, 2, 3]); + + const id2 = '2'; + const name2 = 'name2'; + const age2 = 17; + const networth2 = new Uint8Array([3, 2, 1]); + + const commands: SQLBatchTuple[] = [ + [ + 'INSERT OR REPLACE INTO "User" (id, name, age, networth) VALUES(?, ?, ?, ?)', + [id1, name1, age1, networth1], + ], + [ + 'INSERT OR REPLACE INTO "User" (id, name, age, networth) VALUES(?, ?, ?, ?)', + [[id2, name2, age2, networth2]], + ], + ]; + + console.log('' + JSON.stringify(commands)); + // bomb~ (NOBRIDGE) ERROR Error: Exception in HostFunction: + await db.executeBatch(commands); + + const res = await db.execute('SELECT * FROM User'); + console.log('res:' + JSON.stringify(res)); + }); + it('DumbHostObject allows to write known props', async () => { const id = chance.integer(); const name = chance.name(); diff --git a/src/index.ts b/src/index.ts index a582ef0b..f04ced18 100644 --- a/src/index.ts +++ b/src/index.ts @@ -374,13 +374,45 @@ function enhanceDB(db: InternalDB, options: DBParams): DB { } }; + function sanitizeArrayBuffersInArray( + params?: any[] | any[][] + ): any[] | undefined { + if (!params) { + return params; + } + + return params.map((p) => { + if (Array.isArray(p)) { + return sanitizeArrayBuffersInArray(p); + } + + if (ArrayBuffer.isView(p)) { + return p.buffer; + } + + return p; + }); + } + // spreading the object does not work with HostObjects (db) // We need to manually assign the fields let enhancedDb = { delete: db.delete, attach: db.attach, detach: db.detach, - executeBatch: db.executeBatch, + executeBatch: async ( + commands: SQLBatchTuple[] + ): Promise => { + const sanitizedCommands = commands.map(([query, params]) => { + if (params) { + return [query, sanitizeArrayBuffersInArray(params)]; + } + + return [query]; + }); + + return db.executeBatch(sanitizedCommands as any[]); + }, loadFile: db.loadFile, updateHook: db.updateHook, commitHook: db.commitHook, @@ -394,26 +426,14 @@ function enhanceDB(db: InternalDB, options: DBParams): DB { query: string, params?: Scalar[] ): Promise => { - const sanitizedParams = params?.map((p) => { - if (ArrayBuffer.isView(p)) { - return p.buffer; - } - - return p; - }); + const sanitizedParams = sanitizeArrayBuffersInArray(params); return sanitizedParams ? await db.executeWithHostObjects(query, sanitizedParams as Scalar[]) : await db.executeWithHostObjects(query); }, executeRaw: async (query: string, params?: Scalar[]) => { - const sanitizedParams = params?.map((p) => { - if (ArrayBuffer.isView(p)) { - return p.buffer; - } - - return p; - }); + const sanitizedParams = sanitizeArrayBuffersInArray(params); return db.executeRaw(query, sanitizedParams as Scalar[]); }, @@ -421,24 +441,12 @@ function enhanceDB(db: InternalDB, options: DBParams): DB { // at some point I changed the API but they did not pin their dependency to a specific version // so re-inserting this so it starts working again executeRawAsync: async (query: string, params?: Scalar[]) => { - const sanitizedParams = params?.map((p) => { - if (ArrayBuffer.isView(p)) { - return p.buffer; - } - - return p; - }); + const sanitizedParams = sanitizeArrayBuffersInArray(params); return db.executeRaw(query, sanitizedParams as Scalar[]); }, executeSync: (query: string, params?: Scalar[]): QueryResult => { - const sanitizedParams = params?.map((p) => { - if (ArrayBuffer.isView(p)) { - return p.buffer; - } - - return p; - }); + const sanitizedParams = sanitizeArrayBuffersInArray(params); let intermediateResult = sanitizedParams ? db.executeSync(query, sanitizedParams as Scalar[]) @@ -476,13 +484,7 @@ function enhanceDB(db: InternalDB, options: DBParams): DB { query: string, params?: Scalar[] | undefined ): Promise => { - const sanitizedParams = params?.map((p) => { - if (ArrayBuffer.isView(p)) { - return p.buffer; - } - - return p; - }); + const sanitizedParams = sanitizeArrayBuffersInArray(params); let intermediateResult = await db.execute( query, From 6bad00197e2f5de3c40d17ab831c5e991d5d5d54 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 15 May 2025 16:05:37 +0200 Subject: [PATCH 6/6] Expose offline flag for libsql --- cpp/DBHostObject.cpp | 4 ++-- cpp/DBHostObject.h | 2 +- cpp/bindings.cpp | 13 +++++++++---- cpp/libsql/bridge.cpp | 7 +++++-- cpp/libsql/bridge.h | 3 ++- example/ios/Podfile.lock | 6 +----- example/package.json | 4 ++-- example/src/tests/storage.spec.ts | 4 +--- src/index.ts | 3 ++- 9 files changed, 25 insertions(+), 21 deletions(-) diff --git a/cpp/DBHostObject.cpp b/cpp/DBHostObject.cpp index 33e8e25b..f927e0b4 100644 --- a/cpp/DBHostObject.cpp +++ b/cpp/DBHostObject.cpp @@ -154,11 +154,11 @@ DBHostObject::DBHostObject(jsi::Runtime &rt, std::shared_ptr invoker, std::string &db_name, std::string &path, std::string &url, std::string &auth_token, - int sync_interval) + int sync_interval, bool offline) : db_name(db_name), invoker(std::move(invoker)), rt(rt) { _thread_pool = std::make_shared(); db = opsqlite_libsql_open_sync(db_name, path, url, auth_token, - sync_interval); + sync_interval, offline); create_jsi_functions(); } diff --git a/cpp/DBHostObject.h b/cpp/DBHostObject.h index b665ddd6..13138d29 100644 --- a/cpp/DBHostObject.h +++ b/cpp/DBHostObject.h @@ -54,7 +54,7 @@ class JSI_EXPORT DBHostObject : public jsi::HostObject { // Constructor for a local database with remote sync DBHostObject(jsi::Runtime &rt, std::shared_ptr invoker, std::string &db_name, std::string &path, std::string &url, - std::string &auth_token, int sync_interval); + std::string &auth_token, int sync_interval, bool offline); #endif std::vector getPropertyNames(jsi::Runtime &rt) override; diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 3af52b22..5972658e 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -129,18 +129,23 @@ void install(jsi::Runtime &rt, std::string url = options.getProperty(rt, "url").asString(rt).utf8(rt); std::string auth_token = options.getProperty(rt, "authToken").asString(rt).utf8(rt); + int sync_interval = 0; - if (options.hasProperty(rt, "syncInterval")) { + if (options.hasProperty(rt, "libsqlSyncInterval")) { sync_interval = static_cast( options.getProperty(rt, "syncInterval").asNumber()); } - std::string location; + bool offline = false; + if (options.hasProperty(rt, "libsqlOffline")) { + offline = options.getProperty(rt, "libsqlOffline").asBool(); + } + + std::string location; if (options.hasProperty(rt, "location")) { location = options.getProperty(rt, "location").asString(rt).utf8(rt); } - if (!location.empty()) { if (location == ":memory:") { path = ":memory:"; @@ -152,7 +157,7 @@ void install(jsi::Runtime &rt, } std::shared_ptr db = std::make_shared( - rt, invoker, name, path, url, auth_token, sync_interval); + rt, invoker, name, path, url, auth_token, sync_interval, offline); return jsi::Object::createFromHostObject(rt, db); }); #endif diff --git a/cpp/libsql/bridge.cpp b/cpp/libsql/bridge.cpp index 8d64341b..8c5c7126 100644 --- a/cpp/libsql/bridge.cpp +++ b/cpp/libsql/bridge.cpp @@ -39,7 +39,8 @@ std::string opsqlite_get_db_path(std::string const &db_name, DB opsqlite_libsql_open_sync(std::string const &name, std::string const &base_path, std::string const &url, - std::string const &auth_token, int sync_interval) { + std::string const &auth_token, int sync_interval, + bool offline) { std::string path = opsqlite_get_db_path(name, base_path); int status; @@ -53,7 +54,9 @@ DB opsqlite_libsql_open_sync(std::string const &name, .read_your_writes = '1', .encryption_key = nullptr, .sync_interval = sync_interval, - .with_webpki = '1'}; + .with_webpki = '1', + .offline = offline}; + status = libsql_open_sync_with_config(config, &db, &err); if (status != 0) { throw std::runtime_error(err); diff --git a/cpp/libsql/bridge.h b/cpp/libsql/bridge.h index bb9c75b0..288683d3 100644 --- a/cpp/libsql/bridge.h +++ b/cpp/libsql/bridge.h @@ -40,7 +40,8 @@ DB opsqlite_libsql_open_remote(std::string const &url, DB opsqlite_libsql_open_sync(std::string const &name, std::string const &path, std::string const &url, - std::string const &auth_token, int sync_interval); + std::string const &auth_token, int sync_interval, + bool offline); void opsqlite_libsql_close(DB &db); diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 2a4285a8..187bb1e2 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -14,7 +14,6 @@ PODS: - DoubleConversion - glog - hermes-engine - - OpenSSL-Universal - RCT-Folly (= 2024.01.01.00) - RCTRequired - RCTTypeSafety @@ -34,7 +33,6 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - Yoga - - OpenSSL-Universal (3.3.3001) - RCT-Folly (2024.01.01.00): - boost - DoubleConversion @@ -1640,7 +1638,6 @@ DEPENDENCIES: SPEC REPOS: trunk: - GCDWebServer - - OpenSSL-Universal - SocketRocket EXTERNAL SOURCES: @@ -1788,8 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 06a9c6900587420b90accc394199527c64259db4 - op-sqlite: f41ba334d2e4bb6126c22751e6aa5645c4b5cf46 - OpenSSL-Universal: 6082b0bf950e5636fe0d78def171184e2b3899c2 + op-sqlite: 94ed545f045bdcc18e9daeda43467fe30a5835fe RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fb7d408617e25d7f537940000d766d60149c5fea RCTRequired: 9aaf0ffcc1f41f0c671af863970ef25c422a9920 diff --git a/example/package.json b/example/package.json index 2ac3fc83..a9a463e5 100644 --- a/example/package.json +++ b/example/package.json @@ -66,8 +66,8 @@ "node": ">=18" }, "op-sqlite": { - "libsql": false, - "sqlcipher": true, + "libsql": true, + "sqlcipher": false, "iosSqlite": false, "fts5": true, "rtree": true, diff --git a/example/src/tests/storage.spec.ts b/example/src/tests/storage.spec.ts index f5ee2f31..3220a8f2 100644 --- a/example/src/tests/storage.spec.ts +++ b/example/src/tests/storage.spec.ts @@ -1,6 +1,6 @@ import {Storage} from '@op-engineering/op-sqlite'; import chai from 'chai'; -import {afterEach, beforeEach, describe, it} from './MochaRNAdapter'; +import {beforeEach, describe, it} from './MochaRNAdapter'; const expect = chai.expect; @@ -12,8 +12,6 @@ export function storageTests() { storage = new Storage({encryptionKey: 'test'}); }); - afterEach(() => {}); - it('Can set and get sync', async () => { storage.setItemSync('foo', 'bar'); const res = storage.getItemSync('foo'); diff --git a/src/index.ts b/src/index.ts index a582ef0b..130e1eee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -629,7 +629,8 @@ export const openSync = (params: { authToken: string; name: string; location?: string; - syncInterval?: number; + libsqlSyncInterval?: number; + libsqlOffline?: boolean; }): DB => { if (!isLibsql()) { throw new Error('This function is only available for libsql');