change shadow table rowid to INTEGER PRIMARY KEY AUTOINCREMENT to sur…

…vive vacuuming
asg017 · Jun 30, 2023 · c0ff505 · c0ff505
1 parent 2c3d831
commit c0ff505
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 7 deletions.
diff --git a/src/sqlite-vss.cpp b/src/sqlite-vss.cpp
@@ -337,8 +337,7 @@ static int write_index_insert(faiss::Index *index,
         // INSERT was success, index wasn't written yet, all good to exit
         return SQLITE_OK;
 
-    } else if (sqlite3_extended_errcode(db) != SQLITE_CONSTRAINT_ROWID) {
-
+    } else if (sqlite3_extended_errcode(db) != SQLITE_CONSTRAINT_PRIMARYKEY) {
         // INSERT failed for another unknown reason, bad, return error
         return SQLITE_ERROR;
     }
@@ -390,7 +389,7 @@ static int shadow_data_insert(sqlite3 *db,
     if (rowid == nullptr) {
 
         auto sql = sqlite3_mprintf(
-            "insert into \"%w\".\"%w_data\"(x) values (?)", schema, name);
+            "insert into \"%w\".\"%w_data\"(_) values (?)", schema, name);
 
         int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0);
         sqlite3_free(sql);
@@ -408,7 +407,7 @@ static int shadow_data_insert(sqlite3 *db,
     } else {
 
         auto sql = sqlite3_mprintf(
-            "insert into \"%w\".\"%w_data\"(rowid, x) values (?, ?);", schema,
+            "insert into \"%w\".\"%w_data\"(rowid, _) values (?, ?);", schema,
             name);
 
         int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0);
@@ -471,13 +470,15 @@ static faiss::Index *read_index_select(sqlite3 *db, const char *name, int indexI
     if (rc != SQLITE_OK || stmt == nullptr) {
         sqlite3_finalize(stmt);
         sqlite3_free(sql);
+        printf("zz prepare error\n");
         return nullptr;
     }
 
     sqlite3_bind_int64(stmt, 1, indexId);
-    if (sqlite3_step(stmt) != SQLITE_ROW) {
+    if ((rc = sqlite3_step(stmt)) != SQLITE_ROW) {
         sqlite3_finalize(stmt);
         sqlite3_free(sql);
+        printf("zz step error %d\n", rc);
         return nullptr;
     }
 
@@ -500,7 +501,7 @@ static int create_shadow_tables(sqlite3 *db,
                                 const char *name,
                                 int n) {
 
-    auto sql = sqlite3_mprintf("create table \"%w\".\"%w_index\"(idx)",
+    auto sql = sqlite3_mprintf("create table \"%w\".\"%w_index\"(rowid integer primary key autoincrement, idx)",
                                 schema,
                                 name);
 
@@ -509,7 +510,7 @@ static int create_shadow_tables(sqlite3 *db,
     if (rc != SQLITE_OK)
         return rc;
 
-    sql = sqlite3_mprintf("create table \"%w\".\"%w_data\"(x);",
+    sql = sqlite3_mprintf("create table \"%w\".\"%w_data\"(rowid integer primary key autoincrement, _);",
                           schema,
                           name);
 

diff --git a/tests/test-loadable.py b/tests/test-loadable.py
@@ -498,6 +498,32 @@ def test_vss0_issue_29_upsert(self):
     )
     db.close()
 
+  # Make sure tha VACUUMing a database with vss0 tables still works as expected
+  def test_vss0_vacuum(self):
+    cur = db.cursor()
+    execute_all(cur, "create virtual table x using vss0(a(2));")
+    execute_all(cur, """
+      insert into x(rowid, a)
+        select
+          key + 1000,
+          value
+        from json_each(?);
+      """, ["""
+        [
+          [1, 1],
+          [2, 2],
+          [3, 3]
+        ]
+        """])
+    db.commit()
+
+    db.execute("VACUUM;")
+
+    self.assertEqual(
+      execute_all(db, "select rowid, distance from x where vss_search(a, vss_search_params(?, ?))", ['[0, 0]', 1]),
+      [{'distance': 2.0, 'rowid': 1000}]
+    )
+
 VECTOR_FUNCTIONS = [
   'vector0',
   'vector_debug',