refactor collection for fine grained reactivity (#155)

Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>

Author: samwillis

.changeset/icy-moles-study.md

@@ -0,0 +1,12 @@
+---
+"@tanstack/db-collections": patch
+"@tanstack/react-db": patch
+"@tanstack/vue-db": patch
+"@tanstack/db": patch
+---
+
+A large refactor of the core `Collection` with:
+
+- a change to not use Store internally and emit fine grade changes with `subscribeChanges` and `subscribeKeyChanges` methods.
+- changes to the `Collection` api to be more `Map` like for reads, with `get`, `has`, `size`, `entries`, `keys`, and `values`.
+- renames `config.getId` to `config.getKey` for consistency with the `Map` like api.

README.md

@@ -53,7 +53,7 @@ import { createQueryCollection } from "@tanstack/db-collections"
 const todoCollection = createQueryCollection<Todo>({
   queryKey: ["todos"],
   queryFn: async () => fetch("/api/todos"),
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema, // any standard schema
 })
 ```

docs/overview.md

@@ -173,7 +173,7 @@ If provided, this should be a [Standard Schema](https://standardschema.dev) comp
 const todoCollection = createCollection<Todo>(queryCollectionOptions({
   queryKey: ['todoItems'],
   queryFn: async () => fetch('/api/todos'),
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema // any standard schema
 }))
 ```
@@ -198,7 +198,7 @@ export const todoCollection = createCollection<Todo>(electricCollectionOptions({
       table: 'todos'
     }
   },
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema
 }))
 ```
@@ -208,7 +208,7 @@ The Electric collection requires two Electric-specific options:
 - `shapeOptions` &mdash; the Electric [ShapeStreamOptions](https://electric-sql.com/docs/api/clients/typescript#options) that define the [Shape](https://electric-sql.com/docs/guides/shapes) to sync into the collection; this includes the
   - `url` to your sync engine; and
   - `params` to specify the `table` to sync and any optional `where` clauses, etc.
-- `getId` &mdash; identifies the id for the rows being synced into the collection
+- `getKey` &mdash; identifies the id for the rows being synced into the collection
 
 When you create the collection, sync starts automatically.
 
@@ -228,7 +228,7 @@ export const myPendingTodos = createCollection<Todo>(electricCollectionOptions({
       `
     }
   },
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema
 }))
 ```
@@ -515,7 +515,7 @@ import { queryCollectionOptions } from "@tanstack/db-collections"
 const todoCollection = createCollection<Todo>(queryCollectionOptions({
   queryKey: ["todos"],
   queryFn: async () => fetch("/api/todos"),
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema, // any standard schema
   onInsert: ({ transaction }) => {
     const { changes: newTodo } = transaction.mutations[0]
@@ -528,7 +528,7 @@ const todoCollection = createCollection<Todo>(queryCollectionOptions({
 const listCollection = createCollection<TodoList>(queryCollectionOptions({
   queryKey: ["todo-lists"],
   queryFn: async () => fetch("/api/todo-lists"),
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoListSchema
   onInsert: ({ transaction }) => {
     const { changes: newTodo } = transaction.mutations[0]
@@ -586,7 +586,7 @@ export const todoCollection = createCollection(electricCollectionOptions<Todo>({
       table: 'todos'
     }
   },
-  getId: (item) => item.id,
+  getKey: (item) => item.id,
   schema: todoSchema
 }))
 

examples/react/todo/src/App.tsx

@@ -147,7 +147,7 @@ const createTodoCollection = (type: CollectionType) => {
               timestamptz: (date: string) => new Date(date),
             },
           },
-          getId: (item) => item.id,
+          getKey: (item) => item.id,
           schema: updateTodoSchema,
           onInsert: async ({ transaction }) => {
             const modified = transaction.mutations[0].modified
@@ -201,7 +201,7 @@ const createTodoCollection = (type: CollectionType) => {
                 : undefined,
             }))
           },
-          getId: (item: UpdateTodo) => String(item.id),
+          getKey: (item: UpdateTodo) => String(item.id),
           schema: updateTodoSchema,
           queryClient,
           onInsert: async ({ transaction }) => {
@@ -254,7 +254,7 @@ const createConfigCollection = (type: CollectionType) => {
               },
             },
           },
-          getId: (item: UpdateConfig) => item.id,
+          getKey: (item: UpdateConfig) => item.id,
           schema: updateConfigSchema,
           onInsert: async ({ transaction }) => {
             const modified = transaction.mutations[0].modified
@@ -297,7 +297,7 @@ const createConfigCollection = (type: CollectionType) => {
                 : undefined,
             }))
           },
-          getId: (item: UpdateConfig) => item.id,
+          getKey: (item: UpdateConfig) => item.id,
           schema: updateConfigSchema,
           queryClient,
           onInsert: async ({ transaction }) => {

packages/db-collections/src/electric.ts

@@ -31,7 +31,7 @@ export interface ElectricCollectionConfig<T extends Row<unknown>> {
    */
   id?: string
   schema?: CollectionConfig<T>[`schema`]
-  getId: CollectionConfig<T>[`getId`]
+  getKey: CollectionConfig<T>[`getKey`]
   sync?: CollectionConfig<T>[`sync`]
 
   /**

packages/db-collections/src/query.ts

@@ -54,7 +54,7 @@ export interface QueryCollectionConfig<
 
   // Standard Collection configuration properties
   id?: string
-  getId: CollectionConfig<TItem>[`getId`]
+  getKey: CollectionConfig<TItem>[`getKey`]
   schema?: CollectionConfig<TItem>[`schema`]
   sync?: CollectionConfig<TItem>[`sync`]
 
@@ -115,7 +115,7 @@ export function queryCollectionOptions<
     retry,
     retryDelay,
     staleTime,
-    getId,
+    getKey,
     onInsert,
     onUpdate,
     onDelete,
@@ -136,8 +136,8 @@ export function queryCollectionOptions<
     throw new Error(`[QueryCollection] queryClient must be provided.`)
   }
   // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!getId) {
-    throw new Error(`[QueryCollection] getId must be provided.`)
+  if (!getKey) {
+    throw new Error(`[QueryCollection] getKey must be provided.`)
   }
 
   const internalSync: SyncConfig<TItem>[`sync`] = (params) => {
@@ -184,11 +184,11 @@ export function queryCollectionOptions<
           return
         }
 
-        const currentSyncedItems = new Map(collection.syncedData.state)
-        const newItemsMap = new Map<string, TItem>()
+        const currentSyncedItems = new Map(collection.syncedData)
+        const newItemsMap = new Map<string | number, TItem>()
         newItemsArray.forEach((item) => {
           try {
-            const key = getId(item)
+            const key = getKey(item)
             newItemsMap.set(key, item)
           } catch (e) {
             console.error(
@@ -327,7 +327,7 @@ export function queryCollectionOptions<
 
   return {
     ...baseCollectionConfig,
-    getId,
+    getKey,
     sync: { sync: internalSync },
     onInsert: wrappedOnInsert,
     onUpdate: wrappedOnUpdate,

packages/db-collections/tests/electric.test.ts

@@ -47,7 +47,7 @@ describe(`Electric Integration`, () => {
           table: `test_table`,
         },
       },
-      getId: (item: Row) => item.id,
+      getKey: (item: Row) => item.id as number,
     }
 
     // Get the options with utilities
@@ -75,7 +75,7 @@ describe(`Electric Integration`, () => {
     ])
 
     expect(collection.state).toEqual(
-      new Map([[`KEY::${collection.id}/1`, { id: 1, name: `Test User` }]])
+      new Map([[1, { id: 1, name: `Test User` }]])
     )
   })
 
@@ -107,8 +107,8 @@ describe(`Electric Integration`, () => {
 
     expect(collection.state).toEqual(
       new Map([
-        [`KEY::${collection.id}/1`, { id: 1, name: `Test User` }],
-        [`KEY::${collection.id}/2`, { id: 2, name: `Another User` }],
+        [1, { id: 1, name: `Test User` }],
+        [2, { id: 2, name: `Another User` }],
       ])
     )
   })
@@ -140,7 +140,7 @@ describe(`Electric Integration`, () => {
     ])
 
     expect(collection.state).toEqual(
-      new Map([[`KEY::${collection.id}/1`, { id: 1, name: `Updated User` }]])
+      new Map([[1, { id: 1, name: `Updated User` }]])
     )
   })
 
@@ -161,7 +161,7 @@ describe(`Electric Integration`, () => {
     subscriber([
       {
         key: `1`,
-        value: { id: `1` },
+        value: { id: 1 },
         headers: { operation: `delete` },
       },
       {
@@ -293,7 +293,7 @@ describe(`Electric Integration`, () => {
     it(`should simulate the complete flow`, async () => {
       // Create a fake backend store to simulate server-side storage
       const fakeBackend = {
-        data: new Map<string, { txid: string; value: unknown }>(),
+        data: new Map<number, { txid: string; value: unknown }>(),
         // Simulates persisting data to a backend and returning a txid
         persist: (mutations: Array<PendingMutation>): Promise<string> => {
           const txid = String(Date.now())
@@ -316,7 +316,7 @@ describe(`Electric Integration`, () => {
           fakeBackend.data.forEach((value, key) => {
             if (value.txid === txid) {
               messages.push({
-                key,
+                key: key.toString(),
                 value: value.value as Row,
                 headers: {
                   operation: `insert`,
@@ -371,16 +371,16 @@ describe(`Electric Integration`, () => {
 
       await transaction.isPersisted.promise
 
-      transaction = collection.transactions.state.get(transaction.id)!
+      transaction = collection.transactions.get(transaction.id)!
 
       // Verify the mutation function was called correctly
       expect(testMutationFn).toHaveBeenCalledTimes(1)
 
       // Check that the data was added to the collection
       // Note: In a real implementation, the collection would be updated by the sync process
       // This is just verifying our test setup worked correctly
-      expect(fakeBackend.data.has(`KEY::${collection.id}/1`)).toBe(true)
-      expect(collection.state.has(`KEY::${collection.id}/1`)).toBe(true)
+      expect(fakeBackend.data.has(1)).toBe(true)
+      expect(collection.has(1)).toBe(true)
     })
   })
 
@@ -400,7 +400,7 @@ describe(`Electric Integration`, () => {
             table: `test_table`,
           },
         },
-        getId: (item: Row) => item.id,
+        getKey: (item: Row) => item.id as number,
         onInsert,
         onUpdate,
         onDelete,
@@ -433,7 +433,7 @@ describe(`Electric Integration`, () => {
             table: `test_table`,
           },
         },
-        getId: (item: Row) => item.id,
+        getKey: (item: Row) => item.id as number,
         onInsert,
       }
 
@@ -517,7 +517,7 @@ describe(`Electric Integration`, () => {
             table: `test_table`,
           },
         },
-        getId: (item: Row) => item.id,
+        getKey: (item: Row) => item.id as number,
         onInsert,
       }
 
@@ -530,28 +530,20 @@ describe(`Electric Integration`, () => {
       })
 
       // If awaitTxId wasn't called automatically, this wouldn't be true.
-      expect(testCollection.syncedData.state.size).toEqual(0)
-      expect(
-        testCollection.optimisticOperations.state.filter((o) => o.isActive)
-          .length
-      ).toEqual(1)
+      expect(testCollection.syncedData.size).toEqual(0)
 
       // Verify that our onInsert handler was called
       expect(onInsert).toHaveBeenCalled()
 
       await tx.isPersisted.promise
 
       // Verify that the data was added to the collection via the sync process
-      expect(testCollection.state.has(`KEY::${testCollection.id}/1`)).toBe(true)
-      expect(testCollection.state.get(`KEY::${testCollection.id}/1`)).toEqual({
+      expect(testCollection.has(1)).toBe(true)
+      expect(testCollection.get(1)).toEqual({
         id: 1,
         name: `Direct Persistence User`,
       })
-      expect(testCollection.syncedData.state.size).toEqual(1)
-      expect(
-        testCollection.optimisticOperations.state.filter((o) => o.isActive)
-          .length
-      ).toEqual(0)
+      expect(testCollection.syncedData.size).toEqual(1)
     })
   })
 })