feat(store-sync): sync to RECS (#1197)

.changeset/great-cooks-dream.md

@@ -0,0 +1,26 @@
+---
+"@latticexyz/store-sync": patch
+---
+
+Add RECS sync strategy and corresponding utils
+
+```ts
+import { createPublicClient, http } from 'viem';
+import { syncToRecs } from '@latticexyz/store-sync';
+import storeConfig from 'contracts/mud.config';
+import { defineContractComponents } from './defineContractComponents';
+
+const publicClient = createPublicClient({
+  chain,
+  transport: http(),
+  pollingInterval: 1000,
+});
+
+const { components, singletonEntity, latestBlock$, blockStorageOperations$, waitForTransaction } = await syncToRecs({
+  world,
+  config: storeConfig,
+  address: '0x...',
+  publicClient,
+  components: defineContractComponents(...),
+});
+```

packages/store-sync/package.json

@@ -11,7 +11,8 @@
   "type": "module",
   "exports": {
     ".": "./dist/index.js",
-    "./sqlite": "./dist/sqlite/index.js"
+    "./sqlite": "./dist/sqlite/index.js",
+    "./recs": "./dist/recs/index.js"
   },
   "typesVersions": {
     "*": {
@@ -20,6 +21,9 @@
       ],
       "sqlite": [
         "./src/sqlite/index.ts"
+      ],
+      "recs": [
+        "./src/recs/index.ts"
       ]
     }
   },
@@ -36,13 +40,14 @@
     "@latticexyz/block-logs-stream": "workspace:*",
     "@latticexyz/common": "workspace:*",
     "@latticexyz/protocol-parser": "workspace:*",
+    "@latticexyz/recs": "workspace:*",
     "@latticexyz/schema-type": "workspace:*",
     "@latticexyz/store": "workspace:*",
-    "@latticexyz/store-cache": "workspace:*",
     "better-sqlite3": "^8.4.0",
     "debug": "^4.3.4",
     "drizzle-orm": "^0.27.0",
     "kysely": "^0.26.1",
+    "rxjs": "7.5.5",
     "sql.js": "^1.8.0",
     "superjson": "^1.12.4",
     "viem": "1.3.1"

packages/store-sync/src/blockLogsToStorage.ts

@@ -6,12 +6,10 @@ import {
   abiTypesToSchema,
   TableSchema,
 } from "@latticexyz/protocol-parser";
-import { StoreConfig } from "@latticexyz/store";
+import { StoreConfig, ConfigToKeyPrimitives as Key, ConfigToValuePrimitives as Value } from "@latticexyz/store";
 import { TableId } from "@latticexyz/common";
 import { Address, Hex, decodeAbiParameters, getAddress, parseAbiParameters } from "viem";
 import { debug } from "./debug";
-// TODO: move these type helpers into store?
-import { Key, Value } from "@latticexyz/store-cache";
 import { isDefined } from "@latticexyz/common/utils";
 import { BlockLogs, StorageOperation, Table, TableName, TableNamespace } from "./common";
 
@@ -33,10 +31,14 @@ export type BlockLogsToStorageOptions<TConfig extends StoreConfig = StoreConfig>
   }) => Promise<void>;
 };
 
-export type BlockLogsToStorageResult<TConfig extends StoreConfig = StoreConfig> = (block: BlockLogs) => Promise<{
+export type BlockStorageOperations<TConfig extends StoreConfig = StoreConfig> = {
   blockNumber: BlockLogs["blockNumber"];
   operations: StorageOperation<TConfig>[];
-}>;
+};
+
+export type BlockLogsToStorageResult<TConfig extends StoreConfig = StoreConfig> = (
+  block: BlockLogs
+) => Promise<BlockStorageOperations<TConfig>>;
 
 type TableKey = `${Address}:${TableNamespace}:${TableName}`;
 

packages/store-sync/src/common.ts

@@ -1,28 +1,20 @@
-import { SchemaAbiType, SchemaAbiTypeToPrimitiveType, StaticAbiType } from "@latticexyz/schema-type";
 import { Address, Hex } from "viem";
-// TODO: move these type helpers into store?
-import { Key, Value } from "@latticexyz/store-cache";
-import { GetLogsResult, GroupLogsByBlockNumberResult } from "@latticexyz/block-logs-stream";
-import { StoreEventsAbi, StoreConfig } from "@latticexyz/store";
+import { GetLogsResult, GroupLogsByBlockNumberResult, NonPendingLog } from "@latticexyz/block-logs-stream";
+import {
+  StoreEventsAbi,
+  StoreConfig,
+  KeySchema,
+  ValueSchema,
+  ConfigToKeyPrimitives as Key,
+  ConfigToValuePrimitives as Value,
+} from "@latticexyz/store";
 
 export type ChainId = number;
 export type WorldId = `${ChainId}:${Address}`;
 
 export type TableNamespace = string;
 export type TableName = string;
 
-export type KeySchema = Record<string, StaticAbiType>;
-export type ValueSchema = Record<string, SchemaAbiType>;
-
-export type SchemaToPrimitives<TSchema extends ValueSchema> = {
-  [key in keyof TSchema]: SchemaAbiTypeToPrimitiveType<TSchema[key]>;
-};
-
-export type TableRecord<TKeySchema extends KeySchema = KeySchema, TValueSchema extends ValueSchema = ValueSchema> = {
-  key: SchemaToPrimitives<TKeySchema>;
-  value: SchemaToPrimitives<TValueSchema>;
-};
-
 export type Table = {
   address: Address;
   tableId: Hex;
@@ -36,9 +28,9 @@ export type StoreEventsLog = GetLogsResult<StoreEventsAbi>[number];
 export type BlockLogs = GroupLogsByBlockNumberResult<StoreEventsLog>[number];
 
 export type BaseStorageOperation = {
-  log: StoreEventsLog;
-  namespace: string;
-  name: string;
+  log: NonPendingLog<StoreEventsLog>;
+  namespace: TableNamespace;
+  name: TableName;
 };
 
 export type SetRecordOperation<TConfig extends StoreConfig> = BaseStorageOperation & {

packages/store-sync/src/recs/common.ts

@@ -0,0 +1,13 @@
+import { KeySchema, ValueSchema } from "@latticexyz/store";
+
+export type StoreComponentMetadata = {
+  keySchema: KeySchema;
+  valueSchema: ValueSchema;
+};
+
+export enum SyncStep {
+  INITIALIZE = "initialize",
+  SNAPSHOT = "snapshot",
+  RPC = "rpc",
+  LIVE = "live",
+}

packages/store-sync/src/recs/debug.ts

@@ -0,0 +1,3 @@
+import { debug as parentDebug } from "../debug";
+
+export const debug = parentDebug.extend("recs");

packages/store-sync/src/recs/decodeEntity.ts

@@ -0,0 +1,23 @@
+import { Entity } from "@latticexyz/recs";
+import { StaticAbiType } from "@latticexyz/schema-type";
+import { Hex, decodeAbiParameters } from "viem";
+import { SchemaToPrimitives } from "@latticexyz/store";
+import { entityToHexKeyTuple } from "./entityToHexKeyTuple";
+
+export function decodeEntity<TKeySchema extends Record<string, StaticAbiType>>(
+  keySchema: TKeySchema,
+  entity: Entity
+): SchemaToPrimitives<TKeySchema> {
+  const hexKeyTuple = entityToHexKeyTuple(entity);
+  if (hexKeyTuple.length !== Object.keys(keySchema).length) {
+    throw new Error(
+      `entity key tuple length ${hexKeyTuple.length} does not match key schema length ${Object.keys(keySchema).length}`
+    );
+  }
+  return Object.fromEntries(
+    Object.entries(keySchema).map(([key, type], index) => [
+      key,
+      decodeAbiParameters([{ type }], hexKeyTuple[index] as Hex)[0],
+    ])
+  ) as SchemaToPrimitives<TKeySchema>;
+}

packages/store-sync/src/recs/defineInternalComponents.ts

@@ -0,0 +1,25 @@
+import { World, defineComponent, Type } from "@latticexyz/recs";
+import { Table } from "../common";
+import { StoreComponentMetadata } from "./common";
+
+// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+export function defineInternalComponents(world: World) {
+  return {
+    TableMetadata: defineComponent<{ table: Type.T }, StoreComponentMetadata, Table>(
+      world,
+      { table: Type.T },
+      { metadata: { keySchema: {}, valueSchema: {} } }
+    ),
+    SyncProgress: defineComponent(
+      world,
+      {
+        step: Type.String,
+        message: Type.String,
+        percentage: Type.Number,
+      },
+      {
+        metadata: { keySchema: {}, valueSchema: {} },
+      }
+    ),
+  };
+}

packages/store-sync/src/recs/encodeEntity.ts

@@ -0,0 +1,19 @@
+import { Entity } from "@latticexyz/recs";
+import { StaticAbiType } from "@latticexyz/schema-type";
+import { encodeAbiParameters } from "viem";
+import { SchemaToPrimitives } from "@latticexyz/store";
+import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity";
+
+export function encodeEntity<TKeySchema extends Record<string, StaticAbiType>>(
+  keySchema: TKeySchema,
+  key: SchemaToPrimitives<TKeySchema>
+): Entity {
+  if (Object.keys(keySchema).length !== Object.keys(key).length) {
+    throw new Error(
+      `key length ${Object.keys(key).length} does not match key schema length ${Object.keys(keySchema).length}`
+    );
+  }
+  return hexKeyTupleToEntity(
+    Object.entries(keySchema).map(([keyName, type]) => encodeAbiParameters([{ type }], [key[keyName]]))
+  );
+}

packages/store-sync/src/recs/entityToHexKeyTuple.ts

@@ -0,0 +1,13 @@
+import { Entity } from "@latticexyz/recs";
+import { Hex, sliceHex, size, isHex } from "viem";
+
+export function entityToHexKeyTuple(entity: Entity): readonly Hex[] {
+  if (!isHex(entity)) {
+    throw new Error(`entity ${entity} is not a hex string`);
+  }
+  const length = size(entity);
+  if (length % 32 !== 0) {
+    throw new Error(`entity length ${length} is not a multiple of 32 bytes`);
+  }
+  return new Array(length / 32).fill(0).map((_, index) => sliceHex(entity, index * 32, (index + 1) * 32));
+}

packages/store-sync/src/recs/getTableKey.ts

@@ -0,0 +1,8 @@
+import { Address, getAddress } from "viem";
+import { Table, TableName, TableNamespace } from "../common";
+
+export type TableKey = `${Address}:${TableNamespace}:${TableName}`;
+
+export function getTableKey(table: Pick<Table, "address" | "namespace" | "name">): TableKey {
+  return `${getAddress(table.address)}:${table.namespace}:${table.name}`;
+}

packages/store-sync/src/recs/hexKeyTupleToEntity.ts

@@ -0,0 +1,6 @@
+import { Entity } from "@latticexyz/recs";
+import { Hex, concatHex } from "viem";
+
+export function hexKeyTupleToEntity(hexKeyTuple: readonly Hex[]): Entity {
+  return concatHex(hexKeyTuple as Hex[]) as Entity;
+}

packages/store-sync/src/recs/index.ts

@@ -0,0 +1,7 @@
+export * from "./common";
+export * from "./decodeEntity";
+export * from "./encodeEntity";
+export * from "./entityToHexKeyTuple";
+export * from "./hexKeyTupleToEntity";
+export * from "./recsStorage";
+export * from "./syncToRecs";

packages/store-sync/src/recs/recsStorage.ts

@@ -0,0 +1,96 @@
+import { BlockLogsToStorageOptions } from "../blockLogsToStorage";
+import { StoreConfig } from "@latticexyz/store";
+import { debug } from "./debug";
+import {
+  ComponentValue,
+  Entity,
+  Component as RecsComponent,
+  Schema as RecsSchema,
+  getComponentValue,
+  removeComponent,
+  setComponent,
+  updateComponent,
+} from "@latticexyz/recs";
+import { isDefined } from "@latticexyz/common/utils";
+import { TableId } from "@latticexyz/common";
+import { schemaToDefaults } from "../schemaToDefaults";
+import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity";
+import { defineInternalComponents } from "./defineInternalComponents";
+import { getTableKey } from "./getTableKey";
+import { StoreComponentMetadata } from "./common";
+
+// TODO: should we create components here from config rather than passing them in?
+
+export function recsStorage<TConfig extends StoreConfig = StoreConfig>({
+  components,
+}: {
+  components: ReturnType<typeof defineInternalComponents> &
+    Record<string, RecsComponent<RecsSchema, StoreComponentMetadata>>;
+  config?: TConfig;
+}): BlockLogsToStorageOptions<TConfig> {
+  // TODO: do we need to store block number?
+
+  const componentsByTableId = Object.fromEntries(
+    Object.entries(components).map(([id, component]) => [component.id, component])
+  );
+
+  return {
+    async registerTables({ tables }) {
+      for (const table of tables) {
+        // TODO: check if table exists already and skip/warn?
+        setComponent(components.TableMetadata, getTableKey(table) as Entity, { table });
+      }
+    },
+    async getTables({ tables }) {
+      // TODO: fetch schema from RPC if table not found?
+      return tables
+        .map((table) => getComponentValue(components.TableMetadata, getTableKey(table) as Entity)?.table)
+        .filter(isDefined);
+    },
+    async storeOperations({ operations }) {
+      for (const operation of operations) {
+        const table = getComponentValue(
+          components.TableMetadata,
+          getTableKey({
+            address: operation.log.address,
+            namespace: operation.namespace,
+            name: operation.name,
+          }) as Entity
+        )?.table;
+        if (!table) {
+          debug(
+            `skipping update for unknown table: ${operation.namespace}:${operation.name} at ${operation.log.address}`
+          );
+          continue;
+        }
+
+        const tableId = new TableId(operation.namespace, operation.name).toString();
+        const component = componentsByTableId[operation.log.args.table];
+        if (!component) {
+          debug(`skipping update for unknown component: ${tableId}. Available components: ${Object.keys(components)}`);
+          continue;
+        }
+
+        const entity = hexKeyTupleToEntity(operation.log.args.key);
+
+        if (operation.type === "SetRecord") {
+          debug("setting component", tableId, entity, operation.value);
+          setComponent(component, entity, operation.value as ComponentValue);
+        } else if (operation.type === "SetField") {
+          debug("updating component", tableId, entity, {
+            [operation.fieldName]: operation.fieldValue,
+          });
+          updateComponent(
+            component,
+            entity,
+            { [operation.fieldName]: operation.fieldValue } as ComponentValue,
+            schemaToDefaults(table.valueSchema) as ComponentValue
+          );
+        } else if (operation.type === "DeleteRecord") {
+          debug("deleting component", tableId, entity);
+          removeComponent(component, entity);
+        }
+      }
+    },
+  } as BlockLogsToStorageOptions<TConfig>;
+}