feat(store-sync): add store sync package (#1075)

* feat(block-events-stream): add block events stream package * feat(store-sync): add store sync package * wip anvil test * Revert "wip anvil test" This reverts commit 1952a98. * accidentally left in a store refernence * Update packages/block-events-stream/src/createBlockEventsStream.ts Co-authored-by: alvarius <89248902+alvrs@users.noreply.github.com> * make streams closeable I don't love this design * clean up * add log back in * move comments * refactor with just streams * add README with example * renamed * rename again and take in a tuple as input * fix scope * add TODO * add tests for grouping logs * wip rxjs tests * move fetchLogs to async generator, add tests * add block range tests * get rid of old approach * add note about timers * use concatMap instead of exhaustMap * update readme * feat(schema-type): add isDynamicAbiType (#1096) * update with new block events stream * missed a spot * refine types, move logging to debug * add test * wip config types * refactor as a list of storage operations * getting closer * config is already expanded * fix types * clean up * remove log * pass through log to storage operation * update test snapshot * Create eighty-tigers-argue.md --------- Co-authored-by: alvarius <89248902+alvrs@users.noreply.github.com> Co-authored-by: alvrs <alvarius@lattice.xyz>
latticexyz · Jul 6, 2023 · 904fd7d · 904fd7d
1 parent 0a230b4
commit 904fd7d
Show file tree

Hide file tree

Showing 19 changed files with 567 additions and 6 deletions.
diff --git a/.changeset/eighty-tigers-argue.md b/.changeset/eighty-tigers-argue.md
@@ -0,0 +1,8 @@
+---
+"@latticexyz/block-logs-stream": patch
+"@latticexyz/protocol-parser": patch
+"@latticexyz/store-sync": minor
+"@latticexyz/store": patch
+---
+
+Add store sync package
diff --git a/docs/pages/client-side.mdx b/docs/pages/client-side.mdx
@@ -197,7 +197,9 @@ const multiKey = client.tables.MultiKey.get({ first: "0xDEAD", second: 42 });
 ```tsx
 import { useRow } from "@latticexyz/react";
 function ExampleComponent() {
-  const { network: { storeCache } } = useMUD();
+  const {
+    network: { storeCache },
+  } = useMUD();
   const position = useRow(storeCache, { table: "Position", key: { key: "0x01" } });
   // -> { namespace: "", table: "Position", key: { key: "0x01" }, value: { x: 1, y: 2 } }
   const { value } = position;
@@ -232,7 +234,9 @@ const positionsFiltered = client.tables.Position.scan({
 ```tsx
 import { useRows } from "@latticexyz/react";
 function ExampleComponent() {
-  const { network: { storeCache } } = useMUD();
+  const {
+    network: { storeCache },
+  } = useMUD();
   const positions = useRows(storeCache, { table: "Position" });
   // -> [{key: "0x00", value: {x: 10, y: 32}, namespace: config["namespace"], table: Position}, ...]
   const positionsFiltered = useRows(storeCache, { table: "Position", key: { gt: { key: "0x02" } } });

diff --git a/packages/block-logs-stream/src/groupLogsByBlockNumber.ts b/packages/block-logs-stream/src/groupLogsByBlockNumber.ts
@@ -4,6 +4,12 @@ import { bigIntSort } from "./utils";
 import { isDefined } from "@latticexyz/common/utils";
 import { debug } from "./debug";
 
+export type GroupLogsByBlockNumberResult<TLog extends Log> = {
+  blockNumber: BlockNumber;
+  blockHash: Hex;
+  logs: readonly NonPendingLog<TLog>[];
+}[];
+
 /**
  * Groups logs by their block number.
  *
@@ -17,9 +23,7 @@ import { debug } from "./debug";
  * @returns An array of objects where each object represents a distinct block and includes the block number,
  * the block hash, and an array of logs for that block.
  */
-export function groupLogsByBlockNumber<TLog extends Log>(
-  logs: readonly TLog[]
-): { blockNumber: BlockNumber; blockHash: Hex; logs: readonly NonPendingLog<TLog>[] }[] {
+export function groupLogsByBlockNumber<TLog extends Log>(logs: readonly TLog[]): GroupLogsByBlockNumberResult<TLog> {
   // Pending logs don't have block numbers, so filter them out.
   const nonPendingLogs = logs.filter(isNonPendingLog);
   if (logs.length !== nonPendingLogs.length) {

diff --git a/packages/block-logs-stream/src/index.ts b/packages/block-logs-stream/src/index.ts
@@ -1,6 +1,7 @@
 export * from "./blockRangeToLogs";
 export * from "./createBlockStream";
 export * from "./fetchLogs";
+export * from "./getLogs";
 export * from "./groupLogsByBlockNumber";
 export * from "./isNonPendingBlock";
 export * from "./isNonPendingLog";
diff --git a/packages/protocol-parser/src/decodeField.ts b/packages/protocol-parser/src/decodeField.ts
@@ -0,0 +1,13 @@
+import { Hex } from "viem";
+import { SchemaAbiType, SchemaAbiTypeToPrimitiveType, isDynamicAbiType } from "@latticexyz/schema-type";
+import { decodeDynamicField } from "./decodeDynamicField";
+import { decodeStaticField } from "./decodeStaticField";
+
+export function decodeField<
+  TAbiType extends SchemaAbiType,
+  TPrimitiveType extends SchemaAbiTypeToPrimitiveType<TAbiType>
+>(abiType: TAbiType, data: Hex): TPrimitiveType {
+  return (
+    isDynamicAbiType(abiType) ? decodeDynamicField(abiType, data) : decodeStaticField(abiType, data)
+  ) as TPrimitiveType;
+}
diff --git a/packages/protocol-parser/src/index.ts b/packages/protocol-parser/src/index.ts
@@ -1,6 +1,7 @@
 export * from "./abiTypesToSchema";
 export * from "./common";
 export * from "./decodeDynamicField";
+export * from "./decodeField";
 export * from "./decodeKeyTuple";
 export * from "./decodeRecord";
 export * from "./decodeStaticField";
@@ -9,4 +10,8 @@ export * from "./encodeKeyTuple";
 export * from "./encodeRecord";
 export * from "./errors";
 export * from "./hexToPackedCounter";
+export * from "./hexToSchema";
 export * from "./hexToTableSchema";
+export * from "./schemaIndexToAbiType";
+export * from "./schemaToHex";
+export * from "./staticDataLength";
diff --git a/packages/protocol-parser/src/schemaIndexToAbiType.ts b/packages/protocol-parser/src/schemaIndexToAbiType.ts
@@ -0,0 +1,9 @@
+import { SchemaAbiType } from "@latticexyz/schema-type";
+import { Schema } from "./common";
+
+export function schemaIndexToAbiType(schema: Schema, schemaIndex: number): SchemaAbiType {
+  if (schemaIndex < schema.staticFields.length) {
+    return schema.staticFields[schemaIndex];
+  }
+  return schema.dynamicFields[schemaIndex - schema.staticFields.length];
+}
diff --git a/packages/store-sync/.eslintrc b/packages/store-sync/.eslintrc
@@ -0,0 +1,6 @@
+{
+  "extends": ["../../.eslintrc"],
+  "rules": {
+    "@typescript-eslint/explicit-function-return-type": "error"
+  }
+}
diff --git a/packages/store-sync/.gitignore b/packages/store-sync/.gitignore
@@ -0,0 +1 @@
+dist
diff --git a/packages/store-sync/.npmignore b/packages/store-sync/.npmignore
@@ -0,0 +1,6 @@
+*
+
+!dist/**
+!src/**
+!package.json
+!README.md
diff --git a/packages/store-sync/package.json b/packages/store-sync/package.json
@@ -0,0 +1,45 @@
+{
+  "name": "@latticexyz/store-sync",
+  "version": "1.42.0",
+  "description": "Utilities to sync MUD Store events with a client or cache",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/latticexyz/mud.git",
+    "directory": "packages/store-sync"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "types": "src/index.ts",
+  "scripts": {
+    "build": "pnpm run build:js",
+    "build:js": "tsup",
+    "clean": "pnpm run clean:js",
+    "clean:js": "rimraf dist",
+    "dev": "tsup --watch",
+    "lint": "eslint .",
+    "test": "vitest --run"
+  },
+  "dependencies": {
+    "@latticexyz/block-logs-stream": "workspace:*",
+    "@latticexyz/common": "workspace:*",
+    "@latticexyz/protocol-parser": "workspace:*",
+    "@latticexyz/schema-type": "workspace:*",
+    "@latticexyz/store": "workspace:*",
+    "@latticexyz/store-cache": "workspace:*",
+    "@latticexyz/utils": "workspace:*",
+    "debug": "^4.3.4",
+    "viem": "1.1.7"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.7",
+    "tsup": "^6.7.0",
+    "vitest": "0.31.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "914a1e0ae4a573d685841ca2ea921435057deb8f"
+}
diff --git a/packages/store-sync/src/blockEventsToStorage.test.ts b/packages/store-sync/src/blockEventsToStorage.test.ts
@@ -0,0 +1,153 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { BlockEventsToStorageOptions, blockEventsToStorage } from "./blockEventsToStorage";
+import storeConfig from "@latticexyz/store/mud.config";
+
+const mockedCallbacks = {
+  registerTableSchema: vi.fn<
+    Parameters<BlockEventsToStorageOptions["registerTableSchema"]>,
+    ReturnType<BlockEventsToStorageOptions["registerTableSchema"]>
+  >(),
+  registerTableMetadata: vi.fn<
+    Parameters<BlockEventsToStorageOptions["registerTableMetadata"]>,
+    ReturnType<BlockEventsToStorageOptions["registerTableMetadata"]>
+  >(),
+  getTableSchema: vi.fn<
+    Parameters<BlockEventsToStorageOptions["getTableSchema"]>,
+    ReturnType<BlockEventsToStorageOptions["getTableSchema"]>
+  >(),
+  getTableMetadata: vi.fn<
+    Parameters<BlockEventsToStorageOptions["getTableMetadata"]>,
+    ReturnType<BlockEventsToStorageOptions["getTableMetadata"]>
+  >(),
+};
+
+const mockedDecode = blockEventsToStorage<typeof storeConfig>(mockedCallbacks as any as BlockEventsToStorageOptions);
+
+describe("blockEventsToStorage", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("call setField with data properly decoded", async () => {
+    mockedCallbacks.getTableSchema.mockImplementation(async ({ namespace, name }) => {
+      if (namespace === "mudstore" && name === "StoreMetadata") {
+        return {
+          namespace: "mudstore",
+          name: "StoreMetadata",
+          schema: {
+            keySchema: {
+              staticFields: ["bytes32"],
+              dynamicFields: [],
+            },
+            valueSchema: {
+              staticFields: [],
+              dynamicFields: ["string", "bytes"],
+            },
+          },
+        };
+      }
+
+      if (namespace === "" && name === "Inventory") {
+        return {
+          namespace: "",
+          name: "Inventory",
+          schema: {
+            keySchema: {
+              staticFields: ["address", "uint32", "uint32"],
+              dynamicFields: [],
+            },
+            valueSchema: {
+              staticFields: ["uint32"],
+              dynamicFields: [],
+            },
+          },
+        };
+      }
+    });
+
+    mockedCallbacks.getTableMetadata.mockImplementation(async ({ namespace, name }) => {
+      if (namespace === "" && name === "Inventory") {
+        return {
+          namespace: "",
+          name: "Inventory",
+          keyNames: ["owner", "item", "itemVariant"],
+          valueNames: ["amount"],
+        };
+      }
+    });
+
+    const operations = await mockedDecode({
+      blockNumber: 5448n,
+      blockHash: "0x03e962e7402b2ab295b92feac342a132111dd14b0d1fd4d4a0456fdc77981577",
+      logs: [
+        {
+          address: "0x5fbdb2315678afecb367f032d93f642f64180aa3",
+          topics: ["0xd01f9f1368f831528fc9fe6442366b2b7d957fbfff3bcf7c24d9ab5fe51f8c46"],
+          data: "0x00000000000000000000000000000000496e76656e746f7279000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000796eb990a3f9c431c69149c7a168b91596d87f600000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000040000000800000000000000000000000000000000000000000000000000000000",
+          blockHash: "0x03e962e7402b2ab295b92feac342a132111dd14b0d1fd4d4a0456fdc77981577",
+          blockNumber: 5448n,
+          transactionHash: "0xa6986924609542dc4c2d81c53799d8eab47109ef34ee1e422de595e19ee9bfa4",
+          transactionIndex: 0,
+          logIndex: 0,
+          removed: false,
+          args: {
+            table: "0x00000000000000000000000000000000496e76656e746f727900000000000000",
+            key: [
+              "0x000000000000000000000000796eb990a3f9c431c69149c7a168b91596d87f60",
+              "0x0000000000000000000000000000000000000000000000000000000000000001",
+              "0x0000000000000000000000000000000000000000000000000000000000000001",
+            ],
+            schemaIndex: 0,
+            data: "0x00000008",
+          },
+          eventName: "StoreSetField",
+        },
+      ],
+    });
+
+    expect(operations).toMatchInlineSnapshot(`
+      {
+        "blockHash": "0x03e962e7402b2ab295b92feac342a132111dd14b0d1fd4d4a0456fdc77981577",
+        "blockNumber": 5448n,
+        "operations": [
+          {
+            "keyTuple": {
+              "item": 1,
+              "itemVariant": 1,
+              "owner": "0x796eb990A3F9C431C69149c7a168b91596D87F60",
+            },
+            "log": {
+              "address": "0x5fbdb2315678afecb367f032d93f642f64180aa3",
+              "args": {
+                "data": "0x00000008",
+                "key": [
+                  "0x000000000000000000000000796eb990a3f9c431c69149c7a168b91596d87f60",
+                  "0x0000000000000000000000000000000000000000000000000000000000000001",
+                  "0x0000000000000000000000000000000000000000000000000000000000000001",
+                ],
+                "schemaIndex": 0,
+                "table": "0x00000000000000000000000000000000496e76656e746f727900000000000000",
+              },
+              "blockHash": "0x03e962e7402b2ab295b92feac342a132111dd14b0d1fd4d4a0456fdc77981577",
+              "blockNumber": 5448n,
+              "data": "0x00000000000000000000000000000000496e76656e746f7279000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000003000000000000000000000000796eb990a3f9c431c69149c7a168b91596d87f600000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000040000000800000000000000000000000000000000000000000000000000000000",
+              "eventName": "StoreSetField",
+              "logIndex": 0,
+              "removed": false,
+              "topics": [
+                "0xd01f9f1368f831528fc9fe6442366b2b7d957fbfff3bcf7c24d9ab5fe51f8c46",
+              ],
+              "transactionHash": "0xa6986924609542dc4c2d81c53799d8eab47109ef34ee1e422de595e19ee9bfa4",
+              "transactionIndex": 0,
+            },
+            "name": "Inventory",
+            "namespace": "",
+            "type": "SetField",
+            "value": 8,
+            "valueName": "amount",
+          },
+        ],
+      }
+    `);
+  });
+});