Initial changes for merkle streaming

packages/zkdb/package.json

@@ -28,7 +28,7 @@
     "coverage": "node --experimental-vm-modules --experimental-wasm-threads node_modules/jest/bin/jest.js --coverage",
     "format": "prettier --write --ignore-unknown **/*",
     "prepare": "cd .. && cd .. && husky install packages/zkdb/.husky",
-    "test": "node --experimental-vm-modules --experimental-wasm-threads node_modules/jest/bin/jest.js",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "testw": "node --experimental-vm-modules --experimental-wasm-threads node_modules/jest/bin/jest.js --watch",
     "lint": "npx eslint ./src/**/*.ts --fix",
     "lint-check": "npx eslint ./src/**/*.ts --debug"
@@ -81,15 +81,17 @@
     "@babel/preset-env": "^7.16.4",
     "@babel/preset-typescript": "^7.16.0",
     "@types/jest": "^27.0.3",
+    "@types/mock-fs": "^4.13.3",
     "@typescript-eslint/eslint-plugin": "^5.5.0",
     "@typescript-eslint/parser": "^5.5.0",
     "eslint": "^8.7.0",
     "eslint-import-resolver-typescript": "^3.5.5",
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-o1js": "^0.4.0",
     "husky": "^7.0.1",
-    "jest": "^27.3.1",
+    "jest": "^27.5.1",
     "lint-staged": "^11.0.1",
+    "mock-fs": "^5.2.0",
     "prettier": "^2.3.2",
     "ts-jest": "^27.0.7",
     "typedoc": "^0.24.7",
@@ -105,6 +107,7 @@
     "blockstore-fs": "^1.1.5",
     "bson": "^5.3.0",
     "datastore-fs": "^9.1.4",
-    "helia": "^2.0.1"
+    "helia": "^2.0.1",
+    "jest-worker": "^29.7.0"
   }
 }

packages/zkdb/src/merkle-tree/merkle-tree-base.ts

@@ -255,41 +255,74 @@ export abstract class BaseMerkleTree {
    * as a swap. The Merkle tree can be stored on storage and memory
    * @returns
    */
-  protected static serialize(nodesMap: TMerkleNodesMap): Uint8Array {
-    let matrix: TMerkleNodesStorage = [];
-    for (const level in nodesMap) {
-      for (const nodeIndex in nodesMap[level]) {
-        matrix.push([
-          parseInt(level, 10),
-          parseInt(nodeIndex, 10),
-          nodesMap[level][nodeIndex].toString(),
-        ]);
-      }
-    }
-    return BSON.serialize({ root: matrix });
-  }
+  // protected static serialize(nodesMap: TMerkleNodesMap): Uint8Array {
+  //   let matrix: TMerkleNodesStorage = [];
+  //   for (const level in nodesMap) {
+  //     for (const nodeIndex in nodesMap[level]) {
+  //       matrix.push([
+  //         parseInt(level, 10),
+  //         parseInt(nodeIndex, 10),
+  //         nodesMap[level][nodeIndex].toString(),
+  //       ]);
+  //     }
+  //   }
+  //   return BSON.serialize({ root: matrix });
+  // }
 
   /**
    * @todo Deserialization can be done in a streaming way
    * @param data
    * @returns
    */
-  protected static deserialize(
-    data: Uint8Array
-  ): readonly [number, TMerkleNodesMap] {
-    const matrix = <TMerkleNodesStorage>BSON.deserialize(data).root;
-    const nodesMap: TMerkleNodesMap = {};
-    let height = 0;
-    for (let recordIndex = 0; recordIndex < matrix.length; recordIndex += 1) {
-      const [level, index, value] = matrix[recordIndex];
-      if (typeof nodesMap[level] === 'undefined') {
-        nodesMap[level] = {};
-      }
-      nodesMap[level][index] = Field(value);
-      if (level > height) {
-        height = level;
-      }
-    }
-    return [++height, nodesMap];
+  // protected static deserialize(
+  //   data: Uint8Array
+  // ): readonly [number, TMerkleNodesMap] {
+  //   const matrix = <TMerkleNodesStorage>BSON.deserialize(data).root;
+  //   const nodesMap: TMerkleNodesMap = {};
+  //   let height = 0;
+  //   for (let recordIndex = 0; recordIndex < matrix.length; recordIndex += 1) {
+  //     const [level, index, value] = matrix[recordIndex];
+  //     if (typeof nodesMap[level] === 'undefined') {
+  //       nodesMap[level] = {};
+  //     }
+  //     nodesMap[level][index] = Field(value);
+  //     if (level > height) {
+  //       height = level;
+  //     }
+  //   }
+  //   return [++height, nodesMap];
+  // }
+
+  protected static deserializeNode(
+    buffer: Buffer
+  ): [number, number, string] {
+    // Read the level and index from the buffer
+    const level = buffer.readDoubleLE(0);
+    const index = buffer.readDoubleLE(8);
+
+    // Convert the buffer segment related to the field element back to a hex string
+    const fieldElement = buffer.toString('hex', 16, 48);
+
+    return [level, index, fieldElement];
+  }
+
+  protected static serializeNode(node: [number, number, string]): Buffer {
+    // Extracting the level, index, and field element from the node tuple
+    const [level, index, fieldElement] = node;
+
+    // Convert the field element string to a buffer using hex encoding
+    const fieldElementBuffer = Buffer.from(fieldElement, 'hex');
+
+    // Allocate a buffer to store serialized data. 8 bytes for level, 8 bytes for index, and the rest for the field element
+    const buffer = Buffer.alloc(8 + 8 + fieldElementBuffer.length);
+
+    // Write the level and index to the buffer
+    buffer.writeDoubleLE(level, 0);
+    buffer.writeDoubleLE(index, 8);
+
+    // Copy the field element buffer to our main buffer starting from the 16th byte
+    fieldElementBuffer.copy(buffer, 16);
+
+    return buffer;
   }
 }

packages/zkdb/src/merkle-tree/merkle-tree-gar.ts

@@ -0,0 +1,238 @@
+// import { Poseidon, Field, MerkleTree, Circuit, Bool, Provable } from 'o1js';
+// import { TMerkleNodesMap, BaseMerkleTree } from './merkle-tree-base.js';
+// import crypto from 'crypto';
+// import { MerkleTreeStorage } from './merkle-tree-storage.js';
+// import { StorageEngineLocal } from '../storage-engine/local.js';
+
+// const DEFAULT_HEIGHT = 20;
+
+// (async function run() {
+//   const storage = await StorageEngineLocal.getInstance({
+//     location: './base',
+//   });
+//   const merkleTreeJSON = new MerkleTreeStorage(storage, DEFAULT_HEIGHT);
+//   const expectedMerkleTree = new MerkleTree(DEFAULT_HEIGHT);
+
+//   await getFillMerkleTreeTest(merkleTreeJSON, expectedMerkleTree);
+//   await getNodeTest(merkleTreeJSON, expectedMerkleTree);
+//   await getRootTest(merkleTreeJSON, expectedMerkleTree);
+//   await getSetLeafMerkleTreeTest(merkleTreeJSON, expectedMerkleTree);
+// })();
+
+// function randomInteger(min: number, max: number): number {
+//   const randomValue = Math.random() * (max - min) + min;
+//   return Math.floor(randomValue);
+// }
+
+// function randomBigInt(min: bigint, max: bigint): bigint {
+//   const range = max - min + 1n;
+//   const randomBytes = crypto.getRandomValues(new Uint8Array(8));
+//   const randomValue = BigInt(
+//     '0x' +
+//       Array.from(randomBytes)
+//         .map((b) => b.toString(16).padStart(2, '0'))
+//         .join('')
+//   );
+//   return (randomValue % range) + min;
+// }
+
+// async function getRootTest(
+//   merkleTree: BaseMerkleTree,
+//   expectedMerkleTree: MerkleTree
+// ) {
+//   // Setup
+//   const LEAF_AMOUNT = 15;
+
+//   // Execute
+//   let isPassed = Bool(true);
+
+//   const avaiableIndexies = [0n];
+
+//   for (let i = 0; i < LEAF_AMOUNT; i++) {
+//     const leafIndex = randomBigInt(0n, merkleTree.leafCount);
+//     const leafValue = Poseidon.hash(Field(leafIndex).toFields());
+//     await merkleTree.setLeaf(leafIndex, leafValue);
+//     expectedMerkleTree.setLeaf(leafIndex, leafValue);
+
+//     avaiableIndexies.push(leafIndex);
+
+//     const expectedResult = expectedMerkleTree.getRoot();
+//     const actualResult = await merkleTree.getRoot();
+
+//     if (actualResult === null) {
+//       isPassed = Bool(false);
+//       break;
+//     }
+
+//     isPassed = isPassed.and(actualResult.equals(expectedResult));
+//   }
+
+//   //Verify
+//   Circuit.log('getRoot test is passed', isPassed);
+// }
+
+// async function getNodeTest(
+//   merkleTree: BaseMerkleTree,
+//   expectedMerkleTree: MerkleTree
+// ) {
+//   // Setup
+//   const LEAF_AMOUNT = 30;
+
+//   const zeroes: Field[] = JSON.parse(JSON.stringify(expectedMerkleTree)).zeroes;
+
+//   // Execute
+//   let isPassed = Bool(true);
+
+//   const avaiableIndexies = [1n];
+
+//   for (let i = 0; i < LEAF_AMOUNT; i++) {
+//     console.log('index', i);
+//     const leafIndex = BigInt(i); //randomBigInt(0n, merkleTree.leafCount);
+//     const leafValue = Poseidon.hash(Field(leafIndex).toFields());
+//     await merkleTree.setLeaf(leafIndex, leafValue);
+//     expectedMerkleTree.setLeaf(leafIndex, leafValue);
+
+//     let randomLevel = randomInteger(0, DEFAULT_HEIGHT);
+//     let randomIndex = 0n;
+
+//     if (i % 2 === 0) {
+//       randomIndex = randomBigInt(0n, merkleTree.leafCount);
+//     } else {
+//       const position = randomInteger(1, avaiableIndexies.length);
+//       randomIndex = avaiableIndexies[position];
+//     }
+
+//     avaiableIndexies.push(leafIndex);
+
+//     const expectedResult = await expectedMerkleTree.getNode(
+//       randomLevel,
+//       randomIndex
+//     );
+//     const actualResult = await merkleTree.getNode(randomLevel, randomIndex);
+
+//     let isZero = Bool(false);
+//     for (const zero of zeroes) {
+//       if (expectedResult.equals(Field(zero)).toBoolean()) {
+//         isZero = Bool(true);
+//         break;
+//       }
+//     }
+//     Provable.log('is zero', isZero);
+//     Provable.log('expectedResult', expectedResult);
+//     Provable.log('actualResult', actualResult);
+
+//     isPassed = isPassed.and(expectedResult.equals(actualResult));
+
+//     if (isPassed.not().toBoolean()) {
+//       break;
+//     }
+//   }
+
+//   //Verify
+//   Provable.log('getNodeTest is passed', isPassed);
+// }
+
+// async function getSetLeafMerkleTreeTest(
+//   merkleTree: BaseMerkleTree,
+//   expectedMerkleTree: MerkleTree
+// ) {
+//   console.log('set leaf');
+//   // Setup
+//   const LEAF_AMOUNT = 20;
+
+//   // Execute
+//   for (let i = 0; i < LEAF_AMOUNT; i++) {
+//     const leafIndex = randomBigInt(0n, merkleTree.leafCount);
+//     const leafValue = Poseidon.hash(Field(leafIndex).toFields());
+//     await merkleTree.setLeaf(leafIndex, leafValue);
+//     expectedMerkleTree.setLeaf(leafIndex, leafValue);
+//   }
+
+//   //Verify
+//   const actualResult: TMerkleNodesMap = await merkleTree.getNodes();
+//   const expectedResult: TMerkleNodesMap = JSON.parse(
+//     JSON.stringify(expectedMerkleTree)
+//   ).nodes;
+
+//   Circuit.log('actualResult', actualResult);
+//   Circuit.log('expectedResult', expectedResult);
+
+//   console.log('add leaf is passed', verifyNodes(actualResult, expectedResult));
+// }
+
+// async function getFillMerkleTreeTest(
+//   merkleTree: BaseMerkleTree,
+//   expectedMerkleTree: MerkleTree
+// ) {
+//   // Setup
+//   const LEAF_AMOUNT = 150;
+//   let fields: Field[] = [];
+//   for (let i = 0; i < LEAF_AMOUNT; i++) {
+//     fields = fields.concat(Poseidon.hash(Field(i).toFields()));
+//   }
+
+//   expectedMerkleTree.fill(fields);
+//   const expectedResult: TMerkleNodesMap = JSON.parse(
+//     JSON.stringify(expectedMerkleTree)
+//   ).nodes;
+
+//   // Execute
+//   await merkleTree.fill(fields);
+
+//   //Verify
+//   const actualResult: TMerkleNodesMap = await merkleTree.getNodes();
+
+//   console.log('fill is passed', verifyNodes(actualResult, expectedResult));
+// }
+
+// function verifyNodes(
+//   actualNodeMap: TMerkleNodesMap,
+//   expectedNodeMap: TMerkleNodesMap
+// ): boolean {
+//   const actualLevels = Object.keys(actualNodeMap).sort();
+//   const expectedLevels = Object.keys(expectedNodeMap).sort();
+
+//   if (actualLevels.length !== expectedLevels.length) {
+//     return false;
+//   }
+
+//   for (let i = 0; i < actualLevels.length; i++) {
+//     const level = actualLevels[i];
+//     if (level !== expectedLevels[i]) {
+//       console.log('actualLevels', level);
+//       console.log('expectedLevels', expectedLevels[i]);
+//       return false;
+//     }
+
+//     const actualNodes = actualNodeMap[level as unknown as number];
+//     const expectedNodes = expectedNodeMap[level as unknown as number];
+//     const actualNodeKeys = Object.keys(actualNodes).sort();
+//     const expectedNodeKeys = Object.keys(expectedNodes).sort();
+
+//     if (actualNodeKeys.length !== expectedNodeKeys.length) {
+//       console.log('actualNodeKeys.length', actualNodeKeys.length);
+//       console.log('expectedNodeKeys.length', expectedNodeKeys.length);
+//       return false;
+//     }
+
+//     for (let j = 0; j < actualNodeKeys.length; j++) {
+//       const nodeKey = actualNodeKeys[j];
+//       if (nodeKey !== expectedNodeKeys[j]) {
+//         Provable.log('expected node key', expectedNodeKeys[j]);
+//         Provable.log('actual node key', nodeKey);
+//         return false;
+//       }
+
+//       const actualField = Field(actualNodes[nodeKey]);
+//       const expectedField = expectedNodes[nodeKey];
+
+//       if (!actualField.equals(expectedField)) {
+//         // Circuit.log('expected field', expectedField);
+//         // Circuit.log('actual field', actualField);
+//         return false;
+//       }
+//     }
+//   }
+
+//   return true;
+// }