From 4cda6ec2ccc496ce95262e1f67cef767b35b2988 Mon Sep 17 00:00:00 2001 From: Jarry Xiao <61092285+jarry-xiao@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:55:46 -0500 Subject: [PATCH] Added semi robust backfiller logic (still need snapshot tables) (#105) - Backfiller/Gapfiller v1 prototype --- contracts/sdk/gummyroll/accounts/index.ts | 2 +- contracts/sdk/indexer/backfiller.ts | 165 +++++++ contracts/sdk/indexer/db.ts | 534 +++++++++++++++------ contracts/sdk/indexer/indexer.ts | 85 ++-- contracts/sdk/indexer/indexer/bubblegum.ts | 141 ++++-- contracts/sdk/indexer/indexer/utils.ts | 103 +++- contracts/sdk/indexer/indexerGummyroll.ts | 91 ---- contracts/sdk/indexer/server.ts | 26 +- contracts/sdk/indexer/smokeTest.ts | 4 + 9 files changed, 822 insertions(+), 329 deletions(-) create mode 100644 contracts/sdk/indexer/backfiller.ts delete mode 100644 contracts/sdk/indexer/indexerGummyroll.ts diff --git a/contracts/sdk/gummyroll/accounts/index.ts b/contracts/sdk/gummyroll/accounts/index.ts index 90a54a9865e..6e3112af585 100644 --- a/contracts/sdk/gummyroll/accounts/index.ts +++ b/contracts/sdk/gummyroll/accounts/index.ts @@ -50,7 +50,7 @@ type MerkleRollHeader = { }; type MerkleRoll = { - sequenceNumber: BN; // u128 + sequenceNumber: BN; // u64 activeIndex: number; // u64 bufferSize: number; // u64 changeLogs: ChangeLog[]; diff --git a/contracts/sdk/indexer/backfiller.ts b/contracts/sdk/indexer/backfiller.ts new file mode 100644 index 00000000000..dc349e5be21 --- /dev/null +++ b/contracts/sdk/indexer/backfiller.ts @@ -0,0 +1,165 @@ +import { PublicKey } from "@solana/web3.js"; +import { Connection } from "@solana/web3.js"; +import { decodeMerkleRoll } from "../gummyroll/index"; +import { ParserState, handleLogsAtomic } from "./indexer/utils"; +import { hash, NFTDatabaseConnection } from "./db"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; + +export async function validateTree( + nftDb: NFTDatabaseConnection, + depth: number, + treeId: string, + maxSeq: number | null +) { + let tree = new Map(); + for (const row of await nftDb.getTree(treeId, maxSeq)) { + tree.set(row.node_idx, [row.seq, row.hash]); + } + let nodeIdx = 1; + while (nodeIdx < 1 << depth) { + if (!tree.has(nodeIdx)) { + // Just trust, bro + nodeIdx = 1 << (Math.floor(Math.log2(nodeIdx)) + 1); + continue; + } + let expected = tree.get(nodeIdx)[1]; + let left, right; + if (tree.has(2 * nodeIdx)) { + left = bs58.decode(tree.get(2 * nodeIdx)[1]); + } else { + left = nftDb.emptyNode(depth - Math.floor(Math.log2(2 * nodeIdx))); + } + if (tree.has(2 * nodeIdx + 1)) { + right = bs58.decode(tree.get(2 * nodeIdx + 1)[1]); + } else { + right = nftDb.emptyNode(depth - Math.floor(Math.log2(2 * nodeIdx))); + } + let actual = bs58.encode(hash(left, right)); + if (expected !== actual) { + console.log( + `Node mismatch ${nodeIdx}, expected: ${expected}, actual: ${actual}, left: ${bs58.encode( + left + )}, right: ${bs58.encode(right)}` + ); + return false; + } + ++nodeIdx; + } + return true; +} + +async function plugGapsFromSlot( + connection: Connection, + nftDb: NFTDatabaseConnection, + parserState: ParserState, + treeKey: PublicKey, + slot: number, + startSeq: number, + endSeq: number +) { + const blockData = await connection.getBlock(slot, { + commitment: "confirmed", + }); + for (const tx of blockData.transactions) { + if ( + tx.transaction.message + .programIds() + .every((pk) => !pk.equals(parserState.Bubblegum.programId)) + ) { + continue; + } + if (tx.transaction.message.accountKeys.every((pk) => !pk.equals(treeKey))) { + continue; + } + if (tx.meta.err) { + continue; + } + handleLogsAtomic( + nftDb, + { + err: null, + logs: tx.meta.logMessages, + signature: tx.transaction.signatures[0], + }, + { slot: slot }, + parserState, + startSeq, + endSeq + ); + } +} + +async function plugGaps( + connection: Connection, + nftDb: NFTDatabaseConnection, + parserState: ParserState, + treeId: string, + startSlot: number, + endSlot: number, + startSeq: number, + endSeq: number +) { + const treeKey = new PublicKey(treeId); + for (let slot = startSlot; slot <= endSlot; ++slot) { + await plugGapsFromSlot( + connection, + nftDb, + parserState, + treeKey, + slot, + startSeq, + endSeq + ); + } +} + +export async function fetchAndPlugGaps( + connection: Connection, + nftDb: NFTDatabaseConnection, + minSeq: number, + treeId: string, + parserState: ParserState +) { + let [missingData, maxDbSeq, maxDbSlot] = await nftDb.getMissingData( + minSeq, + treeId + ); + console.log(`Found ${missingData.length} gaps`); + let currSlot = await connection.getSlot("confirmed"); + + let merkleAccount = await connection.getAccountInfo( + new PublicKey(treeId), + "confirmed" + ); + if (!merkleAccount) { + return; + } + let merkleRoll = decodeMerkleRoll(merkleAccount.data); + let merkleSeq = merkleRoll.roll.sequenceNumber.toNumber() - 1; + + if (merkleSeq - maxDbSeq > 1 && maxDbSlot < currSlot) { + console.log("Running forward filler"); + missingData.push({ + prevSeq: maxDbSeq, + currSeq: merkleSeq, + prevSlot: maxDbSlot, + currSlot: currSlot, + }); + } + + for (const { prevSeq, currSeq, prevSlot, currSlot } of missingData) { + console.log(prevSeq, currSeq, prevSlot, currSlot); + await plugGaps( + connection, + nftDb, + parserState, + treeId, + prevSlot, + currSlot, + prevSeq, + currSeq + ); + } + console.log("Done"); + return maxDbSeq; +} diff --git a/contracts/sdk/indexer/db.ts b/contracts/sdk/indexer/db.ts index cfd840ad899..3d89d03a329 100644 --- a/contracts/sdk/indexer/db.ts +++ b/contracts/sdk/indexer/db.ts @@ -7,10 +7,7 @@ import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; import { NewLeafEvent } from "./indexer/bubblegum"; import { BN } from "@project-serum/anchor"; import { bignum } from "@metaplex-foundation/beet"; -import { - Creator, - LeafSchema, -} from "../bubblegum/src/generated"; +import { Creator, LeafSchema } from "../bubblegum/src/generated"; import { ChangeLogEvent } from "./indexer/gummyroll"; let fs = require("fs"); @@ -21,89 +18,99 @@ export function hash(left: Buffer, right: Buffer): Buffer { return Buffer.from(keccak_256.digest(Buffer.concat([left, right]))); } +export type GapInfo = { + prevSeq: number; + currSeq: number; + prevSlot: number; + currSlot: number; +}; export class NFTDatabaseConnection { connection: Database; - tree: Map; emptyNodeCache: Map; constructor(connection: Database) { this.connection = connection; - this.tree = new Map(); this.emptyNodeCache = new Map(); } async beginTransaction() { - return this.connection.run("BEGIN TRANSACTION"); + return await this.connection.run("BEGIN TRANSACTION"); } async rollback() { - return this.connection.run("ROLLBACK"); + return await this.connection.run("ROLLBACK"); } async commit() { - return this.connection.run("COMMIT"); + return await this.connection.run("COMMIT"); } - async upsertRowsFromBackfill(rows: Array<[PathNode, number, number]>) { - this.connection.db.serialize(() => { - this.connection.run("BEGIN TRANSACTION"); - for (const [node, seq, i] of rows) { - this.connection.run( - ` - INSERT INTO - merkle(node_idx, seq, level, hash) - VALUES (?, ?, ?, ?) - `, - node.index, - seq, - i, - node.node.toBase58() - ); - } - this.connection.run("COMMIT"); - }); - } - - async updateChangeLogs(changeLog: ChangeLogEvent, txId: string, treeId: string) { - console.log("Update Change Log"); + async updateChangeLogs( + changeLog: ChangeLogEvent, + txId: string, + slot: number, + treeId: string + ) { if (changeLog.seq == 0) { return; } for (const [i, pathNode] of changeLog.path.entries()) { - this.connection.run( - ` + await this.connection + .run( + ` INSERT INTO - merkle(transaction_id, tree_id, node_idx, seq, level, hash) - VALUES (?, ?, ?, ?, ?, ?) + merkle(transaction_id, slot, tree_id, node_idx, seq, level, hash) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (tree_id, seq, node_idx) + DO UPDATE SET + transaction_id = excluded.transaction_id, + slot = excluded.slot, + tree_id = excluded.tree_id, + level = excluded.level, + hash = excluded.hash `, - txId, - treeId, - pathNode.index, - changeLog.seq, - i, - new PublicKey(pathNode.node).toBase58() - ); + txId, + slot, + treeId, + pathNode.index, + changeLog.seq, + i, + new PublicKey(pathNode.node).toBase58() + ) + .catch((e) => { + console.log("DB error on change log upsert", e); + }); } } - async updateLeafSchema(leafSchema: LeafSchema, leafHash: PublicKey, txId: string, treeId: string) { - console.log("Update Leaf Schema"); - this.connection.run( + async updateLeafSchema( + leafSchema: LeafSchema, + leafHash: PublicKey, + txId: string, + slot: number, + sequenceNumber: number, + treeId: string + ) { + await this.connection.run( ` INSERT INTO leaf_schema( nonce, tree_id, + seq, transaction_id, + slot, owner, delegate, data_hash, creator_hash, leaf_hash ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (nonce, tree_id) DO UPDATE SET + seq = excluded.seq, + transaction_id = excluded.transaction_id, owner = excluded.owner, delegate = excluded.delegate, data_hash = excluded.data_hash, @@ -112,7 +119,9 @@ export class NFTDatabaseConnection { `, (leafSchema.nonce.valueOf() as BN).toNumber(), treeId, + sequenceNumber, txId, + slot, leafSchema.owner.toBase58(), leafSchema.delegate.toBase58(), bs58.encode(leafSchema.dataHash), @@ -121,8 +130,11 @@ export class NFTDatabaseConnection { ); } - async updateNFTMetadata(newLeafEvent: NewLeafEvent, nonce: bignum, treeId: string) { - console.log("Update NFT"); + async updateNFTMetadata( + newLeafEvent: NewLeafEvent, + nonce: bignum, + treeId: string + ) { const uri = newLeafEvent.metadata.uri; const name = newLeafEvent.metadata.name; const symbol = newLeafEvent.metadata.symbol; @@ -141,7 +153,7 @@ export class NFTDatabaseConnection { creators.push(newLeafEvent.metadata.creators[i]); } } - this.connection.run( + await this.connection.run( ` INSERT INTO nft( @@ -217,33 +229,87 @@ export class NFTDatabaseConnection { return result; } - async updateTree() { - let res = await this.connection.all( - ` - SELECT DISTINCT - node_idx, hash, level, max(seq) as seq - FROM merkle - GROUP BY node_idx - ` - ); - for (const row of res) { - this.tree.set(row.node_idx, [row.seq, row.hash]); + async getTree(treeId: string, maxSeq: number | null) { + let res; + if (maxSeq) { + res = await this.connection.all( + ` + SELECT DISTINCT + node_idx, hash, level, max(seq) as seq + FROM merkle + where tree_id = ? and seq <= ? + GROUP BY node_idx + `, + treeId, + maxSeq + ); + } else { + res = await this.connection.all( + ` + SELECT DISTINCT + node_idx, hash, level, max(seq) as seq + FROM merkle + where tree_id = ? + GROUP BY node_idx + `, + treeId + ); } return res; } - async getSequenceNumbers() { - return new Set( - ( - await this.connection.all( - ` - SELECT DISTINCT seq - FROM merkle - ORDER by seq - ` - ) - ).map((x) => x.seq) - ); + async getMissingData(minSeq: number, treeId: string) { + let gaps: Array = []; + let res = await this.connection + .all( + ` + SELECT DISTINCT seq, slot + FROM merkle + where tree_id = ? and seq >= ? + order by seq + `, + treeId, + minSeq + ) + .catch((e) => { + console.log("Failed to make query", e); + return [gaps, null, null]; + }); + for (let i = 0; i < res.length - 1; ++i) { + let [prevSeq, prevSlot] = [res[i].seq, res[i].slot]; + let [currSeq, currSlot] = [res[i + 1].seq, res[i + 1].slot]; + if (currSeq === prevSeq) { + throw new Error( + `Error in DB, encountered identical sequence numbers with different slots: ${prevSlot} ${currSlot}` + ); + } + if (currSeq - prevSeq > 1) { + gaps.push({ prevSeq, currSeq, prevSlot, currSlot }); + } + } + if (res.length > 0) { + return [gaps, res[res.length - 1].seq, res[res.length - 1].slot]; + } + return [gaps, null, null]; + } + + async getTrees() { + let res = await this.connection + .all( + ` + SELECT DISTINCT tree_id, max(level) as depth + FROM merkle + GROUP BY tree_id + ` + ) + .catch((e) => { + console.log("Failed to query table", e); + return []; + }); + + return res.map((x) => { + return [x.tree_id, x.depth]; + }); } async getAllLeaves() { @@ -284,30 +350,145 @@ export class NFTDatabaseConnection { return leafIdxs; } - async getProof(hash: Buffer, treeId: string, check: boolean = true): Promise { + async getMaxSeq(treeId: string) { + let res = await this.connection.get( + ` + SELECT max(seq) as seq + FROM merkle + WHERE tree_id = ? + `, + treeId + ); + if (res) { + return res.seq; + } else { + return null; + } + } + async getInferredProof( + hash: Buffer, + treeId: string, + check: boolean = true + ): Promise { + let latestSeq = await this.getMaxSeq(treeId); + if (!latestSeq) { + return null; + } + let gapIndex = await this.connection.get( + ` + SELECT + m0.seq as seq + FROM merkle m0 + WHERE NOT EXISTS ( + SELECT NULL + FROM merkle m1 + WHERE m1.seq = m0.seq + 1 AND m1.tree_id = ? + ) AND tree_id = ? + ORDER BY m0.seq + LIMIT 1 + `, + treeId, + treeId + ); + if (gapIndex && gapIndex.seq < latestSeq) { + return await this.inferProofWithKnownGap( + hash, + treeId, + gapIndex.seq, + check + ); + } else { + return await this.getProof(hash, treeId, check); + } + } + + async inferProofWithKnownGap( + hash: Buffer, + treeId: string, + seq: number, + check: boolean = true + ) { + let hashString = bs58.encode(hash); + let depth = await this.getDepth(treeId); + if (!depth) { + return null; + } + + let res = await this.connection.get( + ` + SELECT + data_hash as dataHash, + creator_hash as creatorHash, + nonce as nonce, + owner as owner, + delegate as delegate + FROM leaf_schema + WHERE leaf_hash = ? and tree_id = ? + `, + hashString, + treeId + ); + if (res) { + return this.generateProof( + treeId, + (1 << depth) + res.nonce, + hash, + res.dataHash, + res.creatorHash, + res.nonce, + res.owner, + res.delegate, + check, + seq + ); + } else { + return null; + } + } + + async getDepth(treeId: string) { + let res = await this.connection.get( + ` + SELECT max(level) as depth + FROM merkle + WHERE tree_id = ? + `, + treeId + ); + if (res) { + return res.depth; + } else { + return null; + } + } + + async getProof( + hash: Buffer, + treeId: string, + check: boolean = true + ): Promise { let hashString = bs58.encode(hash); let res = await this.connection.all( ` SELECT - DISTINCT m.node_idx as nodeIdx, + m.node_idx as nodeIdx, l.data_hash as dataHash, l.creator_hash as creatorHash, l.nonce as nonce, l.owner as owner, - l.delegate as delegate, - max(m.seq) as seq + l.delegate as delegate FROM merkle m JOIN leaf_schema l - ON m.hash = l.leaf_hash and m.tree_id = l.tree_id + ON m.hash = l.leaf_hash and m.tree_id = l.tree_id and m.seq = l.seq WHERE hash = ? and m.tree_id = ? and level = 0 - GROUP BY node_idx `, hashString, - treeId, + treeId ); if (res.length == 1) { - let data = res[0] + let data = res[0]; return this.generateProof( + treeId, data.nodeIdx, hash, data.dataHash, @@ -323,6 +504,7 @@ export class NFTDatabaseConnection { } async generateProof( + treeId: string, nodeIdx: number, hash: Buffer, dataHash: string, @@ -330,7 +512,8 @@ export class NFTDatabaseConnection { nonce: number, owner: string, delegate: string, - check: boolean = true + check: boolean = true, + maxSequenceNumber: number | null = null ): Promise { let nodes = []; let n = nodeIdx; @@ -343,14 +526,32 @@ export class NFTDatabaseConnection { n >>= 1; } nodes.push(1); - let res = await this.connection.all( - ` - SELECT DISTINCT node_idx, hash, level, max(seq) as seq - FROM merkle where node_idx in (${nodes.join(",")}) - GROUP BY node_idx - ORDER BY level - ` - ); + let res; + if (maxSequenceNumber) { + res = await this.connection.all( + ` + SELECT DISTINCT node_idx, hash, level, max(seq) as seq + FROM merkle WHERE + node_idx in (${nodes.join(",")}) AND tree_id = ? AND seq <= ? + GROUP BY node_idx + ORDER BY level + `, + treeId, + maxSequenceNumber + ); + } else { + res = await this.connection.all( + ` + SELECT DISTINCT node_idx, hash, level, max(seq) as seq + FROM merkle WHERE + node_idx in (${nodes.join(",")}) AND tree_id = ? + GROUP BY node_idx + ORDER BY level + `, + treeId + ); + } + if (res.length < 1) { return null; } @@ -377,6 +578,10 @@ export class NFTDatabaseConnection { owner: owner, delegate: delegate, }; + if (maxSequenceNumber) { + // If this parameter is set, we directly attempt to infer the root value + inferredProof.root = bs58.encode(this.generateRoot(inferredProof)); + } if (check && !this.verifyProof(inferredProof)) { console.log("Proof is invalid"); return null; @@ -384,7 +589,7 @@ export class NFTDatabaseConnection { return inferredProof; } - verifyProof(proof: Proof) { + generateRoot(proof: Proof) { let node = bs58.decode(proof.leaf); let index = proof.index; for (const [i, pNode] of proof.proofNodes.entries()) { @@ -394,6 +599,11 @@ export class NFTDatabaseConnection { node = hash(new PublicKey(pNode).toBuffer(), node); } } + return node; + } + + verifyProof(proof: Proof) { + let node = this.generateRoot(proof); const rehashed = new PublicKey(node).toString(); const received = new PublicKey(proof.root).toString(); return rehashed === received; @@ -481,7 +691,7 @@ export class NFTDatabaseConnection { sellerFeeBasisPoints: metadata.sellerFeeBasisPoints, owner: metadata.owner, delegate: metadata.delegate, - leafHash: metadata.leafHash, + leafHash: metadata.leafHash, creators: creators, }); } @@ -515,66 +725,100 @@ export async function bootstrap( driver: sqlite3.Database, }); + // Allows concurrency in SQLITE + await db.run("PRAGMA journal_mode = WAL;"); + if (create) { - await db.run( - ` - CREATE TABLE IF NOT EXISTS merkle ( - id INTEGER PRIMARY KEY, + db.db.serialize(() => { + db.run("BEGIN TRANSACTION"); + db.run( + ` + CREATE TABLE IF NOT EXISTS merkle ( + tree_id TEXT, + transaction_id TEXT, + slot INT, + node_idx INT, + seq INT, + level INT, + hash TEXT, + PRIMARY KEY (tree_id, seq, node_idx) + ); + ` + ); + db.run( + ` + CREATE INDEX IF NOT EXISTS sequence_number + ON merkle(seq) + ` + ); + db.run( + ` + CREATE INDEX IF NOT EXISTS nodes + ON merkle(node_idx) + ` + ); + db.run( + ` + CREATE TABLE IF NOT EXISTS nft ( tree_id TEXT, - transaction_id TEXT, - node_idx INT, + nonce BIGINT, + name TEXT, + symbol TEXT, + uri TEXT, + seller_fee_basis_points INT, + primary_sale_happened BOOLEAN, + is_mutable BOOLEAN, + creator0 TEXT, + share0 INT, + verified0 BOOLEAN, + creator1 TEXT, + share1 INT, + verified1 BOOLEAN, + creator2 TEXT, + share2 INT, + verified2 BOOLEAN, + creator3 TEXT, + share3 INT, + verified3 BOOLEAN, + creator4 TEXT, + share4 INT, + verified4 BOOLEAN, + PRIMARY KEY (tree_id, nonce) + ); + ` + ); + db.run( + ` + CREATE TABLE IF NOT EXISTS leaf_schema ( + tree_id TEXT, + nonce BIGINT, seq INT, - level INT, - hash TEXT + slot INT, + transaction_id TEXT, + owner TEXT, + delegate TEXT, + data_hash TEXT, + creator_hash TEXT, + leaf_hash TEXT, + PRIMARY KEY (tree_id, nonce) ); - ` - ); - - await db.run( - ` - CREATE TABLE IF NOT EXISTS nft ( - tree_id TEXT, - nonce BIGINT, - name TEXT, - symbol TEXT, - uri TEXT, - seller_fee_basis_points INT, - primary_sale_happened BOOLEAN, - is_mutable BOOLEAN, - creator0 TEXT, - share0 INT, - verified0 BOOLEAN, - creator1 TEXT, - share1 INT, - verified1 BOOLEAN, - creator2 TEXT, - share2 INT, - verified2 BOOLEAN, - creator3 TEXT, - share3 INT, - verified3 BOOLEAN, - creator4 TEXT, - share4 INT, - verified4 BOOLEAN, - PRIMARY KEY (tree_id, nonce) + ` ); - ` - ); - await db.run( - ` - CREATE TABLE IF NOT EXISTS leaf_schema ( - tree_id TEXT, - nonce BIGINT, - transaction_id TEXT, - owner TEXT, - delegate TEXT, - data_hash TEXT, - creator_hash TEXT, - leaf_hash TEXT, - PRIMARY KEY (tree_id, nonce) + db.run( + ` + CREATE TABLE IF NOT EXISTS merkle_snapshot ( + max_seq INT, + tree_id TEXT, + transaction_id TEXT, + node_idx INT, + seq INT, + level INT, + hash TEXT + ); + ` ); - ` - ); + db.run("COMMIT"); + }); } return new NFTDatabaseConnection(db); diff --git a/contracts/sdk/indexer/indexer.ts b/contracts/sdk/indexer/indexer.ts index 952c81216ba..f36a583265e 100644 --- a/contracts/sdk/indexer/indexer.ts +++ b/contracts/sdk/indexer/indexer.ts @@ -1,66 +1,19 @@ import { Keypair } from "@solana/web3.js"; -import { Connection, Context, Logs } from "@solana/web3.js"; +import { Connection } from "@solana/web3.js"; import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../bubblegum/src/generated"; import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../gummyroll/index"; import * as anchor from "@project-serum/anchor"; import { Bubblegum } from "../../target/types/bubblegum"; import { Gummyroll } from "../../target/types/gummyroll"; import NodeWallet from "@project-serum/anchor/dist/cjs/nodewallet"; -import { loadProgram, ParsedLog, parseLogs } from "./indexer/utils"; -import { parseBubblegum } from "./indexer/bubblegum"; -import { bootstrap, NFTDatabaseConnection } from "./db"; +import { loadProgram, handleLogs, handleLogsAtomic } from "./indexer/utils"; +import { bootstrap } from "./db"; +import { fetchAndPlugGaps, validateTree } from "./backfiller"; -const MAX_DEPTH = 20; -const MAX_SIZE = 1024; const localhostUrl = "http://127.0.0.1:8899"; let Bubblegum: anchor.Program; let Gummyroll: anchor.Program; -function indexParsedLog( - db: NFTDatabaseConnection, - txId: string, - parsedLog: ParsedLog | string -) { - if (typeof parsedLog === "string") { - return; - } - if (parsedLog.programId.equals(BUBBLEGUM_PROGRAM_ID)) { - return parseBubblegum( - db, - parsedLog, - { Bubblegum, Gummyroll }, - { txId: txId } - ); - } else { - for (const log of parsedLog.logs) { - indexParsedLog(db, txId, log); - } - } -} - -async function handleLogs( - db: NFTDatabaseConnection, - logs: Logs, - _context: Context -) { - if (logs.err) { - return; - } - const parsedLogs = parseLogs(logs.logs); - if (parsedLogs.length == 0) { - return; - } - db.connection.db.serialize(() => { - db.beginTransaction(); - for (const parsedLog of parsedLogs) { - indexParsedLog(db, logs.signature, parsedLog); - } - console.log("Done executing queries"); - db.commit(); - console.log("Committed"); - }); -} - async function main() { const endpoint = localhostUrl; const connection = new Connection(endpoint, "confirmed"); @@ -83,8 +36,36 @@ async function main() { console.log("loaded programs..."); let subscriptionId = connection.onLogs( BUBBLEGUM_PROGRAM_ID, - async (logs, ctx) => await handleLogs(db, logs, ctx) + (logs, ctx) => handleLogsAtomic(db, logs, ctx, { Gummyroll, Bubblegum }), + "confirmed" ); + while (true) { + try { + const trees = await db.getTrees(); + for (const [treeId, depth] of trees) { + console.log("Scanning for gaps"); + let maxSeq = await fetchAndPlugGaps(connection, db, 0, treeId, { + Gummyroll, + Bubblegum, + }); + console.log("Validation:"); + console.log( + ` Off-chain tree ${treeId} is consistent: ${await validateTree( + db, + depth, + maxSeq, + treeId + )}` + ); + console.log("Moving to next tree"); + } + } catch (e) { + console.log("ERROR"); + console.log(e); + continue; + } + await new Promise((r) => setTimeout(r, 1000)); + } } main(); diff --git a/contracts/sdk/indexer/indexer/bubblegum.ts b/contracts/sdk/indexer/indexer/bubblegum.ts index db201aa6723..ddf111edbff 100644 --- a/contracts/sdk/indexer/indexer/bubblegum.ts +++ b/contracts/sdk/indexer/indexer/bubblegum.ts @@ -20,6 +20,12 @@ function parseIxName(logLine: string): BubblegumIx | null { return logLine.match(ixRegEx)[1] as BubblegumIx; } +function skipTx(sequenceNumber, startSeq, endSeq): boolean { + let left = startSeq !== null ? sequenceNumber <= startSeq : false; + let right = endSeq !== null ? sequenceNumber >= endSeq : false; + return left || right; +} + export type BubblegumIx = | "Redeem" | "Decompress" @@ -35,9 +41,10 @@ export type NewLeafEvent = { nonce: BN; }; -export function parseBubblegum( +export async function parseBubblegum( db: NFTDatabaseConnection, parsedLog: ParsedLog, + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { @@ -45,22 +52,52 @@ export function parseBubblegum( console.log("Bubblegum:", ixName); switch (ixName) { case "CreateTree": - parseBubblegumCreateTree(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumCreateTree( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); break; case "Mint": - parseBubblegumMint(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumMint(db, parsedLog.logs, slot, parser, optionalInfo); break; case "Redeem": - parseBubblegumRedeem(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumRedeem( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); break; case "CancelRedeem": - parseBubblegumCancelRedeem(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumCancelRedeem( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); break; case "Transfer": - parseBubblegumTransfer(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumTransfer( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); break; case "Delegate": - parseBubblegumDelegate(db, parsedLog.logs, parser, optionalInfo); + await parseBubblegumDelegate( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); break; } } @@ -81,9 +118,10 @@ function findGummyrollEvent( return changeLog; } -export function parseBubblegumMint( +export async function parseBubblegumMint( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { @@ -93,19 +131,28 @@ export function parseBubblegumMint( const leafSchema = parseEventFromLog(logs[2] as string, parser.Bubblegum.idl) .data as LeafSchema; let treeId = changeLog.id.toBase58(); - db.updateNFTMetadata(newLeafData, leafSchema.nonce, treeId); - db.updateLeafSchema( + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + await db.updateNFTMetadata(newLeafData, leafSchema.nonce, treeId); + await db.updateLeafSchema( leafSchema, new PublicKey(changeLog.path[0].node), - optionalInfo.txId, - treeId, + txId, + slot, + sequenceNumber, + treeId ); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); } -export function parseBubblegumTransfer( +export async function parseBubblegumTransfer( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { @@ -113,29 +160,39 @@ export function parseBubblegumTransfer( const leafSchema = parseEventFromLog(logs[1] as string, parser.Bubblegum.idl) .data as LeafSchema; let treeId = changeLog.id.toBase58(); - db.updateLeafSchema( + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + await db.updateLeafSchema( leafSchema, new PublicKey(changeLog.path[0].node), - optionalInfo.txId, + txId, + slot, + sequenceNumber, treeId ); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); } -export function parseBubblegumCreateTree( +export async function parseBubblegumCreateTree( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { const changeLog = findGummyrollEvent(logs, parser); let treeId = changeLog.id.toBase58(); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); } -export function parseBubblegumDelegate( +export async function parseBubblegumDelegate( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { @@ -143,29 +200,45 @@ export function parseBubblegumDelegate( const leafSchema = parseEventFromLog(logs[1] as string, parser.Bubblegum.idl) .data as LeafSchema; let treeId = changeLog.id.toBase58(); - db.updateLeafSchema( + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + await db.updateLeafSchema( leafSchema, new PublicKey(changeLog.path[0].node), - optionalInfo.txId, + txId, + slot, + sequenceNumber, treeId ); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); } -export function parseBubblegumRedeem( +export async function parseBubblegumRedeem( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { const changeLog = findGummyrollEvent(logs, parser); + const sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); let treeId = changeLog.id.toBase58(); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, txId, slot, treeId); } -export function parseBubblegumCancelRedeem( +export async function parseBubblegumCancelRedeem( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], + slot: number, parser: ParserState, optionalInfo: OptionalInfo ) { @@ -173,16 +246,24 @@ export function parseBubblegumCancelRedeem( const leafSchema = parseEventFromLog(logs[1] as string, parser.Bubblegum.idl) .data as LeafSchema; let treeId = changeLog.id.toBase58(); - db.updateLeafSchema( + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + await db.updateLeafSchema( leafSchema, new PublicKey(changeLog.path[0].node), - optionalInfo.txId, + txId, + slot, + sequenceNumber, treeId ); - db.updateChangeLogs(changeLog, optionalInfo.txId, treeId); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); } -export function parseBubblegumDecompress( +export async function parseBubblegumDecompress( db: NFTDatabaseConnection, logs: (string | ParsedLog)[], parser: ParserState, diff --git a/contracts/sdk/indexer/indexer/utils.ts b/contracts/sdk/indexer/indexer/utils.ts index ee9e982c9f8..63f058ff7aa 100644 --- a/contracts/sdk/indexer/indexer/utils.ts +++ b/contracts/sdk/indexer/indexer/utils.ts @@ -1,8 +1,11 @@ import * as anchor from "@project-serum/anchor"; -import { PublicKey } from "@solana/web3.js"; +import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../bubblegum/src/generated"; +import { Context, Logs, PublicKey } from "@solana/web3.js"; import { readFileSync } from "fs"; import { Bubblegum } from "../../../target/types/bubblegum"; import { Gummyroll } from "../../../target/types/gummyroll"; +import { NFTDatabaseConnection } from "../db"; +import { parseBubblegum } from "./bubblegum"; const startRegEx = /Program (\w*) invoke \[(\d)\]/; const endRegEx = /Program (\w*) success/; @@ -23,6 +26,9 @@ export type ParsedLog = { export type OptionalInfo = { txId: string; + + startSeq: number | null; + endSeq: number | null; }; /** @@ -117,3 +123,98 @@ export function loadProgram( const IDL = JSON.parse(readFileSync(idlPath).toString()); return new anchor.Program(IDL, programId, provider); } + +/** + * Performs a depth-first traversal of the ParsedLog data structure + * @param db + * @param optionalInfo + * @param slot + * @param parsedState + * @param parsedLog + * @returns + */ +async function indexParsedLog( + db: NFTDatabaseConnection, + optionalInfo: OptionalInfo, + slot: number, + parserState: ParserState, + parsedLog: ParsedLog | string +) { + if (typeof parsedLog === "string") { + return; + } + if (parsedLog.programId.equals(BUBBLEGUM_PROGRAM_ID)) { + return await parseBubblegum(db, parsedLog, slot, parserState, optionalInfo); + } else { + for (const log of parsedLog.logs) { + await indexParsedLog(db, optionalInfo, slot, parserState, log); + } + } +} + +export function handleLogsAtomic( + db: NFTDatabaseConnection, + logs: Logs, + context: Context, + parsedState: ParserState, + startSeq: number | null = null, + endSeq: number | null = null +) { + if (logs.err) { + return; + } + const parsedLogs = parseLogs(logs.logs); + if (parsedLogs.length == 0) { + return; + } + db.connection.db.serialize(() => { + db.beginTransaction(); + for (const parsedLog of parsedLogs) { + indexParsedLog( + db, + { txId: logs.signature, startSeq, endSeq }, + context.slot, + parsedState, + parsedLog + ); + } + db.commit(); + }); +} + +/** + * Processes the logs from a new transaction and searches for the programs + * specified in the ParserState + * @param db + * @param logs + * @param context + * @param parsedState + * @param startSeq + * @param endSeq + * @returns + */ +export async function handleLogs( + db: NFTDatabaseConnection, + logs: Logs, + context: Context, + parsedState: ParserState, + startSeq: number | null = null, + endSeq: number | null = null +) { + if (logs.err) { + return; + } + const parsedLogs = parseLogs(logs.logs); + if (parsedLogs.length == 0) { + return; + } + for (const parsedLog of parsedLogs) { + await indexParsedLog( + db, + { txId: logs.signature, startSeq, endSeq }, + context.slot, + parsedState, + parsedLog + ); + } +} diff --git a/contracts/sdk/indexer/indexerGummyroll.ts b/contracts/sdk/indexer/indexerGummyroll.ts deleted file mode 100644 index 534934c1f47..00000000000 --- a/contracts/sdk/indexer/indexerGummyroll.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { BN, web3 } from "@project-serum/anchor"; -import { PublicKey } from "@solana/web3.js"; -import React from "react"; -import { emptyNode, hash } from "../../tests/merkle-tree"; -import { PathNode, decodeMerkleRoll, OnChainMerkleRoll } from "../gummyroll"; -import { NFTDatabaseConnection } from "./db"; - -export async function updateMerkleRollSnapshot( - connection: web3.Connection, - merkleRollKey: PublicKey, - setMerkleRoll: any -) { - const result = await connection.getAccountInfo(merkleRollKey, "confirmed"); - if (result) { - setMerkleRoll(decodeMerkleRoll(result?.data)); - } -} - -export async function updateMerkleRollLive( - connection: web3.Connection, - merkleRollKey: PublicKey, - setMerkleRoll: any -) { - let subId = connection.onAccountChange( - merkleRollKey, - (result) => { - if (result) { - try { - setMerkleRoll(decodeMerkleRoll(result?.data)); - } catch (e) { - console.log("Failed to deserialize account", e); - } - } - }, - "confirmed" - ); - return subId; -} - -export async function getUpdatedBatch( - merkleRoll: OnChainMerkleRoll, - db: NFTDatabaseConnection -) { - const seq = merkleRoll.roll.sequenceNumber.toNumber(); - let rows: Array<[PathNode, number, number]> = []; - if (seq === 0) { - let nodeIdx = 1 << merkleRoll.header.maxDepth; - for (let i = 0; i < merkleRoll.header.maxDepth; ++i) { - rows.push([ - { - node: new PublicKey(db.emptyNode(i)), - index: nodeIdx, - }, - 0, - i, - ]); - nodeIdx >>= 1; - } - rows.push([ - { - node: new PublicKey(db.emptyNode(merkleRoll.header.maxDepth)), - index: 1, - }, - 0, - merkleRoll.header.maxDepth, - ]); - } else { - const pathNodes = merkleRoll.getChangeLogsWithNodeIndex(); - console.log(`Received Batch! Sequence=${seq}, entries ${pathNodes.length}`); - let data: Array<[number, PathNode[]]> = []; - for (const [i, path] of pathNodes.entries()) { - if (i == seq) { - break; - } - data.push([seq - i, path]); - } - - let sequenceNumbers = await db.getSequenceNumbers(); - for (const [seq, path] of data) { - if (sequenceNumbers.has(seq)) { - continue; - } - for (const [i, node] of path.entries()) { - rows.push([node, seq, i]); - } - } - } - db.upsertRowsFromBackfill(rows); - console.log(`Updated ${rows.length} rows`); - await db.updateTree(); -} diff --git a/contracts/sdk/indexer/server.ts b/contracts/sdk/indexer/server.ts index 09642cf7b60..01cce820dc6 100644 --- a/contracts/sdk/indexer/server.ts +++ b/contracts/sdk/indexer/server.ts @@ -1,10 +1,11 @@ import express from "express"; import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; -import { bootstrap, Proof } from "./db"; +import { bootstrap, NFTDatabaseConnection, Proof } from "./db"; const app = express(); app.use(express.json()); +let nftDb: NFTDatabaseConnection; const port = 4000; type JsonProof = { @@ -37,21 +38,28 @@ function stringifyProof(proof: Proof): string { app.get("/proof", async (req, res) => { const leafHashString = req.query.leafHash; const treeId = req.query.treeId; - console.log("GET request:", leafHashString); - const nftDb = await bootstrap(false); const leafHash: Buffer = bs58.decode(leafHashString); - const proof = await nftDb.getProof(leafHash, treeId, false); - res.send(stringifyProof(proof)); + try { + let proof = await nftDb.getInferredProof(leafHash, treeId, false); + if (proof) { + res.send(stringifyProof(proof)); + } else { + res.send(JSON.stringify({ err: "Failed to fetch proof" })); + } + } catch (e) { + res.send( + JSON.stringify({ err: `Encounter error while fetching proof: ${e}` }) + ); + } }); app.get("/assets", async (req, res) => { const owner = req.query.owner; - console.log("GET request:", owner); - const nftDb = await bootstrap(false); const assets = await nftDb.getAssetsForOwner(owner); res.send(JSON.stringify(assets)); }); -app.listen(port, () => { - console.log(`Example app listening on port ${port}`); +app.listen(port, async () => { + nftDb = await bootstrap(false); + console.log(`Tree server listening on port ${port}`); }); diff --git a/contracts/sdk/indexer/smokeTest.ts b/contracts/sdk/indexer/smokeTest.ts index af7f3695550..8dd12956a7d 100644 --- a/contracts/sdk/indexer/smokeTest.ts +++ b/contracts/sdk/indexer/smokeTest.ts @@ -183,6 +183,10 @@ async function main() { { method: "GET" } ); const proof = await response.json(); + if ("err" in proof) { + console.log(proof) + continue; + } const proofNodes: Array = proof.proofNodes.map((key) => { return { pubkey: new PublicKey(key),