From 9a9048d5f9afbc398a0f0b1f84f5df4aba8e8943 Mon Sep 17 00:00:00 2001 From: Noah Gundotra Date: Tue, 5 Jul 2022 18:50:32 -0400 Subject: [PATCH] Jankle: parse bubblegum via instructions (without logs) (#128) --- contracts/package.json | 2 +- contracts/programs/bubblegum/src/lib.rs | 71 ++- contracts/programs/gumball-machine/src/lib.rs | 48 +- contracts/sdk/bubblegum/src/convenience.ts | 12 + .../bubblegum/src/generated/accounts/Nonce.ts | 2 +- .../src/generated/accounts/Voucher.ts | 7 +- .../src/generated/types/LeafSchema.ts | 55 ++- .../sdk/gumball-machine/instructions/index.ts | 9 +- contracts/sdk/indexer/backfill.ts | 55 +-- contracts/sdk/indexer/backfiller.ts | 282 +++++++++-- contracts/sdk/indexer/db.ts | 92 +++- contracts/sdk/indexer/indexer.ts | 49 +- contracts/sdk/indexer/indexer/README.md | 13 + contracts/sdk/indexer/indexer/bubblegum.ts | 232 --------- contracts/sdk/indexer/indexer/ingester.ts | 147 ++++++ .../indexer/innerInstruction/bubblegum.ts | 215 +++++++++ .../indexer/indexer/instruction/bubblegum.ts | 157 ++++++ .../sdk/indexer/indexer/log/bubblegum.ts | 313 ++++++++++++ .../indexer/indexer/{ => log}/gummyroll.ts | 28 +- contracts/sdk/indexer/indexer/log/utils.ts | 22 + contracts/sdk/indexer/indexer/utils.ts | 417 ++++++++++------ contracts/sdk/indexer/package.json | 1 + contracts/sdk/indexer/scripts/truncate.ts | 454 ++++++++++++++++++ contracts/tests/bubblegum-test.ts | 4 +- contracts/tests/gumball-machine-test.ts | 44 +- contracts/tests/utils.ts | 41 +- contracts/yarn.lock | 39 +- 27 files changed, 2197 insertions(+), 614 deletions(-) create mode 100644 contracts/sdk/indexer/indexer/README.md delete mode 100644 contracts/sdk/indexer/indexer/bubblegum.ts create mode 100644 contracts/sdk/indexer/indexer/ingester.ts create mode 100644 contracts/sdk/indexer/indexer/innerInstruction/bubblegum.ts create mode 100644 contracts/sdk/indexer/indexer/instruction/bubblegum.ts create mode 100644 contracts/sdk/indexer/indexer/log/bubblegum.ts rename contracts/sdk/indexer/indexer/{ => log}/gummyroll.ts (62%) create mode 100644 contracts/sdk/indexer/indexer/log/utils.ts create mode 100644 contracts/sdk/indexer/scripts/truncate.ts diff --git a/contracts/package.json b/contracts/package.json index 561199a493d..f2984d8f879 100644 --- a/contracts/package.json +++ b/contracts/package.json @@ -6,7 +6,7 @@ "@metaplex-foundation/mpl-token-metadata": "2.2.0", "@project-serum/anchor": "^0.21.0", "@solana/spl-token": "^0.1.8", - "@solana/web3.js": "^1.37.0", + "@solana/web3.js": "^1.47.1", "collections": "^5.1.13", "cors": "^2.8.5", "express": "^4.18.1", diff --git a/contracts/programs/bubblegum/src/lib.rs b/contracts/programs/bubblegum/src/lib.rs index 564da7d1fa3..bcec56f2c0e 100644 --- a/contracts/programs/bubblegum/src/lib.rs +++ b/contracts/programs/bubblegum/src/lib.rs @@ -1,30 +1,15 @@ use { + crate::error::BubblegumError, crate::state::metaplex_anchor::MplTokenMetadata, crate::state::{ - NONCE_SIZE, - VOUCHER_PREFIX, - VOUCHER_SIZE, - ASSET_PREFIX, leaf_schema::{LeafSchema, Version}, - metaplex_anchor::{MasterEdition, TokenMetadata}, - Nonce, Voucher, metaplex_adapter::{MetadataArgs, TokenProgramVersion}, - NewNFTEvent, - NFTDecompressionEvent, - }, - gummyroll::{ - program::Gummyroll, - Node, - state::CandyWrapper, - utils::wrap_event, + metaplex_anchor::{MasterEdition, TokenMetadata}, + NFTDecompressionEvent, NewNFTEvent, Nonce, Voucher, ASSET_PREFIX, NONCE_SIZE, + VOUCHER_PREFIX, VOUCHER_SIZE, }, - crate::error::BubblegumError, - crate::utils::{append_leaf, - replace_leaf, - get_asset_id, - cmp_bytes, - cmp_pubkeys, - assert_pubkey_equal, + crate::utils::{ + append_leaf, assert_pubkey_equal, cmp_bytes, cmp_pubkeys, get_asset_id, replace_leaf, }, anchor_lang::{ prelude::*, @@ -36,13 +21,13 @@ use { system_instruction, }, }, + gummyroll::{program::Gummyroll, state::CandyWrapper, utils::wrap_event, Node}, spl_token::state::Mint as SplMint, }; - +pub mod error; pub mod state; pub mod utils; -pub mod error; declare_id!("BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY"); @@ -236,9 +221,9 @@ pub struct DecompressV1<'info> { #[account( mut, seeds = [ - ASSET_PREFIX.as_ref(), - voucher.merkle_slab.as_ref(), - voucher.leaf_schema.nonce().to_le_bytes().as_ref() + ASSET_PREFIX.as_ref(), + voucher.merkle_slab.as_ref(), + voucher.leaf_schema.nonce().to_le_bytes().as_ref(), ], bump )] @@ -395,7 +380,7 @@ pub mod bubblegum { let new_nft = NewNFTEvent { version: Version::V1, metadata: message, - nonce: nonce.count + nonce: nonce.count, }; emit!(new_nft); wrap_event(new_nft.try_to_vec()?, &ctx.accounts.candy_wrapper)?; @@ -483,8 +468,15 @@ pub mod bubblegum { data_hash, creator_hash, ); - let new_leaf = - LeafSchema::new_v0(asset_id, owner, new_delegate, nonce, data_hash, creator_hash); + let new_leaf = LeafSchema::new_v0( + asset_id, + owner, + new_delegate, + nonce, + data_hash, + creator_hash, + ); + wrap_event(new_leaf.try_to_vec()?, &ctx.accounts.candy_wrapper)?; emit!(new_leaf.to_event()); replace_leaf( &merkle_slab.key(), @@ -524,6 +516,7 @@ pub mod bubblegum { ); emit!(previous_leaf.to_event()); let new_leaf = Node::default(); + wrap_event(new_leaf.try_to_vec()?, &ctx.accounts.candy_wrapper)?; replace_leaf( &merkle_slab.key(), *ctx.bumps.get("authority").unwrap(), @@ -555,6 +548,7 @@ pub mod bubblegum { LeafSchema::new_v0(asset_id, owner, delegate, nonce, data_hash, creator_hash); emit!(previous_leaf.to_event()); let new_leaf = Node::default(); + wrap_event(new_leaf.try_to_vec()?, &ctx.accounts.candy_wrapper)?; replace_leaf( &merkle_slab.key(), *ctx.bumps.get("authority").unwrap(), @@ -581,14 +575,19 @@ pub mod bubblegum { ) -> Result<()> { let voucher = &ctx.accounts.voucher; match ctx.accounts.voucher.leaf_schema { - LeafSchema::V1 { owner, .. } => - assert_pubkey_equal(&ctx.accounts.owner.key(), - &owner, - Some(BubblegumError::AssetOwnerMismatch.into()), - ), + LeafSchema::V1 { owner, .. } => assert_pubkey_equal( + &ctx.accounts.owner.key(), + &owner, + Some(BubblegumError::AssetOwnerMismatch.into()), + ), }?; let merkle_slab = ctx.accounts.merkle_slab.to_account_info(); emit!(voucher.leaf_schema.to_event()); + wrap_event( + voucher.leaf_schema.try_to_vec()?, + &ctx.accounts.candy_wrapper, + )?; + replace_leaf( &merkle_slab.key(), *ctx.bumps.get("authority").unwrap(), @@ -598,7 +597,7 @@ pub mod bubblegum { &ctx.accounts.candy_wrapper.to_account_info(), ctx.remaining_accounts, root, - [0; 32], + [0; 32], voucher.leaf_schema.to_node(), voucher.index, ) @@ -627,7 +626,7 @@ pub mod bubblegum { nonce: nonce, }) } - _ => Err(BubblegumError::UnsupportedSchemaVersion) + _ => Err(BubblegumError::UnsupportedSchemaVersion), }?; let voucher = &ctx.accounts.voucher; match metadata.token_program_version { diff --git a/contracts/programs/gumball-machine/src/lib.rs b/contracts/programs/gumball-machine/src/lib.rs index 3317f2709e3..6d90b9b7d15 100644 --- a/contracts/programs/gumball-machine/src/lib.rs +++ b/contracts/programs/gumball-machine/src/lib.rs @@ -10,18 +10,18 @@ use bubblegum::program::Bubblegum; use bubblegum::state::metaplex_adapter::MetadataArgs; use bytemuck::cast_slice_mut; -use gummyroll::{ - program::Gummyroll, state::CandyWrapper, -}; +use gummyroll::{program::Gummyroll, state::CandyWrapper}; use spl_token::native_mint; pub mod state; pub mod utils; -use crate::state::{GumballMachineHeader, EncodeMethod, ZeroCopy}; +use crate::state::{EncodeMethod, GumballMachineHeader, ZeroCopy}; use crate::utils::get_metadata_args; declare_id!("GBALLoMcmimUutWvtNdFFGH5oguS7ghUUV6toQPppuTW"); +const COMPUTE_BUDGET_ADDRESS: &str = "ComputeBudget111111111111111111111111111111"; + #[derive(Accounts)] pub struct InitGumballMachine<'info> { /// CHECK: Validation occurs in instruction @@ -158,16 +158,30 @@ pub struct Destroy<'info> { fn assert_valid_single_instruction_transaction<'info>( instruction_sysvar_account: &AccountInfo<'info>, ) -> Result<()> { - // There should only be one instruction in this transaction (the current call to dispense_...) + // There should only be one non compute-budget instruction + // in this transaction (i.e. the current call to dispense_...) let instruction_sysvar = instruction_sysvar_account.try_borrow_data()?; let mut fixed_data = [0u8; 2]; fixed_data.copy_from_slice(&instruction_sysvar[0..2]); let num_instructions = u16::from_le_bytes(fixed_data); - assert_eq!(num_instructions, 1); - + if num_instructions > 2 { + assert!(false, "Suspicious transaction, failing") + } else if num_instructions == 2 { + let compute_budget_instruction = + load_instruction_at_checked(0, instruction_sysvar_account)?; + + let compute_budget_id: Pubkey = + Pubkey::new(bs58::decode(&COMPUTE_BUDGET_ADDRESS).into_vec().unwrap()[..32].as_ref()); + + assert_eq!(compute_budget_instruction.program_id, compute_budget_id); + let current_instruction = load_instruction_at_checked(1, instruction_sysvar_account)?; + assert_eq!(current_instruction.program_id, id()); + } else if num_instructions == 1 { + let only_instruction = load_instruction_at_checked(0, instruction_sysvar_account)?; + assert_eq!(only_instruction.program_id, id()); + } // We should not be executing dispense... from a CPI - let only_instruction = load_instruction_at_checked(0, instruction_sysvar_account)?; - assert_eq!(only_instruction.program_id, id()); + return Ok(()); } @@ -213,7 +227,7 @@ fn fisher_yates_shuffle_and_fetch_nft_metadata<'info>( gumball_header.creator_address, nft_index, config_line, - EncodeMethod::from(gumball_header.config_line_encode_method) + EncodeMethod::from(gumball_header.config_line_encode_method), ); return Ok(message); } @@ -222,7 +236,7 @@ fn fisher_yates_shuffle_and_fetch_nft_metadata<'info>( // For efficiency, this returns the GumballMachineHeader because it's required to validate // payment parameters. But the main purpose of this function is to determine which config // line to mint to the user, and CPI to bubblegum to actually execute the mint -// Also returns the number of nfts successfully minted, so that the purchaser is charged +// Also returns the number of nfts successfully minted, so that the purchaser is charged // appropriately fn find_and_mint_compressed_nfts<'info>( gumball_machine: &AccountInfo<'info>, @@ -261,9 +275,11 @@ fn find_and_mint_compressed_nfts<'info>( let indices = cast_slice_mut::(indices_data); let num_nfts_to_mint: u64 = (num_items).max(1).min(gumball_header.remaining); - assert!(num_nfts_to_mint > 0, "There are no remaining NFTs to dispense!"); - for _ in 0..num_nfts_to_mint - { + assert!( + num_nfts_to_mint > 0, + "There are no remaining NFTs to dispense!" + ); + for _ in 0..num_nfts_to_mint { let message = fisher_yates_shuffle_and_fetch_nft_metadata( recent_blockhashes, gumball_header, @@ -335,7 +351,7 @@ pub mod gumball_machine { retain_authority: retain_authority.into(), config_line_encode_method: match encode_method { Some(e) => e.to_u8(), - None => EncodeMethod::UTF8.to_u8() + None => EncodeMethod::UTF8.to_u8(), }, _padding: [0; 3], price, @@ -471,7 +487,7 @@ pub mod gumball_machine { } match encode_method { Some(e) => gumball_machine.config_line_encode_method = e.to_u8(), - None => {} + None => {} } match seller_fee_basis_points { Some(s) => gumball_machine.seller_fee_basis_points = s, diff --git a/contracts/sdk/bubblegum/src/convenience.ts b/contracts/sdk/bubblegum/src/convenience.ts index 0510a390bd6..6f706990014 100644 --- a/contracts/sdk/bubblegum/src/convenience.ts +++ b/contracts/sdk/bubblegum/src/convenience.ts @@ -30,6 +30,18 @@ export async function getVoucherPDA(connection: Connection, tree: PublicKey, lea return voucher; } +export async function getLeafAssetId(tree: PublicKey, leafIndex: BN): Promise { + let [assetId] = await PublicKey.findProgramAddress( + [ + Buffer.from("asset", "utf8"), + tree.toBuffer(), + leafIndex.toBuffer("le", 8), + ], + PROGRAM_ID + ); + return assetId +} + export async function getCreateTreeIxs( connection: Connection, maxDepth: number, diff --git a/contracts/sdk/bubblegum/src/generated/accounts/Nonce.ts b/contracts/sdk/bubblegum/src/generated/accounts/Nonce.ts index 83132a7d1c8..e80c37cd720 100644 --- a/contracts/sdk/bubblegum/src/generated/accounts/Nonce.ts +++ b/contracts/sdk/bubblegum/src/generated/accounts/Nonce.ts @@ -17,7 +17,7 @@ export type NonceArgs = { count: beet.bignum } -const nonceDiscriminator = [143, 197, 147, 95, 106, 165, 50, 43] +export const nonceDiscriminator = [143, 197, 147, 95, 106, 165, 50, 43] /** * Holds the data for the {@link Nonce} Account and provides de/serialization * functionality for that data diff --git a/contracts/sdk/bubblegum/src/generated/accounts/Voucher.ts b/contracts/sdk/bubblegum/src/generated/accounts/Voucher.ts index 207f7687264..16c6bbc8844 100644 --- a/contracts/sdk/bubblegum/src/generated/accounts/Voucher.ts +++ b/contracts/sdk/bubblegum/src/generated/accounts/Voucher.ts @@ -21,7 +21,7 @@ export type VoucherArgs = { merkleSlab: web3.PublicKey } -const voucherDiscriminator = [191, 204, 149, 234, 213, 165, 13, 65] +export const voucherDiscriminator = [191, 204, 149, 234, 213, 165, 13, 65] /** * Holds the data for the {@link Voucher} Account and provides de/serialization * functionality for that data @@ -34,7 +34,7 @@ export class Voucher implements VoucherArgs { readonly leafSchema: LeafSchema, readonly index: number, readonly merkleSlab: web3.PublicKey - ) {} + ) { } /** * Creates a {@link Voucher} instance from the provided args. @@ -128,7 +128,7 @@ export class Voucher implements VoucherArgs { */ pretty() { return { - leafSchema: 'LeafSchema.' + LeafSchema[this.leafSchema], + leafSchema: this.leafSchema.__kind, index: this.index, merkleSlab: this.merkleSlab.toBase58(), } @@ -147,6 +147,7 @@ export const voucherBeet = new beet.BeetStruct< >( [ ['accountDiscriminator', beet.uniformFixedSizeArray(beet.u8, 8)], + // @ts-ignore ['leafSchema', leafSchemaBeet], ['index', beet.u32], ['merkleSlab', beetSolana.publicKey], diff --git a/contracts/sdk/bubblegum/src/generated/types/LeafSchema.ts b/contracts/sdk/bubblegum/src/generated/types/LeafSchema.ts index 04c459b4e9b..534d665a5b0 100644 --- a/contracts/sdk/bubblegum/src/generated/types/LeafSchema.ts +++ b/contracts/sdk/bubblegum/src/generated/types/LeafSchema.ts @@ -5,20 +5,63 @@ * See: https://github.com/metaplex-foundation/solita */ +import * as web3 from '@solana/web3.js' import * as beet from '@metaplex-foundation/beet' - +import * as beetSolana from '@metaplex-foundation/beet-solana' /** + * This type is used to derive the {@link LeafSchema} type as well as the de/serializer. + * However don't refer to it in your code but use the {@link LeafSchema} type instead. + * + * @category userTypes * @category enums * @category generated + * @private */ -export enum LeafSchema { - V1, +export type LeafSchemaRecord = { + V1: { + id: web3.PublicKey + owner: web3.PublicKey + delegate: web3.PublicKey + nonce: beet.bignum + dataHash: number[] /* size: 32 */ + creatorHash: number[] /* size: 32 */ + } } +/** + * Union type respresenting the LeafSchema data enum defined in Rust. + * + * NOTE: that it includes a `__kind` property which allows to narrow types in + * switch/if statements. + * Additionally `isLeafSchema*` type guards are exposed below to narrow to a specific variant. + * + * @category userTypes + * @category enums + * @category generated + */ +export type LeafSchema = beet.DataEnumKeyAsKind + +export const isLeafSchemaV1 = ( + x: LeafSchema +): x is LeafSchema & { __kind: 'V1' } => x.__kind === 'V1' + /** * @category userTypes * @category generated */ -export const leafSchemaBeet = beet.fixedScalarEnum( - LeafSchema -) as beet.FixedSizeBeet +export const leafSchemaBeet = beet.dataEnum([ + [ + 'V1', + new beet.BeetArgsStruct( + [ + ['id', beetSolana.publicKey], + ['owner', beetSolana.publicKey], + ['delegate', beetSolana.publicKey], + ['nonce', beet.u64], + ['dataHash', beet.uniformFixedSizeArray(beet.u8, 32)], + ['creatorHash', beet.uniformFixedSizeArray(beet.u8, 32)], + ], + 'LeafSchemaRecord["V1"]' + ), + ], +]); //as beet.FixedSizeBeet; diff --git a/contracts/sdk/gumball-machine/instructions/index.ts b/contracts/sdk/gumball-machine/instructions/index.ts index a817d8bffff..e6d32747540 100644 --- a/contracts/sdk/gumball-machine/instructions/index.ts +++ b/contracts/sdk/gumball-machine/instructions/index.ts @@ -100,12 +100,9 @@ export async function createDispenseNFTForSolIx( merkleRollPubkey: PublicKey, gummyrollProgramId: PublicKey, bubblegumProgramId: PublicKey, - gumballMachine: Program + gumballMachine: Program, ): Promise { - const willyWonkaPDAKey = await getWillyWonkaPDAKey( - gumballMachinePubkey, - gumballMachine.programId - ); + const willyWonkaPDAKey = await getWillyWonkaPDAKey(gumballMachinePubkey, gumballMachine.programId); const bubblegumAuthorityPDAKey = await getBubblegumAuthorityPDA( merkleRollPubkey, ); @@ -167,4 +164,4 @@ export async function createDispenseNFTForTokensIx( args ); return dispenseInstr; -} \ No newline at end of file +} diff --git a/contracts/sdk/indexer/backfill.ts b/contracts/sdk/indexer/backfill.ts index 13ff2c7d4cb..24e6d2aeb72 100644 --- a/contracts/sdk/indexer/backfill.ts +++ b/contracts/sdk/indexer/backfill.ts @@ -1,17 +1,15 @@ -import { Keypair } from "@solana/web3.js"; +import { Keypair, PublicKey } from "@solana/web3.js"; import { Connection } from "@solana/web3.js"; -import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../bubblegum/src/generated"; -import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../gummyroll/index"; import * as anchor from "@project-serum/anchor"; import { Bubblegum } from "../../target/types/bubblegum"; import { Gummyroll } from "../../target/types/gummyroll"; import NodeWallet from "@project-serum/anchor/dist/cjs/nodewallet"; -import { loadProgram, handleLogs, handleLogsAtomic } from "./indexer/utils"; -import { bootstrap, NFTDatabaseConnection } from "./db"; -import { backfillTreeHistory, fetchAndPlugGaps, validateTree } from "./backfiller"; +import { loadPrograms } from "./indexer/utils"; +import { bootstrap } from "./db"; +import { backfillTreeHistory, fillGapsTx, validateTree } from "./backfiller"; -const url = "http://api.internal.mainnet-beta.solana.com"; -// const url = "http://127.0.0.1:8899"; +// const url = "http://api.explorer.mainnet-beta.solana.com"; +const url = "http://127.0.0.1:8899"; let Bubblegum: anchor.Program; let Gummyroll: anchor.Program; @@ -25,42 +23,17 @@ async function main() { }); let db = await bootstrap(); console.log("Finished bootstrapping DB"); - Gummyroll = loadProgram( - provider, - GUMMYROLL_PROGRAM_ID, - "target/idl/gummyroll.json" - ) as anchor.Program; - Bubblegum = loadProgram( - provider, - BUBBLEGUM_PROGRAM_ID, - "target/idl/bubblegum.json" - ) as anchor.Program; + + const parserState = loadPrograms(provider); console.log("loaded programs..."); - console.log("Filling in gaps for tree:", treeId); - // Get first gap - const trees = await db.getTrees(); - const treeInfo = trees.filter((tree) => (tree[0] === treeId)); - let startSeq = 0; - let startSlot: number | null = null; - if (treeInfo) { - let [missingData, maxDbSeq, maxDbSlot] = await db.getMissingData( - 0, - treeId - ); - console.log(missingData, maxDbSeq, maxDbSlot); - if (missingData.length) { - startSlot = missingData[0].prevSlot; - startSeq = missingData[0].prevSeq; - } else { - startSlot = maxDbSlot; - startSeq = maxDbSeq; - } - } + // Fill gaps + console.log("Filling in gaps for tree:", treeId); + let { maxSeq, maxSeqSlot } = await fillGapsTx(connection, db, parserState, treeId); - // Backfill - console.log(`Starting from slot!: ${startSlot} `); - const maxSeq = await backfillTreeHistory(connection, db, { Gummyroll, Bubblegum }, treeId, startSeq, startSlot); + // Backfill to on-chain state, now with a complete db + console.log(`Starting from slot!: ${maxSeqSlot} `); + maxSeq = await backfillTreeHistory(connection, db, parserState, treeId, maxSeq, maxSeqSlot); // Validate console.log("Max SEQUENCE: ", maxSeq); diff --git a/contracts/sdk/indexer/backfiller.ts b/contracts/sdk/indexer/backfiller.ts index 1e630531f98..31e6c0d7617 100644 --- a/contracts/sdk/indexer/backfiller.ts +++ b/contracts/sdk/indexer/backfiller.ts @@ -1,9 +1,11 @@ import { PublicKey, SIGNATURE_LENGTH_IN_BYTES } from "@solana/web3.js"; import { Connection } from "@solana/web3.js"; import { decodeMerkleRoll } from "../gummyroll/index"; -import { ParserState, handleLogsAtomic } from "./indexer/utils"; -import { hash, NFTDatabaseConnection } from "./db"; +import { ParserState, handleInstructionsAtomic } from "./indexer/utils"; +import { handleLogsAtomic } from "./indexer/log/bubblegum"; +import { GapInfo, hash, NFTDatabaseConnection } from "./db"; import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; +import { ParseResult } from "./indexer/utils"; export async function validateTree( nftDb: NFTDatabaseConnection, @@ -48,33 +50,28 @@ export async function validateTree( return true; } -async function plugGapsFromSlot( +/// Inserts data if its in [startSeq, endSeq) +export async function plugGapsFromSlot( connection: Connection, nftDb: NFTDatabaseConnection, parserState: ParserState, - treeKey: PublicKey, slot: number, startSeq: number, - endSeq: number + endSeq: number, + treeKey?: PublicKey, ) { const blockData = await connection.getBlock(slot, { commitment: "confirmed", }); for (const tx of blockData.transactions) { - if ( - tx.transaction.message - .programIds() - .every((pk) => !pk.equals(parserState.Bubblegum.programId)) - ) { - continue; - } - if (tx.transaction.message.accountKeys.every((pk) => !pk.equals(treeKey))) { + if (treeKey && tx.transaction.message.accountKeys.every((pk) => !pk.equals(treeKey) && !pk.equals(parserState.Bubblegum.programId))) { continue; } if (tx.meta.err) { continue; } - handleLogsAtomic( + + const parseResult = handleLogsAtomic( nftDb, { err: null, @@ -86,6 +83,64 @@ async function plugGapsFromSlot( startSeq, endSeq ); + if (parseResult === ParseResult.LogTruncated) { + const instructionInfo = { + accountKeys: tx.transaction.message.accountKeys, + instructions: tx.transaction.message.instructions, + innerInstructions: tx.meta.innerInstructions, + } + handleInstructionsAtomic( + nftDb, + instructionInfo, + tx.transaction.signatures[0], + { slot: slot }, + parserState, + startSeq, + endSeq + ); + } + } +} + +type BatchedGapRequest = { + slot: number, + startSeq: number, + endSeq: number +}; + +async function plugGapsFromSlotBatched( + connection: Connection, + nftDb: NFTDatabaseConnection, + parserState: ParserState, + treeId: string, + requests: BatchedGapRequest[], + batchSize: number = 20, +) { + const treeKey = new PublicKey(treeId); + let idx = 0; + while (idx < requests.length) { + const batchJobs = []; + for (let i = 0; i < batchSize; i += 1) { + const requestIdx = idx + i; + if (requestIdx >= requests.length) { break } + const request = requests[requestIdx]; + batchJobs.push( + plugGapsFromSlot( + connection, + nftDb, + parserState, + request.slot, + request.startSeq, + request.endSeq, + treeKey, + ) + .catch((e) => { + console.error(`Failed to plug gap from slot: ${request.slot}`, e); + }) + ); + } + await Promise.all(batchJobs); + idx += batchJobs.length; } } @@ -101,15 +156,19 @@ async function plugGaps( ) { const treeKey = new PublicKey(treeId); for (let slot = startSlot; slot <= endSlot; ++slot) { - await plugGapsFromSlot( - connection, - nftDb, - parserState, - treeKey, - slot, - startSeq, - endSeq - ); + try { + await plugGapsFromSlot( + connection, + nftDb, + parserState, + slot, + startSeq, + endSeq, + treeKey, + ); + } catch (e) { + console.error(`Failed to plug gap from slot: ${slot}`, e); + } } } @@ -120,11 +179,12 @@ function onlyUnique(value, index, self) { export async function getAllTreeSlots( connection: Connection, treeId: string, - afterSig?: string + afterSig?: string, + untilSig?: string, ): Promise { const treeAddress = new PublicKey(treeId); // todo: paginate - let lastAddress: string | null = null; + let lastAddress: string | null = untilSig; let done = false; const history: number[] = []; @@ -133,22 +193,27 @@ export async function getAllTreeSlots( let opts = lastAddress ? { before: lastAddress } : {}; const finalOpts = { ...baseOpts, ...opts }; console.log(finalOpts); - const sigs = await connection.getSignaturesForAddress(treeAddress, finalOpts); + const rawSigs = (await connection.getSignaturesForAddress(treeAddress, finalOpts)) + if (rawSigs.length === 0) { + return []; + } else if (rawSigs.length < 1000) { + done = true; + } + console.log(rawSigs); + const sigs = rawSigs.filter((confirmedSig) => !confirmedSig.err); + console.log(sigs); console.log(sigs[sigs.length - 1]); lastAddress = sigs[sigs.length - 1].signature; sigs.map((sigInfo) => { history.push(sigInfo.slot); }) - - if (sigs.length < 1000) { - done = true; - } } return history.reverse().filter(onlyUnique); } /// Returns tree history in chronological order (oldest first) +/// Backfill gaps, then checks for recent transactions since gapfill export async function backfillTreeHistory( connection: Connection, nftDb: NFTDatabaseConnection, @@ -160,12 +225,12 @@ export async function backfillTreeHistory( const treeAddress = new PublicKey(treeId); const merkleRoll = decodeMerkleRoll(await (await connection.getAccountInfo(treeAddress)).data); const maxSeq = merkleRoll.roll.sequenceNumber.toNumber(); - // Sequence number on-chain is ready to setup the + + // When synced up, on-chain seq # is going to be maxSeq + 1 if (startSeq === maxSeq - 1) { return startSeq; } const earliestTxId = await nftDb.getTxIdForSlot(treeId, fromSlot); - console.log("Tx id:", earliestTxId); const treeHistory = await getAllTreeSlots(connection, treeId, earliestTxId); console.log("Retrieved tree history!", treeHistory); @@ -183,10 +248,10 @@ export async function backfillTreeHistory( connection, nftDb, parserState, - treeAddress, treeHistory[historyIndex], 0, - maxSeq, + maxSeq + 1, + treeAddress, ) ) } @@ -197,12 +262,49 @@ export async function backfillTreeHistory( return maxSeq; } +async function plugGapsBatched( + batchSize: number, + missingData, + connection: Connection, + nftDb: NFTDatabaseConnection, + parserState: ParserState, + treeId: string, +) { + let numProcessed = 0; + while (numProcessed < missingData.length) { + const batchJobs = []; + for (let i = 0; i < batchSize; i++) { + const index = numProcessed + i; + if (index >= missingData.length) { + break; + } + batchJobs.push( + plugGaps( + connection, + nftDb, + parserState, + treeId, + missingData[index].prevSlot, + missingData[index].currSlot, + missingData[index].prevSeq, + missingData[index].currSeq + ) + ) + } + numProcessed += batchJobs.length; + await Promise.all(batchJobs); + } + + console.log("num processed: ", numProcessed); +} + export async function fetchAndPlugGaps( connection: Connection, nftDb: NFTDatabaseConnection, minSeq: number, treeId: string, - parserState: ParserState + parserState: ParserState, + batchSize?: number, ) { let [missingData, maxDbSeq, maxDbSlot] = await nftDb.getMissingData( minSeq, @@ -231,19 +333,103 @@ export async function fetchAndPlugGaps( }); } - for (const { prevSeq, currSeq, prevSlot, currSlot } of missingData) { - console.log(prevSeq, currSeq, prevSlot, currSlot); - await plugGaps( - connection, - nftDb, - parserState, - treeId, - prevSlot, - currSlot, - prevSeq, - currSeq - ); - } + await plugGapsBatched( + batchSize ?? 1, + missingData, + connection, + nftDb, + parserState, + treeId, + ); console.log("Done"); return maxDbSeq; } + +async function findMissingTxSlots( + connection: Connection, + treeId: string, + missingData: any[] +): Promise { + const mostRecentGap = missingData[missingData.length - 1]; + const txSlots = await getAllTreeSlots( + connection, + treeId, + missingData[0].prevTxId, + mostRecentGap.currTxId + ); + + const missingTxSlots = []; + let gapIdx = 0; + let txSlotIdx = 0; + while (txSlotIdx < txSlots.length && gapIdx < missingData.length) { + const slot = txSlots[txSlotIdx]; + const currGap = missingData[gapIdx]; + console.log(slot, currGap) + if (slot > currGap.currSlot) { + gapIdx += 1; + } else if (slot < currGap.prevSlot) { + // This can happen if there are too many tx's that have been returned + txSlotIdx += 1; + // throw new Error("tx slot is beneath current gap slot range, very likely that something is not sorted properly") + } else { + txSlotIdx += 1; + missingTxSlots.push({ slot, startSeq: currGap.prevSeq, endSeq: currGap.currSeq }); + } + } + + return missingTxSlots; +} + + +function calculateMissingData(missingData: GapInfo[]) { + let missingSlots = 0; + let missingSeqs = 0; + for (const gap of missingData) { + missingSlots += gap.currSlot - gap.prevSlot; + missingSeqs += gap.currSeq - gap.prevSeq; + } + return { missingSlots, missingSeqs }; +} + +/// Fills in gaps for a given tree +/// by asychronously batching +export async function fillGapsTx( + connection: Connection, + db: NFTDatabaseConnection, + parserState: ParserState, + treeId: string, +) { + const trees = await db.getTrees(); + const treeInfo = trees.filter((tree) => (tree[0] === treeId)); + let startSeq = 0; + let startSlot: number | null = null; + if (treeInfo) { + let [missingData, maxDbSeq, maxDbSlot] = await db.getMissingDataWithTx( + 0, + treeId + ); + const { missingSeqs, missingSlots } = calculateMissingData(missingData); + console.log("Missing seqs:", missingSeqs); + console.log("Missing slots:", missingSlots); + + missingData.prevSlot + if (missingData.length) { + const txIdSlotPairs = await findMissingTxSlots(connection, treeId, missingData); + console.log("Num slots to fetch:", txIdSlotPairs.length); + await plugGapsFromSlotBatched( + connection, + db, + parserState, + treeId, + txIdSlotPairs, + ); + } else { + console.log("No gaps found!") + } + startSlot = maxDbSlot; + startSeq = maxDbSeq; + } + + return { maxSeq: startSeq, maxSeqSlot: startSlot } +} + diff --git a/contracts/sdk/indexer/db.ts b/contracts/sdk/indexer/db.ts index a0d43f05f87..af77a58912b 100644 --- a/contracts/sdk/indexer/db.ts +++ b/contracts/sdk/indexer/db.ts @@ -1,14 +1,11 @@ import sqlite3 from "sqlite3"; import { open, Database, Statement } from "sqlite"; -import { PathNode } from "../gummyroll"; import { PublicKey, SystemProgram } from "@solana/web3.js"; import { keccak_256 } from "js-sha3"; import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; -import { LeafSchemaEvent, NewLeafEvent } from "./indexer/bubblegum"; import { BN } from "@project-serum/anchor"; -import { bignum } from "@metaplex-foundation/beet"; import { Creator } from "../bubblegum/src/generated"; -import { ChangeLogEvent } from "./indexer/gummyroll"; +import { LeafSchemaEvent, NewLeafEvent, ChangeLogEvent } from "./indexer/ingester"; let fs = require("fs"); /** @@ -24,6 +21,23 @@ export type GapInfo = { prevSlot: number; currSlot: number; }; + +export type GapTxInfo = GapInfo & { + prevTxId: string, + currTxId: string, +} + +export type AssetInfo = { + treeId: string, + assetId: string, + owner: string, + nonce: BN, + dataHash: string, + leafHash: string, + creatorHash: string, + compressed: number, +} + export class NFTDatabaseConnection { connection: Database; emptyNodeCache: Map; @@ -83,6 +97,7 @@ export class NFTDatabaseConnection { } } + async updateLeafSchema( leafSchemaRecord: LeafSchemaEvent, leafHash: PublicKey, @@ -122,6 +137,7 @@ export class NFTDatabaseConnection { creator_hash = excluded.creator_hash, leaf_hash = excluded.leaf_hash, compressed = excluded.compressed + WHERE seq <= excluded.seq `, leafSchema.id.toBase58(), (leafSchema.nonce.valueOf() as BN).toNumber(), @@ -278,6 +294,42 @@ export class NFTDatabaseConnection { return res; } + async getMissingDataWithTx(minSeq: number, treeId: string) { + let gaps: Array = []; + let res = await this.connection + .all( + ` + SELECT DISTINCT seq, slot, transaction_id + FROM merkle + where tree_id = ? and seq >= ? + order by seq + `, + treeId, + minSeq + ) + .catch((e) => { + console.log("Failed to make query", e); + return [gaps, null, null]; + }); + for (let i = 0; i < res.length - 1; ++i) { + let [prevSeq, prevSlot, prevTxId] = [res[i].seq, res[i].slot, res[i].transaction_id]; + let [currSeq, currSlot, currTxId] = [res[i + 1].seq, res[i + 1].slot, res[i + 1].transaction_id]; + if (currSeq === prevSeq) { + throw new Error( + `Error in DB, encountered identical sequence numbers with different slots: ${prevSlot} ${currSlot}` + ); + } + if (currSeq - prevSeq > 1) { + gaps.push({ prevSeq, currSeq, prevSlot, currSlot, prevTxId, currTxId }); + } + } + if (res.length > 0) { + return [gaps, res[res.length - 1].seq, res[res.length - 1].slot]; + } + return [gaps, null, null]; + } + + async getMissingData(minSeq: number, treeId: string) { let gaps: Array = []; let res = await this.connection @@ -657,6 +709,38 @@ export class NFTDatabaseConnection { return transactionId.length ? transactionId[0].transaction_id as string : null; } + async getAssetInfo(assetId: string): Promise { + const query = ` + SELECT + tree_id, + asset_id, + nonce, + owner, + data_hash, + leaf_hash, + creator_hash, + compressed + FROM leaf_schema + WHERE asset_id = ? + `; + const rawAssetInfo = await this.connection.all(query, assetId); + if (rawAssetInfo.length) { + const rawAsset = rawAssetInfo[0]; + return { + treeId: rawAsset.tree_id, + assetId: rawAsset.asset_id, + owner: rawAsset.owner, + nonce: new BN(rawAsset.nonce), + dataHash: rawAsset.data_hash, + creatorHash: rawAsset.creator_hash, + leafHash: rawAsset.leaf_hash, + compressed: rawAsset.compressed + } + } else { + return null + } + } + async getAssetsForOwner(owner: string, treeId?: string) { const query = ` SELECT diff --git a/contracts/sdk/indexer/indexer.ts b/contracts/sdk/indexer/indexer.ts index 74f3bc7ecf5..f4206445c4a 100644 --- a/contracts/sdk/indexer/indexer.ts +++ b/contracts/sdk/indexer/indexer.ts @@ -1,19 +1,28 @@ -import { Keypair } from "@solana/web3.js"; -import { Connection } from "@solana/web3.js"; +import { Keypair, Logs, Connection, Context } from "@solana/web3.js"; import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../bubblegum/src/generated"; -import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../gummyroll/index"; import * as anchor from "@project-serum/anchor"; -import { Bubblegum } from "../../target/types/bubblegum"; -import { Gummyroll } from "../../target/types/gummyroll"; import NodeWallet from "@project-serum/anchor/dist/cjs/nodewallet"; -import { loadProgram, handleLogs, handleLogsAtomic } from "./indexer/utils"; -import { bootstrap } from "./db"; -import { fetchAndPlugGaps, validateTree } from "./backfiller"; +import { handleLogsAtomic } from "./indexer/log/bubblegum"; +import { loadPrograms, ParseResult, ParserState } from "./indexer/utils"; +import { bootstrap, NFTDatabaseConnection } from "./db"; +import { fetchAndPlugGaps, plugGapsFromSlot, validateTree } from "./backfiller"; -// const url = "http://api.internal.mainnet-beta.solana.com"; +// const url = "http://api.explorer.mainnet-beta.solana.com"; const url = "http://127.0.0.1:8899"; -let Bubblegum: anchor.Program; -let Gummyroll: anchor.Program; + +async function handleLogSubscription( + connection: Connection, + db: NFTDatabaseConnection, + logs: Logs, + ctx: Context, + parserState: ParserState, +) { + const result = handleLogsAtomic(db, logs, ctx, parserState); + if (result === ParseResult.LogTruncated) { + console.log("\t\tLOG TRUNCATED\n\n\n\n") + await plugGapsFromSlot(connection, db, parserState, ctx.slot, 0, Number.MAX_SAFE_INTEGER); + } +} async function main() { const endpoint = url; @@ -24,20 +33,11 @@ async function main() { }); let db = await bootstrap(); console.log("Finished bootstrapping DB"); - Gummyroll = loadProgram( - provider, - GUMMYROLL_PROGRAM_ID, - "target/idl/gummyroll.json" - ) as anchor.Program; - Bubblegum = loadProgram( - provider, - BUBBLEGUM_PROGRAM_ID, - "target/idl/bubblegum.json" - ) as anchor.Program; + const parserState = loadPrograms(provider); console.log("loaded programs..."); let subscriptionId = connection.onLogs( BUBBLEGUM_PROGRAM_ID, - (logs, ctx) => handleLogsAtomic(db, logs, ctx, { Gummyroll, Bubblegum }), + (logs, ctx) => handleLogSubscription(connection, db, logs, ctx, parserState), "confirmed" ); while (true) { @@ -45,10 +45,7 @@ async function main() { const trees = await db.getTrees(); for (const [treeId, depth] of trees) { console.log("Scanning for gaps"); - await fetchAndPlugGaps(connection, db, 0, treeId, { - Gummyroll, - Bubblegum, - }); + let maxSeq = await fetchAndPlugGaps(connection, db, 0, treeId, parserState, 5); console.log("Validation:"); console.log( ` Off-chain tree ${treeId} is consistent: ${await validateTree( diff --git a/contracts/sdk/indexer/indexer/README.md b/contracts/sdk/indexer/indexer/README.md new file mode 100644 index 00000000000..771494b3b74 --- /dev/null +++ b/contracts/sdk/indexer/indexer/README.md @@ -0,0 +1,13 @@ +# Quick and Dirty: Typescript Indexer + +Query to check backfill script by removing all the most recent +merkle tree nodes +```sql +DELETE FROM merkle +WHERE seq in +(SELECT seq FROM + (SELECT node_idx, MAX(seq) AS seq + FROM merkle + WHERE level = 0 group by node_idx +)); +``` diff --git a/contracts/sdk/indexer/indexer/bubblegum.ts b/contracts/sdk/indexer/indexer/bubblegum.ts deleted file mode 100644 index 355ffbca469..00000000000 --- a/contracts/sdk/indexer/indexer/bubblegum.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { - ParsedLog, - ParserState, - ixRegEx, - parseEventFromLog, - OptionalInfo, - dataRegEx, - decodeEvent, -} from "./utils"; -import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../../gummyroll"; -import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../bubblegum/src/generated"; -import { ChangeLogEvent, parseEventGummyroll } from "./gummyroll"; -import { - TokenProgramVersion, - MetadataArgs, -} from "../../bubblegum/src/generated/types"; -import { BN, Event } from "@project-serum/anchor"; -import { NFTDatabaseConnection } from "../db"; -import { PublicKey } from "@solana/web3.js"; -import { IdlEvent } from "@project-serum/anchor/dist/cjs/idl"; - -function parseIxName(logLine: string): BubblegumIx | null { - return logLine.match(ixRegEx)[1] as BubblegumIx; -} - -function skipTx(sequenceNumber, startSeq, endSeq): boolean { - let left = startSeq !== null ? sequenceNumber <= startSeq : false; - let right = endSeq !== null ? sequenceNumber >= endSeq : false; - return left || right; -} - -export type BubblegumIx = - | "Redeem" - | "DecompressV1" - | "Transfer" - | "CreateTree" - | "MintV1" - | "Burn" - | "CancelRedeem" - | "Delegate"; - -export type NewLeafEvent = { - version: TokenProgramVersion; - metadata: MetadataArgs; - nonce: BN; -}; - -export type LeafSchemaEvent = { - schema: { - v1: { - id: PublicKey; - owner: PublicKey; - delegate: PublicKey; - nonce: BN; - dataHash: number[] /* size: 32 */; - creatorHash: number[] /* size: 32 */; - }; - }; -}; - -export async function parseBubblegum( - db: NFTDatabaseConnection, - parsedLog: ParsedLog, - slot: number, - parser: ParserState, - optionalInfo: OptionalInfo -) { - const ixName = parseIxName(parsedLog.logs[0] as string); - console.log("Bubblegum:", ixName); - switch (ixName) { - case "CreateTree": - await parseBubblegumCreateTree( - db, - parsedLog.logs, - slot, - parser, - optionalInfo - ); - break; - case "MintV1": - await parseBubblegumMint(db, parsedLog.logs, slot, parser, optionalInfo); - break; - case "Redeem": - await parseReplaceLeaf( - db, - parsedLog.logs, - slot, - parser, - optionalInfo, - false - ); - break; - case "CancelRedeem": - await parseReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); - break; - case "Burn": - await parseReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); - break; - case "Transfer": - await parseReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); - break; - case "Delegate": - await parseReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); - break; - } -} - -function findGummyrollEvent( - logs: (string | ParsedLog)[], - parser: ParserState -): ChangeLogEvent | null { - let changeLog: ChangeLogEvent | null; - for (const log of logs) { - if (typeof log !== "string" && log.programId.equals(GUMMYROLL_PROGRAM_ID)) { - changeLog = parseEventGummyroll(log, parser.Gummyroll); - } - } - if (!changeLog) { - console.log("Failed to find gummyroll changelog"); - } - return changeLog; -} - -function findBubblegumEvents( - logs: (string | ParsedLog)[], - parser: ParserState -): Array { - let events = []; - for (const log of logs) { - if (typeof log !== "string") { - continue; - } - let data = log.match(dataRegEx); - if (data && data.length > 1) { - events.push(decodeEvent(data[1], parser.Bubblegum.idl)); - } - } - return events; -} - -export async function parseBubblegumMint( - db: NFTDatabaseConnection, - logs: (string | ParsedLog)[], - slot: number, - parser: ParserState, - optionalInfo: OptionalInfo -) { - const changeLog = findGummyrollEvent(logs, parser); - const events = findBubblegumEvents(logs, parser); - if (events.length !== 2) { - return; - } - const newLeafData = events[0].data as NewLeafEvent; - const leafSchema = events[1].data as LeafSchemaEvent; - let treeId = changeLog.id.toBase58(); - let sequenceNumber = changeLog.seq; - let { startSeq, endSeq, txId } = optionalInfo; - if (skipTx(sequenceNumber, startSeq, endSeq)) { - return; - } - console.log(`Sequence Number: ${sequenceNumber}`); - await db.updateNFTMetadata(newLeafData, leafSchema.schema.v1.id.toBase58()); - await db.updateLeafSchema( - leafSchema, - new PublicKey(changeLog.path[0].node), - txId, - slot, - sequenceNumber, - treeId - ); - await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); -} - - - -export async function parseReplaceLeaf( - db: NFTDatabaseConnection, - logs: (string | ParsedLog)[], - slot: number, - parser: ParserState, - optionalInfo: OptionalInfo, - compressed: boolean = true -) { - const changeLog = findGummyrollEvent(logs, parser); - const events = findBubblegumEvents(logs, parser); - if (events.length !== 1) { - return; - } - const leafSchema = events[0].data as LeafSchemaEvent; - let treeId = changeLog.id.toBase58(); - let sequenceNumber = changeLog.seq; - let { startSeq, endSeq, txId } = optionalInfo; - if (skipTx(sequenceNumber, startSeq, endSeq)) { - return; - } - console.log(`Sequence Number: ${sequenceNumber}`); - await db.updateLeafSchema( - leafSchema, - new PublicKey(changeLog.path[0].node), - txId, - slot, - sequenceNumber, - treeId, - compressed - ); - await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); -} - -export async function parseBubblegumCreateTree( - db: NFTDatabaseConnection, - logs: (string | ParsedLog)[], - slot: number, - parser: ParserState, - optionalInfo: OptionalInfo -) { - const changeLog = findGummyrollEvent(logs, parser); - const sequenceNumber = changeLog.seq; - let { startSeq, endSeq, txId } = optionalInfo; - if (skipTx(sequenceNumber, startSeq, endSeq)) { - return; - } - console.log(`Sequence Number: ${sequenceNumber}`); - let treeId = changeLog.id.toBase58(); - await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); -} - -export async function parseBubblegumDecompress( - db: NFTDatabaseConnection, - logs: (string | ParsedLog)[], - parser: ParserState, - optionalInfo: OptionalInfo -) {} diff --git a/contracts/sdk/indexer/indexer/ingester.ts b/contracts/sdk/indexer/indexer/ingester.ts new file mode 100644 index 00000000000..da94429326e --- /dev/null +++ b/contracts/sdk/indexer/indexer/ingester.ts @@ -0,0 +1,147 @@ +import { + ParserState, + OptionalInfo, + decodeEvent, +} from "./utils"; +import { ParsedLog } from "./log/utils"; +import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID, PathNode } from "../../gummyroll"; +import { + TokenProgramVersion, + MetadataArgs, +} from "../../bubblegum/src/generated/types"; +import { BN, } from "@project-serum/anchor"; +import { NFTDatabaseConnection } from "../db"; +import { PublicKey } from "@solana/web3.js"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; + +function skipTx(sequenceNumber, startSeq, endSeq): boolean { + let left = startSeq !== null ? sequenceNumber <= startSeq : false; + let right = endSeq !== null ? sequenceNumber >= endSeq : false; + return left || right; +} + +export type BubblegumIx = + | "Redeem" + | "DecompressV1" + | "Transfer" + | "CreateTree" + | "MintV1" + | "Burn" + | "CancelRedeem" + | "Delegate"; + +export type ChangeLogEvent = { + id: PublicKey, + path: PathNode[], + seq: number, + index: number, +}; + +export type NewLeafEvent = { + version: TokenProgramVersion; + metadata: MetadataArgs; + nonce: BN; +}; + +export type LeafSchemaEvent = { + schema: { + v1: { + id: PublicKey; + owner: PublicKey; + delegate: PublicKey; + nonce: BN; + dataHash: number[] /* size: 32 */; + creatorHash: number[] /* size: 32 */; + }; + }; +}; + + +export async function ingestBubblegumMint( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + changeLog: ChangeLogEvent, + newLeafData: NewLeafEvent, + leafSchema: LeafSchemaEvent, +) { + let treeId = changeLog.id.toBase58(); + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + const schema = leafSchema.schema.v1; + console.log("Leaf Schema:", { + schema: { + id: schema.id.toString(), + owner: schema.owner.toString(), + delegate: schema.delegate.toString(), + nonce: schema.nonce.toNumber(), + }, + leafHash: new PublicKey(changeLog.path[0].node).toString(), + dataHash: bs58.encode(leafSchema.schema.v1.dataHash), + creatorHash: bs58.encode(leafSchema.schema.v1.creatorHash), + }); + await db.updateNFTMetadata(newLeafData, leafSchema.schema.v1.id.toBase58()); + await db.updateLeafSchema( + leafSchema, + new PublicKey(changeLog.path[0].node), + txId, + slot, + sequenceNumber, + treeId + ); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); +} + +export async function ingestBubblegumReplaceLeaf( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + changeLog: ChangeLogEvent, + leafSchema: LeafSchemaEvent, + compressed: boolean = true +) { + let treeId = changeLog.id.toBase58(); + let sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + await db.updateLeafSchema( + leafSchema, + new PublicKey(changeLog.path[0].node), + txId, + slot, + sequenceNumber, + treeId, + compressed + ); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); +} + +export async function ingestBubblegumCreateTree( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + changeLog: ChangeLogEvent +) { + const sequenceNumber = changeLog.seq; + let { startSeq, endSeq, txId } = optionalInfo; + if (skipTx(sequenceNumber, startSeq, endSeq)) { + return; + } + console.log(`Sequence Number: ${sequenceNumber}`); + let treeId = changeLog.id.toBase58(); + await db.updateChangeLogs(changeLog, optionalInfo.txId, slot, treeId); +} + +export async function ingestBubblegumDecompress( + db: NFTDatabaseConnection, + logs: (string | ParsedLog)[], + parser: ParserState, + optionalInfo: OptionalInfo +) { } diff --git a/contracts/sdk/indexer/indexer/innerInstruction/bubblegum.ts b/contracts/sdk/indexer/indexer/innerInstruction/bubblegum.ts new file mode 100644 index 00000000000..b351499f6d3 --- /dev/null +++ b/contracts/sdk/indexer/indexer/innerInstruction/bubblegum.ts @@ -0,0 +1,215 @@ +import { NFTDatabaseConnection } from "../../db" +import { + ParserState, + OptionalInfo, + decodeEventInstructionData, + leafSchemaFromLeafData, + destructureBubblegumMintAccounts, + findWrapInstructions +} from "../utils" +import { PublicKey, CompiledInstruction } from "@solana/web3.js" +import { BorshInstructionCoder } from "@project-serum/anchor"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; +import { ChangeLogEvent, ingestBubblegumCreateTree, ingestBubblegumMint, ingestBubblegumReplaceLeaf, LeafSchemaEvent, NewLeafEvent } from "../ingester"; + +/** + * This kind of difficult because there is no depth associated with the inner instructions + */ +export async function parseBubblegumInnerInstructions( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + innerInstructions: CompiledInstruction[], +) { + let i = 0; + while (i < innerInstructions.length) { + const programId = accountKeys[innerInstructions[i].programIdIndex]; + if (programId.equals(parser.Bubblegum.programId)) { + i = await parseBubblegumExecutionContext(db, slot, parser, optionalInfo, accountKeys, innerInstructions, i); + } + i++; + } +} + + +async function parseBubblegumCreateTreeInstructions( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + currentIndex: number +): Promise { + const [found, count] = findWrapInstructions( + accountKeys, + instructions.slice(currentIndex), + 1 + ); + const changeLogEvent = decodeEventInstructionData( + parser.Gummyroll.idl, + "ChangeLogEvent", + found[0].data + ).data as ChangeLogEvent; + await ingestBubblegumCreateTree( + db, + slot, + optionalInfo, + changeLogEvent + ); + return currentIndex + count; +} + +async function parseBubblegumMintInstructions( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + currentIndex: number +): Promise { + const [found, count] = findWrapInstructions( + accountKeys, + instructions.slice(currentIndex + 1), + 2 + ); + const newLeafData = decodeEventInstructionData( + parser.Bubblegum.idl, + "NewNFTEvent", + found[0].data + ).data as NewLeafEvent; + const changeLogEvent = decodeEventInstructionData( + parser.Gummyroll.idl, + "ChangeLogEvent", + found[1].data + ).data as ChangeLogEvent; + + const { owner, delegate, merkleSlab } = destructureBubblegumMintAccounts( + accountKeys, + instructions[currentIndex] + ); + const leafSchema = await leafSchemaFromLeafData(owner, delegate, merkleSlab, newLeafData); + + await ingestBubblegumMint( + db, + slot, + optionalInfo, + changeLogEvent, + newLeafData, + leafSchema + ); + return currentIndex + count +} + +/// Untested +/// Todo: test +async function parseBubblegumReplaceLeafInstructions( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + currentIndex: number, + compressed: boolean = true +): Promise { + const [found, count] = findWrapInstructions( + accountKeys, + instructions, + 2 + ); + const leafSchema = decodeEventInstructionData( + parser.Bubblegum.idl, + "LeafSchemaEvent", + found[0].data + ).data as LeafSchemaEvent; + const changeLogEvent = decodeEventInstructionData( + parser.Gummyroll.idl, + "ChangeLogEvent", + found[1].data + ).data as ChangeLogEvent; + await ingestBubblegumReplaceLeaf( + db, + slot, + optionalInfo, + changeLogEvent, + leafSchema, + compressed + ) + return currentIndex + count +} + +/** + * Here we know that instructions at current index may actually CPIs from bubblegum + */ +async function parseBubblegumExecutionContext( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + currentIndex: number +): Promise { + const coder = new BorshInstructionCoder(parser.Bubblegum.idl); + const instruction = instructions[currentIndex]; + const decodedIx = coder.decode(bs58.decode(instruction.data)); + if (decodedIx) { + const name = decodedIx.name.charAt(0).toUpperCase() + decodedIx.name.slice(1); + console.log(`Found: ${name}`); + switch (name) { + case "CreateTree": + return await parseBubblegumCreateTreeInstructions( + db, + slot, + parser, + optionalInfo, + accountKeys, + instructions, + currentIndex + ); + case "MintV1": + return await parseBubblegumMintInstructions( + db, + slot, + parser, + optionalInfo, + accountKeys, + instructions, + currentIndex + ); + /// TODO(ngundotra): add tests for the following leaf-replacements + case "Redeem": + return await parseBubblegumReplaceLeafInstructions( + db, + slot, + parser, + optionalInfo, + accountKeys, + instructions, + currentIndex, + false + ) + case "Burn": + case "CancelRedeem": + case "Delegate": + case "Transfer": + return await parseBubblegumReplaceLeafInstructions( + db, + slot, + parser, + optionalInfo, + accountKeys, + instructions, + currentIndex + ) + default: + break + } + } + return currentIndex +} + diff --git a/contracts/sdk/indexer/indexer/instruction/bubblegum.ts b/contracts/sdk/indexer/indexer/instruction/bubblegum.ts new file mode 100644 index 00000000000..583d7655bf1 --- /dev/null +++ b/contracts/sdk/indexer/indexer/instruction/bubblegum.ts @@ -0,0 +1,157 @@ +import { hash, NFTDatabaseConnection } from "../../db" +import { ParserState, OptionalInfo } from "../utils" +import { PublicKey, CompiledInstruction, CompiledInnerInstruction } from "@solana/web3.js" +import { BorshInstructionCoder } from "@project-serum/anchor"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; +import { ChangeLogEvent, ingestBubblegumCreateTree, ingestBubblegumMint, ingestBubblegumReplaceLeaf, LeafSchemaEvent, NewLeafEvent } from "../ingester"; +import { findWrapInstructions, decodeEventInstructionData, destructureBubblegumMintAccounts, leafSchemaFromLeafData } from "../utils"; + +/// Copied from https://github.com/solana-labs/solana/blob/d07b0798504f757340868d15c199aba9bd00ba5d/explorer/src/utils/anchor.tsx#L57 +export async function parseBubblegumInstruction( + db: NFTDatabaseConnection, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + accountKeys: PublicKey[], + instruction: CompiledInstruction, + innerInstructions: CompiledInnerInstruction[], +) { + const coder = new BorshInstructionCoder(parser.Bubblegum.idl); + const decodedIx = coder.decode(bs58.decode(instruction.data)); + if (decodedIx) { + const name = decodedIx.name.charAt(0).toUpperCase() + decodedIx.name.slice(1); + console.log(`Found: ${name}`); + switch (name) { + case "CreateTree": + await parseBubblegumCreateTree( + db, + slot, + optionalInfo, + parser, + accountKeys, + innerInstructions + ) + break; + case "MintV1": + await parseBubblegumMint( + db, + slot, + optionalInfo, + parser, + accountKeys, + instruction, + innerInstructions + ) + break; + case "Redeem": + await parseBubblegumReplaceLeafInstruction( + db, + slot, + optionalInfo, + parser, + accountKeys, + innerInstructions, + false + ); + break + case "Burn": + case "CancelRedeem": + case "Delegate": + case "Transfer": + await parseBubblegumReplaceLeafInstruction( + db, + slot, + optionalInfo, + parser, + accountKeys, + innerInstructions, + ) + break; + default: + break + } + } else { + console.error("Could not decode Bubblegum found in slot:", slot); + } +} + +async function parseBubblegumCreateTree( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + parser: ParserState, + accountKeys: PublicKey[], + innerInstructions: CompiledInnerInstruction[], +) { + let changeLogEvent: ChangeLogEvent | null = null; + for (const innerInstruction of innerInstructions) { + const [wrapIxs] = findWrapInstructions(accountKeys, innerInstruction.instructions, 1); + changeLogEvent = decodeEventInstructionData(parser.Gummyroll.idl, "ChangeLogEvent", wrapIxs[0].data).data as ChangeLogEvent; + } + + await ingestBubblegumCreateTree( + db, + slot, + optionalInfo, + changeLogEvent, + ); +} + + + +async function parseBubblegumMint( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + parser: ParserState, + accountKeys: PublicKey[], + instruction: CompiledInstruction, + innerInstructions: CompiledInnerInstruction[], +) { + let newLeafData: NewLeafEvent; + let changeLogEvent: ChangeLogEvent; + for (const innerInstruction of innerInstructions) { + const [wrapIxs] = findWrapInstructions(accountKeys, innerInstruction.instructions, 2); + newLeafData = decodeEventInstructionData(parser.Bubblegum.idl, "NewNFTEvent", wrapIxs[0].data).data as NewLeafEvent; + changeLogEvent = decodeEventInstructionData(parser.Gummyroll.idl, "ChangeLogEvent", wrapIxs[1].data).data as ChangeLogEvent; + } + + const { owner, delegate, merkleSlab } = destructureBubblegumMintAccounts(accountKeys, instruction); + const leafSchema = await leafSchemaFromLeafData(owner, delegate, merkleSlab, newLeafData); + + await ingestBubblegumMint( + db, + slot, + optionalInfo, + changeLogEvent, + newLeafData, + leafSchema, + ) +} + +async function parseBubblegumReplaceLeafInstruction( + db: NFTDatabaseConnection, + slot: number, + optionalInfo: OptionalInfo, + parser: ParserState, + accountKeys: PublicKey[], + innerInstructions: CompiledInnerInstruction[], + compressed: boolean = true +) { + let leafSchema: LeafSchemaEvent; + let changeLogEvent: ChangeLogEvent; + for (const innerInstruction of innerInstructions) { + const [wrapIxs] = findWrapInstructions(accountKeys, innerInstruction.instructions, 2); + leafSchema = decodeEventInstructionData(parser.Bubblegum.idl, "LeafSchemaEvent", wrapIxs[0].data).data as LeafSchemaEvent; + changeLogEvent = decodeEventInstructionData(parser.Gummyroll.idl, "ChangeLogEvent", wrapIxs[1].data).data as ChangeLogEvent; + } + + await ingestBubblegumReplaceLeaf( + db, + slot, + optionalInfo, + changeLogEvent, + leafSchema, + compressed + ) +} diff --git a/contracts/sdk/indexer/indexer/log/bubblegum.ts b/contracts/sdk/indexer/indexer/log/bubblegum.ts new file mode 100644 index 00000000000..f1a2ffbabf2 --- /dev/null +++ b/contracts/sdk/indexer/indexer/log/bubblegum.ts @@ -0,0 +1,313 @@ +import { PublicKey, Logs, Context } from "@solana/web3.js"; +import * as anchor from '@project-serum/anchor'; +import { NFTDatabaseConnection } from "../../db"; +import { ParserState, ParseResult, OptionalInfo, decodeEvent } from "../utils"; +import { BubblegumIx, LeafSchemaEvent, NewLeafEvent, ingestBubblegumCreateTree, ingestBubblegumMint, ingestBubblegumReplaceLeaf } from "../ingester"; +import { ParsedLog, endRegEx, startRegEx, ixRegEx, dataRegEx } from './utils'; +import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../../bubblegum/src/generated"; +import { findGummyrollEvent } from './gummyroll'; + +function findBubblegumEvents( + logs: (string | ParsedLog)[], + parser: ParserState +): Array { + let events = []; + for (const log of logs) { + if (typeof log !== "string") { + continue; + } + let data = log.match(dataRegEx); + if (data && data.length > 1) { + events.push(decodeEvent(data[1], parser.Bubblegum.idl)); + } + } + return events; +} + +/** + * Recursively parses the logs of a program instruction execution + * @param programId + * @param depth + * @param logs + * @returns + */ +function parseInstructionLog( + programId: PublicKey, + depth: number, + logs: string[] +) { + const parsedLog: ParsedLog = { + programId, + depth, + logs: [], + }; + let instructionComplete = false; + while (!instructionComplete) { + const logLine = logs[0]; + logs = logs.slice(1); + let result = logLine.match(endRegEx); + if (result) { + if (result[1] != programId.toString()) { + throw Error(`Unexpected program id finished: ${result[1]}`); + } + instructionComplete = true; + } else { + result = logLine.match(startRegEx); + if (result) { + const programId = new PublicKey(result[1]); + const depth = Number(result[2]) - 1; + const parsedInfo = parseInstructionLog(programId, depth, logs); + parsedLog.logs.push(parsedInfo.parsedLog); + logs = parsedInfo.logs; + } else { + parsedLog.logs.push(logLine); + } + } + } + return { parsedLog, logs }; +} + +/** + * Parses logs so that emitted event data can be tied to its execution context + * @param logs + * @returns + */ +export function parseLogs(logs: string[]): ParsedLog[] { + let parsedLogs: ParsedLog[] = []; + while (logs && logs.length) { + const logLine = logs[0]; + logs = logs.slice(1); + const result = logLine.match(startRegEx); + const programId = new PublicKey(result[1]); + const depth = Number(result[2]) - 1; + const parsedInfo = parseInstructionLog(programId, depth, logs); + parsedLogs.push(parsedInfo.parsedLog); + logs = parsedInfo.logs; + } + return parsedLogs; +} + + +/** + * Performs a depth-first traversal of the ParsedLog data structure + * @param db + * @param optionalInfo + * @param slot + * @param parsedState + * @param parsedLog + * @returns + */ +async function indexParsedLog( + db: NFTDatabaseConnection, + optionalInfo: OptionalInfo, + slot: number, + parserState: ParserState, + parsedLog: ParsedLog | string +) { + if (typeof parsedLog === "string") { + return; + } + if (parsedLog.programId.equals(BUBBLEGUM_PROGRAM_ID)) { + return await parseBubblegumLog(db, parsedLog, slot, parserState, optionalInfo); + } else { + for (const log of parsedLog.logs) { + await indexParsedLog(db, optionalInfo, slot, parserState, log); + } + } +} + +function isLogTruncated(logs: string[]): boolean { + // Loops backward + for (let i = logs.length - 1; i >= 0; i--) { + if (logs[i].startsWith("Log truncated")) { + return true; + } + } + return false; +} + +/** + * Returns false if log is truncated + */ +export function handleLogsAtomic( + db: NFTDatabaseConnection, + logs: Logs, + context: Context, + parsedState: ParserState, + startSeq: number | null = null, + endSeq: number | null = null +): ParseResult { + if (logs.err) { + return ParseResult.TransactionError; + } + if (isLogTruncated(logs.logs)) { + return ParseResult.LogTruncated; + } + const parsedLogs = parseLogs(logs.logs); + if (parsedLogs.length === 0) { + return ParseResult.Success; + } + db.connection.db.serialize(() => { + db.beginTransaction(); + for (const parsedLog of parsedLogs) { + indexParsedLog( + db, + { txId: logs.signature, startSeq, endSeq }, + context.slot, + parsedState, + parsedLog + ); + } + db.commit(); + }); + return ParseResult.Success; +} + +/** + * Processes the logs from a new transaction and searches for the programs + * specified in the ParserState + * @param db + * @param logs + * @param context + * @param parsedState + * @param startSeq + * @param endSeq + * @returns + */ +export async function handleLogs( + db: NFTDatabaseConnection, + logs: Logs, + context: Context, + parsedState: ParserState, + startSeq: number | null = null, + endSeq: number | null = null +) { + if (logs.err) { + return; + } + const parsedLogs = parseLogs(logs.logs); + if (parsedLogs.length == 0) { + return; + } + for (const parsedLog of parsedLogs) { + await indexParsedLog( + db, + { txId: logs.signature, startSeq, endSeq }, + context.slot, + parsedState, + parsedLog + ); + } +} + +function parseIxName(logLine: string): BubblegumIx | null { + return logLine.match(ixRegEx)[1] as BubblegumIx; +} + +export async function parseBubblegumLog( + db: NFTDatabaseConnection, + parsedLog: ParsedLog, + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo +) { + const ixName = parseIxName(parsedLog.logs[0] as string); + console.log("Bubblegum:", ixName); + switch (ixName) { + case "CreateTree": + await parseBubblegumCreateTree( + db, + parsedLog.logs, + slot, + parser, + optionalInfo + ); + break; + case "MintV1": + await parseBubblegumMint(db, parsedLog.logs, slot, parser, optionalInfo); + break; + case "Redeem": + await parseBubblegumReplaceLeaf( + db, + parsedLog.logs, + slot, + parser, + optionalInfo, + false + ); + break; + case "CancelRedeem": + await parseBubblegumReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); + break; + case "Burn": + await parseBubblegumReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); + break; + case "Transfer": + await parseBubblegumReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); + break; + case "Delegate": + await parseBubblegumReplaceLeaf(db, parsedLog.logs, slot, parser, optionalInfo); + break; + } +} + +async function parseBubblegumCreateTree( + db: NFTDatabaseConnection, + logs: (string | ParsedLog)[], + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo +) { + const changeLog = findGummyrollEvent(logs, parser); + await ingestBubblegumCreateTree(db, slot, optionalInfo, changeLog); +} + +async function parseBubblegumMint( + db: NFTDatabaseConnection, + logs: (string | ParsedLog)[], + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo +) { + const changeLog = findGummyrollEvent(logs, parser); + const events = findBubblegumEvents(logs, parser); + if (events.length !== 2) { + console.error("Could not find enough Bubblegum events for Bubblegum mint"); + return; + } + const newLeafData = events[0].data as NewLeafEvent; + const leafSchema = events[1].data as LeafSchemaEvent; + await ingestBubblegumMint( + db, + slot, + optionalInfo, + changeLog, + newLeafData, + leafSchema + ) +} + +async function parseBubblegumReplaceLeaf( + db: NFTDatabaseConnection, + logs: (string | ParsedLog)[], + slot: number, + parser: ParserState, + optionalInfo: OptionalInfo, + compressed: boolean = true +) { + const changeLog = findGummyrollEvent(logs, parser); + const events = findBubblegumEvents(logs, parser); + if (events.length !== 1) { + console.error("Could not find leafSchema event for Bubblegum replace"); + return; + } + const leafSchema = events[0].data as LeafSchemaEvent; + await ingestBubblegumReplaceLeaf( + db, + slot, + optionalInfo, + changeLog, + leafSchema, + compressed + ) +} diff --git a/contracts/sdk/indexer/indexer/gummyroll.ts b/contracts/sdk/indexer/indexer/log/gummyroll.ts similarity index 62% rename from contracts/sdk/indexer/indexer/gummyroll.ts rename to contracts/sdk/indexer/indexer/log/gummyroll.ts index ece2312f27c..93446131662 100644 --- a/contracts/sdk/indexer/indexer/gummyroll.ts +++ b/contracts/sdk/indexer/indexer/log/gummyroll.ts @@ -1,18 +1,14 @@ import * as anchor from '@project-serum/anchor'; import { PublicKey } from '@solana/web3.js'; -import { Gummyroll, PathNode } from "../../gummyroll" +import { Gummyroll } from "../../../gummyroll" +import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from '../../../gummyroll'; import { parseEventFromLog, ParsedLog, ixRegEx } from './utils'; +import { ParserState } from '../utils'; +import { ChangeLogEvent } from '../ingester'; export type GummyrollIx = 'InitEmptyGummyroll' | 'InitEmptyGummyrollWithRoot' | 'Replace' | 'Append' | 'InsertOrAppend' | 'VerifyLeaf' | 'TransferAuthority'; -export type ChangeLogEvent = { - id: PublicKey, - path: PathNode[], - seq: number, - index: number, -}; - function parseIxName(logLine: string): GummyrollIx | null { return logLine.match(ixRegEx)[1] as GummyrollIx } @@ -34,3 +30,19 @@ export function parseEventGummyroll(parsedLog: ParsedLog, gummyroll: anchor.Prog function parseChangelogEvent(logs: string[], gummyroll: anchor.Program): ChangeLogEvent | null { return parseEventFromLog(logs[logs.length - 2], gummyroll.idl).data as ChangeLogEvent; } + +export function findGummyrollEvent( + logs: (string | ParsedLog)[], + parser: ParserState +): ChangeLogEvent | null { + let changeLog: ChangeLogEvent | null; + for (const log of logs) { + if (typeof log !== "string" && log.programId.equals(GUMMYROLL_PROGRAM_ID)) { + changeLog = parseEventGummyroll(log, parser.Gummyroll); + } + } + if (!changeLog) { + console.log("Failed to find gummyroll changelog"); + } + return changeLog; +} diff --git a/contracts/sdk/indexer/indexer/log/utils.ts b/contracts/sdk/indexer/indexer/log/utils.ts new file mode 100644 index 00000000000..337c0742789 --- /dev/null +++ b/contracts/sdk/indexer/indexer/log/utils.ts @@ -0,0 +1,22 @@ +import { PublicKey } from '@solana/web3.js'; +import * as anchor from '@project-serum/anchor'; +import { decodeEvent } from '../utils'; + +export const startRegEx = /Program (\w*) invoke \[(\d)\]/; +export const endRegEx = /Program (\w*) success/; +export const dataRegEx = + /Program data: ((?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{3}=|[A-Za-z\d+/]{2}==)?$)/; +export const ixRegEx = /Program log: Instruction: (\w+)/; + +export function parseEventFromLog( + log: string, + idl: anchor.Idl +): anchor.Event | null { + return decodeEvent(log.match(dataRegEx)[1], idl); +} + +export type ParsedLog = { + programId: PublicKey; + logs: (string | ParsedLog)[]; + depth: number; +}; diff --git a/contracts/sdk/indexer/indexer/utils.ts b/contracts/sdk/indexer/indexer/utils.ts index 5c3910cb41a..e62b038c0b6 100644 --- a/contracts/sdk/indexer/indexer/utils.ts +++ b/contracts/sdk/indexer/indexer/utils.ts @@ -1,106 +1,44 @@ import * as anchor from "@project-serum/anchor"; -import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../bubblegum/src/generated"; -import { Context, Logs, PublicKey } from "@solana/web3.js"; +import { CompiledInnerInstruction, CompiledInstruction, Context, Logs, PublicKey } from "@solana/web3.js"; import { readFileSync } from "fs"; import { Bubblegum } from "../../../target/types/bubblegum"; import { Gummyroll } from "../../../target/types/gummyroll"; import { NFTDatabaseConnection } from "../db"; -import { parseBubblegum } from "./bubblegum"; +import { parseBubblegumInstruction } from "./instruction/bubblegum"; +import { parseBubblegumInnerInstructions } from "./innerInstruction/bubblegum"; +import { Idl, IdlTypeDef } from '@project-serum/anchor/dist/cjs/idl'; +import { IdlCoder } from '@project-serum/anchor/dist/cjs/coder/borsh/idl'; +import { Layout } from "buffer-layout"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; +import { + Creator, + MetadataArgs, + metadataArgsBeet, + TokenProgramVersion, + TokenStandard, +} from "../../bubblegum/src/generated"; +import { NewLeafEvent, LeafSchemaEvent } from "./ingester"; +import { keccak_256 } from "js-sha3"; +import { getLeafAssetId } from "../../bubblegum/src/convenience"; +import * as beetSolana from '@metaplex-foundation/beet-solana' +import * as beet from '@metaplex-foundation/beet' -const startRegEx = /Program (\w*) invoke \[(\d)\]/; -const endRegEx = /Program (\w*) success/; -export const dataRegEx = - /Program data: ((?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{3}=|[A-Za-z\d+/]{2}==)?$)/; -export const ixRegEx = /Program log: Instruction: (\w+)/; +import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../../gummyroll"; +import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../bubblegum/src/generated"; +import { GumballMachine, PROGRAM_ID as GUMBALL_MACHINE_ID } from "../../gumball-machine"; +import { CANDY_WRAPPER_PROGRAM_ID } from "../../utils"; export type ParserState = { Gummyroll: anchor.Program; Bubblegum: anchor.Program; }; -export type ParsedLog = { - programId: PublicKey; - logs: (string | ParsedLog)[]; - depth: number; -}; - export type OptionalInfo = { txId: string; - startSeq: number | null; endSeq: number | null; }; -/** - * Recursively parses the logs of a program instruction execution - * @param programId - * @param depth - * @param logs - * @returns - */ -function parseInstructionLog( - programId: PublicKey, - depth: number, - logs: string[] -) { - const parsedLog: ParsedLog = { - programId, - depth, - logs: [], - }; - let instructionComplete = false; - while (!instructionComplete) { - const logLine = logs[0]; - logs = logs.slice(1); - let result = logLine.match(endRegEx); - if (result) { - if (result[1] != programId.toString()) { - throw Error(`Unexpected program id finished: ${result[1]}`); - } - instructionComplete = true; - } else { - result = logLine.match(startRegEx); - if (result) { - const programId = new PublicKey(result[1]); - const depth = Number(result[2]) - 1; - const parsedInfo = parseInstructionLog(programId, depth, logs); - parsedLog.logs.push(parsedInfo.parsedLog); - logs = parsedInfo.logs; - } else { - parsedLog.logs.push(logLine); - } - } - } - return { parsedLog, logs }; -} - -/** - * Parses logs so that emitted event data can be tied to its execution context - * @param logs - * @returns - */ -export function parseLogs(logs: string[]): ParsedLog[] { - let parsedLogs: ParsedLog[] = []; - while (logs && logs.length) { - const logLine = logs[0]; - logs = logs.slice(1); - const result = logLine.match(startRegEx); - const programId = new PublicKey(result[1]); - const depth = Number(result[2]) - 1; - const parsedInfo = parseInstructionLog(programId, depth, logs); - parsedLogs.push(parsedInfo.parsedLog); - logs = parsedInfo.logs; - } - return parsedLogs; -} - -export function parseEventFromLog( - log: string, - idl: anchor.Idl -): anchor.Event | null { - return decodeEvent(log.match(dataRegEx)[1], idl); -} - /** * Example: * ``` @@ -124,97 +62,268 @@ export function loadProgram( return new anchor.Program(IDL, programId, provider); } -/** - * Performs a depth-first traversal of the ParsedLog data structure - * @param db - * @param optionalInfo - * @param slot - * @param parsedState - * @param parsedLog - * @returns - */ -async function indexParsedLog( +export enum ParseResult { + Success, + LogTruncated, + TransactionError +}; + +function indexZippedInstruction( db: NFTDatabaseConnection, - optionalInfo: OptionalInfo, + context: { txId: string, startSeq: number, endSeq: number }, slot: number, parserState: ParserState, - parsedLog: ParsedLog | string + accountKeys: PublicKey[], + zippedInstruction: ZippedInstruction, ) { - if (typeof parsedLog === "string") { - return; - } - if (parsedLog.programId.equals(BUBBLEGUM_PROGRAM_ID)) { - return await parseBubblegum(db, parsedLog, slot, parserState, optionalInfo); + const { instruction, innerInstructions } = zippedInstruction; + const programId = accountKeys[instruction.programIdIndex]; + if (programId.equals(BUBBLEGUM_PROGRAM_ID)) { + console.log("Found bubblegum"); + parseBubblegumInstruction( + db, + slot, + parserState, + context, + accountKeys, + instruction, + innerInstructions + ); } else { - for (const log of parsedLog.logs) { - await indexParsedLog(db, optionalInfo, slot, parserState, log); + if (innerInstructions.length) { + parseBubblegumInnerInstructions( + db, + slot, + parserState, + context, + accountKeys, + innerInstructions[0].instructions, + ) } } } -export function handleLogsAtomic( - db: NFTDatabaseConnection, - logs: Logs, - context: Context, - parsedState: ParserState, - startSeq: number | null = null, - endSeq: number | null = null +export function decodeEventInstructionData( + idl: Idl, + eventName: string, + base58String: string, ) { - if (logs.err) { - return; + const rawLayouts: [string, Layout][] = idl.events.map((event) => { + let eventTypeDef: IdlTypeDef = { + name: event.name, + type: { + kind: "struct", + fields: event.fields.map((f) => { + return { name: f.name, type: f.type }; + }), + }, + }; + return [event.name, IdlCoder.typeDefLayout(eventTypeDef, idl.types)]; + }); + const layouts = new Map(rawLayouts); + const buffer = bs58.decode(base58String); + const layout = layouts.get(eventName); + if (!layout) { + console.error("Could not find corresponding layout for event:", eventName); } - const parsedLogs = parseLogs(logs.logs); - if (parsedLogs.length == 0) { - return; + const data = layout.decode(buffer); + return { data, name: eventName }; +} + +export function destructureBubblegumMintAccounts( + accountKeys: PublicKey[], + instruction: CompiledInstruction +) { + return { + owner: accountKeys[instruction.accounts[4]], + delegate: accountKeys[instruction.accounts[5]], + merkleSlab: accountKeys[instruction.accounts[6]], } - db.connection.db.serialize(() => { - db.beginTransaction(); - for (const parsedLog of parsedLogs) { - indexParsedLog( - db, - { txId: logs.signature, startSeq, endSeq }, - context.slot, - parsedState, - parsedLog - ); - } - db.commit(); - }); } -/** - * Processes the logs from a new transaction and searches for the programs - * specified in the ParserState - * @param db - * @param logs - * @param context - * @param parsedState - * @param startSeq - * @param endSeq - * @returns - */ -export async function handleLogs( + +type ZippedInstruction = { + instructionIndex: number, + instruction: CompiledInstruction, + innerInstructions: CompiledInnerInstruction[], +} + +/// Similar to `order_instructions` in `/nft_ingester/src/utils/instructions.rs` +function zipInstructions( + instructions: CompiledInstruction[], + innerInstructions: CompiledInnerInstruction[], +): ZippedInstruction[] { + const zippedIxs: ZippedInstruction[] = []; + let innerIxIndex = 0; + const innerIxMap: Map = new Map(); + for (const innerIx of innerInstructions) { + innerIxMap.set(innerIx.index, innerIx); + } + for (const [instructionIndex, instruction] of instructions.entries()) { + zippedIxs.push({ + instructionIndex, + instruction, + innerInstructions: innerIxMap.has(instructionIndex) ? [innerIxMap.get(instructionIndex)] : [] + }) + } + return zippedIxs; +} + +export function handleInstructionsAtomic( db: NFTDatabaseConnection, - logs: Logs, + instructionInfo: { + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + innerInstructions: CompiledInnerInstruction[], + }, + txId: string, context: Context, parsedState: ParserState, startSeq: number | null = null, endSeq: number | null = null ) { - if (logs.err) { - return; - } - const parsedLogs = parseLogs(logs.logs); - if (parsedLogs.length == 0) { - return; - } - for (const parsedLog of parsedLogs) { - await indexParsedLog( + const { accountKeys, instructions, innerInstructions } = instructionInfo; + + const zippedInstructions = zipInstructions(instructions, innerInstructions); + for (const zippedInstruction of zippedInstructions) { + indexZippedInstruction( db, - { txId: logs.signature, startSeq, endSeq }, + { txId, startSeq, endSeq }, context.slot, parsedState, - parsedLog - ); + accountKeys, + zippedInstruction, + ) + } +} + +export function loadPrograms(provider: anchor.Provider) { + const Gummyroll = loadProgram( + provider, + GUMMYROLL_PROGRAM_ID, + "target/idl/gummyroll.json" + ) as anchor.Program; + const Bubblegum = loadProgram( + provider, + BUBBLEGUM_PROGRAM_ID, + "target/idl/bubblegum.json" + ) as anchor.Program; + const GumballMachine = loadProgram( + provider, + GUMBALL_MACHINE_ID, + "target/idl/gumball_machine.json" + ) as anchor.Program; + return { Gummyroll, Bubblegum, GumballMachine }; +} + +export function hashMetadata(message: MetadataArgs) { + // Todo: fix Solita - This is an issue with beet serializing complex enums + message.tokenStandard = getTokenStandard(message.tokenStandard); + message.tokenProgramVersion = getTokenProgramVersion(message.tokenProgramVersion); + + const [serialized, byteSize] = metadataArgsBeet.serialize(message); + if (byteSize < 20) { + console.log(serialized.length); + console.error("Unable to serialize metadata args properly") + } + return digest(serialized) +} + +type UnverifiedCreator = { + address: PublicKey, + share: number +}; + +export const unverifiedCreatorBeet = new beet.BeetArgsStruct( + [ + ['address', beetSolana.publicKey], + ['share', beet.u8], + ], + 'UnverifiedCreator' +) + +export function hashCreators(creators: Creator[]) { + const bytes = []; + for (const creator of creators) { + const unverifiedCreator = { + address: creator.address, + share: creator.share + } + const [buffer, _byteSize] = unverifiedCreatorBeet.serialize(unverifiedCreator); + bytes.push(buffer); + } + return digest(Buffer.concat(bytes)); +} + +export async function leafSchemaFromLeafData( + owner: PublicKey, + delegate: PublicKey, + treeId: PublicKey, + newLeafData: NewLeafEvent +): Promise { + const id = await getLeafAssetId(treeId, newLeafData.nonce); + return { + schema: { + v1: { + id, + owner, + delegate, + dataHash: [...hashMetadata(newLeafData.metadata)], + creatorHash: [...hashCreators(newLeafData.metadata.creators)], + nonce: newLeafData.nonce, + } + } + } +} + +export function digest(input: Buffer): Buffer { + return Buffer.from(keccak_256.digest(input)) +} + + +function getTokenProgramVersion(object: Object): TokenProgramVersion { + if (Object.keys(object).includes("original")) { + return TokenProgramVersion.Original + } else if (Object.keys(object).includes("token2022")) { + return TokenProgramVersion.Token2022 + } else { + return object as TokenProgramVersion; + } +} + +function getTokenStandard(object: Object): TokenStandard { + if (!object) { return null }; + const keys = Object.keys(object); + if (keys.includes("nonFungible")) { + return TokenStandard.NonFungible + } else if (keys.includes("fungible")) { + return TokenStandard.Fungible + } else if (keys.includes("fungibleAsset")) { + return TokenStandard.FungibleAsset + } else if (keys.includes("nonFungibleEdition")) { + return TokenStandard.NonFungibleEdition + } else { + return object as TokenStandard; + } +} + +/// Returns number of instructions read through +export function findWrapInstructions( + accountKeys: PublicKey[], + instructions: CompiledInstruction[], + amount: number, +): [CompiledInstruction[], number] { + let count = 0; + let found: CompiledInstruction[] = []; + while (found.length < amount && count < instructions.length) { + const ix = instructions[count]; + if (accountKeys[ix.programIdIndex].equals(CANDY_WRAPPER_PROGRAM_ID)) { + found.push(ix); + } + count += 1; + } + if (found.length < amount) { + throw new Error(`Unable to find ${amount} wrap instructions: found ${found.length}`) } + return [found, count]; } diff --git a/contracts/sdk/indexer/package.json b/contracts/sdk/indexer/package.json index ba9dd14dc4d..959fb692b46 100644 --- a/contracts/sdk/indexer/package.json +++ b/contracts/sdk/indexer/package.json @@ -6,6 +6,7 @@ "dependencies": { "@project-serum/anchor": "0.21", "@solana/web3.js": "^1.43.6", + "borsh": "^0.7.0", "express": "^4.18.1", "sqlite": "^4.1.1", "sqlite3": "^5.0.8" diff --git a/contracts/sdk/indexer/scripts/truncate.ts b/contracts/sdk/indexer/scripts/truncate.ts new file mode 100644 index 00000000000..e030f431d77 --- /dev/null +++ b/contracts/sdk/indexer/scripts/truncate.ts @@ -0,0 +1,454 @@ +import { + Keypair, + Connection, + TransactionResponse, + TransactionInstruction, + PublicKey, + SYSVAR_SLOT_HASHES_PUBKEY, + SYSVAR_INSTRUCTIONS_PUBKEY, + LAMPORTS_PER_SOL, + SystemProgram, + ComputeBudgetProgram, +} from "@solana/web3.js"; +import * as anchor from '@project-serum/anchor'; +import NodeWallet from "@project-serum/anchor/dist/cjs/nodewallet"; +import { CANDY_WRAPPER_PROGRAM_ID } from "../../utils"; +import { getBubblegumAuthorityPDA, getCreateTreeIxs, getLeafAssetId } from "../../bubblegum/src/convenience"; +import { addProof, getMerkleRollAccountSize, PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from '../../gummyroll'; +import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../../bubblegum/src/generated"; +import { + TokenStandard, + MetadataArgs, + TokenProgramVersion, + createTransferInstruction, + createMintV1Instruction, + LeafSchema, + leafSchemaBeet, +} from "../../bubblegum/src/generated"; +import { execute, num32ToBuffer } from "../../../tests/utils"; +import { hashCreators, hashMetadata } from "../indexer/utils"; +import { BN } from "@project-serum/anchor"; +import { bs58 } from "@project-serum/anchor/dist/cjs/utils/bytes"; +import fetch from "node-fetch"; +import { keccak_256 } from 'js-sha3'; +import { BinaryWriter } from 'borsh'; +import { createAddConfigLinesInstruction, createInitializeGumballMachineIxs, decodeGumballMachine, EncodeMethod, GumballMachine, gumballMachineHeaderBeet, InitializeGumballMachineInstructionArgs } from "../../gumball-machine"; +import { getWillyWonkaPDAKey } from "../../gumball-machine"; +import { createDispenseNFTForSolIx } from "../../gumball-machine"; +import { loadPrograms } from "../indexer/utils"; +import { strToByteArray } from "../../utils"; +import { NATIVE_MINT } from "@solana/spl-token"; + +// const url = "http://api.explorer.mainnet-beta.solana.com"; +const url = "http://127.0.0.1:8899"; + +function keypairFromString(seed: string) { + const spaces = " "; + const buffer = Buffer.from(`${seed}${spaces}`.slice(0, 32));; + return Keypair.fromSeed(Uint8Array.from(buffer)); +} + +const MAX_BUFFER_SIZE = 256; +const MAX_DEPTH = 20; +const CANOPY_DEPTH = 5; + +/** + * Truncates logs by sending too many append instructions + * This forces the indexer to go into gap-filling mode + * and use the WRAP CPI args to complete the database. + */ +async function main() { + const endpoint = url; + const connection = new Connection(endpoint, "confirmed"); + const payer = keypairFromString('bubblegum-mini-milady'); + const provider = new anchor.Provider(connection, new NodeWallet(payer), { + commitment: "confirmed", + }); + + // // TODO: add gumball-machine version of truncate(test cpi indexing using instruction data) + // let { txId, tx } = await truncateViaBubblegum(connection, provider, payer); + // checkTxTruncated(tx); + + // // TOOD: add this after gumball-machine mints + // let results = await testWithBubblegumTransfers(connection, provider, payer); + // results.txs.map((tx) => { + // checkTxTruncated(tx); + // }) + + const { GumballMachine } = loadPrograms(provider); + await truncateWithGumball( + connection, provider, payer, GumballMachine + ); +} + +function checkTxTruncated(tx: TransactionResponse) { + if (tx.meta.logMessages) { + let logsTruncated = false; + for (const log of tx.meta.logMessages) { + if (log.startsWith('Log truncated')) { + logsTruncated = true; + } + } + console.log(`Logs truncated: ${logsTruncated}`); + } else { + console.error("NO LOG MESSAGES FOUND AT ALL...error!!!") + } +} + +function getMetadata(num: number): MetadataArgs { + return { + name: `${num}`, + symbol: `MILADY`, + uri: "http://remilia.org", + sellerFeeBasisPoints: 0, + primarySaleHappened: false, + isMutable: false, + uses: null, + collection: null, + creators: [], + tokenProgramVersion: TokenProgramVersion.Original, + tokenStandard: TokenStandard.NonFungible, + editionNonce: 0, + } +} + +async function truncateViaBubblegum( + connection: Connection, + provider: anchor.Provider, + payer: Keypair, +) { + const bgumTree = keypairFromString("bubblegum-mini-tree"); + const authority = await getBubblegumAuthorityPDA(bgumTree.publicKey); + + const acctInfo = await connection.getAccountInfo(bgumTree.publicKey, "confirmed"); + let createIxs = []; + if (!acctInfo || acctInfo.lamports === 0) { + console.log("Creating tree:", bgumTree.publicKey.toBase58()); + console.log("Requesting airdrop:", await connection.requestAirdrop(payer.publicKey, 5e10)); + createIxs = await getCreateTreeIxs(connection, MAX_DEPTH, MAX_BUFFER_SIZE, CANOPY_DEPTH, payer.publicKey, bgumTree.publicKey, payer.publicKey); + console.log(""); + } else { + console.log("Bubblegum tree already exists:", bgumTree.publicKey.toBase58()); + } + + const mintIxs = []; + for (let i = 0; i < 6; i++) { + const metadata = getMetadata(i); + mintIxs.push(createMintV1Instruction( + { + owner: payer.publicKey, + delegate: payer.publicKey, + authority, + candyWrapper: CANDY_WRAPPER_PROGRAM_ID, + gummyrollProgram: GUMMYROLL_PROGRAM_ID, + mintAuthority: payer.publicKey, + merkleSlab: bgumTree.publicKey, + }, + { message: metadata } + )); + } + console.log("Sending multiple mint ixs in a transaction"); + const ixs = createIxs.concat(mintIxs); + const txId = await execute(provider, ixs, [payer, bgumTree], true); + console.log(`Executed multiple mint ixs here: ${txId}`); + const tx = await connection.getTransaction(txId, { commitment: 'confirmed' }); + return { txId, tx }; +} + +type ProofResult = { + dataHash: number[], + creatorHash: number[], + root: number[], + proofNodes: Buffer[], + nonce: number, + index: number, +} + +async function getTransferInfoFromServer(leafHash: Buffer, treeId: PublicKey): Promise { + const proofServerUrl = "http://127.0.0.1:4000/proof"; + const hash = bs58.encode(leafHash); + const url = `${proofServerUrl}?leafHash=${hash}&treeId=${treeId.toString()}`; + const response = await fetch( + url, + { method: "GET" } + ); + const proof = await response.json(); + return { + dataHash: [...bs58.decode(proof.dataHash as string)], + creatorHash: [...bs58.decode(proof.creatorHash as string)], + root: [...bs58.decode(proof.root as string)], + proofNodes: (proof.proofNodes as string[]).map((node) => bs58.decode(node)), + nonce: proof.nonce, + index: proof.index, + }; +} + +// todo: expose somewhere in utils +function digest(input: Buffer): Buffer { + return Buffer.from(keccak_256.digest(input)) +} + +/// Typescript impl of LeafSchema::to_node() +function hashLeafSchema(leafSchema: LeafSchema, dataHash: Buffer, creatorHash: Buffer): Buffer { + // Fix issue with solita, the following code should work, but doesn't seem to + // const result = leafSchemaBeet.toFixedFromValue(leafSchema); + // const buffer = Buffer.alloc(result.byteSize); + // result.write(buffer, 0, leafSchema); + + const writer = new BinaryWriter(); + // When we have versions other than V1, we definitely want to use solita + writer.writeU8(1); + writer.writeFixedArray(leafSchema.id.toBuffer()); + writer.writeFixedArray(leafSchema.owner.toBuffer()); + writer.writeFixedArray(leafSchema.delegate.toBuffer()); + writer.writeFixedArray(new BN(leafSchema.nonce).toBuffer('le', 8)); + writer.writeFixedArray(dataHash); + writer.writeFixedArray(creatorHash); + const buf = Buffer.from(writer.toArray()); + return digest(buf); +} + +async function testWithBubblegumTransfers( + connection: Connection, + provider: anchor.Provider, + payer: Keypair, +) { + const bgumTree = keypairFromString("bubblegum-mini-tree"); + const authority = await getBubblegumAuthorityPDA(bgumTree.publicKey); + + // const acctInfo = await connection.getAccountInfo(bgumTree.publicKey, "confirmed"); + // const merkleRoll = decodeMerkleRoll(acctInfo.data); + // const root = Array.from(merkleRoll.roll.changeLogs[merkleRoll.roll.activeIndex].root.toBytes()); + + const txIds = []; + const txs = []; + const finalDestination = keypairFromString("bubblegum-final-destination"); + for (let i = 0; i < 6; i++) { + const metadata = getMetadata(i); + const computedDataHash = hashMetadata(metadata); + const computedCreatorHash = hashCreators(metadata.creators); + const leafSchema: LeafSchema = { + __kind: "V1", + id: await getLeafAssetId(bgumTree.publicKey, new BN(i)), + owner: payer.publicKey, + delegate: payer.publicKey, + nonce: new BN(i), + dataHash: [...computedDataHash], + creatorHash: [...computedCreatorHash], + }; + const leafHash = hashLeafSchema(leafSchema, computedDataHash, computedCreatorHash); + console.log("Data hash:", bs58.encode(computedDataHash)); + console.log("Creator hash:", bs58.encode(computedCreatorHash)); + console.log("schema:", { + id: leafSchema.id.toString(), + owner: leafSchema.owner.toString(), + delegate: leafSchema.owner.toString(), + nonce: new BN(i), + }); + const { root, dataHash, creatorHash, proofNodes, nonce, index } = await getTransferInfoFromServer(leafHash, bgumTree.publicKey); + const transferIx = addProof(createTransferInstruction({ + authority, + candyWrapper: CANDY_WRAPPER_PROGRAM_ID, + gummyrollProgram: GUMMYROLL_PROGRAM_ID, + owner: payer.publicKey, + delegate: payer.publicKey, + newOwner: finalDestination.publicKey, + merkleSlab: bgumTree.publicKey, + }, { + dataHash, + creatorHash, + nonce, + root, + index, + }), proofNodes.slice(0, MAX_DEPTH - CANOPY_DEPTH)); + txIds.push(await execute(provider, [transferIx], [payer], true)); + txs.push(await connection.getTransaction(txIds[txIds.length - 1], { commitment: 'confirmed' })); + } + console.log(`Transferred all NFTs to ${finalDestination.publicKey.toString()}`); + console.log(`Executed multiple transfer ixs here: ${txIds}`); + return { txIds, txs }; +} + +async function initializeGumballMachine( + payer: Keypair, + creator: Keypair, + gumballMachineAcctKeypair: Keypair, + gumballMachineAcctSize: number, + merkleRollKeypair: Keypair, + merkleRollAccountSize: number, + gumballMachineInitArgs: InitializeGumballMachineInstructionArgs, + mint: PublicKey, + gumballMachine: anchor.Program, +) { + const initializeGumballMachineInstrs = + await createInitializeGumballMachineIxs( + payer, + gumballMachineAcctKeypair, + gumballMachineAcctSize, + merkleRollKeypair, + merkleRollAccountSize, + gumballMachineInitArgs, + mint, + creator.publicKey, + GUMMYROLL_PROGRAM_ID, + BUBBLEGUM_PROGRAM_ID, + gumballMachine + ); + await execute( + gumballMachine.provider, + initializeGumballMachineInstrs, + [payer, gumballMachineAcctKeypair, merkleRollKeypair, creator], + true + ); +} + +async function addConfigLines( + provider: anchor.Provider, + authority: Keypair, + gumballMachineAcctKey: PublicKey, + configLinesToAdd: Uint8Array, +) { + const addConfigLinesInstr = createAddConfigLinesInstruction( + { + gumballMachine: gumballMachineAcctKey, + authority: authority.publicKey, + }, + { + newConfigLinesData: configLinesToAdd, + } + ); + await execute( + provider, + [addConfigLinesInstr], + [authority] + ) +} + +async function dispenseCompressedNFTForSol( + numNFTs: BN, + payer: Keypair, + receiver: PublicKey, + gumballMachineAcctKeypair: Keypair, + merkleRollKeypair: Keypair, + gumballMachine: anchor.Program, +) { + const additionalComputeBudgetInstruction = ComputeBudgetProgram.requestUnits({ + units: 1000000, + additionalFee: 0, + }); + const dispenseInstr = await createDispenseNFTForSolIx( + { numItems: numNFTs }, + payer.publicKey, + receiver, + gumballMachineAcctKeypair.publicKey, + merkleRollKeypair.publicKey, + GUMMYROLL_PROGRAM_ID, + BUBBLEGUM_PROGRAM_ID, + gumballMachine + ); + const txId = await execute( + gumballMachine.provider, + [additionalComputeBudgetInstruction, dispenseInstr], + [payer], + true + ); +} + +async function truncateWithGumball( + connection: Connection, + provider: anchor.Provider, + payer: Keypair, + gumballMachine: anchor.Program, +) { + const EXTENSION_LEN = 28; + const MAX_MINT_SIZE = 10; + const GUMBALL_MACHINE_ACCT_CONFIG_INDEX_ARRAY_SIZE = 1000; + const GUMBALL_MACHINE_ACCT_CONFIG_LINES_SIZE = 7000; + const GUMBALL_MACHINE_ACCT_SIZE = + gumballMachineHeaderBeet.byteSize + + GUMBALL_MACHINE_ACCT_CONFIG_INDEX_ARRAY_SIZE + + GUMBALL_MACHINE_ACCT_CONFIG_LINES_SIZE; + const MERKLE_ROLL_ACCT_SIZE = getMerkleRollAccountSize(3, 8); + + const creatorAddress = keypairFromString('gumball-machine-creat0r'); + const gumballMachineAcctKeypair = keypairFromString('gumball-machine-acct') + const merkleRollKeypair = keypairFromString("gumball-machine-tree"); + const nftBuyer = keypairFromString("gumball-machine-buyer") + const botWallet = Keypair.generate(); + + // Give creator enough funds to produce accounts for gumball-machine + await connection.requestAirdrop( + creatorAddress.publicKey, + 4 * LAMPORTS_PER_SOL, + ); + await connection.requestAirdrop( + payer.publicKey, + 11 * LAMPORTS_PER_SOL, + ); + await connection.requestAirdrop( + nftBuyer.publicKey, + 11 * LAMPORTS_PER_SOL, + ); + console.log("airdrop successfull") + + const baseGumballMachineInitProps = { + maxDepth: 3, + maxBufferSize: 8, + urlBase: strToByteArray("https://arweave.net", 64), + nameBase: strToByteArray("Milady", 32), + symbol: strToByteArray("MILADY", 8), + sellerFeeBasisPoints: 100, + isMutable: true, + retainAuthority: true, + encodeMethod: EncodeMethod.Base58Encode, + price: new BN(0.1), + goLiveDate: new BN(1234.0), + botWallet: botWallet.publicKey, + // receiver: creatorReceiverTokenAccount.address, + receiver: creatorAddress.publicKey, + authority: creatorAddress.publicKey, + collectionKey: SystemProgram.programId, // 0x0 -> no collection key + extensionLen: new BN(EXTENSION_LEN), + maxMintSize: new BN(MAX_MINT_SIZE), + maxItems: new BN(250), + }; + + await initializeGumballMachine( + payer, + creatorAddress, + gumballMachineAcctKeypair, + GUMBALL_MACHINE_ACCT_SIZE, + merkleRollKeypair, + MERKLE_ROLL_ACCT_SIZE, + baseGumballMachineInitProps, + NATIVE_MINT, + gumballMachine + ); + console.log('init`d'); + + // add 10 config lines + let arr: number[] = []; + const buffers = []; + for (let i = 0; i < MAX_MINT_SIZE + 1; i++) { + const str = `url-${i} `.slice(0, EXTENSION_LEN); + arr = arr.concat(strToByteArray(str)); + buffers.push(Buffer.from(str)); + } + await addConfigLines( + provider, + creatorAddress, + gumballMachineAcctKeypair.publicKey, + Buffer.from(arr), + ); + + await dispenseCompressedNFTForSol( + new BN(6), + nftBuyer, + creatorAddress.publicKey, + gumballMachineAcctKeypair, + merkleRollKeypair, + gumballMachine + ) +} + +main(); diff --git a/contracts/tests/bubblegum-test.ts b/contracts/tests/bubblegum-test.ts index 05d7c6dcf46..c6d50fd373b 100644 --- a/contracts/tests/bubblegum-test.ts +++ b/contracts/tests/bubblegum-test.ts @@ -38,9 +38,9 @@ import { TOKEN_PROGRAM_ID, Token, } from "@solana/spl-token"; -import { bufferToArray } from "./utils"; +import { bufferToArray, execute } from "./utils"; import { TokenProgramVersion, Version } from "../sdk/bubblegum/src/generated"; -import { CANDY_WRAPPER_PROGRAM_ID, execute, logTx } from "../sdk/utils"; +import { CANDY_WRAPPER_PROGRAM_ID } from "../sdk/utils"; import { getBubblegumAuthorityPDA, getCreateTreeIxs, getNonceCount, getVoucherPDA } from "../sdk/bubblegum/src/convenience"; // @ts-ignore diff --git a/contracts/tests/gumball-machine-test.ts b/contracts/tests/gumball-machine-test.ts index 094f606587d..79a66910dd2 100644 --- a/contracts/tests/gumball-machine-test.ts +++ b/contracts/tests/gumball-machine-test.ts @@ -8,6 +8,7 @@ import { Transaction, Connection as web3Connection, LAMPORTS_PER_SOL, + ComputeBudgetProgram, } from "@solana/web3.js"; import { assert } from "chai"; @@ -37,7 +38,6 @@ import { val, strToByteArray, strToByteUint8Array, - logTx } from "../sdk/utils/index"; import { GumballMachineHeader, @@ -51,7 +51,7 @@ import { getAccount, } from "../../deps/solana-program-library/token/js/src"; import { NATIVE_MINT } from "@solana/spl-token"; -import { num32ToBuffer, arrayEquals } from "./utils"; +import { num32ToBuffer, arrayEquals, execute, logTx } from "./utils"; import { EncodeMethod } from "../sdk/gumball-machine/src/generated/types/EncodeMethod"; import { getBubblegumAuthorityPDA } from "../sdk/bubblegum/src/convenience"; @@ -423,13 +423,12 @@ describe("gumball-machine", () => { BubblegumProgramId, GumballMachine ); - const tx = new Transaction().add(dispenseInstr); - let txId = await GumballMachine.provider.send(tx, [payer], { - commitment: "confirmed", - }); - if (verbose) { - await logTx(GumballMachine.provider, txId); - } + const txId = await execute( + GumballMachine.provider, + [dispenseInstr], + [payer], + true + ); } async function dispenseCompressedNFTForTokens( @@ -543,7 +542,7 @@ describe("gumball-machine", () => { botWallet: Keypair.generate().publicKey, receiver: creatorPaymentWallet.publicKey, authority: creatorAddress.publicKey, - collectionKey: SystemProgram.programId, + collectionKey: SystemProgram.programId, // 0x0 -> no collection key extensionLen: new BN(28), maxMintSize: new BN(10), maxItems: new BN(250), @@ -567,6 +566,7 @@ describe("gumball-machine", () => { baseGumballMachineInitProps, NATIVE_MINT ); + await addConfigLines( creatorAddress, gumballMachineAcctKeypair.publicKey, @@ -626,37 +626,43 @@ describe("gumball-machine", () => { const tx = new Transaction() .add(dispenseNFTForSolInstr) .add(dummyInstr); + let confirmedTxId: string; try { - await GumballMachine.provider.send( + confirmedTxId = await GumballMachine.provider.send( tx, [nftBuyer, payer, dummyNewAcctKeypair], { commitment: "confirmed", } ); + } catch (e) { } + + if (confirmedTxId) assert( false, "Dispense should fail when part of transaction with multiple instructions, but it succeeded" ); - } catch (e) { } }); it("Cannot dispense NFT for SOL with prior instructions in transaction", async () => { const tx = new Transaction() .add(dummyInstr) .add(dispenseNFTForSolInstr); + let confirmedTxId: string; try { - await GumballMachine.provider.send( + confirmedTxId = await GumballMachine.provider.send( tx, [nftBuyer, payer, dummyNewAcctKeypair], { commitment: "confirmed", } ); + } catch (e) { } + + if (confirmedTxId) assert( false, "Dispense should fail when part of transaction with multiple instructions, but it succeeded" ); - } catch (e) { } }); }); it("Can dispense single NFT paid in sol", async () => { @@ -968,15 +974,15 @@ describe("gumball-machine", () => { ); // Since there were only two config lines added, we should have only successfully minted (and paid for) two NFTs - const newExpectedCreatorTokenBalance = Number(creatorReceiverTokenAccount.amount) - + (val(baseGumballMachineInitProps.price).toNumber() * 2); + const newExpectedCreatorTokenBalance = Number(creatorReceiverTokenAccount.amount) + + (val(baseGumballMachineInitProps.price).toNumber() * 2); assert( Number(newCreatorTokenAccount.amount) === newExpectedCreatorTokenBalance, "The creator did not receive their payment as expected" ); - const newExpectedBuyerTokenBalance = Number(buyerTokenAccount.amount) - - (val(baseGumballMachineInitProps.price).toNumber() * 2); + const newExpectedBuyerTokenBalance = Number(buyerTokenAccount.amount) + - (val(baseGumballMachineInitProps.price).toNumber() * 2); assert( Number(newBuyerTokenAccount.amount) === newExpectedBuyerTokenBalance, "The nft buyer did not pay for the nft as expected" @@ -993,7 +999,7 @@ describe("gumball-machine", () => { merkleRollKeypair ); assert(false, "Dispense unexpectedly succeeded with no NFTs remaining"); - } catch(e) {} + } catch (e) { } }); }); }); diff --git a/contracts/tests/utils.ts b/contracts/tests/utils.ts index 778cfa4d861..f1358208176 100644 --- a/contracts/tests/utils.ts +++ b/contracts/tests/utils.ts @@ -1,7 +1,38 @@ +import { Provider } from "@project-serum/anchor"; +import { TransactionInstruction, Transaction, Signer } from "@solana/web3.js"; + +/// Wait for a transaction of a certain id to confirm and optionally log its messages +export async function logTx(provider: Provider, txId: string, verbose: boolean = true) { + await provider.connection.confirmTransaction(txId, "confirmed"); + if (verbose) { + console.log( + (await provider.connection.getConfirmedTransaction(txId, "confirmed")).meta + .logMessages + ); + } +}; + +/// Execute a series of instructions in a txn +export async function execute( + provider: Provider, + instructions: TransactionInstruction[], + signers: Signer[], + skipPreflight: boolean = false +): Promise { + let tx = new Transaction(); + instructions.map((ix) => { tx = tx.add(ix) }); + const txid = await provider.send(tx, signers, { + commitment: "confirmed", + skipPreflight, + }); + await logTx(provider, txid, true); + return txid; +} + /// Convert a 32 bit number to a buffer of bytes export function num32ToBuffer(num: number) { - const isU32 = (num >= 0 && num < Math.pow(2,32)); - const isI32 = (num >= -1*Math.pow(2, 31) && num < Math.pow(2,31)) + const isU32 = (num >= 0 && num < Math.pow(2, 32)); + const isI32 = (num >= -1 * Math.pow(2, 31) && num < Math.pow(2, 31)) if (!isU32 || !isI32) { throw new Error("Attempted to convert non 32 bit integer to byte array") } @@ -15,9 +46,9 @@ export function num32ToBuffer(num: number) { /// Check if two Array types contain the same values in order export function arrayEquals(a, b) { return Array.isArray(a) && - Array.isArray(b) && - a.length === b.length && - a.every((val, index) => val === b[index]); + Array.isArray(b) && + a.length === b.length && + a.every((val, index) => val === b[index]); } /// Convert Buffer to Uint8Array diff --git a/contracts/yarn.lock b/contracts/yarn.lock index 12fd08f6c51..01a8253772f 100644 --- a/contracts/yarn.lock +++ b/contracts/yarn.lock @@ -16,6 +16,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.17.2": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.6.tgz#6a1ef59f838debd670421f8c7f2cbb8da9751580" + integrity sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ== + dependencies: + regenerator-runtime "^0.13.4" + "@ethersproject/bytes@^5.5.0": version "5.5.0" resolved "https://registry.npmjs.org/@ethersproject/bytes/-/bytes-5.5.0.tgz" @@ -426,22 +433,24 @@ superstruct "^0.14.2" tweetnacl "^1.0.0" -"@solana/web3.js@^1.37.0": - version "1.37.0" - resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.37.0.tgz" - integrity sha512-O2iCcgkGdi2FXwVLztPIZHcBuZXdhbVLavMsG+RdEyFGzFD0tQN1rOJ+Xb5eaexjqtgcqRN+Fyg3wAhLcHJbiA== +"@solana/web3.js@^1.47.1": + version "1.47.1" + resolved "https://registry.yarnpkg.com/@solana/web3.js/-/web3.js-1.47.1.tgz#21243d4a19836584bc3429d3cee602191646f845" + integrity sha512-snsEZDR4smySxY6ZBNJUPStTVKkbgBiel14VZDjRC79Pcol3nbxjlg6HGg4rvfCSkSbpDPJPPwoTImigflEBBg== dependencies: "@babel/runtime" "^7.12.5" "@ethersproject/sha2" "^5.5.0" "@solana/buffer-layout" "^4.0.0" + bigint-buffer "^1.1.5" bn.js "^5.0.0" borsh "^0.7.0" bs58 "^4.0.1" buffer "6.0.1" - cross-fetch "^3.1.4" + fast-stable-stringify "^1.0.0" jayson "^3.4.4" js-sha3 "^0.8.0" - rpc-websockets "^7.4.2" + node-fetch "2" + rpc-websockets "^7.5.0" secp256k1 "^4.0.2" superstruct "^0.14.2" tweetnacl "^1.0.0" @@ -2545,6 +2554,19 @@ rpc-websockets@^7.4.2: bufferutil "^4.0.1" utf-8-validate "^5.0.2" +rpc-websockets@^7.5.0: + version "7.5.0" + resolved "https://registry.yarnpkg.com/rpc-websockets/-/rpc-websockets-7.5.0.tgz#bbeb87572e66703ff151e50af1658f98098e2748" + integrity sha512-9tIRi1uZGy7YmDjErf1Ax3wtqdSSLIlnmL5OtOzgd5eqPKbsPpwDP5whUDO2LQay3Xp0CcHlcNSGzacNRluBaQ== + dependencies: + "@babel/runtime" "^7.17.2" + eventemitter3 "^4.0.7" + uuid "^8.3.2" + ws "^8.5.0" + optionalDependencies: + bufferutil "^4.0.1" + utf-8-validate "^5.0.2" + rxjs@^7.1.0: version "7.5.5" resolved "https://registry.npmjs.org/rxjs/-/rxjs-7.5.5.tgz" @@ -3144,6 +3166,11 @@ ws@^7.4.5: resolved "https://registry.npmjs.org/ws/-/ws-7.5.6.tgz" integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA== +ws@^8.5.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.0.tgz#8e71c75e2f6348dbf8d78005107297056cb77769" + integrity sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ== + ws@~8.2.3: version "8.2.3" resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba"