Add backfill script to Jankle (solana-labs#119)

* jankle update 0

* add backfill script

* server update

* cleanup backfiller

* add txid to assets query

* add txid to assets query

* add txid to assets query

* update txId

* add compressed field to assets query

Co-authored-by: Noah Gundotra <noahgundotra@noahs-mbp.mynetworksettings.com>

contracts/package.json

@@ -8,6 +8,7 @@
     "@solana/spl-token": "^0.1.8",
     "@solana/web3.js": "^1.37.0",
     "collections": "^5.1.13",
+    "cors": "^2.8.5",
     "express": "^4.18.1",
     "pg-promise": "^10.11.1",
     "react": "^18.1.0",
@@ -16,14 +17,14 @@
     "typescript-collections": "^1.3.3"
   },
   "devDependencies": {
+    "@metaplex-foundation/rustbin": "^0.3.1",
+    "@metaplex-foundation/solita": "^0.10.0",
     "@types/chai": "^4.3.0",
     "@types/mocha": "^9.0.0",
     "chai": "^4.3.4",
     "mocha": "^9.0.3",
     "ts-mocha": "^10.0.0",
-    "typescript": "^4.3.5",
-    "@metaplex-foundation/rustbin": "^0.3.1",
-    "@metaplex-foundation/solita": "^0.10.0"
+    "typescript": "^4.3.5"
   },
   "scripts": {
     "update-deps": "cd deps/metaplex-program-library; git fetch; git checkout 7e2810a; cd ../solana-program-library; git fetch; cd ../..",

contracts/sdk/bubblegum/idl/bubblegum.json

@@ -893,6 +893,12 @@
           },
           {
             "name": "DecompressV1"
+          },
+          {
+            "name": "Compress"
+          },
+          {
+            "name": "Burn"
           }
         ]
       }
@@ -951,6 +957,33 @@
           "index": false
         }
       ]
+    },
+    {
+      "name": "NFTDecompressionEvent",
+      "fields": [
+        {
+          "name": "version",
+          "type": {
+            "defined": "Version"
+          },
+          "index": false
+        },
+        {
+          "name": "id",
+          "type": "publicKey",
+          "index": false
+        },
+        {
+          "name": "treeId",
+          "type": "publicKey",
+          "index": false
+        },
+        {
+          "name": "nonce",
+          "type": "u64",
+          "index": false
+        }
+      ]
     }
   ],
   "errors": [
@@ -963,6 +996,16 @@
       "code": 6001,
       "name": "PublicKeyMismatch",
       "msg": "PublicKeyMismatch"
+    },
+    {
+      "code": 6002,
+      "name": "HashingMismatch",
+      "msg": "Hashing Mismatch Within Leaf Schema"
+    },
+    {
+      "code": 6003,
+      "name": "UnsupportedSchemaVersion",
+      "msg": "Unsupported Schema Version"
     }
   ],
   "metadata": {

contracts/sdk/bubblegum/src/generated/errors/index.ts

@@ -57,6 +57,52 @@ createErrorFromNameLookup.set(
   () => new PublicKeyMismatchError()
 )
 
+/**
+ * HashingMismatch: 'Hashing Mismatch Within Leaf Schema'
+ *
+ * @category Errors
+ * @category generated
+ */
+export class HashingMismatchError extends Error {
+  readonly code: number = 0x1772
+  readonly name: string = 'HashingMismatch'
+  constructor() {
+    super('Hashing Mismatch Within Leaf Schema')
+    if (typeof Error.captureStackTrace === 'function') {
+      Error.captureStackTrace(this, HashingMismatchError)
+    }
+  }
+}
+
+createErrorFromCodeLookup.set(0x1772, () => new HashingMismatchError())
+createErrorFromNameLookup.set(
+  'HashingMismatch',
+  () => new HashingMismatchError()
+)
+
+/**
+ * UnsupportedSchemaVersion: 'Unsupported Schema Version'
+ *
+ * @category Errors
+ * @category generated
+ */
+export class UnsupportedSchemaVersionError extends Error {
+  readonly code: number = 0x1773
+  readonly name: string = 'UnsupportedSchemaVersion'
+  constructor() {
+    super('Unsupported Schema Version')
+    if (typeof Error.captureStackTrace === 'function') {
+      Error.captureStackTrace(this, UnsupportedSchemaVersionError)
+    }
+  }
+}
+
+createErrorFromCodeLookup.set(0x1773, () => new UnsupportedSchemaVersionError())
+createErrorFromNameLookup.set(
+  'UnsupportedSchemaVersion',
+  () => new UnsupportedSchemaVersionError()
+)
+
 /**
  * Attempts to resolve a custom program error from the provided error code.
  * @category Errors

contracts/sdk/bubblegum/src/generated/types/InstructionName.ts

@@ -18,6 +18,8 @@ export enum InstructionName {
   Transfer,
   Delegate,
   DecompressV1,
+  Compress,
+  Burn,
 }
 
 /**

contracts/sdk/bubblegum/src/generated/types/LeafSchema.ts

@@ -49,6 +49,7 @@ export const isLeafSchemaV1 = (
  * @category userTypes
  * @category generated
  */
+// @ts-ignore
 export const leafSchemaBeet = beet.dataEnum<LeafSchemaRecord>([
   [
     'V1',
@@ -64,4 +65,4 @@ export const leafSchemaBeet = beet.dataEnum<LeafSchemaRecord>([
       'LeafSchemaRecord["V1"]'
     ),
   ],
-]) as beet.FixedSizeBeet<LeafSchema> | any
+]) as beet.FixedSizeBeet<LeafSchema>

contracts/sdk/indexer/backfill.ts

@@ -0,0 +1,81 @@
+import { Keypair } from "@solana/web3.js";
+import { Connection } from "@solana/web3.js";
+import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "../bubblegum/src/generated";
+import { PROGRAM_ID as GUMMYROLL_PROGRAM_ID } from "../gummyroll/index";
+import * as anchor from "@project-serum/anchor";
+import { Bubblegum } from "../../target/types/bubblegum";
+import { Gummyroll } from "../../target/types/gummyroll";
+import NodeWallet from "@project-serum/anchor/dist/cjs/nodewallet";
+import { loadProgram, handleLogs, handleLogsAtomic } from "./indexer/utils";
+import { bootstrap, NFTDatabaseConnection } from "./db";
+import { backfillTreeHistory, fetchAndPlugGaps, validateTree } from "./backfiller";
+
+const url = "http://api.internal.mainnet-beta.solana.com";
+// const url = "http://127.0.0.1:8899";
+let Bubblegum: anchor.Program<Bubblegum>;
+let Gummyroll: anchor.Program<Gummyroll>;
+
+async function main() {
+    const treeId = process.argv[2];
+    const endpoint = url;
+    const connection = new Connection(endpoint, "confirmed");
+    const payer = Keypair.generate();
+    const provider = new anchor.Provider(connection, new NodeWallet(payer), {
+        commitment: "confirmed",
+    });
+    let db = await bootstrap();
+    console.log("Finished bootstrapping DB");
+    Gummyroll = loadProgram(
+        provider,
+        GUMMYROLL_PROGRAM_ID,
+        "target/idl/gummyroll.json"
+    ) as anchor.Program<Gummyroll>;
+    Bubblegum = loadProgram(
+        provider,
+        BUBBLEGUM_PROGRAM_ID,
+        "target/idl/bubblegum.json"
+    ) as anchor.Program<Bubblegum>;
+    console.log("loaded programs...");
+    console.log("Filling in gaps for tree:", treeId);
+
+    // Get first gap
+    const trees = await db.getTrees();
+    const treeInfo = trees.filter((tree) => (tree[0] === treeId));
+    let startSeq = 0;
+    let startSlot: number | null = null;
+    if (treeInfo) {
+        let [missingData, maxDbSeq, maxDbSlot] = await db.getMissingData(
+            0,
+            treeId
+        );
+        console.log(missingData, maxDbSeq, maxDbSlot);
+        if (missingData.length) {
+            startSlot = missingData[0].prevSlot;
+            startSeq = missingData[0].prevSeq;
+        } else {
+            startSlot = maxDbSlot;
+            startSeq = maxDbSeq;
+        }
+    }
+
+    // Backfill
+    console.log(`Starting from slot!: ${startSlot} `);
+    const maxSeq = await backfillTreeHistory(connection, db, { Gummyroll, Bubblegum }, treeId, startSeq, startSlot);
+
+    // Validate
+    console.log("Max SEQUENCE: ", maxSeq);
+    const depth = await db.getDepth(treeId);
+    console.log(`Tree ${treeId} has ${depth}`);
+    console.log("Validating")
+    console.log(
+        `    Off - chain tree ${treeId} is consistent: ${await validateTree(
+            db,
+            depth,
+            treeId,
+            maxSeq,
+        )
+        } `
+    );
+}
+
+main();

contracts/sdk/indexer/backfiller.ts

@@ -1,4 +1,4 @@
-import { PublicKey } from "@solana/web3.js";
+import { PublicKey, SIGNATURE_LENGTH_IN_BYTES } from "@solana/web3.js";
 import { Connection } from "@solana/web3.js";
 import { decodeMerkleRoll } from "../gummyroll/index";
 import { ParserState, handleLogsAtomic } from "./indexer/utils";
@@ -113,6 +113,90 @@ async function plugGaps(
   }
 }
 
+function onlyUnique(value, index, self) {
+  return self.indexOf(value) === index;
+}
+
+export async function getAllTreeSlots(
+  connection: Connection,
+  treeId: string,
+  afterSig?: string
+): Promise<number[]> {
+  const treeAddress = new PublicKey(treeId);
+  // todo: paginate
+  let lastAddress: string | null = null;
+  let done = false;
+  const history: number[] = [];
+
+  const baseOpts = afterSig ? { until: afterSig } : {};
+  while (!done) {
+    let opts = lastAddress ? { before: lastAddress } : {};
+    const finalOpts = { ...baseOpts, ...opts };
+    console.log(finalOpts);
+    const sigs = await connection.getSignaturesForAddress(treeAddress, finalOpts);
+    console.log(sigs[sigs.length - 1]);
+    lastAddress = sigs[sigs.length - 1].signature;
+    sigs.map((sigInfo) => {
+      history.push(sigInfo.slot);
+    })
+
+    if (sigs.length < 1000) {
+      done = true;
+    }
+  }
+
+  return history.reverse().filter(onlyUnique);
+}
+
+/// Returns tree history in chronological order (oldest first)
+export async function backfillTreeHistory(
+  connection: Connection,
+  nftDb: NFTDatabaseConnection,
+  parserState: ParserState,
+  treeId: string,
+  startSeq: number,
+  fromSlot: number | null,
+): Promise<number> {
+  const treeAddress = new PublicKey(treeId);
+  const merkleRoll = decodeMerkleRoll(await (await connection.getAccountInfo(treeAddress)).data);
+  const maxSeq = merkleRoll.roll.sequenceNumber.toNumber();
+  // Sequence number on-chain is ready to setup the 
+  if (startSeq === maxSeq - 1) {
+    return startSeq;
+  }
+  const earliestTxId = await nftDb.getTxIdForSlot(treeId, fromSlot);
+  console.log("Tx id:", earliestTxId);
+  const treeHistory = await getAllTreeSlots(connection, treeId, earliestTxId);
+  console.log("Retrieved tree history!", treeHistory);
+
+  let numProcessed = 0;
+  let batchSize = 20;
+  while (numProcessed < treeHistory.length) {
+    const batchJobs = [];
+    for (let i = 0; i < batchSize; i++) {
+      const historyIndex = numProcessed + i;
+      if (historyIndex >= treeHistory.length) {
+        break;
+      }
+      batchJobs.push(
+        plugGapsFromSlot(
+          connection,
+          nftDb,
+          parserState,
+          treeAddress,
+          treeHistory[historyIndex],
+          0,
+          maxSeq,
+        )
+      )
+    }
+    await Promise.all(batchJobs);
+    numProcessed += batchJobs.length;
+    console.log("num processed: ", numProcessed);
+  }
+  return maxSeq;
+}
+
 export async function fetchAndPlugGaps(
   connection: Connection,
   nftDb: NFTDatabaseConnection,