From 937aed139f712069f39d2941ad163195d1da874c Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Thu, 23 Nov 2023 20:25:39 +0530 Subject: [PATCH] Fix sync status update in historical processing job (#488) * Fill watcher start block only in active mode * Fix sync status update in historical processing * Upgrade package versions * Upgrade package versions in codegen template * Fix graph-node test indexer --- lerna.json | 2 +- packages/cache/package.json | 2 +- packages/cli/package.json | 12 +++---- packages/cli/src/server.ts | 33 ++++++++++--------- packages/codegen/package.json | 4 +-- .../templates/database-template.handlebars | 6 ++++ .../src/templates/indexer-template.handlebars | 4 +++ .../src/templates/package-template.handlebars | 10 +++--- packages/graph-node/package.json | 10 +++--- packages/graph-node/test/utils/indexer.ts | 6 ++++ packages/ipld-eth-client/package.json | 6 ++-- packages/peer/package.json | 2 +- packages/rpc-eth-client/package.json | 8 ++--- packages/solidity-mapper/package.json | 2 +- packages/test/package.json | 2 +- packages/tracing-client/package.json | 2 +- packages/util/package.json | 8 ++--- packages/util/src/database.ts | 9 +++++ packages/util/src/indexer.ts | 17 ++++++++++ packages/util/src/job-runner.ts | 13 +++++--- packages/util/src/types.ts | 2 ++ 21 files changed, 104 insertions(+), 56 deletions(-) diff --git a/lerna.json b/lerna.json index 16ac4d1e9..9578bfcf1 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "0.2.74", + "version": "0.2.75", "npmClient": "yarn", "useWorkspaces": true, "command": { diff --git a/packages/cache/package.json b/packages/cache/package.json index f0c076cc4..df0d9ca07 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cache", - "version": "0.2.74", + "version": "0.2.75", "description": "Generic object cache", "main": "dist/index.js", "scripts": { diff --git a/packages/cli/package.json b/packages/cli/package.json index e026c5ebb..12b64e1de 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cli", - "version": "0.2.74", + "version": "0.2.75", "main": "dist/index.js", "license": "AGPL-3.0", "scripts": { @@ -12,13 +12,13 @@ }, "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.74", - "@cerc-io/ipld-eth-client": "^0.2.74", + "@cerc-io/cache": "^0.2.75", + "@cerc-io/ipld-eth-client": "^0.2.75", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.74", - "@cerc-io/rpc-eth-client": "^0.2.74", - "@cerc-io/util": "^0.2.74", + "@cerc-io/peer": "^0.2.75", + "@cerc-io/rpc-eth-client": "^0.2.75", + "@cerc-io/util": "^0.2.75", "@ethersproject/providers": "^5.4.4", "@graphql-tools/utils": "^9.1.1", "@ipld/dag-cbor": "^8.0.0", diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 8ffe131cb..97eec8cab 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -285,25 +285,26 @@ export class ServerCmd { assert(indexer); assert(eventWatcher); - const syncStatus = await indexer.getSyncStatus(); - if (!syncStatus) { - const contracts = await this.database.getContracts(); - const startBlock = getStartBlock(contracts); - await fillBlocks( - jobQueue, - indexer, - eventWatcher, - config.jobQueue.blockDelayInMilliSecs, - { - startBlock, - endBlock: startBlock - } - ); - } - if (config.server.kind === KIND_ACTIVE) { // Delete all active and pending (before completed) jobs to prevent creating jobs after completion of processing previous block await jobQueue.deleteAllJobs('completed'); + + const syncStatus = await indexer.getSyncStatus(); + if (!syncStatus) { + const contracts = await this.database.getContracts(); + const startBlock = getStartBlock(contracts); + await fillBlocks( + jobQueue, + indexer, + eventWatcher, + config.jobQueue.blockDelayInMilliSecs, + { + startBlock, + endBlock: startBlock + } + ); + } + await eventWatcher.start(); } diff --git a/packages/codegen/package.json b/packages/codegen/package.json index fdcb7a35b..7bbd322cb 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/codegen", - "version": "0.2.74", + "version": "0.2.75", "description": "Code generator", "private": true, "main": "index.js", @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/util": "^0.2.74", + "@cerc-io/util": "^0.2.75", "@graphql-tools/load-files": "^6.5.2", "@npmcli/package-json": "^5.0.0", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 58f685c5f..5fe2deab5 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -265,6 +265,12 @@ export class Database implements DatabaseInterface { return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError); } + async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial): Promise { + const repo = queryRunner.manager.getRepository(SyncStatus); + + return this._baseDatabase.updateSyncStatus(repo, syncStatus); + } + async getSyncStatus (queryRunner: QueryRunner): Promise { const repo = queryRunner.manager.getRepository(SyncStatus); diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 637efa977..fa32b83b6 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -673,6 +673,10 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError); } + async updateSyncStatus (syncStatus: DeepPartial): Promise { + return this._baseIndexer.updateSyncStatus(syncStatus); + } + async getEvent (id: string): Promise { return this._baseIndexer.getEvent(id); } diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index d9b847d89..5a68a6e5d 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -41,12 +41,12 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.74", - "@cerc-io/ipld-eth-client": "^0.2.74", - "@cerc-io/solidity-mapper": "^0.2.74", - "@cerc-io/util": "^0.2.74", + "@cerc-io/cli": "^0.2.75", + "@cerc-io/ipld-eth-client": "^0.2.75", + "@cerc-io/solidity-mapper": "^0.2.75", + "@cerc-io/util": "^0.2.75", {{#if (subgraphPath)}} - "@cerc-io/graph-node": "^0.2.74", + "@cerc-io/graph-node": "^0.2.75", {{/if}} "@ethersproject/providers": "^5.4.4", "debug": "^4.3.1", diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index cb14df82e..67309ae8d 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -1,10 +1,10 @@ { "name": "@cerc-io/graph-node", - "version": "0.2.74", + "version": "0.2.75", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { - "@cerc-io/solidity-mapper": "^0.2.74", + "@cerc-io/solidity-mapper": "^0.2.75", "@ethersproject/providers": "^5.4.4", "@graphprotocol/graph-ts": "^0.22.0", "@nomiclabs/hardhat-ethers": "^2.0.2", @@ -51,9 +51,9 @@ "dependencies": { "@apollo/client": "^3.3.19", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", - "@cerc-io/cache": "^0.2.74", - "@cerc-io/ipld-eth-client": "^0.2.74", - "@cerc-io/util": "^0.2.74", + "@cerc-io/cache": "^0.2.75", + "@cerc-io/ipld-eth-client": "^0.2.75", + "@cerc-io/util": "^0.2.75", "@types/json-diff": "^0.5.2", "@types/yargs": "^17.0.0", "bn.js": "^4.11.9", diff --git a/packages/graph-node/test/utils/indexer.ts b/packages/graph-node/test/utils/indexer.ts index 771408a1a..c11c4a4cc 100644 --- a/packages/graph-node/test/utils/indexer.ts +++ b/packages/graph-node/test/utils/indexer.ts @@ -190,6 +190,12 @@ export class Indexer implements IndexerInterface { return undefined; } + async updateSyncStatus (syncStatus: SyncStatusInterface): Promise { + assert(syncStatus); + + return {} as SyncStatusInterface; + } + async markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise { assert(blocks); diff --git a/packages/ipld-eth-client/package.json b/packages/ipld-eth-client/package.json index baa219f18..8de68882d 100644 --- a/packages/ipld-eth-client/package.json +++ b/packages/ipld-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/ipld-eth-client", - "version": "0.2.74", + "version": "0.2.75", "description": "IPLD ETH Client", "main": "dist/index.js", "scripts": { @@ -20,8 +20,8 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.74", - "@cerc-io/util": "^0.2.74", + "@cerc-io/cache": "^0.2.75", + "@cerc-io/util": "^0.2.75", "cross-fetch": "^3.1.4", "debug": "^4.3.1", "ethers": "^5.4.4", diff --git a/packages/peer/package.json b/packages/peer/package.json index 6a68b17a6..d9c9e9a59 100644 --- a/packages/peer/package.json +++ b/packages/peer/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/peer", - "version": "0.2.74", + "version": "0.2.75", "description": "libp2p module", "main": "dist/index.js", "exports": "./dist/index.js", diff --git a/packages/rpc-eth-client/package.json b/packages/rpc-eth-client/package.json index f99ce2c50..bcd03e741 100644 --- a/packages/rpc-eth-client/package.json +++ b/packages/rpc-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/rpc-eth-client", - "version": "0.2.74", + "version": "0.2.75", "description": "RPC ETH Client", "main": "dist/index.js", "scripts": { @@ -19,9 +19,9 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/cache": "^0.2.74", - "@cerc-io/ipld-eth-client": "^0.2.74", - "@cerc-io/util": "^0.2.74", + "@cerc-io/cache": "^0.2.75", + "@cerc-io/ipld-eth-client": "^0.2.75", + "@cerc-io/util": "^0.2.75", "chai": "^4.3.4", "ethers": "^5.4.4", "left-pad": "^1.3.0", diff --git a/packages/solidity-mapper/package.json b/packages/solidity-mapper/package.json index 81f9f618d..7fc47334d 100644 --- a/packages/solidity-mapper/package.json +++ b/packages/solidity-mapper/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/solidity-mapper", - "version": "0.2.74", + "version": "0.2.75", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { diff --git a/packages/test/package.json b/packages/test/package.json index 6df37cbc3..1d82ab164 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/test", - "version": "0.2.74", + "version": "0.2.75", "main": "dist/index.js", "license": "AGPL-3.0", "private": true, diff --git a/packages/tracing-client/package.json b/packages/tracing-client/package.json index 50146f7a0..5d4a0fa31 100644 --- a/packages/tracing-client/package.json +++ b/packages/tracing-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/tracing-client", - "version": "0.2.74", + "version": "0.2.75", "description": "ETH VM tracing client", "main": "dist/index.js", "scripts": { diff --git a/packages/util/package.json b/packages/util/package.json index 0fe701ff9..da5ff43aa 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,13 +1,13 @@ { "name": "@cerc-io/util", - "version": "0.2.74", + "version": "0.2.75", "main": "dist/index.js", "license": "AGPL-3.0", "dependencies": { "@apollo/utils.keyvaluecache": "^1.0.1", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.74", - "@cerc-io/solidity-mapper": "^0.2.74", + "@cerc-io/peer": "^0.2.75", + "@cerc-io/solidity-mapper": "^0.2.75", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@ethersproject/properties": "^5.7.0", "@ethersproject/providers": "^5.4.4", @@ -52,7 +52,7 @@ "yargs": "^17.0.1" }, "devDependencies": { - "@cerc-io/cache": "^0.2.74", + "@cerc-io/cache": "^0.2.75", "@nomiclabs/hardhat-waffle": "^2.0.1", "@types/bunyan": "^1.8.8", "@types/express": "^4.17.14", diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 3095bf596..c4ff77b16 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -211,6 +211,15 @@ export class Database { return repo.save(entity); } + async updateSyncStatus (repo: Repository, syncStatus: DeepPartial): Promise { + const entity = await repo.findOne(); + + return await repo.save({ + ...entity, + ...syncStatus + }); + } + async getBlockProgress (repo: Repository, blockHash: string): Promise { return repo.findOne({ where: { blockHash } }); } diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index f03d40277..7825a466a 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -291,6 +291,23 @@ export class Indexer { return res; } + async updateSyncStatus (syncStatus: DeepPartial): Promise { + const dbTx = await this._db.createTransactionRunner(); + let res; + + try { + res = await this._db.updateSyncStatus(dbTx, syncStatus); + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + return res; + } + async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise> { assert(blockFilter.blockHash || blockFilter.blockNumber); const blocks = await this._ethClient.getFullBlocks(blockFilter); diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index 1dd102ff1..3b2264e59 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -237,11 +237,14 @@ export class JobRunner { // Update sync status canonical, indexed and chain head block to end block // Update with zero hash as they won't be used during historical processing - await Promise.all([ - this._indexer.updateSyncStatusCanonicalBlock(constants.HashZero, endBlock, true), - this._indexer.updateSyncStatusIndexedBlock(constants.HashZero, endBlock, true), - this._indexer.updateSyncStatusChainHead(constants.HashZero, endBlock, true) - ]); + await this._indexer.updateSyncStatus({ + latestCanonicalBlockHash: constants.HashZero, + latestIndexedBlockHash: constants.HashZero, + chainHeadBlockHash: constants.HashZero, + latestCanonicalBlockNumber: endBlock, + latestIndexedBlockNumber: endBlock, + chainHeadBlockNumber: endBlock + }); log(`Sync status canonical, indexed and chain head block updated to ${endBlock}`); this._historicalProcessingCompletedUpto = endBlock; diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index edfbd4001..654ea44dd 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -195,6 +195,7 @@ export interface IndexerInterface { updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise + updateSyncStatus (syncStatus: DeepPartial): Promise; markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise saveEventEntity (dbEvent: EventInterface): Promise saveEvents (dbEvents: DeepPartial[]): Promise @@ -258,6 +259,7 @@ export interface DatabaseInterface { updateSyncStatusCanonicalBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force?: boolean): Promise; updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise; updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force?: boolean): Promise; + updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial): Promise; saveEvents (queryRunner: QueryRunner, events: DeepPartial[]): Promise; saveBlockWithEvents (queryRunner: QueryRunner, block: DeepPartial, events: DeepPartial[]): Promise; saveEventEntity (queryRunner: QueryRunner, entity: EventInterface): Promise;