From 9438e4bd8f8c9387f0cfe7b43ef31f41d4169ae0 Mon Sep 17 00:00:00 2001 From: wighawag Date: Thu, 8 Feb 2024 14:14:40 +0000 Subject: [PATCH] forgot to commit --- examples/mud/CHANGELOG.md | 10 ++ examples/mud/package.json | 2 +- examples/web-demo/CHANGELOG.md | 10 ++ examples/web-demo/package.json | 2 +- .../ethereum-indexer-browser/CHANGELOG.md | 8 ++ .../ethereum-indexer-browser/package.json | 3 +- .../src/storage/state/OnIndexedDB.ts | 98 +++++++++++++++++-- packages/ethereum-indexer-cli/CHANGELOG.md | 8 ++ packages/ethereum-indexer-cli/package.json | 2 +- packages/ethereum-indexer-cli/src/cli.ts | 6 +- packages/ethereum-indexer-cli/src/index.ts | 26 +++-- packages/ethereum-indexer-cli/src/types.ts | 2 +- packages/ethereum-indexer-server/CHANGELOG.md | 7 ++ packages/ethereum-indexer-server/package.json | 2 +- .../ethereum-indexer-streams/CHANGELOG.md | 7 ++ .../ethereum-indexer-streams/package.json | 2 +- packages/ethereum-indexer-utils/CHANGELOG.md | 6 ++ packages/ethereum-indexer-utils/package.json | 2 +- .../ethereum-indexer-utils/src/indexer.ts | 12 ++- packages/ethereum-indexer-utils/tsconfig.json | 2 +- pnpm-lock.yaml | 3 + 21 files changed, 190 insertions(+), 30 deletions(-) diff --git a/examples/mud/CHANGELOG.md b/examples/mud/CHANGELOG.md index a22d736..27c0d7f 100644 --- a/examples/mud/CHANGELOG.md +++ b/examples/mud/CHANGELOG.md @@ -1,5 +1,15 @@ # web-demo +## 0.1.61 + +### Patch Changes + +- Updated dependencies + - ethereum-indexer-browser@0.6.30 + - event-processor-bleeps@0.0.54 + - event-processor-conquest-eth@0.0.54 + - event-processor-nfts@0.0.54 + ## 0.1.60 ### Patch Changes diff --git a/examples/mud/package.json b/examples/mud/package.json index fbd21c9..17c894b 100644 --- a/examples/mud/package.json +++ b/examples/mud/package.json @@ -1,7 +1,7 @@ { "name": "mud-demo", "private": true, - "version": "0.1.60", + "version": "0.1.61", "type": "module", "scripts": { "dev": "vite", diff --git a/examples/web-demo/CHANGELOG.md b/examples/web-demo/CHANGELOG.md index e3b2534..2b1b420 100644 --- a/examples/web-demo/CHANGELOG.md +++ b/examples/web-demo/CHANGELOG.md @@ -1,5 +1,15 @@ # web-demo +## 0.1.61 + +### Patch Changes + +- Updated dependencies + - ethereum-indexer-browser@0.6.30 + - event-processor-bleeps@0.0.54 + - event-processor-conquest-eth@0.0.54 + - event-processor-nfts@0.0.54 + ## 0.1.60 ### Patch Changes diff --git a/examples/web-demo/package.json b/examples/web-demo/package.json index e9d1ea3..7d9d798 100644 --- a/examples/web-demo/package.json +++ b/examples/web-demo/package.json @@ -1,7 +1,7 @@ { "name": "web-demo", "private": true, - "version": "0.1.60", + "version": "0.1.61", "type": "module", "scripts": { "dev": "vite", diff --git a/packages/ethereum-indexer-browser/CHANGELOG.md b/packages/ethereum-indexer-browser/CHANGELOG.md index bee58e0..b9c6a6e 100644 --- a/packages/ethereum-indexer-browser/CHANGELOG.md +++ b/packages/ethereum-indexer-browser/CHANGELOG.md @@ -1,5 +1,13 @@ # ethereum-indexer-browser +## 0.6.30 + +### Patch Changes + +- support folder export with lastSync + allow fetch lastSync first to get latest sync +- Updated dependencies + - ethereum-indexer-utils@0.6.12 + ## 0.6.29 ### Patch Changes diff --git a/packages/ethereum-indexer-browser/package.json b/packages/ethereum-indexer-browser/package.json index b5c71dc..5df4d75 100644 --- a/packages/ethereum-indexer-browser/package.json +++ b/packages/ethereum-indexer-browser/package.json @@ -1,6 +1,6 @@ { "name": "ethereum-indexer-browser", - "version": "0.6.29", + "version": "0.6.30", "publishConfig": { "access": "public" }, @@ -18,6 +18,7 @@ }, "dependencies": { "ethereum-indexer": "workspace:*", + "ethereum-indexer-utils": "workspace:*", "idb-keyval": "^6.2.1", "named-logs": "^0.2.2", "sveltore": "^0.0.2", diff --git a/packages/ethereum-indexer-browser/src/storage/state/OnIndexedDB.ts b/packages/ethereum-indexer-browser/src/storage/state/OnIndexedDB.ts index 9cee446..9e1258b 100644 --- a/packages/ethereum-indexer-browser/src/storage/state/OnIndexedDB.ts +++ b/packages/ethereum-indexer-browser/src/storage/state/OnIndexedDB.ts @@ -1,4 +1,5 @@ import {Abi, AllData, simple_hash, LastSync, ProcessorContext} from 'ethereum-indexer'; +import {contextFilenames} from 'ethereum-indexer-utils'; import {get, set, del} from 'idb-keyval'; function getStorageID(name: string, chainId: string, config: ProcessorConfig) { @@ -8,23 +9,100 @@ function getStorageID(name: string, chainId: string type StateData = AllData; -export function keepStateOnIndexedDB(name: string, remote?: string) { +export type IndexedStateLocation = {url: string} | {prefix: string}; + +function getURL(remote: IndexedStateLocation | string, context: ProcessorContext, lastSync = false) { + let url: string; + if (typeof remote === 'string') { + url = remote; + } else if ('url' in remote) { + url = remote.url; + } else { + const {stateFile, lastSyncFile} = contextFilenames(context); + if (lastSync) { + url = remote.prefix + lastSyncFile; + } else { + url = remote.prefix + stateFile; + } + } + return url; +} + +export function keepStateOnIndexedDB( + name: string, + remote?: IndexedStateLocation | string | IndexedStateLocation[], +) { return { fetch: async (context: ProcessorContext) => { const storageID = getStorageID(name, context.source.chainId, 'config' in context ? context.config : undefined); const existingState = await get>(storageID); - let remoteState: StateData | undefined; + let remoteState: StateData | undefined; if (remote) { - try { - const response = await fetch(remote); - const json = await response.json(); - remoteState = json; - } catch(err) { - console.error(`failed to fetch remote-state`, err); + if (Array.isArray(remote)) { + + let latest: {index: number; lastSync?: LastSync} | undefined; + for (let i = 0; i < remote.length; i++) { + if (typeof remote[i] === 'string' || 'url' in remote[i]) { + // console.log(`raw url, do not fetch lastSync`); + continue; + } + const urlOfLastSync = getURL(remote[i], context, true); + try { + const response = await fetch(urlOfLastSync); + const json: LastSync = await response.json(); + if (!latest || !latest.lastSync || json.lastToBlock > latest.lastSync.lastToBlock) { + latest = { + index: i, + lastSync: json + } + } + } catch (err) { + console.error(`failed to fetch remote lastSync`, err); + } + } + + if (existingState && latest && latest.lastSync && latest.lastSync.lastToBlock < existingState.lastSync.lastToBlock) { + // console.log(`Existing State`) + return existingState; + } + + if (!latest) { + console.error(`could not fetch any valid lastSync, still continue with first`); + latest = { + index: 0 + } + } + const url = getURL(remote[latest.index], context); + try { + const response = await fetch(url); + const json = await response.json(); + remoteState = json; + } catch (err) { + console.error(`failed to fetch remote-state, try second`, err); + + const url = getURL(remote[(latest.index + 1) % remote.length], context); + try { + const response = await fetch(url); + const json = await response.json(); + remoteState = json; + } catch(err) { + console.error(`failed to fetch second remote-state`, err); + // TODO more than 2 + } + } + } else { + const url = getURL(remote, context); + try { + const response = await fetch(url); + const json = await response.json(); + remoteState = json; + } catch (err) { + console.error(`failed to fetch remote-state`, err); + } } } - + if (!existingState) { return remoteState; } else { @@ -39,7 +117,7 @@ export function keepStateOnIndexedDB; - } + }, ) => { const storageID = getStorageID(name, context.source.chainId, 'config' in context ? context.config : undefined); await set(storageID, {...all, __VERSION__: context.version}); diff --git a/packages/ethereum-indexer-cli/CHANGELOG.md b/packages/ethereum-indexer-cli/CHANGELOG.md index b6d37fc..5478818 100644 --- a/packages/ethereum-indexer-cli/CHANGELOG.md +++ b/packages/ethereum-indexer-cli/CHANGELOG.md @@ -1,5 +1,13 @@ # ethereum-indexer-cli +## 0.6.26 + +### Patch Changes + +- support folder export with lastSync + allow fetch lastSync first to get latest sync +- Updated dependencies + - ethereum-indexer-utils@0.6.12 + ## 0.6.25 ### Patch Changes diff --git a/packages/ethereum-indexer-cli/package.json b/packages/ethereum-indexer-cli/package.json index cd4283b..12050ad 100644 --- a/packages/ethereum-indexer-cli/package.json +++ b/packages/ethereum-indexer-cli/package.json @@ -1,6 +1,6 @@ { "name": "ethereum-indexer-cli", - "version": "0.6.25", + "version": "0.6.26", "description": "", "keywords": [], "author": "", diff --git a/packages/ethereum-indexer-cli/src/cli.ts b/packages/ethereum-indexer-cli/src/cli.ts index 1038429..53ba45c 100644 --- a/packages/ethereum-indexer-cli/src/cli.ts +++ b/packages/ethereum-indexer-cli/src/cli.ts @@ -13,15 +13,15 @@ program .name('ei') .version(pkg.version) .usage(`ei -p [-d -n http://localhost:8545]`) - .description('Run The Indexer as Server') + .description('Run The Indexer And Write To File') .requiredOption( '-p, --processor ', `path to the event processor module (need to export a field named "createProcessor")` ) - .requiredOption('-f, --file ', 'file to start from') + .requiredOption('-f, --folder ', 'folder to read and write to') .option( '-d, --deployments ', - "path the folder containing contract deployments, use hardhat-deploy format, optional if processor's module provide it" + "path the folder containing contract deployments, use hardhat-deploy/rocketh format, optional if processor's module provide it" ) .option( '--rps ', diff --git a/packages/ethereum-indexer-cli/src/index.ts b/packages/ethereum-indexer-cli/src/index.ts index bd64da8..e84edca 100644 --- a/packages/ethereum-indexer-cli/src/index.ts +++ b/packages/ethereum-indexer-cli/src/index.ts @@ -14,7 +14,7 @@ import {JSONRPCHTTPProvider} from 'eip-1193-jsonrpc-provider'; import {EIP1193ProviderWithoutEvents} from 'eip-1193'; import fs from 'fs'; import path from 'path'; -import {loadContracts} from 'ethereum-indexer-utils'; +import {contextFilenames, loadContracts} from 'ethereum-indexer-utils'; import {bnReplacer, bnReviver} from './utils/bn'; import {createRequire} from 'module'; @@ -24,6 +24,11 @@ type ProcessorWithKeepState = { keepState(keeper: KeepState): void; }; +function filepaths(folder: string, context: ProcessorContext) { + const {stateFile, lastSyncFile} = contextFilenames(context) + return {stateFile: path.join(folder, stateFile),lastSyncFile:path.join(folder, lastSyncFile)} +} + // TODO ethereum-indexer-server could reuse export async function init(options: Options) { let processor: EventProcessor | undefined; @@ -69,10 +74,16 @@ export async function init(options: Options) if (!(processor as any).keepState) { throw new Error(`this processor do not support "keepState" config`); } + + + (processor as unknown as ProcessorWithKeepState).keepState({ fetch: async (context: ProcessorContext) => { + const {stateFile} = filepaths(options.folder, context); + console.log({reading: stateFile}); try { - const content = fs.readFileSync(options.file, 'utf-8'); + + const content = fs.readFileSync(stateFile, 'utf-8'); const json = JSON.parse(content, bnReviver); return { state: json.state, @@ -80,16 +91,20 @@ export async function init(options: Options) history: json.history, }; } catch { + console.log(`no ${stateFile}`); return undefined as any; // TODO fix type in KeepState to allow undefined } }, save: async (context, all) => { + const {stateFile,lastSyncFile} = filepaths(options.folder, context); + console.log({saving: stateFile, sync: lastSyncFile}); const data = {lastSync: all.lastSync, state: all.state, history: all.history}; - const dirname = path.dirname(options.file); + const dirname = path.dirname(stateFile); if (!fs.existsSync(dirname)) { fs.mkdirSync(dirname, {recursive: true}); } - fs.writeFileSync(options.file, JSON.stringify(data, bnReplacer, 2)); + fs.writeFileSync(lastSyncFile, JSON.stringify(data.lastSync, bnReplacer, 2)); + fs.writeFileSync(stateFile, JSON.stringify(data, bnReplacer, 2)); }, clear: async () => {}, }); @@ -170,7 +185,4 @@ export async function run(options: Options) { while (newLastSync.lastToBlock < newLastSync.latestBlock) { newLastSync = await indexer.indexMore(); } - - // const data = {lastSync: newLastSync, state}; - // fs.writeFileSync(options.file, JSON.stringify(data, bnReplacer, 2)); } diff --git a/packages/ethereum-indexer-cli/src/types.ts b/packages/ethereum-indexer-cli/src/types.ts index 197edfc..1703979 100644 --- a/packages/ethereum-indexer-cli/src/types.ts +++ b/packages/ethereum-indexer-cli/src/types.ts @@ -1,7 +1,7 @@ export type Options = { processor: string; nodeUrl: string; - file: string; + folder: string; deployments?: string; rps?: number; }; diff --git a/packages/ethereum-indexer-server/CHANGELOG.md b/packages/ethereum-indexer-server/CHANGELOG.md index f23afd9..9d5e10b 100644 --- a/packages/ethereum-indexer-server/CHANGELOG.md +++ b/packages/ethereum-indexer-server/CHANGELOG.md @@ -1,5 +1,12 @@ # ethereum-indexer-server +## 0.6.29 + +### Patch Changes + +- Updated dependencies + - ethereum-indexer-utils@0.6.12 + ## 0.6.28 ### Patch Changes diff --git a/packages/ethereum-indexer-server/package.json b/packages/ethereum-indexer-server/package.json index 8953530..5284fb2 100644 --- a/packages/ethereum-indexer-server/package.json +++ b/packages/ethereum-indexer-server/package.json @@ -1,6 +1,6 @@ { "name": "ethereum-indexer-server", - "version": "0.6.28", + "version": "0.6.29", "description": "", "keywords": [], "author": "", diff --git a/packages/ethereum-indexer-streams/CHANGELOG.md b/packages/ethereum-indexer-streams/CHANGELOG.md index 7dd4d99..cb56fc2 100644 --- a/packages/ethereum-indexer-streams/CHANGELOG.md +++ b/packages/ethereum-indexer-streams/CHANGELOG.md @@ -1,5 +1,12 @@ # ethereum-indexer-server +## 0.6.25 + +### Patch Changes + +- Updated dependencies + - ethereum-indexer-utils@0.6.12 + ## 0.6.24 ### Patch Changes diff --git a/packages/ethereum-indexer-streams/package.json b/packages/ethereum-indexer-streams/package.json index 5b4d40e..3728a3f 100644 --- a/packages/ethereum-indexer-streams/package.json +++ b/packages/ethereum-indexer-streams/package.json @@ -1,6 +1,6 @@ { "name": "ethereum-indexer-streams", - "version": "0.6.24", + "version": "0.6.25", "description": "", "keywords": [], "author": "", diff --git a/packages/ethereum-indexer-utils/CHANGELOG.md b/packages/ethereum-indexer-utils/CHANGELOG.md index b932146..cdcad68 100644 --- a/packages/ethereum-indexer-utils/CHANGELOG.md +++ b/packages/ethereum-indexer-utils/CHANGELOG.md @@ -1,5 +1,11 @@ # ethereum-indexer-utils +## 0.6.12 + +### Patch Changes + +- support folder export with lastSync + allow fetch lastSync first to get latest sync + ## 0.6.11 ### Patch Changes diff --git a/packages/ethereum-indexer-utils/package.json b/packages/ethereum-indexer-utils/package.json index cf9815d..0c75438 100644 --- a/packages/ethereum-indexer-utils/package.json +++ b/packages/ethereum-indexer-utils/package.json @@ -1,6 +1,6 @@ { "name": "ethereum-indexer-utils", - "version": "0.6.11", + "version": "0.6.12", "description": "", "publishConfig": { "access": "public" diff --git a/packages/ethereum-indexer-utils/src/indexer.ts b/packages/ethereum-indexer-utils/src/indexer.ts index 452e2f3..077651a 100644 --- a/packages/ethereum-indexer-utils/src/indexer.ts +++ b/packages/ethereum-indexer-utils/src/indexer.ts @@ -1,6 +1,16 @@ -import {Abi, LastSync} from 'ethereum-indexer'; +import {Abi, LastSync, ProcessorContext, simple_hash} from 'ethereum-indexer'; import {filterOutFieldsFromObject} from './javascript'; export function formatLastSync(lastSync: LastSync): any { return filterOutFieldsFromObject(lastSync, ['_rev', '_id', 'batch']); } + + +export function contextFilenames(context: ProcessorContext) { + const configHash = 'config' in context && context.config ? simple_hash(context.config) : undefined; + const networkString = `${context.source.chainId }${(context.source.chainId == '1337' || context.source.chainId == '31337') && context.source.genesisHash ? `-${context.source.genesisHash}`: ''}` + const prefix = `${networkString}${configHash ? `-${configHash}`: ``}${context.version ? `-${context.version}`: ``}`; + const stateFile = `${prefix}-state.json`; + const lastSyncFile = `${prefix}-lastSync.json`; + return {stateFile,lastSyncFile} +} \ No newline at end of file diff --git a/packages/ethereum-indexer-utils/tsconfig.json b/packages/ethereum-indexer-utils/tsconfig.json index 6033ae3..275ce21 100644 --- a/packages/ethereum-indexer-utils/tsconfig.json +++ b/packages/ethereum-indexer-utils/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "moduleResolution": "node", "lib": ["ES2020"], - "target": "ES6", + "target": "ES2020", "declaration": true, "declarationMap": true, "sourceMap": true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a73c3c..8578807 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -334,6 +334,9 @@ importers: ethereum-indexer: specifier: workspace:* version: link:../ethereum-indexer + ethereum-indexer-utils: + specifier: workspace:* + version: link:../ethereum-indexer-utils idb-keyval: specifier: ^6.2.1 version: 6.2.1