forgot to commit

examples/mud/CHANGELOG.md

@@ -1,5 +1,15 @@
 # web-demo
 
+## 0.1.61
+
+### Patch Changes
+
+- Updated dependencies
+  - ethereum-indexer-browser@0.6.30
+  - event-processor-bleeps@0.0.54
+  - event-processor-conquest-eth@0.0.54
+  - event-processor-nfts@0.0.54
+
 ## 0.1.60
 
 ### Patch Changes

examples/mud/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "mud-demo",
 	"private": true,
-	"version": "0.1.60",
+	"version": "0.1.61",
 	"type": "module",
 	"scripts": {
 		"dev": "vite",

examples/web-demo/CHANGELOG.md

@@ -1,5 +1,15 @@
 # web-demo
 
+## 0.1.61
+
+### Patch Changes
+
+- Updated dependencies
+  - ethereum-indexer-browser@0.6.30
+  - event-processor-bleeps@0.0.54
+  - event-processor-conquest-eth@0.0.54
+  - event-processor-nfts@0.0.54
+
 ## 0.1.60
 
 ### Patch Changes

examples/web-demo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "web-demo",
 	"private": true,
-	"version": "0.1.60",
+	"version": "0.1.61",
 	"type": "module",
 	"scripts": {
 		"dev": "vite",

packages/ethereum-indexer-browser/CHANGELOG.md

@@ -1,5 +1,13 @@
 # ethereum-indexer-browser
 
+## 0.6.30
+
+### Patch Changes
+
+- support folder export with lastSync + allow fetch lastSync first to get latest sync
+- Updated dependencies
+  - ethereum-indexer-utils@0.6.12
+
 ## 0.6.29
 
 ### Patch Changes

packages/ethereum-indexer-browser/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ethereum-indexer-browser",
-	"version": "0.6.29",
+	"version": "0.6.30",
 	"publishConfig": {
 		"access": "public"
 	},
@@ -18,6 +18,7 @@
 	},
 	"dependencies": {
 		"ethereum-indexer": "workspace:*",
+		"ethereum-indexer-utils": "workspace:*",
 		"idb-keyval": "^6.2.1",
 		"named-logs": "^0.2.2",
 		"sveltore": "^0.0.2",

packages/ethereum-indexer-browser/src/storage/state/OnIndexedDB.ts

@@ -1,4 +1,5 @@
 import {Abi, AllData, simple_hash, LastSync, ProcessorContext} from 'ethereum-indexer';
+import {contextFilenames} from 'ethereum-indexer-utils';
 import {get, set, del} from 'idb-keyval';
 
 function getStorageID<ProcessorConfig = undefined>(name: string, chainId: string, config: ProcessorConfig) {
@@ -8,23 +9,100 @@ function getStorageID<ProcessorConfig = undefined>(name: string, chainId: string
 
 type StateData<ABI extends Abi, ProcessResultType, Extra> = AllData<ABI, ProcessResultType, Extra>;
 
-export function keepStateOnIndexedDB<ABI extends Abi, ProcessResultType, ProcessorConfig>(name: string, remote?: string) {
+export type IndexedStateLocation = {url: string} | {prefix: string};
+
+function getURL(remote: IndexedStateLocation | string, context: ProcessorContext<Abi, any>, lastSync = false) {
+	let url: string;
+	if (typeof remote === 'string') {
+		url = remote;
+	} else if ('url' in remote) {
+		url = remote.url;
+	} else {
+		const {stateFile, lastSyncFile} = contextFilenames(context);
+		if (lastSync) {
+			url = remote.prefix + lastSyncFile;
+		} else {
+			url = remote.prefix + stateFile;
+		}
+	}
+	return url;
+}
+
+export function keepStateOnIndexedDB<ABI extends Abi, ProcessResultType, ProcessorConfig>(
+	name: string,
+	remote?: IndexedStateLocation | string | IndexedStateLocation[],
+) {
 	return {
 		fetch: async (context: ProcessorContext<ABI, ProcessorConfig>) => {
 			const storageID = getStorageID(name, context.source.chainId, 'config' in context ? context.config : undefined);
 			const existingState = await get<StateData<ABI, ProcessResultType, unknown>>(storageID);
 
-			let remoteState: StateData<ABI, ProcessResultType, unknown> | undefined; 
+			let remoteState: StateData<ABI, ProcessResultType, unknown> | undefined;
 			if (remote) {
-				try {
-					const response = await fetch(remote);
-					const json = await response.json();
-					remoteState = json;
-				} catch(err) {
-					console.error(`failed to fetch remote-state`, err);
+				if (Array.isArray(remote)) {
+
+					let latest: {index: number; lastSync?: LastSync<Abi>} | undefined;
+					for (let i = 0; i < remote.length; i++) {
+						if (typeof remote[i] === 'string' || 'url' in remote[i]) {
+							// console.log(`raw url, do not fetch lastSync`);
+							continue;
+						}
+						const urlOfLastSync = getURL(remote[i], context, true);
+						try {
+							const response = await fetch(urlOfLastSync);
+							const json: LastSync<Abi> = await response.json();
+							if (!latest || !latest.lastSync || json.lastToBlock > latest.lastSync.lastToBlock) {
+								latest = {
+									index: i,
+									lastSync: json
+								}
+							}
+						} catch (err) {
+							console.error(`failed to fetch remote lastSync`, err);
+						}
+					}
+
+					if (existingState && latest && latest.lastSync && latest.lastSync.lastToBlock < existingState.lastSync.lastToBlock) {
+						// console.log(`Existing State`)
+						return existingState;
+					}
+
+					if (!latest) {
+						console.error(`could not fetch any valid lastSync, still continue with first`);
+						latest = {
+							index: 0
+						}
+					}
+					const url = getURL(remote[latest.index], context);
+					try {
+						const response = await fetch(url);
+						const json = await response.json();
+						remoteState = json;
+					} catch (err) {
+						console.error(`failed to fetch remote-state, try second`, err);
+
+						const url = getURL(remote[(latest.index + 1) % remote.length], context);
+						try {
+							const response = await fetch(url);
+							const json = await response.json();
+							remoteState = json;
+						} catch(err) {
+							console.error(`failed to fetch second remote-state`, err);
+							// TODO more than 2
+						}
+					}
+				} else {
+					const url = getURL(remote, context);
+					try {
+						const response = await fetch(url);
+						const json = await response.json();
+						remoteState = json;
+					} catch (err) {
+						console.error(`failed to fetch remote-state`, err);
+					}
 				}
 			}
-			
+
 			if (!existingState) {
 				return remoteState;
 			} else {
@@ -39,7 +117,7 @@ export function keepStateOnIndexedDB<ABI extends Abi, ProcessResultType, Process
 			all: {
 				state: ProcessResultType;
 				lastSync: LastSync<ABI>;
-			}
+			},
 		) => {
 			const storageID = getStorageID(name, context.source.chainId, 'config' in context ? context.config : undefined);
 			await set(storageID, {...all, __VERSION__: context.version});

packages/ethereum-indexer-cli/CHANGELOG.md

@@ -1,5 +1,13 @@
 # ethereum-indexer-cli
 
+## 0.6.26
+
+### Patch Changes
+
+- support folder export with lastSync + allow fetch lastSync first to get latest sync
+- Updated dependencies
+  - ethereum-indexer-utils@0.6.12
+
 ## 0.6.25
 
 ### Patch Changes

packages/ethereum-indexer-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ethereum-indexer-cli",
-  "version": "0.6.25",
+  "version": "0.6.26",
   "description": "",
   "keywords": [],
   "author": "",

packages/ethereum-indexer-cli/src/cli.ts

@@ -13,15 +13,15 @@ program
 	.name('ei')
 	.version(pkg.version)
 	.usage(`ei -p <processor's path> [-d <deployment folder> -n http://localhost:8545]`)
-	.description('Run The Indexer as Server')
+	.description('Run The Indexer And Write To File')
 	.requiredOption(
 		'-p, --processor <path>',
 		`path to the event processor module (need to export a field named "createProcessor")`
 	)
-	.requiredOption('-f, --file <value>', 'file to start from')
+	.requiredOption('-f, --folder <value>', 'folder to read and write to')
 	.option(
 		'-d, --deployments <value>',
-		"path the folder containing contract deployments, use hardhat-deploy format, optional if processor's module provide it"
+		"path the folder containing contract deployments, use hardhat-deploy/rocketh format, optional if processor's module provide it"
 	)
 	.option(
 		'--rps <value>',

packages/ethereum-indexer-cli/src/index.ts

@@ -14,7 +14,7 @@ import {JSONRPCHTTPProvider} from 'eip-1193-jsonrpc-provider';
 import {EIP1193ProviderWithoutEvents} from 'eip-1193';
 import fs from 'fs';
 import path from 'path';
-import {loadContracts} from 'ethereum-indexer-utils';
+import {contextFilenames, loadContracts} from 'ethereum-indexer-utils';
 import {bnReplacer, bnReviver} from './utils/bn';
 import {createRequire} from 'module';
 
@@ -24,6 +24,11 @@ type ProcessorWithKeepState<ABI extends Abi> = {
 	keepState(keeper: KeepState<ABI, any, {history: any}, any>): void;
 };
 
+function filepaths(folder: string, context: ProcessorContext<Abi, any>) {
+	const {stateFile, lastSyncFile} = contextFilenames(context)
+	return {stateFile: path.join(folder, stateFile),lastSyncFile:path.join(folder, lastSyncFile)}
+}
+
 // TODO ethereum-indexer-server could reuse
 export async function init<ABI extends Abi, ProcessResultType>(options: Options) {
 	let processor: EventProcessor<ABI, ProcessResultType> | undefined;
@@ -69,27 +74,37 @@ export async function init<ABI extends Abi, ProcessResultType>(options: Options)
 	if (!(processor as any).keepState) {
 		throw new Error(`this processor do not support "keepState" config`);
 	}
+
+
+
 	(processor as unknown as ProcessorWithKeepState<ABI>).keepState({
 		fetch: async (context: ProcessorContext<ABI, any>) => {
+			const {stateFile} = filepaths(options.folder, context);
+			console.log({reading: stateFile});
 			try {
-				const content = fs.readFileSync(options.file, 'utf-8');
+
+				const content = fs.readFileSync(stateFile, 'utf-8');
 				const json = JSON.parse(content, bnReviver);
 				return {
 					state: json.state,
 					lastSync: json.lastSync,
 					history: json.history,
 				};
 			} catch {
+				console.log(`no ${stateFile}`);
 				return undefined as any; // TODO fix type in KeepState to allow undefined
 			}
 		},
 		save: async (context, all) => {
+			const {stateFile,lastSyncFile} = filepaths(options.folder, context);
+			console.log({saving: stateFile, sync: lastSyncFile});
 			const data = {lastSync: all.lastSync, state: all.state, history: all.history};
-			const dirname = path.dirname(options.file);
+			const dirname = path.dirname(stateFile);
 			if (!fs.existsSync(dirname)) {
 				fs.mkdirSync(dirname, {recursive: true});
 			}
-			fs.writeFileSync(options.file, JSON.stringify(data, bnReplacer, 2));
+			fs.writeFileSync(lastSyncFile, JSON.stringify(data.lastSync, bnReplacer, 2));
+			fs.writeFileSync(stateFile, JSON.stringify(data, bnReplacer, 2));
 		},
 		clear: async () => {},
 	});
@@ -170,7 +185,4 @@ export async function run(options: Options) {
 	while (newLastSync.lastToBlock < newLastSync.latestBlock) {
 		newLastSync = await indexer.indexMore();
 	}
-
-	// const data = {lastSync: newLastSync, state};
-	// fs.writeFileSync(options.file, JSON.stringify(data, bnReplacer, 2));
 }

packages/ethereum-indexer-cli/src/types.ts

@@ -1,7 +1,7 @@
 export type Options = {
 	processor: string;
 	nodeUrl: string;
-	file: string;
+	folder: string;
 	deployments?: string;
 	rps?: number;
 };

packages/ethereum-indexer-server/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ethereum-indexer-server
 
+## 0.6.29
+
+### Patch Changes
+
+- Updated dependencies
+  - ethereum-indexer-utils@0.6.12
+
 ## 0.6.28
 
 ### Patch Changes

packages/ethereum-indexer-server/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ethereum-indexer-server",
-	"version": "0.6.28",
+	"version": "0.6.29",
 	"description": "",
 	"keywords": [],
 	"author": "",

packages/ethereum-indexer-streams/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ethereum-indexer-server
 
+## 0.6.25
+
+### Patch Changes
+
+- Updated dependencies
+  - ethereum-indexer-utils@0.6.12
+
 ## 0.6.24
 
 ### Patch Changes

packages/ethereum-indexer-streams/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ethereum-indexer-streams",
-	"version": "0.6.24",
+	"version": "0.6.25",
 	"description": "",
 	"keywords": [],
 	"author": "",

packages/ethereum-indexer-utils/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ethereum-indexer-utils
 
+## 0.6.12
+
+### Patch Changes
+
+- support folder export with lastSync + allow fetch lastSync first to get latest sync
+
 ## 0.6.11
 
 ### Patch Changes

packages/ethereum-indexer-utils/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ethereum-indexer-utils",
-	"version": "0.6.11",
+	"version": "0.6.12",
 	"description": "",
 	"publishConfig": {
 		"access": "public"