diff --git a/.changeset/cuddly-schools-swim.md b/.changeset/cuddly-schools-swim.md new file mode 100644 index 0000000000..a5aebca2c9 --- /dev/null +++ b/.changeset/cuddly-schools-swim.md @@ -0,0 +1,7 @@ +--- +'demo-parse-server-migration': minor +'@moralisweb3/streams': minor +'@moralisweb3/parse-server': minor +--- + +Incuded parse server package and added document b diff --git a/.changeset/red-trees-hunt.md b/.changeset/red-trees-hunt.md new file mode 100644 index 0000000000..01f2302828 --- /dev/null +++ b/.changeset/red-trees-hunt.md @@ -0,0 +1,7 @@ +--- +'demo-parse-server-migration': minor +'@moralisweb3/streams': minor +'@moralisweb3/parse-server': minor +--- + +Included parse server package and added document builder logic to streams package diff --git a/demos/parse-server-migration/package.json b/demos/parse-server-migration/package.json index 49069e2b7c..fb9ecd93a3 100644 --- a/demos/parse-server-migration/package.json +++ b/demos/parse-server-migration/package.json @@ -4,8 +4,9 @@ "main": "dist/index.js", "private": true, "dependencies": { + "@moralisweb3/parse-server": "2.6.7", "@codemirror/language": "^0.20.0", - "@moralisweb3/core": "^2.2.0", + "@moralisweb3/core": "^2.6.7", "@types/node": "^18.7.15", "dotenv": "^16.0.1", "envalid": "7.3.1", diff --git a/demos/parse-server-migration/src/index.ts b/demos/parse-server-migration/src/index.ts index 853e6194a3..88cab78124 100644 --- a/demos/parse-server-migration/src/index.ts +++ b/demos/parse-server-migration/src/index.ts @@ -6,6 +6,7 @@ import { parseServer } from './parseServer'; // @ts-ignore import ParseServer from 'parse-server'; import http from 'http'; +import { streamsSync } from '@moralisweb3/parse-server'; export const app = express(); @@ -18,7 +19,14 @@ app.use(express.json()); app.use(cors()); -app.use(`/server`, parseServer); +app.use( + streamsSync(parseServer, { + apiKey: config.MORALIS_API_KEY, + webhookUrl: '/streams', + }), +); + +app.use(`/server`, parseServer.app); const httpServer = http.createServer(app); httpServer.listen(config.PORT, () => { diff --git a/demos/parse-server-migration/src/parseServer.ts b/demos/parse-server-migration/src/parseServer.ts index 26f63e7aa5..32ccc073e5 100644 --- a/demos/parse-server-migration/src/parseServer.ts +++ b/demos/parse-server-migration/src/parseServer.ts @@ -1,5 +1,5 @@ // @ts-ignore -import { ParseServer } from 'parse-server'; +import ParseServer from 'parse-server'; import config from './config'; import MoralisEthAdapter from './auth/MoralisEthAdapter'; diff --git a/packages/common/core/src/version.ts b/packages/common/core/src/version.ts new file mode 100644 index 0000000000..1af19cd074 --- /dev/null +++ b/packages/common/core/src/version.ts @@ -0,0 +1 @@ +export const LIB_VERSION = '2.6.7'; diff --git a/packages/parseServer/.eslintrc.js b/packages/parseServer/.eslintrc.js new file mode 100644 index 0000000000..92cea72b04 --- /dev/null +++ b/packages/parseServer/.eslintrc.js @@ -0,0 +1,5 @@ +module.exports = { + extends: ['@moralisweb3'], + plugins: ['jest'], + ignorePatterns: ['**/lib/**/*', '**/*.test.ts', '**/dist/**/*', '**/build/**/*', '**/generated/**/*'], +}; diff --git a/packages/parseServer/README.md b/packages/parseServer/README.md new file mode 100644 index 0000000000..591516370f --- /dev/null +++ b/packages/parseServer/README.md @@ -0,0 +1,106 @@ +# @moralisweb3/parse-server + +# Parse Server Moralis Streams + +This Plugin adapts parse-server to support [streams](https://github.com/MoralisWeb3/streams-beta) + +# Usage + +Since parse server is runs on express, this plugin is a middleware that can be added to the express app. + +## Installations + +First add parse-server to your express app: + +```bash +yarn add parse-server +``` + +Then add moralis parse server plugin: + +```bash +yarn add @moralisweb3/parse-server +``` + +## Setup parse server + +Initialize parse server in your express app: + +```javascript +import ParseServer from 'parse-server'; +import config from './config'; + +export const parseServer = new ParseServer({ + databaseURI: config.DATABASE_URI, + cloud: config.CLOUD_PATH, + appId: config.APPLICATION_ID, + masterKey: config.MASTER_KEY, + serverURL: config.SERVER_URL, +}); +``` + +## Setup moralis parse server plugin + +Then add the plugin to your express app: + +```typescript +import { initializeStreams } from '@moralisweb3/parse-server'; + +``` + +the initializeStreams function takes the following options: +- the parse server instance +- Other options + +```typescript +interface StreamOptions { + webhookUrl?: string; + apiKey: string; +} +``` + +- `apiKey`: Your Moralis API key +- `webhookUrl` - the url of choice to receive the stream data (optional). default path is `/streams` + + +## Putting all together + +```typescript +import Moralis from 'moralis'; +import express from 'express'; +import config from './config'; +import { streamsSync } from '@moralisweb3/parse-server'; + +const expressApp = express(); + +Moralis.start({ + apiKey: config.MORALIS_API_KEY, +}); + +expressApp.use(express.urlencoded({ extended: true })); +expressApp.use(express.json()); + +expressApp.use(cors()); + +expressApp.use( + streamsSync(parseServer, { + apiKey: config.MORALIS_API_KEY, + webhookUrl: '/streams-webhook', + }), +); + +expressApp.use(`/${config.SERVER_ENDPOINT}`, parseServer.app); +expressApp.use(errorHandler); + +app.listen(config.PORT, () => { + console.log(`${config.APP_NAME} is running on port ${config.PORT}`); +}); +``` + +The endpoint to receive webhooks is `YOUR_EXPRESSAPP_URL/SET_WEBHOOKURL`. This is the URL that you should use when setting up a stream. + +# Done! + +After you have configured the plugin and created a stream you can see the data in the dashboard. Note that the stream tag will be concatenated with `Txs` and `Logs` meaning if you have a tag called "MyStream" you will have two collections in DB called "MyStreamTxs" and "MyStreamLogs", which will contain the transactions and logs respectively. + +Full example can be found [here](https://github.com/MoralisWeb3/Moralis-JS-SDK/tree/main/demos/parse-server-migration) \ No newline at end of file diff --git a/packages/parseServer/jest.config.js b/packages/parseServer/jest.config.js new file mode 100644 index 0000000000..1c13fbaa33 --- /dev/null +++ b/packages/parseServer/jest.config.js @@ -0,0 +1,4 @@ +/* eslint-disable global-require */ +module.exports = { + ...require('../../jest.config'), +}; diff --git a/packages/parseServer/package.json b/packages/parseServer/package.json new file mode 100644 index 0000000000..ccf10b0f70 --- /dev/null +++ b/packages/parseServer/package.json @@ -0,0 +1,32 @@ +{ + "name": "@moralisweb3/parse-server", + "author": "Moralis", + "version": "2.6.7", + "license": "MIT", + "private": false, + "main": "./lib/index.js", + "types": "./lib/index.d.ts", + "sideEffects": false, + "files": [ + "lib/*" + ], + "scripts": { + "lint": "eslint . --ext .js,.ts,.tsx,jsx", + "clean": "rm -rf lib && rm -rf tsconfig.tsbuildinfo && rm -rf ./node_modules/.cache/nx", + "build": "tsc", + "dev": "tsc --watch" + }, + "devDependencies": { + "@types/parse": "^2.18.18", + "@types/express": "4.17.14", + "prettier": "^2.5.1", + "typescript": "^4.5.5" + }, + "dependencies": { + "@moralisweb3/streams": "^2.6.7", + "moralis": "^2.6.7", + "express": "^4.18.1", + "parse": "3.4.4", + "body-parser": "^1.20.0" + } +} diff --git a/packages/parseServer/src/index.ts b/packages/parseServer/src/index.ts new file mode 100644 index 0000000000..d1a032c26b --- /dev/null +++ b/packages/parseServer/src/index.ts @@ -0,0 +1 @@ +export * from './streams'; diff --git a/packages/parseServer/src/streams/index.ts b/packages/parseServer/src/streams/index.ts new file mode 100644 index 0000000000..d7e04ee8c1 --- /dev/null +++ b/packages/parseServer/src/streams/index.ts @@ -0,0 +1 @@ +export * from './processor'; diff --git a/packages/parseServer/src/streams/processor.ts b/packages/parseServer/src/streams/processor.ts new file mode 100644 index 0000000000..4deea28aa1 --- /dev/null +++ b/packages/parseServer/src/streams/processor.ts @@ -0,0 +1,19 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { webhookRouter } from './webbhook'; +import { MoralisCore } from 'moralis/core'; +import { MoralisStreams } from '@moralisweb3/streams'; +import { MoralisApiUtils } from '@moralisweb3/api-utils'; + +interface StreamOptions { + webhookUrl?: string; + apiKey: string; +} + +export const streamsSync = (parseInstance: any, options: StreamOptions) => { + const core = MoralisCore.create(); + const streams = MoralisStreams.create(core); + const apiUtils = MoralisApiUtils.create(core); + core.registerModules([streams, apiUtils]); + core.start({ apiKey: options.apiKey }); + return webhookRouter(parseInstance, options?.webhookUrl || '/streams-webhook', streams); +}; diff --git a/packages/parseServer/src/streams/upsert.ts b/packages/parseServer/src/streams/upsert.ts new file mode 100644 index 0000000000..5a9ada298c --- /dev/null +++ b/packages/parseServer/src/streams/upsert.ts @@ -0,0 +1,55 @@ +import { Document, Update } from '@moralisweb3/streams'; +import Parse from 'parse/node'; + +export class Upsert { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(private parseServer: any) { + Parse.initialize(this.parseServer.config.appId); + Parse.serverURL = this.parseServer.config.serverURL; + Parse.masterKey = this.parseServer.config.masterKey; + } + + async execute(path: string, filter: Record, update: Update) { + return this.upsert(update.collectionName + path, filter, update.document); + } + + private async upsert(className: string, filter: Record, update: Document) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const results = await this.lazyUpsert(className, filter, update as any); + await Parse.Object.saveAll(results, { useMasterKey: true }); + } + + private async lazyUpsert(className: string, filter: Record, update: Record) { + delete update.id; + const query = new Parse.Query(className); + + for (const key in filter) { + if (Object.prototype.hasOwnProperty.call(filter, key)) { + query.equalTo(key, filter[key]); + } + } + + const results = await query.find({ useMasterKey: true }); + + if (results.length > 0) { + for (let i = 0; i < results.length; i++) { + for (const updateKey in update) { + if (Object.prototype.hasOwnProperty.call(update, updateKey)) { + results[i].set(updateKey, update[updateKey]); + } + } + } + + return results; + } + + const objectClass = Parse.Object.extend(className); + const object = new objectClass(); + // eslint-disable-next-line guard-for-in + for (const updateKey in update) { + object.set(updateKey, update[updateKey]); + } + + return [object]; + } +} diff --git a/packages/parseServer/src/streams/webbhook.ts b/packages/parseServer/src/streams/webbhook.ts new file mode 100644 index 0000000000..25f05f5608 --- /dev/null +++ b/packages/parseServer/src/streams/webbhook.ts @@ -0,0 +1,99 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { IWebhook } from '@moralisweb3/streams-typings'; +import express, { Request } from 'express'; +import MoralisStreams, { + LogsProcessor, + TxsProcessor, + CollectionNameBuilder, + InternalTxsProcessor, + Update, +} from '@moralisweb3/streams'; +import bodyParser from 'body-parser'; +import { Upsert } from './upsert'; + +export const tagsMap = new Map(); + +const collectionNameBuilder = new CollectionNameBuilder(); +const logsProcessor = new LogsProcessor(collectionNameBuilder); +const txsProcessor = new TxsProcessor(collectionNameBuilder); +const internalTxProcessor = new InternalTxsProcessor(collectionNameBuilder); + +const verifySignature = (req: Request, streams: MoralisStreams) => { + const providedSignature = req.headers['x-signature']; + if (!providedSignature) { + throw new Error('Signature not provided'); + } + streams.verifySignature({ + body: req.body, + signature: providedSignature as string, + }); +}; + +export const webhookRouter = (parseObject: any, webhookUrl: string, streams: MoralisStreams) => { + return express.Router().post(webhookUrl, bodyParser.json({ limit: '50mb' }), async (req, res) => { + try { + verifySignature(req, streams); + } catch (e) { + return res.status(401).json({ message: e.message }); + } + try { + const updates: Record = {}; + const batch = req.body as IWebhook; + + const logUpdates = logsProcessor.process(batch); + const txUpdates = txsProcessor.process(batch); + const internalTxUpdates = internalTxProcessor.process(batch); + + // Prepare updates + if (!updates['Logs']) { + updates['Logs'] = []; + } + updates['Logs'].push(prepareUpdate(logUpdates, ['logIndex', 'transactionHash'])); + + if (!updates['Txs']) { + updates['Txs'] = []; + } + updates['Txs'].push(prepareUpdate(txUpdates, ['transactionIndex'])); + + if (!updates['TxsInternal']) { + updates['TxsInternal'] = []; + } + updates['TxsInternal'].push(prepareUpdate(internalTxUpdates, ['hash'])); + + const results: unknown[] = []; + const upsert = new Upsert(parseObject); + // eslint-disable-next-line guard-for-in + for (const tableName in updates) { + for (let index = 0; index < updates[tableName].length; index++) { + const data = updates[tableName][index]; + data.forEach(({ filter, update }: any) => { + results.push(upsert.execute(tableName, filter, update)); + }); + } + } + await Promise.all(results); + } catch (e) { + // eslint-disable-next-line no-console + console.log('error while inserting logs', e.message); + return res.status(500).json({ message: 'error while inserting logs' }); + } + + return res.status(200).json({ message: 'ok' }); + }); +}; + +const prepareUpdate = (updates: Update[], filters: string[]) => { + const results: unknown[] = []; + for (const update of updates) { + results.push({ + filter: filters.reduce((acc: Record, filter: string) => { + // @ts-ignore + acc[filter] = update.document[filter]; + return acc; + }, {}), + update, + upsert: true, + }); + } + return results; +}; diff --git a/packages/parseServer/tsconfig.json b/packages/parseServer/tsconfig.json new file mode 100644 index 0000000000..0adb6ec955 --- /dev/null +++ b/packages/parseServer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.package.json", + "compilerOptions": { + "outDir": "./lib/", + "declarationDir": "./lib/", + "rootDir": "./src", + "lib": ["ESNext", "DOM"] + }, + "include": ["src/**/*", "types/**/*"] +} diff --git a/packages/streams/src/index.ts b/packages/streams/src/index.ts index a6a47e3238..7875b8d52c 100644 --- a/packages/streams/src/index.ts +++ b/packages/streams/src/index.ts @@ -11,5 +11,6 @@ export * from './methods/types'; export * from './resolvers/evmStreams/types'; export * from './resolvers/history/types'; export * from './resolvers/project/types'; +export * from './mapping'; export default MoralisStreams; diff --git a/packages/streams/src/mapping/core/CollectionNameBuilder.test.ts b/packages/streams/src/mapping/core/CollectionNameBuilder.test.ts new file mode 100644 index 0000000000..fc60616548 --- /dev/null +++ b/packages/streams/src/mapping/core/CollectionNameBuilder.test.ts @@ -0,0 +1,19 @@ +import { CollectionNameBuilder } from './CollectionNameBuilder'; + +describe('CollectionNameBuilder', () => { + const builder = new CollectionNameBuilder(); + + function test(name: string, expectedTableName: string) { + it(name, () => { + expect(builder.build(name)).toBe(expectedTableName); + }); + } + + test('lorem ipsum', 'LoremIpsum'); + test('test-smth', 'TestSmth'); + test(' LoremIpsum ', 'Loremipsum'); + test(' hóhó ', 'HH'); + test('100Ω', '100'); + test('SOME TEST', 'SomeTest'); + test('some_test', 'Some_test'); +}); diff --git a/packages/streams/src/mapping/core/CollectionNameBuilder.ts b/packages/streams/src/mapping/core/CollectionNameBuilder.ts new file mode 100644 index 0000000000..ec57da2a35 --- /dev/null +++ b/packages/streams/src/mapping/core/CollectionNameBuilder.ts @@ -0,0 +1,30 @@ +export class CollectionNameBuilder { + private readonly cache: Record = {}; + private cacheLimit = 256; + + public build(tag: string): string { + let result = this.cache[tag]; + if (!result) { + result = this.process(tag); + if (this.cacheLimit > 0) { + // Simple anti DDOS protection. + this.cache[tag] = result; + this.cacheLimit--; + } + } + return result; + } + + private process(tag: string): string { + const parts = tag + .split(/[^a-zA-Z0-9_]/) + .filter((p) => !!p) + .map((p) => { + return p.substring(0, 1).toUpperCase() + p.substring(1).toLowerCase(); + }); + if (parts.length < 1) { + throw new Error(`Cannot build table name from value "${tag}"`); + } + return parts.join(''); + } +} diff --git a/packages/streams/src/mapping/core/index.ts b/packages/streams/src/mapping/core/index.ts new file mode 100644 index 0000000000..c767e9fe85 --- /dev/null +++ b/packages/streams/src/mapping/core/index.ts @@ -0,0 +1 @@ +export * from './CollectionNameBuilder'; diff --git a/packages/streams/src/mapping/index.ts b/packages/streams/src/mapping/index.ts new file mode 100644 index 0000000000..e8267b8f2b --- /dev/null +++ b/packages/streams/src/mapping/index.ts @@ -0,0 +1,5 @@ +export * from './core'; +export * from './storage'; +export * from './internal-txs-processor'; +export * from './logs-processor'; +export * from './txs-processor'; diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.test.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.test.ts new file mode 100644 index 0000000000..c5b373fbdf --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.test.ts @@ -0,0 +1,34 @@ +import { Block, InternalTransaction } from '@moralisweb3/streams-typings'; +import { InternalTxDocumentBuilder } from './InternalTxDocumentBuilder'; + +describe('TxDocumentBuilder', () => { + it('builds correctly', () => { + const internalTx: InternalTransaction = { + from: '0x7a250d5630b4cf539739df2c5dacb4c659f2488d', + to: '0xd5cd71b64657d77e8e31b95f50a7950f647671ef', + value: '3687039609867276', + gas: '22202', + transactionHash: '0xa9a129f20710ea9d8b11e87a2e68c3682eabf5690aa1d1881d8ac8acd044aa87', + }; + const block: Block = { + number: '15766768', + hash: '0x85eb63e4717089625c30fd50751fc79557c06c5d9d1185222150ce76ef3e7e52', + timestamp: '1665996863', + }; + + const doc = InternalTxDocumentBuilder.build(internalTx, block, false, '0x20'); + + expect(doc).toBeDefined(); + expect(doc.id).toBe('0x51cce9e0ed5373ec3bb3dc7c8bb381e76896f5606cef161745da6554d2a3944d'); + expect(doc.hash).toBe('0xa9a129f20710ea9d8b11e87a2e68c3682eabf5690aa1d1881d8ac8acd044aa87'); + expect(doc.chainId).toBe(32); + expect(doc.from).toBe('0x7a250d5630b4cf539739df2c5dacb4c659f2488d'); + expect(doc.to).toBe('0xd5cd71b64657d77e8e31b95f50a7950f647671ef'); + expect(doc.value).toBe('3687039609867276'); + expect(doc.gas).toBe(22202); + expect(doc.blockHash).toBe('0x85eb63e4717089625c30fd50751fc79557c06c5d9d1185222150ce76ef3e7e52'); + expect(doc.blockNumber).toBe(15766768); + expect(doc.blockTimestamp).toBe(1665996863); + expect(doc.confirmed).toBe(false); + }); +}); diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.ts new file mode 100644 index 0000000000..3611f26f80 --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentBuilder.ts @@ -0,0 +1,37 @@ +import { Block, InternalTransaction } from '@moralisweb3/streams-typings'; +import { Document } from '../storage/Update'; + +import { InternalTxDocumentId } from './InternalTxDocumentId'; + +export interface InternalTxDocument extends Document { + id: string; + hash: string; + chainId: number; + from: string; + to: string; + value: string; + gas: number; + blockHash: string; + blockTimestamp: number; + blockNumber: number; + confirmed: boolean; +} + +export class InternalTxDocumentBuilder { + public static build(tx: InternalTransaction, block: Block, confirmed: boolean, chainId: string): InternalTxDocument { + const chain = Number(chainId); + return { + id: InternalTxDocumentId.create(chain, tx.transactionHash), + hash: tx.transactionHash, + chainId: chain, + from: tx.from as string, + to: tx.to as string, + value: tx.value as string, + gas: parseInt(tx.gas || '0', 10), + blockHash: block.hash, + blockTimestamp: parseInt(block.timestamp, 10), + blockNumber: parseInt(block.number, 10), + confirmed, + }; + } +} diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.test.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.test.ts new file mode 100644 index 0000000000..85109e80c8 --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.test.ts @@ -0,0 +1,14 @@ +import { InternalTxDocumentId } from './InternalTxDocumentId'; + +describe('InternalTxDocumentId', () => { + it('returns id', () => { + const id1 = InternalTxDocumentId.create(0x1, '0x7e3e589a63a13a33a2f02a500fee07b30e20808732fb0161af5b617a1c168640'); + expect(id1).toEqual('0x0013ba8a43e8cacbe545aff7bf7eb408e93e319aa8f3f6624e8b8e8e46422fac'); + + const id16 = InternalTxDocumentId.create( + 0x16, + '0x7e3e589a63a13a33a2f02a500fee07b30e20808732fb0161af5b617a1c168640', + ); + expect(id16).toEqual('0xf7e77fb3ce9c2f67072b746cdcc7decc8c430b8a6ee3210355e5a789c8600ce5'); + }); +}); diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.ts new file mode 100644 index 0000000000..14e6c21e0f --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxDocumentId.ts @@ -0,0 +1,9 @@ +import { ethers } from 'ethers'; + +export class InternalTxDocumentId { + public static create(chainId: number, transactionHash: string): string { + const safeTransactionHash = transactionHash.toLowerCase(); + const rawId = ethers.utils.toUtf8Bytes(`${chainId};${safeTransactionHash}`); + return ethers.utils.sha256(rawId); + } +} diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.test.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.test.ts new file mode 100644 index 0000000000..b8dd884818 --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.test.ts @@ -0,0 +1,60 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; +import { InternalTxsProcessor } from './InternalTxsProcessor'; + +describe('InternalTxsProcessor', () => { + const batch: IWebhook = { + confirmed: true, + chainId: '0x1', + abi: [], + streamId: '554c0507-94a8-4c99-bce3-d99b04210801', + tag: 'Uniswap Deployer', + retries: 0, + block: { + number: '15766768', + hash: '0x85eb63e4717089625c30fd50751fc79557c06c5d9d1185222150ce76ef3e7e52', + timestamp: '1665996863', + }, + logs: [], + txs: [], + txsInternal: [ + { + from: '0x7a250d5630b4cf539739df2c5dacb4c659f2488d', + to: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + value: '50000000000000000', + gas: '607563', + transactionHash: '0x84337d2d7aacd6324a9d1b2070099de10af84a8bf815864a38e8ce517fc82651', + }, + { + from: '0x7a250d5630b4cf539739df2c5dacb4c659f2488d', + to: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + value: '180000000000000000', + gas: '116327', + transactionHash: '0x0ab569b93b477c0e944e3dff31b0e13a7161c723dcd79ca22c3d3c6287a3e4ab', + }, + { + from: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + to: '0x7a250d5630b4cf539739df2c5dacb4c659f2488d', + value: '9405305756444464', + gas: '2300', + transactionHash: '0x7fbc05b12369e364a827352cfd741a7ecd4cf655c21e6039de45aa8c03fc4548', + }, + ], + erc20Transfers: [], + erc20Approvals: [], + nftApprovals: { + ERC1155: [], + ERC721: [], + }, + nftTransfers: [], + }; + + it('processes correctly', () => { + const processor = new InternalTxsProcessor(new CollectionNameBuilder()); + + const updates = processor.process(batch); + + expect(updates.length).toEqual(3); + expect(updates[0].collectionName).toEqual('UniswapDeployer'); + }); +}); diff --git a/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.ts b/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.ts new file mode 100644 index 0000000000..8706bd8a05 --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/InternalTxsProcessor.ts @@ -0,0 +1,28 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; +import { Update } from '../storage/Update'; +import { InternalTxDocument, InternalTxDocumentBuilder } from './InternalTxDocumentBuilder'; + +export class InternalTxsProcessor { + public constructor(private readonly collectionNameBuilder: CollectionNameBuilder) {} + + public process(batch: IWebhook): InternalTxDocumentUpdate[] { + const updates: InternalTxDocumentUpdate[] = []; + + for (const internalTx of batch.txsInternal) { + const document = InternalTxDocumentBuilder.build(internalTx, batch.block, batch.confirmed, batch.chainId); + + updates.push({ + collectionName: this.collectionNameBuilder.build(batch.tag), + document, + }); + } + + return updates; + } +} + +export interface InternalTxDocumentUpdate extends Update { + collectionName: string; + document: InternalTxDocument; +} diff --git a/packages/streams/src/mapping/internal-txs-processor/index.ts b/packages/streams/src/mapping/internal-txs-processor/index.ts new file mode 100644 index 0000000000..e9778d520e --- /dev/null +++ b/packages/streams/src/mapping/internal-txs-processor/index.ts @@ -0,0 +1,3 @@ +export * from './InternalTxsProcessor'; +export * from './InternalTxDocumentBuilder'; +export * from './InternalTxDocumentId'; diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.test.ts b/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.test.ts new file mode 100644 index 0000000000..5ac5017cd4 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.test.ts @@ -0,0 +1,56 @@ +import { LogDocumentBuilder } from './LogDocumentBuilder'; +import { BigNumber } from 'ethers'; +import { Block, Log } from '@moralisweb3/streams-typings'; +import { ParsedLog } from './LogParser'; + +describe('LogDocumentBuilder', () => { + it('builds row correctly', () => { + const log: Log = { + logIndex: '56', + transactionHash: '0xb8f5496884cc69154cc38d133b6ace6923f9430e829e2f2cd1bc07bde9306388', + address: '0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce', + data: '0x0000000000000000000000000000000000000000000edd1be4934b422c1c0000', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x0000000000000000000000006d17cc023df5156efbc726946ce5d04fe484ef39', + topic2: '0x00000000000000000000000034189c75cbb13bdb4f5953cda6c3045cfca84a9e', + topic3: null, + }; + const parsedLog: ParsedLog = { + name: 'Transfer', + params: { + from: { + value: '0x6D17CC023Df5156EFBC726946cE5d04fE484eF39', + type: 'address', + }, + to: { + value: '0x34189c75Cbb13Bdb4F5953CDa6c3045CFcA84a9e', + type: 'address', + }, + value: { + value: BigNumber.from('0x0edd1be4934b422c1c0000'), + type: 'uint256', + }, + }, + }; + const block: Block = { + number: '15631019', + hash: '0x76f9e7f08bc0c4f0621fc12af6730352578a8f980a06baffda8615235574fc75', + timestamp: '1664358227', + }; + + const doc = LogDocumentBuilder.build(log, parsedLog, block, true, '0x1'); + + expect(doc.id).toBe('0x16b85919120eab4e52fb2d23d71a389ce6fdbd57016600d24033c8d00e9bd5a9'); + expect(doc.address).toBe('0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce'); + expect(doc.blockHash).toBe('0x76f9e7f08bc0c4f0621fc12af6730352578a8f980a06baffda8615235574fc75'); + expect(doc.blockNumber).toBe(15631019); + expect(doc.blockTimestamp).toBe(1664358227); + expect(doc.confirmed).toBe(true); + expect(doc.logIndex).toBe(56); + expect(doc.name).toBe('Transfer'); + expect(doc.chainId).toBe(1); + expect(doc['from']).toBe('0x6d17cc023df5156efbc726946ce5d04fe484ef39'); + expect(doc['to']).toBe('0x34189c75cbb13bdb4f5953cda6c3045cfca84a9e'); + expect(doc['value']).toBe('17969119000000000000000000'); + }); +}); diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.ts b/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.ts new file mode 100644 index 0000000000..d92aa4e5bf --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentBuilder.ts @@ -0,0 +1,75 @@ +import { Block, Log } from '@moralisweb3/streams-typings'; + +import { ParsedLog } from './LogParser'; +import { LogDocumentId } from './LogDocumentId'; +import { LogDocumentValueFormatter } from './LogDocumentValueFormatter'; +import { ParamNameResolver } from './ParamNameResolver'; +import { Document } from '../storage/Update'; + +interface BaseLogDocument { + id: string; + name: string; + logIndex: number; + transactionHash: string; + address: string; + blockHash: string; + blockTimestamp: number; + blockNumber: number; + confirmed: boolean; + chainId: number; +} + +export interface LogDocument extends BaseLogDocument, Document { + [paramName: string]: LogDocumentValue; +} + +export type LogDocumentValue = string | number | boolean; + +const paramNames: (keyof BaseLogDocument)[] = [ + 'id', + 'name', + 'logIndex', + 'transactionHash', + 'address', + 'blockHash', + 'blockTimestamp', + 'blockNumber', + 'confirmed', + 'chainId', +]; + +const restrictedParamNames: string[] = [ + ...paramNames, + // Some extra names + '_id', + 'uniqueId', + 'updatedAt', + 'createdAt', + 'user', + 'userId', +]; + +export class LogDocumentBuilder { + public static build(log: Log, parsedLog: ParsedLog, block: Block, confirmed: boolean, chainId: string): LogDocument { + const nameResolver = new ParamNameResolver(restrictedParamNames); + const chain = Number(chainId); + + const document: LogDocument = { + id: LogDocumentId.create(chain, log.transactionHash, log.logIndex), + name: parsedLog.name, + logIndex: parseInt(log.logIndex, 10), + transactionHash: log.transactionHash, + address: log.address, + blockHash: block.hash, + blockTimestamp: parseInt(block.timestamp, 10), + blockNumber: parseInt(block.number, 10), + confirmed, + chainId: chain, + }; + + nameResolver.iterate(parsedLog.params, (safeParamName, paramValue) => { + document[safeParamName] = LogDocumentValueFormatter.format(paramValue); + }); + return document; + } +} diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentId.test.ts b/packages/streams/src/mapping/logs-processor/LogDocumentId.test.ts new file mode 100644 index 0000000000..814d73786c --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentId.test.ts @@ -0,0 +1,23 @@ +import { LogDocumentId } from './LogDocumentId'; + +describe('LogDocumentId', () => { + const transactionHash = '0x32A8EC048252e5e01c10ce34b68dd2c09d93c7f5fc7870d17108f5d09615d4B1'; + + it('creates correct id', () => { + const logIndex = '11'; + const id1 = LogDocumentId.create(0x1, transactionHash, logIndex); + const id2 = LogDocumentId.create(0x1, transactionHash.toUpperCase(), logIndex); + const id3 = LogDocumentId.create(0x1, transactionHash.toLowerCase(), logIndex); + + const expectedId = '0xd41b67934bd263462e1721e5a2f04656254d78f442562013b69c9667b6f4cc91'; + expect(id1).toEqual(expectedId); + expect(id2).toEqual(expectedId); + expect(id3).toEqual(expectedId); + }); + + it('logs with different logIndex have different id', () => { + expect(LogDocumentId.create(0x512, transactionHash, '1')).not.toBe( + LogDocumentId.create(0x512, transactionHash, '2'), + ); + }); +}); diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentId.ts b/packages/streams/src/mapping/logs-processor/LogDocumentId.ts new file mode 100644 index 0000000000..868560d636 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentId.ts @@ -0,0 +1,9 @@ +import { ethers } from 'ethers'; + +export class LogDocumentId { + public static create(chainId: number, transactionHash: string, logIndex: string): string { + const safeTransactionHash = transactionHash.toLowerCase(); + const rawId = ethers.utils.toUtf8Bytes(`${chainId};${safeTransactionHash};${logIndex}`); + return ethers.utils.sha256(rawId); + } +} diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.test.ts b/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.test.ts new file mode 100644 index 0000000000..5d798063bd --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.test.ts @@ -0,0 +1,20 @@ +import { BigNumber } from 'ethers'; +import { LogParamValue } from './LogParser'; +import { LogDocumentValueFormatter } from './LogDocumentValueFormatter'; + +describe('LogDocumentValueFormatter', () => { + function test(type: string, value: LogParamValue, expectedValue: unknown) { + it(`formats ${type} value to ${expectedValue}`, () => { + expect(LogDocumentValueFormatter.format({ type, value })).toBe(expectedValue); + }); + } + + test('string', 'test', 'test'); + test('address', '0x351228872BD3fd72f64596623d0f5e8e8014F801', '0x351228872bd3fd72f64596623d0f5e8e8014f801'); + test('uint256', BigNumber.from(0x100), '256'); + test( + 'bytes32', + '0xe8ceaa0b6368653b230114926c9cb015f1dafa62a3f7ac854a2a246accf2061c', + '0xe8ceaa0b6368653b230114926c9cb015f1dafa62a3f7ac854a2a246accf2061c', + ); +}); diff --git a/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.ts b/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.ts new file mode 100644 index 0000000000..28c51eb756 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogDocumentValueFormatter.ts @@ -0,0 +1,20 @@ +import { BigNumber } from 'ethers'; + +import { LogParam } from './LogParser'; +import { LogDocumentValue } from './LogDocumentBuilder'; + +export class LogDocumentValueFormatter { + public static format(param: LogParam): LogDocumentValue { + switch (param.type) { + case 'string': + return param.value as string; + case 'address': + return (param.value as string).toLowerCase(); + default: + if (BigNumber.isBigNumber(param.value)) { + return (param.value as BigNumber).toString(); + } + return param.value.toString(); + } + } +} diff --git a/packages/streams/src/mapping/logs-processor/LogParser.test.ts b/packages/streams/src/mapping/logs-processor/LogParser.test.ts new file mode 100644 index 0000000000..ee8dc4e529 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogParser.test.ts @@ -0,0 +1,231 @@ +import { BigNumber } from 'ethers'; + +import { AbiItem, Log } from '@moralisweb3/streams-typings'; + +import { LogParam, LogParser } from './LogParser'; + +describe('LogParser', () => { + function expectBigNumber(param: LogParam, expectedValue: string) { + expect(BigNumber.isBigNumber(param.value)).toEqual(true); + expect((param.value as BigNumber).toString()).toEqual(expectedValue); + } + + it('reads Transfer event correctly', () => { + const log: Log = { + logIndex: '1', + transactionHash: '0x13330587c90eb5efe8cd49a1da7314660d51cc0de35b97b6d423584459a5a643', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000a79e31c40', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000beefbabeea323f07c59926295205d3b7a17e8638', + topic2: '0x00000000000000000000000088e6a0c2ddd26feeb64f039a2c41296fcb3f5640', + topic3: null, + }; + const abiItems: AbiItem[] = [ + { + anonymous: false, + inputs: [ + { + indexed: true, + name: 'owner', + type: 'address', + }, + { + indexed: true, + name: 'spender', + type: 'address', + }, + { + indexed: false, + name: 'value', + type: 'uint256', + }, + ], + name: 'Approval', + type: 'event', + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + name: 'from', + type: 'address', + }, + { + indexed: true, + name: 'to', + type: 'address', + }, + { + indexed: false, + name: 'value', + type: 'uint256', + }, + ], + name: 'Transfer', + type: 'event', + }, + ]; + + const { name, params } = new LogParser(abiItems).read(log); + + expect(name).toBe('Transfer'); + + expect(params['from'].type).toEqual('address'); + expect(typeof params['from'].value).toBe('string'); + expect(params['from'].value).toEqual('0xBEEFBaBEeA323F07c59926295205d3b7a17E8638'); + + expect(params['to'].type).toEqual('address'); + expect(typeof params['to'].value).toBe('string'); + expect(params['to'].value).toEqual('0x88e6A0c2dDD26FEEb64F039a2c41296FcB3f5640'); + + expect(params['value'].type).toEqual('uint256'); + expectBigNumber(params['value'], '44994600000'); + }); + + it('reads Swap event correctly', () => { + const log: Log = { + logIndex: '242', + transactionHash: '0xeaad25e6cc05765d8e8a4358aee9593f578495b22890fcc10000a0ea4f37c444', + address: '0xed92bfe08de542bbb40fdbe0a27ca66313c0c457', + data: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001b1c830beb6d62ce6300000000000000000000000000000000000000000000000000b521481ba091890000000000000000000000000000000000000000000000000000000000000000', + topic0: '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822', + topic1: '0x000000000000000000000000def1c0ded9bec7f1a1670819833240f027b25eff', + topic2: '0x000000000000000000000000def1c0ded9bec7f1a1670819833240f027b25eff', + topic3: null, + }; + const abiItems: AbiItem[] = [ + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: 'address', + name: 'sender', + type: 'address', + }, + { + indexed: false, + internalType: 'uint256', + name: 'amount0In', + type: 'uint256', + }, + { + indexed: false, + internalType: 'uint256', + name: 'amount1In', + type: 'uint256', + }, + { + indexed: false, + internalType: 'uint256', + name: 'amount0Out', + type: 'uint256', + }, + { + indexed: false, + internalType: 'uint256', + name: 'amount1Out', + type: 'uint256', + }, + { + indexed: true, + internalType: 'address', + name: 'to', + type: 'address', + }, + ], + name: 'Swap', + type: 'event', + }, + ]; + + const { name, params } = new LogParser(abiItems).read(log); + + expect(name).toEqual('Swap'); + + expect(params['amount0In'].type).toEqual('uint256'); + expectBigNumber(params['amount0In'], '0'); + + expect(params['amount1In'].type).toEqual('uint256'); + expectBigNumber(params['amount1In'], '500116588950949383779'); + + expect(params['amount0Out'].type).toEqual('uint256'); + expectBigNumber(params['amount0Out'], '50983564369498505'); + + expect(params['amount1Out'].type).toEqual('uint256'); + expectBigNumber(params['amount1Out'], '0'); + }); + + it('reads log with string argument correctly', () => { + const log: Log = { + logIndex: '111', + transactionHash: '0x7306fc7e0f7d88e791219464acf1aae0d18573d8b11f03e836f51460bb5f2ed9', + address: '0x283af0b28c62c092c9727f1ee09c02ca627eb7f5', + data: '0x0000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000d9c2fea2771f200000000000000000000000000000000000000000000000000000000652569e30000000000000000000000000000000000000000000000000000000000000006626974636f730000000000000000000000000000000000000000000000000000', + topic0: '0xca6abbe9d7f11422cb6ca7629fbf6fe9efb1c621f71ce8f02b9f2a230097404f', + topic1: '0xe8ceaa0b6368653b230114926c9cb015f1dafa62a3f7ac854a2a246accf2061c', + topic2: '0x000000000000000000000000909189c920e1c4c3bb4d29f3c6ab63030219a965', + topic3: null, + }; + const abiItems: AbiItem[] = [ + { + anonymous: false, + inputs: [ + { + indexed: false, + internalType: 'string', + name: 'name', + type: 'string', + }, + { + indexed: true, + internalType: 'bytes32', + name: 'label', + type: 'bytes32', + }, + { + indexed: true, + internalType: 'address', + name: 'owner', + type: 'address', + }, + { + indexed: false, + internalType: 'uint256', + name: 'cost', + type: 'uint256', + }, + { + indexed: false, + internalType: 'uint256', + name: 'expires', + type: 'uint256', + }, + ], + name: 'NameRegistered', + type: 'event', + }, + ]; + + const { name, params } = new LogParser(abiItems).read(log); + + expect(name).toBe('NameRegistered'); + + expect(params['name'].type).toBe('string'); + expect(params['name'].value).toBe('bitcos'); + + expect(params['label'].type).toBe('bytes32'); + expect(params['label'].value).toBe('0xe8ceaa0b6368653b230114926c9cb015f1dafa62a3f7ac854a2a246accf2061c'); + + expect(params['owner'].type).toBe('address'); + expect(params['owner'].value).toBe('0x909189C920E1c4c3Bb4D29f3c6aB63030219a965'); + + expect(params['cost'].type).toBe('uint256'); + expectBigNumber(params['cost'], '3830904303088114'); + + expect(params['expires'].type).toBe('uint256'); + expectBigNumber(params['expires'], '1696950755'); + }); +}); diff --git a/packages/streams/src/mapping/logs-processor/LogParser.ts b/packages/streams/src/mapping/logs-processor/LogParser.ts new file mode 100644 index 0000000000..68ac467c3e --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogParser.ts @@ -0,0 +1,48 @@ +import { BigNumber } from 'ethers'; + +import { AbiItem, Log } from '@moralisweb3/streams-typings'; +import { JsonFragment, Interface } from '@ethersproject/abi'; + +export interface ParsedLog { + name: string; + params: Record; +} + +export interface LogParam { + value: LogParamValue; + type: string; +} + +export type LogParamValue = BigNumber | string; + +export class LogParser { + private readonly abiInterface: Interface; + + public constructor(abiItems: AbiItem[]) { + this.abiInterface = new Interface(abiItems as JsonFragment[]); + } + + public read(log: Log): ParsedLog { + // Solidity supports max 3 topics. https://docs.soliditylang.org/en/latest/contracts.html#events + const topics = [log.topic0, log.topic1, log.topic2, log.topic3].filter((t) => t !== null) as string[]; + + const result = this.abiInterface.parseLog({ + data: log.data, + topics, + }); + + const params: Record = {}; + + for (const input of result.eventFragment.inputs) { + params[input.name] = { + type: input.type, + value: result.args[input.name], + }; + } + + return { + name: result.name, + params, + }; + } +} diff --git a/packages/streams/src/mapping/logs-processor/LogsProcessor.test.ts b/packages/streams/src/mapping/logs-processor/LogsProcessor.test.ts new file mode 100644 index 0000000000..ecc7ddbb0c --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogsProcessor.test.ts @@ -0,0 +1,398 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; + +import { LogsProcessor } from './LogsProcessor'; + +const batch: IWebhook = { + confirmed: false, + chainId: '0x1', + abi: [ + { + anonymous: false, + inputs: [ + { + indexed: true, + name: 'owner', + type: 'address', + }, + { + indexed: true, + name: 'spender', + type: 'address', + }, + { + indexed: false, + name: 'value', + type: 'uint256', + }, + ], + name: 'Approval', + type: 'event', + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + name: 'from', + type: 'address', + }, + { + indexed: true, + name: 'to', + type: 'address', + }, + { + indexed: false, + name: 'value', + type: 'uint256', + }, + ], + name: 'Transfer', + type: 'event', + }, + ], + retries: 0, + block: { + number: '15695815', + hash: '0x771a48ac8bc09f19bf6cbeea98095650d988b61238fb14716991e59bdcfc1477', + timestamp: '1665140723', + }, + logs: [ + { + logIndex: '3', + transactionHash: '0xa7840a0693ec502b0df42b6e7f75ad7a873dda95ee829cff01272e9e6fd5a37c', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000004a515efa', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000b4e16d0168e52d35cacd2c6185b44281ec28c9dc', + topic2: '0x00000000000000000000000088ff79eb2bc5850f27315415da8685282c7610f9', + topic3: null, + }, + { + logIndex: '19', + transactionHash: '0xfdfd919547092b1473611c454c6c93e87471f26dea898a8e12c6dda89be4041c', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000008f0d180', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x0000000000000000000000006ce8f00a80b9fa56d57cc5f49f3398a90463192e', + topic2: '0x00000000000000000000000040ec5b33f54e0e8a33a975908c5ba1c14e5bbbdf', + topic3: null, + }, + { + logIndex: '23', + transactionHash: '0x2b518aa06948c3fa84301979ad088517a9e8735d80647aaf1caf77a844a6c61d', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000020b20546f', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000397ff1542f962076d0bfe58ea045ffa2d347aca0', + topic2: '0x000000000000000000000000beefbabeea323f07c59926295205d3b7a17e8638', + topic3: null, + }, + { + logIndex: '53', + transactionHash: '0xff1a200c2dfd77882d35f96594fe20f031c0bdbeef4de55aa52ea0e523d15568', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000003f83114c', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000397ff1542f962076d0bfe58ea045ffa2d347aca0', + topic2: '0x0000000000000000000000001111111254fb6c44bac0bed2854e76f90643097d', + topic3: null, + }, + { + logIndex: '56', + transactionHash: '0xff1a200c2dfd77882d35f96594fe20f031c0bdbeef4de55aa52ea0e523d15568', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000003f83114c', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x0000000000000000000000001111111254fb6c44bac0bed2854e76f90643097d', + topic2: '0x00000000000000000000000054f660940a32ce6cdf833e30926b2523497032ed', + topic3: null, + }, + { + logIndex: '63', + transactionHash: '0x0bf4e1f42097117f502eba2cf2392d1cb969f26ab0647222637395de2410c026', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000007641700', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x00000000000000000000000097f16b00d436fcd49d5911e68002f1cd4d5e47c5', + topic2: '0x000000000000000000000000def171fe48cf0115b1d80b88dc8eab59176fee57', + topic3: null, + }, + { + logIndex: '64', + transactionHash: '0x0bf4e1f42097117f502eba2cf2392d1cb969f26ab0647222637395de2410c026', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000007641700', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000def171fe48cf0115b1d80b88dc8eab59176fee57', + topic2: '0x000000000000000000000000b3c839dbde6b96d37c56ee4f9dad3390d49310aa', + topic3: null, + }, + { + logIndex: '95', + transactionHash: '0x560e6161c0d09bb265af281c6964e0316ef0c5357172eb375e6dba15c6b0e09e', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000000000000', + topic0: '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', + topic1: '0x000000000000000000000000cd231d4ba7b15a4722ac057419d9cd7689e7b8db', + topic2: '0x00000000000000000000000028cba71583e5f7149e59d59ee9294e204fa24741', + topic3: null, + }, + { + logIndex: '140', + transactionHash: '0x91fed188f3f3f94f4f00b376561644b1fd3eda11fbcd8d22e397813a2e7fa9c4', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000012a05f200', + topic0: '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', + topic1: '0x0000000000000000000000006ac0bf4b95985636d6f9a5c5e28e611cf8004683', + topic2: '0x00000000000000000000000068b3465833fb72a70ecdf485e0e4c7bd8665fc45', + topic3: null, + }, + { + logIndex: '142', + transactionHash: '0x91fed188f3f3f94f4f00b376561644b1fd3eda11fbcd8d22e397813a2e7fa9c4', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x000000000000000000000000000000000000000000000000000000012a05f200', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x0000000000000000000000006ac0bf4b95985636d6f9a5c5e28e611cf8004683', + topic2: '0x00000000000000000000000088e6a0c2ddd26feeb64f039a2c41296fcb3f5640', + topic3: null, + }, + { + logIndex: '220', + transactionHash: '0x161d8c3641b9e648905580f1288fb3e8935136652518eb71a1facdcd4765de3a', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x00000000000000000000000000000000000000000000000000000003ba6b31bc', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000cdec510974a17fcf3156efb104990de7f12cbcfe', + topic2: '0x00000000000000000000000023ddd3e3692d1861ed57ede224608875809e127f', + topic3: null, + }, + { + logIndex: '289', + transactionHash: '0xaa22a7eef71898b7bf413cfb1941538afc2e7b874d092394c2efd2e6e81c8258', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x00000000000000000000000000000000000000000000000000000001dc3db980', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x0000000000000000000000007d6636635b7c00e59363b544ed518c6c18c2c8eb', + topic2: '0x00000000000000000000000042261e4c83d41f86cd603513957bb26f9b4c9663', + topic3: null, + }, + { + logIndex: '302', + transactionHash: '0x2b98a4b51c0860b6eb5a68d8a98c3ca402b8a12e55a8e6095d2ea071ce6aaff8', + address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + data: '0x0000000000000000000000000000000000000000000000000000000000d67044', + topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: '0x000000000000000000000000b4e16d0168e52d35cacd2c6185b44281ec28c9dc', + topic2: '0x0000000000000000000000001266faebc05d5c96e941a0d715228ba68efada86', + topic3: null, + }, + ], + txs: [], + txsInternal: [], + erc20Transfers: [ + { + transactionHash: '0xa7840a0693ec502b0df42b6e7f75ad7a873dda95ee829cff01272e9e6fd5a37c', + logIndex: '3', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc', + to: '0x88ff79eb2bc5850f27315415da8685282c7610f9', + value: '1246846714', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '1246.846714', + }, + { + transactionHash: '0xfdfd919547092b1473611c454c6c93e87471f26dea898a8e12c6dda89be4041c', + logIndex: '19', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x6ce8f00a80b9fa56d57cc5f49f3398a90463192e', + to: '0x40ec5b33f54e0e8a33a975908c5ba1c14e5bbbdf', + value: '150000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '150', + }, + { + transactionHash: '0x2b518aa06948c3fa84301979ad088517a9e8735d80647aaf1caf77a844a6c61d', + logIndex: '23', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x397ff1542f962076d0bfe58ea045ffa2d347aca0', + to: '0xbeefbabeea323f07c59926295205d3b7a17e8638', + value: '8776602735', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '8776.602735', + }, + { + transactionHash: '0xff1a200c2dfd77882d35f96594fe20f031c0bdbeef4de55aa52ea0e523d15568', + logIndex: '53', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x397ff1542f962076d0bfe58ea045ffa2d347aca0', + to: '0x1111111254fb6c44bac0bed2854e76f90643097d', + value: '1065554252', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '1065.554252', + }, + { + transactionHash: '0xff1a200c2dfd77882d35f96594fe20f031c0bdbeef4de55aa52ea0e523d15568', + logIndex: '56', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x1111111254fb6c44bac0bed2854e76f90643097d', + to: '0x54f660940a32ce6cdf833e30926b2523497032ed', + value: '1065554252', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '1065.554252', + }, + { + transactionHash: '0x0bf4e1f42097117f502eba2cf2392d1cb969f26ab0647222637395de2410c026', + logIndex: '63', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x97f16b00d436fcd49d5911e68002f1cd4d5e47c5', + to: '0xdef171fe48cf0115b1d80b88dc8eab59176fee57', + value: '124000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '124', + }, + { + transactionHash: '0x0bf4e1f42097117f502eba2cf2392d1cb969f26ab0647222637395de2410c026', + logIndex: '64', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0xdef171fe48cf0115b1d80b88dc8eab59176fee57', + to: '0xb3c839dbde6b96d37c56ee4f9dad3390d49310aa', + value: '124000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '124', + }, + { + transactionHash: '0x91fed188f3f3f94f4f00b376561644b1fd3eda11fbcd8d22e397813a2e7fa9c4', + logIndex: '142', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x6ac0bf4b95985636d6f9a5c5e28e611cf8004683', + to: '0x88e6a0c2ddd26feeb64f039a2c41296fcb3f5640', + value: '5000000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '5000', + }, + { + transactionHash: '0x161d8c3641b9e648905580f1288fb3e8935136652518eb71a1facdcd4765de3a', + logIndex: '220', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0xcdec510974a17fcf3156efb104990de7f12cbcfe', + to: '0x23ddd3e3692d1861ed57ede224608875809e127f', + value: '16012489148', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '16012.489148', + }, + { + transactionHash: '0xaa22a7eef71898b7bf413cfb1941538afc2e7b874d092394c2efd2e6e81c8258', + logIndex: '289', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0x7d6636635b7c00e59363b544ed518c6c18c2c8eb', + to: '0x42261e4c83d41f86cd603513957bb26f9b4c9663', + value: '7990000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '7990', + }, + { + transactionHash: '0x2b98a4b51c0860b6eb5a68d8a98c3ca402b8a12e55a8e6095d2ea071ce6aaff8', + logIndex: '302', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + from: '0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc', + to: '0x1266faebc05d5c96e941a0d715228ba68efada86', + value: '14053444', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '14.053444', + }, + ], + erc20Approvals: [ + { + transactionHash: '0x560e6161c0d09bb265af281c6964e0316ef0c5357172eb375e6dba15c6b0e09e', + logIndex: '95', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + owner: '0xcd231d4ba7b15a4722ac057419d9cd7689e7b8db', + spender: '0x28cba71583e5f7149e59d59ee9294e204fa24741', + value: '0', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '0', + }, + { + transactionHash: '0x91fed188f3f3f94f4f00b376561644b1fd3eda11fbcd8d22e397813a2e7fa9c4', + logIndex: '140', + contract: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + owner: '0x6ac0bf4b95985636d6f9a5c5e28e611cf8004683', + spender: '0x68b3465833fb72a70ecdf485e0e4c7bd8665fc45', + value: '5000000000', + tokenName: 'USD Coin', + tokenSymbol: 'USDC', + tokenDecimals: '6', + valueWithDecimals: '5000', + }, + ], + nftApprovals: { + ERC1155: [], + ERC721: [], + }, + nftTransfers: [], + tag: 'Some Tag', + streamId: 'ba3b3c52-3dd3-4eb7-a2b7-4b61d3439c5e', +}; + +describe('LogsProcessor', () => { + const processor = new LogsProcessor(new CollectionNameBuilder()); + + it('processes correctly', () => { + const updates = processor.process(batch); + + expect(updates.length).toEqual(13); + + const update0 = updates[0]; + expect(update0.collectionName).toEqual('SomeTag'); + expect(update0.document.name).toEqual('Transfer'); + expect(update0.document.chainId).toEqual(1); + expect(update0.document['from']).toEqual('0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc'); + expect(update0.document['to']).toEqual('0x88ff79eb2bc5850f27315415da8685282c7610f9'); + expect(update0.document['value']).toEqual('1246846714'); + + const update8 = updates[8]; + expect(update8.collectionName).toEqual('SomeTag'); + expect(update8.document.name).toEqual('Approval'); + expect(update0.document.chainId).toEqual(1); + expect(update8.document['spender']).toEqual('0x68b3465833fb72a70ecdf485e0e4c7bd8665fc45'); + expect(update8.document['owner']).toEqual('0x6ac0bf4b95985636d6f9a5c5e28e611cf8004683'); + expect(update8.document['value']).toEqual('5000000000'); + }); + + it('returns empty array when no abi', () => { + const batchWithNoAbi: IWebhook = Object.assign({}, batch, { + abi: [], + }); + + const updates = processor.process(batchWithNoAbi); + + expect(updates.length).toEqual(0); + }); +}); diff --git a/packages/streams/src/mapping/logs-processor/LogsProcessor.ts b/packages/streams/src/mapping/logs-processor/LogsProcessor.ts new file mode 100644 index 0000000000..e4a17a45e4 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/LogsProcessor.ts @@ -0,0 +1,37 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; + +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; +import { LogParser } from './LogParser'; +import { LogDocument, LogDocumentBuilder } from './LogDocumentBuilder'; +import { Update } from '../storage/Update'; + +export class LogsProcessor { + public constructor(private readonly collectionNameBuilder: CollectionNameBuilder) {} + + public process(batch: IWebhook): LogDocumentUpdate[] { + const updates: LogDocumentUpdate[] = []; + + if (batch.abi.length < 1) { + return updates; + } + + const logParser = new LogParser(batch.abi); + + for (const log of batch.logs) { + const logParams = logParser.read(log); + const document = LogDocumentBuilder.build(log, logParams, batch.block, batch.confirmed, batch.chainId); + + updates.push({ + collectionName: this.collectionNameBuilder.build(batch.tag), + document, + }); + } + + return updates; + } +} + +export interface LogDocumentUpdate extends Update { + collectionName: string; + document: LogDocument; +} diff --git a/packages/streams/src/mapping/logs-processor/ParamNameResolver.test.ts b/packages/streams/src/mapping/logs-processor/ParamNameResolver.test.ts new file mode 100644 index 0000000000..b698e618e2 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/ParamNameResolver.test.ts @@ -0,0 +1,18 @@ +import { ParamNameResolver } from './ParamNameResolver'; + +describe('ParamNameResolver', () => { + it('resolves correctly', () => { + const resolver = new ParamNameResolver(['alfa', 'beta']); + + expect(resolver.resolve('foo')).toEqual('foo'); + expect(resolver.resolve('bar')).toEqual('bar'); + + expect(resolver.resolve('beta')).toEqual('_beta'); + + expect(resolver.resolve('alfa')).toEqual('_alfa'); + expect(resolver.resolve('_alfa')).toEqual('__alfa'); + expect(resolver.resolve('__alfa')).toEqual('___alfa'); + expect(resolver.resolve('____alfa')).toEqual('____alfa'); + expect(resolver.resolve('___alfa')).toEqual('_____alfa'); + }); +}); diff --git a/packages/streams/src/mapping/logs-processor/ParamNameResolver.ts b/packages/streams/src/mapping/logs-processor/ParamNameResolver.ts new file mode 100644 index 0000000000..37b4cc6e39 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/ParamNameResolver.ts @@ -0,0 +1,31 @@ +export class ParamNameResolver { + private readonly usedNames: string[] = []; + + public constructor(private readonly restrictedNames: string[]) {} + + public iterate(object: Record, callback: (safeName: string, value: Value) => void) { + // We need to always keep parameters in the same order + // because the RowParamNameResolver is order-sensitive. + const sortedNames = Object.keys(object).sort((a, b) => a.localeCompare(b)); + + sortedNames.forEach((name) => { + const safeName = this.resolve(name); + callback(safeName, object[name]); + }); + } + + public resolve(name: string): string { + if (this.isUsed(name)) { + do { + name = `_${name}`; + } while (this.isUsed(name)); + } + + this.usedNames.push(name); + return name; + } + + private isUsed(name: string): boolean { + return this.restrictedNames.includes(name) || this.usedNames.includes(name); + } +} diff --git a/packages/streams/src/mapping/logs-processor/index.ts b/packages/streams/src/mapping/logs-processor/index.ts new file mode 100644 index 0000000000..e8db8f8286 --- /dev/null +++ b/packages/streams/src/mapping/logs-processor/index.ts @@ -0,0 +1,6 @@ +export * from './LogDocumentBuilder'; +export * from './LogDocumentId'; +export * from './LogDocumentValueFormatter'; +export * from './LogParser'; +export * from './LogsProcessor'; +export * from './ParamNameResolver'; diff --git a/packages/streams/src/mapping/storage/Update.ts b/packages/streams/src/mapping/storage/Update.ts new file mode 100644 index 0000000000..98837a2bb8 --- /dev/null +++ b/packages/streams/src/mapping/storage/Update.ts @@ -0,0 +1,9 @@ +export interface Update { + collectionName: string; + document: Doc; +} + +export interface Document { + id: string; + confirmed: boolean; +} diff --git a/packages/streams/src/mapping/storage/index.ts b/packages/streams/src/mapping/storage/index.ts new file mode 100644 index 0000000000..9a9d180ac5 --- /dev/null +++ b/packages/streams/src/mapping/storage/index.ts @@ -0,0 +1 @@ +export * from './Update'; diff --git a/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.test.ts b/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.test.ts new file mode 100644 index 0000000000..0065735eae --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.test.ts @@ -0,0 +1,55 @@ +import { Block, Transaction } from '@moralisweb3/streams-typings'; +import { TxDocumentBuilder } from './TxDocumentBuilder'; + +describe('TxDocumentBuilder', () => { + it('builds correctly', () => { + const tx: Transaction = { + hash: '0x91e4046c7768132aa614c6e0d4773b8cd20cee1948b2508e570d3ce630415588', + gas: '207128', + gasPrice: '8652938136', + nonce: '4262103', + input: '0x', + transactionIndex: '75', + fromAddress: '0xdfd5293d8e347dfe59e90efd55b2956a1343963d', + toAddress: '0xc2bacad6632903dc03e190bf33a6e8c7ea0881c1', + value: '10000000000000000', + type: '2', + v: '0', + r: '8522664046905990578164143534491536109331129839629903737002530670138249739539', + s: '48306749826871999834738509867081969620532520428036344972909508655681719515828', + receiptCumulativeGasUsed: '6692052', + receiptGasUsed: '21000', + receiptContractAddress: null, + receiptRoot: null, + receiptStatus: '1', + }; + const block: Block = { + number: '15666916', + hash: '0x1af899d66c7847ba6d1faa4d9ca1b37e4f337232662a8e60163a6718a234d9ff', + timestamp: '1664791547', + }; + + const doc = TxDocumentBuilder.build(tx, block, false, '0x100'); + + expect(doc).toBeDefined(); + expect(doc.id).toBe('0xd5c8346479d45c9ddb2bceed5ba23e44b5f10676494542b7b494401f78c295d4'); + expect(doc.hash).toBe('0x91e4046c7768132aa614c6e0d4773b8cd20cee1948b2508e570d3ce630415588'); + expect(doc.chainId).toBe(256); + expect(doc.transactionIndex).toBe(75); + expect(doc.gas).toBe(207128); + expect(doc.gasPrice).toBe(8652938136); + expect(doc.nonce).toBe(4262103); + expect(doc.fromAddress).toBe('0xdfd5293d8e347dfe59e90efd55b2956a1343963d'); + expect(doc.toAddress).toBe('0xc2bacad6632903dc03e190bf33a6e8c7ea0881c1'); + expect(doc.value).toBe('10000000000000000'); + expect(doc.input).toBe('0x'); + expect(doc.type).toBe(2); + expect(doc.receiptStatus).toBe(1); + expect(doc.receiptGasUsed).toBe(21000); + expect(doc.receiptCumulativeGasUsed).toBe(6692052); + expect(doc.blockHash).toBe('0x1af899d66c7847ba6d1faa4d9ca1b37e4f337232662a8e60163a6718a234d9ff'); + expect(doc.blockTimestamp).toBe(1664791547); + expect(doc.blockNumber).toBe(15666916); + expect(doc.confirmed).toBe(false); + }); +}); diff --git a/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.ts b/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.ts new file mode 100644 index 0000000000..b2590e6408 --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxDocumentBuilder.ts @@ -0,0 +1,53 @@ +import { Block, Transaction } from '@moralisweb3/streams-typings'; +import { Document } from '../storage/Update'; + +import { TxDocumentId } from './TxDocumentId'; + +export interface TxDocument extends Document { + id: string; + hash: string; + chainId: number; + transactionIndex: number; + gas: number; + gasPrice: number; + nonce: number; + fromAddress: string; + toAddress: string | null; + value: string; + input: string | null; + type: number; + receiptStatus: number; + receiptGasUsed: number; + receiptCumulativeGasUsed: number; + blockHash: string; + blockTimestamp: number; + blockNumber: number; + confirmed: boolean; +} + +export class TxDocumentBuilder { + public static build(tx: Transaction, block: Block, confirmed: boolean, chainId: string): TxDocument { + const chain = Number(chainId); + return { + id: TxDocumentId.create(chain, tx.hash), + hash: tx.hash, + chainId: chain, + transactionIndex: parseInt(tx.transactionIndex, 10), + gas: parseInt(tx.gas as string, 10), + gasPrice: parseInt(tx.gasPrice as string, 10), + nonce: parseInt(tx.nonce as string, 10), + fromAddress: tx.fromAddress, + toAddress: tx.toAddress, + value: tx.value || '0', + input: tx.input, + type: parseInt(tx.type as string, 10), + receiptStatus: parseInt(tx.receiptStatus as string, 10), + receiptGasUsed: parseInt(tx.receiptGasUsed as string, 10), + receiptCumulativeGasUsed: parseInt(tx.receiptCumulativeGasUsed as string, 10), + blockHash: block.hash, + blockTimestamp: parseInt(block.timestamp, 10), + blockNumber: parseInt(block.number, 10), + confirmed, + }; + } +} diff --git a/packages/streams/src/mapping/txs-processor/TxDocumentId.test.ts b/packages/streams/src/mapping/txs-processor/TxDocumentId.test.ts new file mode 100644 index 0000000000..358ebeaca8 --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxDocumentId.test.ts @@ -0,0 +1,11 @@ +import { TxDocumentId } from './TxDocumentId'; + +describe('TxDocumentId', () => { + it('returns id', () => { + const id1 = TxDocumentId.create(1, '0xbe0deb85f7c6a31c017d6ce442bb019614b292c5db1d389eb745beeee28e561c'); + expect(id1).toEqual('0x554b5817ff0c728a6f3b365ef1b90c821c17029155b32cbaa29f5f00245a8144'); + + const id16 = TxDocumentId.create(16, '0xbe0deb85f7c6a31c017d6ce442bb019614b292c5db1d389eb745beeee28e561c'); + expect(id16).toEqual('0xb97e2cb8d3bcf46395fc1751eb0524a3bc6777e5735a6120cbe02ab8871a4965'); + }); +}); diff --git a/packages/streams/src/mapping/txs-processor/TxDocumentId.ts b/packages/streams/src/mapping/txs-processor/TxDocumentId.ts new file mode 100644 index 0000000000..a9583d9cc9 --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxDocumentId.ts @@ -0,0 +1,9 @@ +import { ethers } from 'ethers'; + +export class TxDocumentId { + public static create(chainId: number, transactionHash: string): string { + const safeTransactionHash = transactionHash.toLowerCase(); + const rawId = ethers.utils.toUtf8Bytes(`${chainId};${safeTransactionHash}`); + return ethers.utils.sha256(rawId); + } +} diff --git a/packages/streams/src/mapping/txs-processor/TxsProcessor.test.ts b/packages/streams/src/mapping/txs-processor/TxsProcessor.test.ts new file mode 100644 index 0000000000..d5dee2da5c --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxsProcessor.test.ts @@ -0,0 +1,122 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; +import { TxsProcessor } from './TxsProcessor'; + +const batch: IWebhook = { + confirmed: false, + chainId: '0x1', + abi: [], + retries: 0, + block: { + number: '15696081', + hash: '0x9c3932399ae5b425e81c16b61cd527d6300082a9c8e4c276db3081460b5e646f', + timestamp: '1665143927', + }, + logs: [], + txs: [ + { + hash: '0xb94e25c6907c753fcd046ba36b95cd9eec15a4843607c3ff87f45b98c5293645', + gas: '207128', + gasPrice: '7288122076', + nonce: '4289830', + input: + '0xa9059cbb000000000000000000000000ad98dec1d73c09a12b81c16dedd90c0a8a03682800000000000000000000000000000000000000000000003c4fa4cb806161c000', + transactionIndex: '38', + fromAddress: '0xdfd5293d8e347dfe59e90efd55b2956a1343963d', + toAddress: '0xb62132e35a6c13ee1ee0f84dc5d40bad8d815206', + value: '0', + type: '2', + v: '1', + r: '64538019474906420811295350502897238449661938464225305515979147407366116831170', + s: '10718135606581849754509921163526511868200012476577348382484516618973216691114', + receiptCumulativeGasUsed: '2632519', + receiptGasUsed: '52487', + receiptContractAddress: null, + receiptRoot: null, + receiptStatus: '1', + }, + { + hash: '0x016f82df5bb05188fe00e5e9b11b68a830898b4a5403671ae79fd049f19483a6', + gas: '207128', + gasPrice: '7288122076', + nonce: '4289831', + input: + '0xa9059cbb0000000000000000000000003c77eae09ffd1f3fe857a2c8fa4a36ddcba3e755000000000000000000000000000000000000000000000000000000000362b300', + transactionIndex: '41', + fromAddress: '0xdfd5293d8e347dfe59e90efd55b2956a1343963d', + toAddress: '0xdac17f958d2ee523a2206206994597c13d831ec7', + value: '0', + type: '2', + v: '1', + r: '18561203549462965384619437500420904478230644993644522205793218440961750463538', + s: '26339235603347869000913182429117335400121947312504648590472646624428804989463', + receiptCumulativeGasUsed: '2779925', + receiptGasUsed: '63197', + receiptContractAddress: null, + receiptRoot: null, + receiptStatus: '1', + }, + { + hash: '0xf5db9667a130cefed66c2617ef6e36bbebeec22fb5dc489188dd6d630eae520d', + gas: '207128', + gasPrice: '7288122076', + nonce: '4289832', + input: '0x', + transactionIndex: '45', + fromAddress: '0xdfd5293d8e347dfe59e90efd55b2956a1343963d', + toAddress: '0x225e5444f67aea8eef93a49046a2a813f75c37e6', + value: '71348490000000000', + type: '2', + v: '0', + r: '77968559538229211287527738410611582872919079253133065181840806520047253155548', + s: '30953370743478203461278091867739471110266432453462233258707690109034018562240', + receiptCumulativeGasUsed: '2894449', + receiptGasUsed: '21000', + receiptContractAddress: null, + receiptRoot: null, + receiptStatus: '1', + }, + { + hash: '0x849572d6a1cad43061894e38b60b410108e7beb809358200befc75668c7278d5', + gas: '207128', + gasPrice: '7288122076', + nonce: '4289833', + input: + '0xa9059cbb00000000000000000000000021d01291867fc0f0b9a3fde458b8cf4adcb245960000000000000000000000000000000000000000000010692b696884e5121c00', + transactionIndex: '47', + fromAddress: '0xdfd5293d8e347dfe59e90efd55b2956a1343963d', + toAddress: '0x7420b4b9a0110cdc71fb720908340c03f9bc03ec', + value: '0', + type: '2', + v: '0', + r: '13669456909385815475383308339656995316617376836527050720134288100409117781602', + s: '47620834695764436165872449721956560965570145671771472844801055415916529176252', + receiptCumulativeGasUsed: '2992398', + receiptGasUsed: '51864', + receiptContractAddress: null, + receiptRoot: null, + receiptStatus: '1', + }, + ], + txsInternal: [], + erc20Transfers: [], + erc20Approvals: [], + nftApprovals: { + ERC1155: [], + ERC721: [], + }, + nftTransfers: [], + tag: 'Transaction', + streamId: 'ba3b3c52-3dd3-4eb7-a2b7-4b61d3439c5e', +}; + +describe('TxsProcessor', () => { + it('builds correctly', () => { + const processor = new TxsProcessor(new CollectionNameBuilder()); + + const updates = processor.process(batch); + + expect(updates.length).toBe(4); + expect(updates[0].collectionName).toBe('Transaction'); + }); +}); diff --git a/packages/streams/src/mapping/txs-processor/TxsProcessor.ts b/packages/streams/src/mapping/txs-processor/TxsProcessor.ts new file mode 100644 index 0000000000..f9d3ea07f9 --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/TxsProcessor.ts @@ -0,0 +1,29 @@ +import { IWebhook } from '@moralisweb3/streams-typings'; + +import { CollectionNameBuilder } from '../core/CollectionNameBuilder'; +import { Update } from '../storage/Update'; +import { TxDocument, TxDocumentBuilder } from './TxDocumentBuilder'; + +export class TxsProcessor { + public constructor(private readonly collectionNameBuilder: CollectionNameBuilder) {} + + public process(batch: IWebhook): TxDocumentUpdate[] { + const updates: TxDocumentUpdate[] = []; + + for (const tx of batch.txs) { + const document = TxDocumentBuilder.build(tx, batch.block, batch.confirmed, batch.chainId); + + updates.push({ + collectionName: this.collectionNameBuilder.build(batch.tag), + document, + }); + } + + return updates; + } +} + +export interface TxDocumentUpdate extends Update { + collectionName: string; + document: TxDocument; +} diff --git a/packages/streams/src/mapping/txs-processor/index.ts b/packages/streams/src/mapping/txs-processor/index.ts new file mode 100644 index 0000000000..aed2fc0639 --- /dev/null +++ b/packages/streams/src/mapping/txs-processor/index.ts @@ -0,0 +1,3 @@ +export * from './TxDocumentBuilder'; +export * from './TxDocumentId'; +export * from './TxsProcessor'; diff --git a/yarn.lock b/yarn.lock index 91b7a88a2b..b1e24ea5e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1208,6 +1208,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@7.18.0": + version "7.18.0" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.0.tgz#6d77142a19cb6088f0af662af1ada37a604d34ae" + integrity sha512-YMQvx/6nKEaucl0MY56mwIG483xk8SDNdlUwb2Ts6FUpr7fm85DxEmsY18LXBNhcTz6tO6JwZV8w1W06v8UKeg== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/runtime@7.18.3": version "7.18.3" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4" @@ -6194,16 +6201,7 @@ "@types/qs" "*" "@types/serve-static" "*" -"@types/express@4.17.3": - version "4.17.3" - resolved "https://registry.npmjs.org/@types/express/-/express-4.17.3.tgz" - integrity sha512-I8cGRJj3pyOLs/HndoP+25vOqhqWkAZsWMEmq1qXy/b/M3ppufecUwaK2/TVDVxcV61/iSdhykUjQQ2DLSrTdg== - dependencies: - "@types/body-parser" "*" - "@types/express-serve-static-core" "*" - "@types/serve-static" "*" - -"@types/express@^4.17.14": +"@types/express@4.17.14", "@types/express@^4.17.14": version "4.17.14" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.14.tgz#143ea0557249bc1b3b54f15db4c81c3d4eb3569c" integrity sha512-TEbt+vaPFQ+xpxFLFssxUDXj5cWCxZJjIcB7Yg0k0GMHGtgtQgpvx/MUQUeAkNbA9AAGrwkAsoeItdTgS7FMyg== @@ -6213,6 +6211,15 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/express@4.17.3": + version "4.17.3" + resolved "https://registry.npmjs.org/@types/express/-/express-4.17.3.tgz" + integrity sha512-I8cGRJj3pyOLs/HndoP+25vOqhqWkAZsWMEmq1qXy/b/M3ppufecUwaK2/TVDVxcV61/iSdhykUjQQ2DLSrTdg== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "*" + "@types/serve-static" "*" + "@types/fs-capacitor@*": version "2.0.0" resolved "https://registry.npmjs.org/@types/fs-capacitor/-/fs-capacitor-2.0.0.tgz" @@ -6446,6 +6453,13 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/parse@^2.18.18": + version "2.18.18" + resolved "https://registry.yarnpkg.com/@types/parse/-/parse-2.18.18.tgz#13e716595a49b6bd82a0f04d89c8c6f5930bc861" + integrity sha512-uAkXKOnZfKcWt22Akjt0W0ikWhqfIHi3wu/gCaL3m5Dq4/YFnvZX8zOnwxq8AKb/+MjDDQjsK2LKuY0heb5L3w== + dependencies: + "@types/node" "*" + "@types/pbkdf2@^3.0.0": version "3.1.0" resolved "https://registry.npmjs.org/@types/pbkdf2/-/pbkdf2-3.1.0.tgz" @@ -9060,6 +9074,24 @@ body-parser@1.20.0, body-parser@^1.18.3: type-is "~1.6.18" unpipe "1.0.0" +body-parser@^1.20.0: + version "1.20.1" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" + integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== + dependencies: + bytes "3.1.2" + content-type "~1.0.4" + debug "2.6.9" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" + iconv-lite "0.4.24" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.1" + type-is "~1.6.18" + unpipe "1.0.0" + bonjour-service@^1.0.11: version "1.0.14" resolved "https://registry.yarnpkg.com/bonjour-service/-/bonjour-service-1.0.14.tgz#c346f5bc84e87802d08f8d5a60b93f758e514ee7" @@ -18612,6 +18644,21 @@ parse@3.4.2: optionalDependencies: crypto-js "4.1.1" +parse@3.4.4: + version "3.4.4" + resolved "https://registry.yarnpkg.com/parse/-/parse-3.4.4.tgz#6358f2a59df4dfac6f6f76f1589d92942249fd43" + integrity sha512-6mnFGD3FroQtCYImRXOFF9+BohGWGwaOIo7htD0HxGVooMR1Iyb69eYj+AoLr+o3sBBGCCGmNmjNjmvUIOnSaw== + dependencies: + "@babel/runtime" "7.18.0" + "@babel/runtime-corejs3" "7.17.8" + idb-keyval "6.0.3" + react-native-crypto-js "1.0.0" + uuid "3.4.0" + ws "8.6.0" + xmlhttprequest "1.8.0" + optionalDependencies: + crypto-js "4.1.1" + parseurl@^1.3.2, parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz" @@ -19803,18 +19850,18 @@ qs@6.10.3: dependencies: side-channel "^1.0.4" -qs@6.9.6: - version "6.9.6" - resolved "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz" - integrity sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ== - -qs@^6.10.3: +qs@6.11.0, qs@^6.10.3: version "6.11.0" resolved "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz" integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== dependencies: side-channel "^1.0.4" +qs@6.9.6: + version "6.9.6" + resolved "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz" + integrity sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ== + qs@~6.5.2: version "6.5.3" resolved "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz" @@ -22826,6 +22873,11 @@ typescript@^4.8.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.3.tgz#d59344522c4bc464a65a730ac695007fdb66dd88" integrity sha512-goMHfm00nWPa8UvR/CPSvykqf6dVV8x/dp0c5mFTMTIu0u0FlGWRioyy7Nn0PGAdHxpJZnuO/ut+PpQ8UiHAig== +typescript@^4.8.4: + version "4.8.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.4.tgz#c464abca159669597be5f96b8943500b238e60e6" + integrity sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ== + typical@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz" @@ -24188,6 +24240,11 @@ ws@8.2.3, ws@~8.2.3: resolved "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz" integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== +ws@8.6.0: + version "8.6.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.6.0.tgz#e5e9f1d9e7ff88083d0c0dd8281ea662a42c9c23" + integrity sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw== + ws@>=7.4.6, ws@^8.3.0, ws@^8.4.2, ws@^8.5.0: version "8.8.1" resolved "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz"