From 28e9864844a22994205f44fc279be6b019d4b019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Mon, 30 Sep 2024 09:07:27 -0600 Subject: [PATCH] feat: add principal cache etag to account endpoints (#2097) * feat: add principal cache * fix: also consider sponsor transactions * chore: import sponsors in migration * fix: add down migration --- ...727465879167_principal-stx-txs-sponsors.js | 21 +++ src/api/controllers/cache-controller.ts | 83 +++------ src/api/routes/address.ts | 21 ++- src/api/routes/v2/addresses.ts | 9 +- src/datastore/pg-store.ts | 40 +++++ src/datastore/pg-write-store.ts | 1 + .../importers/new-block-importer.ts | 1 + tests/api/cache-control.test.ts | 170 +++++++++++++----- 8 files changed, 236 insertions(+), 110 deletions(-) create mode 100644 migrations/1727465879167_principal-stx-txs-sponsors.js diff --git a/migrations/1727465879167_principal-stx-txs-sponsors.js b/migrations/1727465879167_principal-stx-txs-sponsors.js new file mode 100644 index 0000000000..4b969e2ebd --- /dev/null +++ b/migrations/1727465879167_principal-stx-txs-sponsors.js @@ -0,0 +1,21 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = pgm => { + pgm.sql(` + INSERT INTO principal_stx_txs + (principal, tx_id, block_height, index_block_hash, microblock_hash, microblock_sequence, + tx_index, canonical, microblock_canonical) + ( + SELECT + sponsor_address AS principal, tx_id, block_height, index_block_hash, microblock_hash, + microblock_sequence, tx_index, canonical, microblock_canonical + FROM txs + WHERE sponsor_address IS NOT NULL + ) + ON CONFLICT ON CONSTRAINT unique_principal_tx_id_index_block_hash_microblock_hash DO NOTHING + `); +}; + +exports.down = pgm => {}; diff --git a/src/api/controllers/cache-controller.ts b/src/api/controllers/cache-controller.ts index e6ee858a5e..dd0ee51767 100644 --- a/src/api/controllers/cache-controller.ts +++ b/src/api/controllers/cache-controller.ts @@ -2,17 +2,13 @@ import * as prom from 'prom-client'; import { normalizeHashString } from '../../helpers'; import { PgStore } from '../../datastore/pg-store'; import { logger } from '../../logger'; -import { sha256 } from '@hirosystems/api-toolkit'; +import { + CACHE_CONTROL_MUST_REVALIDATE, + parseIfNoneMatchHeader, + sha256, +} from '@hirosystems/api-toolkit'; import { FastifyReply, FastifyRequest } from 'fastify'; -/** - * A `Cache-Control` header used for re-validation based caching. - * * `public` == allow proxies/CDNs to cache as opposed to only local browsers. - * * `no-cache` == clients can cache a resource but should revalidate each time before using it. - * * `must-revalidate` == somewhat redundant directive to assert that cache must be revalidated, required by some CDNs - */ -const CACHE_CONTROL_MUST_REVALIDATE = 'public, no-cache, must-revalidate'; - /** * Describes a key-value to be saved into a request's locals, representing the current * state of the chain depending on the type of information being requested by the endpoint. @@ -25,6 +21,8 @@ enum ETagType { mempool = 'mempool', /** ETag based on the status of a single transaction across the mempool or canonical chain. */ transaction = 'transaction', + /** Etag based on the confirmed balance of a single principal (STX address or contract id) */ + principal = 'principal', } /** Value that means the ETag did get calculated but it is empty. */ @@ -75,52 +73,6 @@ function getETagMetrics(): ETagCacheMetrics { return _eTagMetrics; } -/** - * Parses the etag values from a raw `If-None-Match` request header value. - * The wrapping double quotes (if any) and validation prefix (if any) are stripped. - * The parsing is permissive to account for commonly non-spec-compliant clients, proxies, CDNs, etc. - * E.g. the value: - * ```js - * `"a", W/"b", c,d, "e", "f"` - * ``` - * Would be parsed and returned as: - * ```js - * ['a', 'b', 'c', 'd', 'e', 'f'] - * ``` - * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#syntax - * ``` - * If-None-Match: "etag_value" - * If-None-Match: "etag_value", "etag_value", ... - * If-None-Match: * - * ``` - * @param ifNoneMatchHeaderValue - raw header value - * @returns an array of etag values - */ -export function parseIfNoneMatchHeader( - ifNoneMatchHeaderValue: string | undefined -): string[] | undefined { - if (!ifNoneMatchHeaderValue) { - return undefined; - } - // Strip wrapping double quotes like `"hello"` and the ETag validation-prefix like `W/"hello"`. - // The API returns compliant, strong-validation ETags (double quoted ASCII), but can't control what - // clients, proxies, CDNs, etc may provide. - const normalized = /^(?:"|W\/")?(.*?)"?$/gi.exec(ifNoneMatchHeaderValue.trim())?.[1]; - if (!normalized) { - // This should never happen unless handling a buggy request with something like `If-None-Match: ""`, - // or if there's a flaw in the above code. Log warning for now. - logger.warn(`Normalized If-None-Match header is falsy: ${ifNoneMatchHeaderValue}`); - return undefined; - } else if (normalized.includes(',')) { - // Multiple etag values provided, likely irrelevant extra values added by a proxy/CDN. - // Split on comma, also stripping quotes, weak-validation prefixes, and extra whitespace. - return normalized.split(/(?:W\/"|")?(?:\s*),(?:\s*)(?:W\/"|")?/gi); - } else { - // Single value provided (the typical case) - return [normalized]; - } -} - async function calculateETag( db: PgStore, etagType: ETagType, @@ -155,7 +107,7 @@ async function calculateETag( } return digest.result.digest; } catch (error) { - logger.error(error, 'Unable to calculate mempool'); + logger.error(error, 'Unable to calculate mempool etag'); return; } @@ -178,7 +130,20 @@ async function calculateETag( ]; return sha256(elements.join(':')); } catch (error) { - logger.error(error, 'Unable to calculate transaction'); + logger.error(error, 'Unable to calculate transaction etag'); + return; + } + + case ETagType.principal: + try { + const params = req.params as { address?: string; principal?: string }; + const principal = params.address ?? params.principal; + if (!principal) return ETAG_EMPTY; + const activity = await db.getPrincipalLastActivityTxIds(principal); + const text = `${activity.stx_tx_id}:${activity.ft_tx_id}:${activity.nft_tx_id}`; + return sha256(text); + } catch (error) { + logger.error(error, 'Unable to calculate principal etag'); return; } } @@ -224,3 +189,7 @@ export async function handleMempoolCache(request: FastifyRequest, reply: Fastify export async function handleTransactionCache(request: FastifyRequest, reply: FastifyReply) { return handleCache(ETagType.transaction, request, reply); } + +export async function handlePrincipalCache(request: FastifyRequest, reply: FastifyReply) { + return handleCache(ETagType.principal, request, reply); +} diff --git a/src/api/routes/address.ts b/src/api/routes/address.ts index ad66b7cdce..6b38f30798 100644 --- a/src/api/routes/address.ts +++ b/src/api/routes/address.ts @@ -15,7 +15,12 @@ import { } from '../controllers/db-controller'; import { InvalidRequestError, InvalidRequestErrorType, NotFoundError } from '../../errors'; import { decodeClarityValueToRepr } from 'stacks-encoding-native-js'; -import { handleChainTipCache, handleMempoolCache } from '../controllers/cache-controller'; +import { + handleChainTipCache, + handleMempoolCache, + handlePrincipalCache, + handleTransactionCache, +} from '../controllers/cache-controller'; import { PgStore } from '../../datastore/pg-store'; import { logger } from '../../logger'; import { has0xPrefix } from '@hirosystems/api-toolkit'; @@ -86,7 +91,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/stx', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_stx_balance', summary: 'Get account STX balance', @@ -142,7 +147,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/balances', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_balance', summary: 'Get account balances', @@ -234,7 +239,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/transactions', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { deprecated: true, operationId: 'get_account_transactions', @@ -307,7 +312,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/:tx_id/with_transfers', { - preHandler: handleChainTipCache, + preHandler: handleTransactionCache, schema: { deprecated: true, operationId: 'get_single_transaction_with_transfers', @@ -373,7 +378,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/transactions_with_transfers', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { deprecated: true, operationId: 'get_account_transactions_with_transfers', @@ -485,7 +490,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/assets', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_assets', summary: 'Get account assets', @@ -533,7 +538,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/stx_inbound', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_inbound', summary: 'Get inbound STX transfers', diff --git a/src/api/routes/v2/addresses.ts b/src/api/routes/v2/addresses.ts index 62410b9cce..f36d665826 100644 --- a/src/api/routes/v2/addresses.ts +++ b/src/api/routes/v2/addresses.ts @@ -1,4 +1,7 @@ -import { handleChainTipCache } from '../../../api/controllers/cache-controller'; +import { + handlePrincipalCache, + handleTransactionCache, +} from '../../../api/controllers/cache-controller'; import { AddressParamsSchema, AddressTransactionParamsSchema } from './schemas'; import { parseDbAddressTransactionTransfer, parseDbTxWithAccountTransferSummary } from './helpers'; import { InvalidRequestError, NotFoundError } from '../../../errors'; @@ -23,7 +26,7 @@ export const AddressRoutesV2: FastifyPluginAsync< fastify.get( '/:address/transactions', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_address_transactions', summary: 'Get address transactions', @@ -71,7 +74,7 @@ export const AddressRoutesV2: FastifyPluginAsync< fastify.get( '/:address/transactions/:tx_id/events', { - preHandler: handleChainTipCache, + preHandler: handleTransactionCache, schema: { operationId: 'get_address_transaction_events', summary: 'Get events for an address transaction', diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 9cc7b62c0b..9a494adb3c 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -4404,4 +4404,44 @@ export class PgStore extends BasePgStore { } return result; } + + /** Retrieves the last transaction IDs with STX, FT and NFT activity for a principal */ + async getPrincipalLastActivityTxIds( + principal: string + ): Promise<{ stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }> { + const result = await this.sql< + { stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }[] + >` + WITH last_stx AS ( + SELECT tx_id + FROM principal_stx_txs + WHERE principal = ${principal} AND canonical = true AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC + LIMIT 1 + ), + last_ft AS ( + SELECT tx_id + FROM ft_events + WHERE (sender = ${principal} OR recipient = ${principal}) + AND canonical = true + AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC + LIMIT 1 + ), + last_nft AS ( + SELECT tx_id + FROM nft_events + WHERE (sender = ${principal} OR recipient = ${principal}) + AND canonical = true + AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC + LIMIT 1 + ) + SELECT + (SELECT tx_id FROM last_stx) AS stx_tx_id, + (SELECT tx_id FROM last_ft) AS ft_tx_id, + (SELECT tx_id FROM last_nft) AS nft_tx_id + `; + return result[0]; + } } diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index 50d8648778..a1b79e599c 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -1180,6 +1180,7 @@ export class PgWriteStore extends PgStore { tx.token_transfer_recipient_address, tx.contract_call_contract_id, tx.smart_contract_contract_id, + tx.sponsor_address, ].filter((p): p is string => !!p) ); for (const event of stxEvents) { diff --git a/src/event-replay/parquet-based/importers/new-block-importer.ts b/src/event-replay/parquet-based/importers/new-block-importer.ts index d34f0c8458..7caec49504 100644 --- a/src/event-replay/parquet-based/importers/new-block-importer.ts +++ b/src/event-replay/parquet-based/importers/new-block-importer.ts @@ -251,6 +251,7 @@ const populateBatchInserters = (db: PgWriteStore) => { entry.tx.token_transfer_recipient_address, entry.tx.contract_call_contract_id, entry.tx.smart_contract_contract_id, + entry.tx.sponsor_address, ] .filter((p): p is string => !!p) .forEach(p => principals.add(p)); diff --git a/tests/api/cache-control.test.ts b/tests/api/cache-control.test.ts index 92b30122af..78f06774f5 100644 --- a/tests/api/cache-control.test.ts +++ b/tests/api/cache-control.test.ts @@ -10,7 +10,6 @@ import { } from '../../src/datastore/common'; import { startApiServer, ApiServer } from '../../src/api/init'; import { I32_MAX } from '../../src/helpers'; -import { parseIfNoneMatchHeader } from '../../src/api/controllers/cache-controller'; import { TestBlockBuilder, testMempoolTx } from '../utils/test-builders'; import { PgWriteStore } from '../../src/datastore/pg-write-store'; import { PgSqlClient, bufferToHex } from '@hirosystems/api-toolkit'; @@ -38,47 +37,6 @@ describe('cache-control tests', () => { await migrate('down'); }); - test('parse if-none-match header', () => { - // Test various combinations of etags with and without weak-validation prefix, with and without - // wrapping quotes, without and without spaces after commas. - const vectors: { - input: string | undefined; - output: string[] | undefined; - }[] = [ - { input: '""', output: undefined }, - { input: '', output: undefined }, - { input: undefined, output: undefined }, - { - input: '"bfc13a64729c4290ef5b2c2730249c88ca92d82d"', - output: ['bfc13a64729c4290ef5b2c2730249c88ca92d82d'], - }, - { input: 'W/"67ab43", "54ed21", "7892dd"', output: ['67ab43', '54ed21', '7892dd'] }, - { input: '"fail space" ', output: ['fail space'] }, - { input: 'W/"5e15153d-120f"', output: ['5e15153d-120f'] }, - { - input: '"", "" , "asdf"', - output: ['', '', 'asdf'], - }, - { - input: '"","","asdf"', - output: ['', '', 'asdf'], - }, - { - input: 'W/"","","asdf"', - output: ['', '', 'asdf'], - }, - { - input: '"",W/"", W/"asdf", "abcd","123"', - output: ['', '', 'asdf', 'abcd', '123'], - }, - ]; - expect(vectors).toBeTruthy(); - for (const entry of vectors) { - const result = parseIfNoneMatchHeader(entry.input); - expect(result).toEqual(entry.output); - } - }); - test('block chaintip cache control', async () => { const addr1 = 'ST28D4Q6RCQSJ6F7TEYWQDS4N1RXYEP9YBWMYSB97'; const addr2 = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; @@ -644,4 +602,132 @@ describe('cache-control tests', () => { const etag5 = request11.headers['etag']; expect(etag2).not.toBe(etag5); }); + + test('principal cache control', async () => { + const sender_address = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; + const url = `/extended/v2/addresses/${sender_address}/transactions`; + await db.update( + new TestBlockBuilder({ + block_height: 1, + index_block_hash: '0x01', + parent_index_block_hash: '0x00', + }).build() + ); + + // ETag zero. + const request1 = await supertest(api.server).get(url); + expect(request1.status).toBe(200); + expect(request1.type).toBe('application/json'); + const etag0 = request1.headers['etag']; + + // Add STX txs. + await db.update( + new TestBlockBuilder({ + block_height: 2, + index_block_hash: '0x02', + parent_index_block_hash: '0x01', + }) + .addTx({ tx_id: '0x0001', sender_address, token_transfer_amount: 200n }) + .addTxStxEvent({ sender: sender_address, amount: 200n }) + .build() + ); + + // Valid ETag. + const request2 = await supertest(api.server).get(url); + expect(request2.status).toBe(200); + expect(request2.type).toBe('application/json'); + expect(request2.headers['etag']).toBeTruthy(); + const etag1 = request2.headers['etag']; + expect(etag1).not.toEqual(etag0); + + // Cache works with valid ETag. + const request3 = await supertest(api.server).get(url).set('If-None-Match', etag1); + expect(request3.status).toBe(304); + expect(request3.text).toBe(''); + + // Add FT tx. + await db.update( + new TestBlockBuilder({ + block_height: 3, + index_block_hash: '0x03', + parent_index_block_hash: '0x02', + }) + .addTx({ tx_id: '0x0002' }) + .addTxFtEvent({ recipient: sender_address }) + .build() + ); + + // Cache is now a miss. + const request4 = await supertest(api.server).get(url).set('If-None-Match', etag1); + expect(request4.status).toBe(200); + expect(request4.type).toBe('application/json'); + expect(request4.headers['etag']).not.toEqual(etag1); + const etag2 = request4.headers['etag']; + + // Cache works with new ETag. + const request5 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request5.status).toBe(304); + expect(request5.text).toBe(''); + + // Add NFT tx. + await db.update( + new TestBlockBuilder({ + block_height: 4, + index_block_hash: '0x04', + parent_index_block_hash: '0x03', + }) + .addTx({ tx_id: '0x0003' }) + .addTxNftEvent({ recipient: sender_address }) + .build() + ); + + // Cache is now a miss. + const request6 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request6.status).toBe(200); + expect(request6.type).toBe('application/json'); + expect(request6.headers['etag']).not.toEqual(etag2); + const etag3 = request6.headers['etag']; + + // Cache works with new ETag. + const request7 = await supertest(api.server).get(url).set('If-None-Match', etag3); + expect(request7.status).toBe(304); + expect(request7.text).toBe(''); + + // Add sponsored tx. + await db.update( + new TestBlockBuilder({ + block_height: 5, + index_block_hash: '0x05', + parent_index_block_hash: '0x04', + }) + .addTx({ tx_id: '0x0004', sponsor_address: sender_address }) + .build() + ); + + // Cache is now a miss. + const request8 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request8.status).toBe(200); + expect(request8.type).toBe('application/json'); + expect(request8.headers['etag']).not.toEqual(etag3); + const etag4 = request8.headers['etag']; + + // Cache works with new ETag. + const request9 = await supertest(api.server).get(url).set('If-None-Match', etag4); + expect(request9.status).toBe(304); + expect(request9.text).toBe(''); + + // Advance chain with no changes to this address. + await db.update( + new TestBlockBuilder({ + block_height: 6, + index_block_hash: '0x06', + parent_index_block_hash: '0x05', + }).build() + ); + + // Cache still works. + const request10 = await supertest(api.server).get(url).set('If-None-Match', etag4); + expect(request10.status).toBe(304); + expect(request10.text).toBe(''); + }); });