Skip to content

Commit 3f6bc8e

Browse files
majectyjoojis
majecty
authored and
joojis
committed
Use an index to find rows in the next page in the Tx query
The indexer was using SQL's `skip` for the pagination. The DB should scan the number of skipped rows, to get the page result. For the performance enhancement, we concluded that change the pagination method. Finding a particular row using an index and get the following `n` rows is much faster than using `skip`. This commit uses `firstEvaluatedKey` and `lastEvaluatedKey` for the pagination in the Tx query. The Tx query will find a row using `firstEvaluatedKey` or `lastEvaluatedKey` and returns next `n` rows.
1 parent d788959 commit 3f6bc8e

File tree

5 files changed

+335
-41
lines changed

5 files changed

+335
-41
lines changed

‎src/models/logic/transaction.ts

Lines changed: 171 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ import * as _ from "lodash";
1010
import * as Sequelize from "sequelize";
1111
import { Transaction } from "sequelize";
1212
import * as Exception from "../../exception";
13+
import { txPagination } from "../../routers/pagination";
1314
import models from "../index";
14-
import { TransactionInstance } from "../transaction";
15+
import { TransactionAttribute, TransactionInstance } from "../transaction";
1516
import { createAddressLog } from "./addressLog";
1617
import { updateAssetScheme } from "./assetscheme";
1718
import { createChangeAssetScheme } from "./changeAssetScheme";
@@ -395,17 +396,45 @@ async function getHashesByPlatformAddress(params: {
395396
address: string;
396397
page: number;
397398
itemsPerPage: number;
399+
firstEvaluatedKey: [number, number] | null;
400+
lastEvaluatedKey: [number, number] | null;
398401
}): Promise<string[]> {
399-
const { address, page, itemsPerPage } = params;
402+
const {
403+
address,
404+
page,
405+
itemsPerPage,
406+
firstEvaluatedKey,
407+
lastEvaluatedKey
408+
} = params;
409+
410+
const whereCond: any[] = [
411+
{
412+
address
413+
}
414+
];
415+
if (firstEvaluatedKey || lastEvaluatedKey) {
416+
whereCond.push(
417+
txPagination.where({
418+
firstEvaluatedKey,
419+
lastEvaluatedKey
420+
})
421+
);
422+
}
400423
try {
401424
return models.AddressLog.findAll({
402425
attributes: ["transactionHash"],
403426
where: {
404-
address
427+
[Sequelize.Op.and]: whereCond
405428
},
406-
order: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
429+
order: txPagination.orderby({
430+
firstEvaluatedKey,
431+
lastEvaluatedKey
432+
}),
407433
limit: itemsPerPage,
408-
offset: (page - 1) * itemsPerPage
434+
offset:
435+
firstEvaluatedKey || lastEvaluatedKey
436+
? 0
437+
: (page - 1) * itemsPerPage
409438
}).map(r => r.get("transactionHash"));
410439
} catch (err) {
411440
console.error(err);
@@ -418,18 +447,48 @@ async function getHashesByAssetAddress(params: {
418447
assetType?: string | null;
419448
page: number;
420449
itemsPerPage: number;
450+
firstEvaluatedKey: [number, number] | null;
451+
lastEvaluatedKey: [number, number] | null;
421452
}): Promise<string[]> {
422-
const { address, assetType, page, itemsPerPage } = params;
453+
const {
454+
address,
455+
assetType,
456+
page,
457+
itemsPerPage,
458+
firstEvaluatedKey,
459+
lastEvaluatedKey
460+
} = params;
461+
462+
const whereCond: any[] = [
463+
{
464+
address,
465+
...(assetType && { assetType })
466+
}
467+
];
468+
if (firstEvaluatedKey || lastEvaluatedKey) {
469+
whereCond.push(
470+
txPagination.where({
471+
firstEvaluatedKey,
472+
lastEvaluatedKey
473+
})
474+
);
475+
}
476+
423477
try {
424478
return models.AssetAddressLog.findAll({
425479
attributes: ["transactionHash"],
426480
where: {
427-
address,
428-
...(assetType && { assetType })
481+
[Sequelize.Op.and]: whereCond
429482
},
430-
order: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
483+
order: txPagination.orderby({
484+
firstEvaluatedKey,
485+
lastEvaluatedKey
486+
}),
431487
limit: itemsPerPage,
432-
offset: (page - 1) * itemsPerPage
488+
offset:
489+
firstEvaluatedKey || lastEvaluatedKey
490+
? 0
491+
: (page - 1) * itemsPerPage
433492
}).map(r => r.get("transactionHash"));
434493
} catch (err) {
435494
console.error(err);
@@ -441,17 +500,46 @@ async function getHashesByAssetType(params: {
441500
assetType: string;
442501
page: number;
443502
itemsPerPage: number;
503+
firstEvaluatedKey: [number, number] | null;
504+
lastEvaluatedKey: [number, number] | null;
444505
}): Promise<string[]> {
445-
const { assetType, page, itemsPerPage } = params;
506+
const {
507+
assetType,
508+
page,
509+
itemsPerPage,
510+
firstEvaluatedKey,
511+
lastEvaluatedKey
512+
} = params;
513+
514+
const whereCond: any[] = [
515+
{
516+
assetType
517+
}
518+
];
519+
if (firstEvaluatedKey || lastEvaluatedKey) {
520+
whereCond.push(
521+
txPagination.where({
522+
firstEvaluatedKey,
523+
lastEvaluatedKey
524+
})
525+
);
526+
}
527+
446528
try {
447529
return models.AssetTypeLog.findAll({
448530
attributes: ["transactionHash"],
449531
where: {
450-
assetType
532+
[Sequelize.Op.and]: whereCond
451533
},
452-
order: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
534+
order: txPagination.orderby({
535+
firstEvaluatedKey,
536+
lastEvaluatedKey
537+
}),
453538
limit: itemsPerPage,
454-
offset: (page - 1) * itemsPerPage
539+
offset:
540+
firstEvaluatedKey || lastEvaluatedKey
541+
? 0
542+
: (page - 1) * itemsPerPage
455543
}).map(r => r.get("transactionHash"));
456544
} catch (err) {
457545
console.error(err);
@@ -464,38 +552,86 @@ async function getHashes(params: {
464552
assetType?: string | null;
465553
page: number;
466554
itemsPerPage: number;
555+
firstEvaluatedKey: [number, number] | null;
556+
lastEvaluatedKey: [number, number] | null;
467557
type?: string[] | null;
468558
includePending?: boolean | null;
469559
}): Promise<string[]> {
470-
const { address, assetType, page, itemsPerPage } = params;
560+
const {
561+
address,
562+
assetType,
563+
page,
564+
itemsPerPage,
565+
firstEvaluatedKey,
566+
lastEvaluatedKey
567+
} = params;
471568
if (address != null && assetType != null) {
472569
return getHashesByAssetAddress({
473570
address,
474571
assetType,
475572
page,
476-
itemsPerPage
573+
itemsPerPage,
574+
firstEvaluatedKey,
575+
lastEvaluatedKey
477576
});
478577
} else if (address != null) {
479578
if (AssetAddress.check(address)) {
480-
return getHashesByAssetAddress({ address, page, itemsPerPage });
579+
return getHashesByAssetAddress({
580+
address,
581+
page,
582+
itemsPerPage,
583+
firstEvaluatedKey,
584+
lastEvaluatedKey
585+
});
481586
} else if (PlatformAddress.check(address)) {
482-
return getHashesByPlatformAddress({ address, page, itemsPerPage });
587+
return getHashesByPlatformAddress({
588+
address,
589+
page,
590+
itemsPerPage,
591+
firstEvaluatedKey,
592+
lastEvaluatedKey
593+
});
483594
}
484595
throw Error(`Invalid address: ${address}`);
485596
} else if (assetType != null) {
486-
return getHashesByAssetType({ assetType, page, itemsPerPage });
597+
return getHashesByAssetType({
598+
assetType,
599+
page,
600+
itemsPerPage,
601+
firstEvaluatedKey,
602+
lastEvaluatedKey
603+
});
487604
}
488-
return models.Transaction.findAll({
489-
attributes: ["hash"],
490-
where: {
605+
const whereCond: any[] = [
606+
{
491607
...(params.type != null
492608
? { type: { [Sequelize.Op.in]: params.type } }
493609
: {}),
494610
...(params.includePending !== true ? { isPending: false } : {})
611+
}
612+
];
613+
if (firstEvaluatedKey || lastEvaluatedKey) {
614+
whereCond.push(
615+
txPagination.where({
616+
firstEvaluatedKey,
617+
lastEvaluatedKey
618+
})
619+
);
620+
}
621+
return models.Transaction.findAll({
622+
attributes: ["hash"],
623+
where: {
624+
[Sequelize.Op.and]: whereCond
495625
},
496-
order: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
626+
order: txPagination.orderby({
627+
firstEvaluatedKey,
628+
lastEvaluatedKey
629+
}),
497630
limit: itemsPerPage,
498-
offset: (page - 1) * itemsPerPage
631+
offset:
632+
firstEvaluatedKey || lastEvaluatedKey
633+
? 0
634+
: (page - 1) * itemsPerPage
499635
}).map(result => result.get("hash"));
500636
}
501637

@@ -505,11 +641,13 @@ export async function getTransactions(params: {
505641
type?: string[] | null;
506642
page: number;
507643
itemsPerPage: number;
644+
firstEvaluatedKey: [number, number] | null;
645+
lastEvaluatedKey: [number, number] | null;
508646
includePending?: boolean | null;
509647
onlyConfirmed?: boolean | null;
510648
confirmThreshold?: number | null;
511649
}) {
512-
const { itemsPerPage } = params;
650+
const { itemsPerPage, firstEvaluatedKey, lastEvaluatedKey } = params;
513651
try {
514652
// TODO: Querying twice will waste IO bandwidth and take longer time as long as the response time
515653
// Find a way to merge these queries.
@@ -524,7 +662,10 @@ export async function getTransactions(params: {
524662
where: {
525663
hash: hashes
526664
},
527-
order: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
665+
order: txPagination.orderby({
666+
firstEvaluatedKey,
667+
lastEvaluatedKey
668+
}),
528669
include: [...fullIncludeArray]
529670
});
530671
} catch (err) {
@@ -670,3 +811,7 @@ export async function getRegularKeyOwnerByPublicKey(
670811
}
671812
return tx.get("signer");
672813
}
814+
815+
export function createTxEvaluatedKey(tx: TransactionAttribute) {
816+
return JSON.stringify([tx.blockNumber, tx.transactionIndex]);
817+
}

‎src/routers/pagination.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,3 +220,39 @@ export const aggsUTXOPagination = {
220220
}
221221
}
222222
};
223+
224+
export const txPagination = {
225+
forwardOrder: [["blockNumber", "DESC"], ["transactionIndex", "DESC"]],
226+
reverseOrder: [["blockNumber", "ASC"], ["transactionIndex", "ASC"]],
227+
orderby: (params: {
228+
firstEvaluatedKey?: number[] | null;
229+
lastEvaluatedKey?: number[] | null;
230+
}) => {
231+
const order = queryOrder(params);
232+
if (order === "forward") {
233+
return txPagination.forwardOrder;
234+
} else if (order === "reverse") {
235+
return txPagination.reverseOrder;
236+
}
237+
},
238+
where: (params: {
239+
firstEvaluatedKey?: number[] | null;
240+
lastEvaluatedKey?: number[] | null;
241+
}) => {
242+
const order = queryOrder(params);
243+
const { firstEvaluatedKey, lastEvaluatedKey } = params;
244+
if (order === "forward") {
245+
const blockNumber = lastEvaluatedKey![0];
246+
const transactionIndex = lastEvaluatedKey![1];
247+
return Sequelize.literal(
248+
`("blockNumber", "transactionIndex")<(${blockNumber}, ${transactionIndex})`
249+
);
250+
} else if (order === "reverse") {
251+
const blockNumber = firstEvaluatedKey![0];
252+
const transactionIndex = firstEvaluatedKey![1];
253+
return Sequelize.literal(
254+
`("blockNumber", "transactionIndex")>(${blockNumber}, ${transactionIndex})`
255+
);
256+
}
257+
}
258+
};

0 commit comments

Comments
 (0)