Use an index to find rows in the next page in the UTXO query

The indexer was using SQL's `skip` for the pagination. The DB should
scan the number of skipped rows, to get the page result. For the
performance enhancement, we concluded that change the pagination
method. Finding a particular row using an index and get the following
`n` rows is much faster than using `skip`.

This commit uses `lastEvaluatedKey` for the pagination in the UTXO
query. The UTXO query will find a row using `lastEvaluatedKey` and
returns next `n` rows.

Author: Park Juhyung

src/models/logic/utils/middleware.ts

@@ -1,4 +1,4 @@
-import { RequestHandler } from "express";
+import { NextFunction, Request, RequestHandler, Response } from "express";
 import { SERVICE_UNAVAILABLE } from "http-status-codes";
 import { IndexerContext } from "../../../context";
 
@@ -19,3 +19,18 @@ export function syncIfNeeded(context: IndexerContext): RequestHandler {
         next();
     };
 }
+
+export function parseLastEvaluatedKey(
+    req: Request,
+    _: Response,
+    next: NextFunction
+): any {
+    try {
+        if (req.query.lastEvaluatedKey) {
+            req.query.lastEvaluatedKey = JSON.parse(req.query.lastEvaluatedKey);
+        }
+        next();
+    } catch (e) {
+        next(e);
+    }
+}

src/models/logic/utxo.ts

@@ -148,13 +148,15 @@ async function getUTXOQuery(params: {
     shardId?: number | null;
     onlyConfirmed?: boolean | null;
     confirmThreshold?: number | null;
+    lastEvaluatedKey?: number[] | null;
 }) {
     const {
         address,
         shardId,
         onlyConfirmed,
         confirmThreshold,
-        assetType
+        assetType,
+        lastEvaluatedKey
     } = params;
     const query = [];
     if (address) {
@@ -197,6 +199,18 @@ async function getUTXOQuery(params: {
             usedTransactionHash: null
         });
     }
+
+    if (lastEvaluatedKey) {
+        const lastBlockNumber = lastEvaluatedKey[0];
+        const lastTransactionIndex = lastEvaluatedKey[1];
+        const lastTransactionOutputIndex = lastEvaluatedKey[2];
+        query.push(
+            Sequelize.literal(
+                `("UTXO"."blockNumber", "UTXO"."transactionIndex", "UTXO"."transactionOutputIndex")<(${lastBlockNumber}, ${lastTransactionIndex}, ${lastTransactionOutputIndex})`
+            )
+        );
+    }
+
     return query;
 }
 
@@ -206,6 +220,7 @@ export async function getUTXO(params: {
     shardId?: number | null;
     page?: number | null;
     itemsPerPage?: number | null;
+    lastEvaluatedKey?: number[] | null;
     onlyConfirmed?: boolean | null;
     confirmThreshold?: number | null;
 }) {
@@ -215,6 +230,7 @@ export async function getUTXO(params: {
         shardId,
         page = 1,
         itemsPerPage = 15,
+        lastEvaluatedKey,
         onlyConfirmed = false,
         confirmThreshold = 0
     } = params;
@@ -223,7 +239,8 @@ export async function getUTXO(params: {
         assetType,
         shardId,
         onlyConfirmed,
-        confirmThreshold
+        confirmThreshold,
+        lastEvaluatedKey
     });
     let includeArray: any = [
         {
@@ -269,7 +286,7 @@ export async function getUTXO(params: {
                 ["transactionOutputIndex", "DESC"]
             ],
             limit: itemsPerPage!,
-            offset: (page! - 1) * itemsPerPage!,
+            offset: lastEvaluatedKey ? 0 : (page! - 1) * itemsPerPage!,
             include: includeArray
         });
     } catch (err) {

src/routers/asset.ts

@@ -6,12 +6,16 @@ import * as Exception from "../exception";
 import * as AssetImageModel from "../models/logic/assetimage";
 import * as AssetSchemeModel from "../models/logic/assetscheme";
 import * as BlockModel from "../models/logic/block";
-import { syncIfNeeded } from "../models/logic/utils/middleware";
+import {
+    parseLastEvaluatedKey,
+    syncIfNeeded
+} from "../models/logic/utils/middleware";
 import * as UTXOModel from "../models/logic/utxo";
 import {
     assetTypeSchema,
     paginationSchema,
     snapshotSchema,
+    utxoPaginationSchema,
     utxoSchema,
     validate
 } from "./validator";
@@ -84,6 +88,11 @@ export function handle(context: IndexerContext, router: Router) {
      *         in: query
      *         required: false
      *         type: number
+     *       - name: lastEvaluatedKey
+     *         description: the evaulated key of the last item in the previous page. It will be used for the pagination
+     *         in: query
+     *         required: false
+     *         type: string
      *       - name: itemsPerPage
      *         description: items per page for the pagination (default 15)
      *         in: query
@@ -114,10 +123,12 @@ export function handle(context: IndexerContext, router: Router) {
      */
     router.get(
         "/utxo",
+        parseLastEvaluatedKey,
         validate({
             query: {
                 ...utxoSchema,
-                ...paginationSchema
+                ...paginationSchema,
+                ...utxoPaginationSchema
             }
         }),
         syncIfNeeded(context),
@@ -128,11 +139,15 @@ export function handle(context: IndexerContext, router: Router) {
                 req.query.shardId && parseInt(req.query.shardId, 10);
             const page = req.query.page && parseInt(req.query.page, 10);
             const itemsPerPage =
-                req.query.itemsPerPage && parseInt(req.query.itemsPerPage, 10);
+                (req.query.itemsPerPage &&
+                    parseInt(req.query.itemsPerPage, 10)) ||
+                15;
             const onlyConfirmed = req.query.onlyConfirmed;
             const confirmThreshold =
                 req.query.confirmThreshold &&
                 parseInt(req.query.confirmThreshold, 10);
+            const lastEvaluatedKey = req.query.lastEvaluatedKey;
+
             let assetType;
             try {
                 if (assetTypeString) {
@@ -143,12 +158,29 @@ export function handle(context: IndexerContext, router: Router) {
                     assetType,
                     shardId,
                     page,
-                    itemsPerPage,
+                    itemsPerPage: itemsPerPage + 1,
+                    lastEvaluatedKey,
                     onlyConfirmed,
                     confirmThreshold
                 });
                 const utxo = utxoInsts.map(inst => inst.get({ plain: true }));
-                res.json(utxo);
+                const hasNextPage = utxo.length === itemsPerPage + 1;
+                if (hasNextPage) {
+                    utxo.pop();
+                }
+                const lastItem = utxo[utxo.length - 1];
+
+                res.json({
+                    data: utxo,
+                    lastEvaluatedKey: lastItem
+                        ? JSON.stringify([
+                              lastItem.blockNumber,
+                              lastItem.transactionIndex,
+                              lastItem.transactionOutputIndex
+                          ])
+                        : null,
+                    hasNextPage
+                });
             } catch (e) {
                 next(e);
             }

src/routers/validator.ts

@@ -69,6 +69,14 @@ export const paginationSchema = {
         .max(100)
 };
 
+export const utxoPaginationSchema = {
+    lastEvaluatedKey: Joi.array().items(
+        Joi.number(),
+        Joi.number(),
+        Joi.number()
+    )
+};
+
 export const txSchema = {
     address,
     assetType: assetTypeSchema,