Improve the pagination performance of the Block API

Author: joojis

src/models/logic/block.ts

@@ -3,7 +3,8 @@ import { Block, H256, U64 } from "codechain-sdk/lib/core/classes";
 import * as _ from "lodash";
 import * as Sequelize from "sequelize";
 import * as Exception from "../../exception";
-import { BlockInstance } from "../block";
+import { blockPagination } from "../../routers/pagination";
+import { BlockAttribute, BlockInstance } from "../block";
 import models from "../index";
 import * as AddressLogModel from "./addressLog";
 import * as AssetAddressLogModel from "./assetAddressLog";
@@ -127,27 +128,46 @@ export async function getBlocks(params: {
     address?: string;
     page?: number | null;
     itemsPerPage?: number | null;
+    firstEvaluatedKey?: [number] | null;
+    lastEvaluatedKey?: [number] | null;
 }) {
-    const { page = 1, itemsPerPage = 15, address } = params;
-    let query = {};
-    if (address) {
-        query = {
-            author: address
-        };
-    }
+    const {
+        address,
+        page = 1,
+        itemsPerPage = 15,
+        firstEvaluatedKey,
+        lastEvaluatedKey
+    } = params;
     try {
         return await models.Block.findAll({
-            order: [["number", "DESC"]],
+            order: blockPagination.orderby({
+                firstEvaluatedKey,
+                lastEvaluatedKey
+            }),
             limit: itemsPerPage!,
-            offset: (page! - 1) * itemsPerPage!,
-            where: query
+            offset:
+                firstEvaluatedKey || lastEvaluatedKey
+                    ? 0
+                    : (page! - 1) * itemsPerPage!,
+            where: {
+                ...(address && { author: address }),
+                ...((firstEvaluatedKey || lastEvaluatedKey) &&
+                    blockPagination.where({
+                        firstEvaluatedKey,
+                        lastEvaluatedKey
+                    }))
+            }
         });
     } catch (err) {
         console.log(err);
         throw Exception.DBError();
     }
 }
 
+export function createBlockEvaluatedKey(block: BlockAttribute): string {
+    return JSON.stringify([block.number]);
+}
+
 export async function getNumberOfBlocks(params: { address?: string }) {
     const { address } = params;
     try {

src/routers/block.ts

@@ -4,8 +4,13 @@ import * as _ from "lodash";
 import { IndexerContext } from "../context";
 import * as BlockModel from "../models/logic/block";
 import * as TransactionModel from "../models/logic/transaction";
-import { syncIfNeeded } from "../models/logic/utils/middleware";
 import {
+    parseEvaluatedKey,
+    syncIfNeeded
+} from "../models/logic/utils/middleware";
+import { createPaginationResult } from "./pagination";
+import {
+    blockPaginationSchema,
     blockSchema,
     paginationSchema,
     syncSchema,
@@ -304,6 +309,16 @@ export function handle(context: IndexerContext, router: Router) {
      *         in: query
      *         required: false
      *         type: number
+     *       - name: firstEvaluatedKey
+     *         description: the evaulated key of the first item in the previous page. It will be used for the pagination
+     *         in: query
+     *         required: false
+     *         type: string
+     *       - name: lastEvaluatedKey
+     *         description: the evaulated key of the last item in the previous page. It will be used for the pagination
+     *         in: query
+     *         required: false
+     *         type: string
      *       - name: itemsPerPage
      *         description: items per page for the pagination (default 15)
      *         in: query
@@ -324,29 +339,44 @@ export function handle(context: IndexerContext, router: Router) {
      */
     router.get(
         "/block",
+        parseEvaluatedKey,
         validate({
             query: {
                 ...blockSchema,
-                ...paginationSchema
+                ...paginationSchema,
+                ...blockPaginationSchema
             }
         }),
         syncIfNeeded(context),
         async (req, res, next) => {
             const address = req.query.address;
             const page = req.query.page && parseInt(req.query.page, 10);
             const itemsPerPage =
-                req.query.itemsPerPage && parseInt(req.query.itemsPerPage, 10);
-
+                req.query.itemsPerPage &&
+                parseInt(req.query.itemsPerPage, 10) + 1;
+            const lastEvaluatedKey = req.query.lastEvaluatedKey;
+            const firstEvaluatedKey = req.query.firstEvaluatedKey;
             try {
-                const blockInsts = await BlockModel.getBlocks({
+                const blocks = await BlockModel.getBlocks({
                     address,
                     page,
-                    itemsPerPage
-                });
-                const blocks = blockInsts.map(blockInst =>
-                    blockInst.get({ plain: true })
+                    itemsPerPage,
+                    firstEvaluatedKey,
+                    lastEvaluatedKey
+                }).then(instances =>
+                    instances.map(i => i.get({ plain: true }))
+                );
+                res.json(
+                    createPaginationResult({
+                        query: {
+                            firstEvaluatedKey,
+                            lastEvaluatedKey
+                        },
+                        rows: blocks,
+                        getEvaluatedKey: BlockModel.createBlockEvaluatedKey,
+                        itemsPerPage
+                    })
                 );
-                res.json(blocks);
             } catch (e) {
                 next(e);
             }

src/routers/pagination.ts

@@ -109,3 +109,37 @@ export const utxoPagination = {
         }
     }
 };
+
+type BlockEvaluationKey = [number];
+export const blockPagination = {
+    orderby: (params: {
+        firstEvaluatedKey?: BlockEvaluationKey | null;
+        lastEvaluatedKey?: BlockEvaluationKey | null;
+    }) => {
+        switch (queryOrder(params)) {
+            case "forward":
+                return [["number", "DESC"]];
+            case "reverse":
+                return [["number", "ASC"]];
+            default:
+                throw Error("Unreachable");
+        }
+    },
+    where: (params: {
+        firstEvaluatedKey?: BlockEvaluationKey | null;
+        lastEvaluatedKey?: BlockEvaluationKey | null;
+    }) => {
+        switch (queryOrder(params)) {
+            case "forward": {
+                const [blockNumber] = params.lastEvaluatedKey!;
+                return { number: { [Sequelize.Op.lt]: blockNumber } };
+            }
+            case "reverse": {
+                const [blockNumber] = params.firstEvaluatedKey!;
+                return { number: { [Sequelize.Op.gt]: blockNumber } };
+            }
+            default:
+                throw Error("Unreachable");
+        }
+    }
+};

src/routers/validator.ts

@@ -82,6 +82,12 @@ export const utxoPaginationSchema = {
     )
 };
 
+const blockEvaluationKey = Joi.array().items(Joi.number());
+export const blockPaginationSchema = {
+    lastEvaluatedKey: blockEvaluationKey,
+    firstEvaluatedKey: blockEvaluationKey
+};
+
 export const txSchema = {
     address,
     assetType: assetTypeSchema,