feat(notion): add rate limit support and process documents concurrently

Author: kerwanp

examples/node-script-notion/contentlayer.config.ts

@@ -1,40 +1,13 @@
 import { makeSource, defineDatabase } from 'contentlayer-source-notion'
-import * as notion from '@notionhq/client'
 
-const client = new notion.Client({
-  auth: process.env.NOTION_TOKEN,
-})
-
-const Category = defineDatabase(() => ({
-  name: 'Category',
-  databaseId: '8bf1821701f14ab8bbbfaad0309ed09c',
-  importContent: false,
-}))
-
-const Post = defineDatabase(() => ({
+export const Post = defineDatabase(() => ({
   name: 'Post',
   databaseId: 'fe26b972ec3f4b32a1882230915fe111',
-  properties: {
-    email: {
-      name: 'Email',
-      required: true,
-    },
-    category: {
-      type: 'relation',
-      name: 'Category',
-      relation: Category,
-      single: true,
-    },
-  },
-  computedFields: {
-    slug: {
-      type: 'string',
-      resolve: (_) => 'test',
-    },
-  },
 }))
 
 export default makeSource({
-  client,
-  databaseTypes: [Post, Category],
+  clientOptions: {
+    auth: process.env.NOTION_TOKEN,
+  },
+  databaseTypes: [Post],
 })

packages/@contentlayer/source-notion/package.json

@@ -22,6 +22,8 @@
     "@notion-render/client": "1.0.0-rc.1",
     "@notionhq/client": "^2.2.3",
     "contentful-management": "7.22.4",
+    "p-queue": "^7.3.4",
+    "p-retry": "^5.1.2",
     "slugify": "^1.6.5"
   }
 }

packages/@contentlayer/source-notion/src/index.ts

@@ -5,6 +5,7 @@ import { NotionRenderer } from '@notion-render/client'
 import * as notion from '@notionhq/client'
 
 import { fetchAllDocuments } from './fetchData/fetchAllDocuments.js'
+import { fetchNotion } from './notion/fetchNotion.js'
 import { provideSchema } from './schema/provideSchema.js'
 import { flattendDatabaseTypeDef } from './schema/utils/flattenDatabaseTypeDef.js'
 import { NotionClient, NotionRenderer as NotionRendererTag } from './services.js'
@@ -23,8 +24,11 @@ export const makeSource: core.MakeSourcePlugin<PluginOptions & core.PartialArgs>
     _.def(),
   )
 
-  const client = rest.client ?? new notion.Client()
-  const renderer = rest.renderer ?? new NotionRenderer({ client })
+  const client = new notion.Client({
+    fetch: fetchNotion,
+    ...rest.clientOptions,
+  })
+  const renderer = new NotionRenderer({ client, ...rest.rendererOptions })
 
   return {
     type: 'notion',

packages/@contentlayer/source-notion/src/notion/fetchDatabasePages.ts

@@ -15,16 +15,20 @@ export const fetchDatabasePages = ({ databaseTypeDef }: FetchDatabasePagesArgs)
     S.service(NotionClient),
     S.chain((client) =>
       S.async<unknown, unknown, PageObjectResponse[]>(async (emit) => {
-        let nextCursor = undefined
+        let nextCursor: string | undefined = undefined
+        console.log(`Querying pages for ${databaseTypeDef.name}`)
 
         do {
+          console.log(`Querying pages for ${databaseTypeDef.name} with cursor ${nextCursor}`)
           const res = await client.databases.query({
             database_id: databaseTypeDef.databaseId,
+            start_cursor: nextCursor ?? undefined,
             filter: databaseTypeDef.query?.filter,
             sorts: databaseTypeDef.query?.sorts,
           })
 
-          nextCursor = res.next_cursor
+          console.log(`Found ${res.results.length} pages`)
+          nextCursor = res.next_cursor as string | undefined // NOTE: Throw type error and make res any if not typed, why???
           emit.single(res.results as PageObjectResponse[])
         } while (nextCursor)
 

packages/@contentlayer/source-notion/src/notion/fetchNotion.ts

@@ -0,0 +1,22 @@
+import PQueue from 'p-queue'
+import pRetry from 'p-retry'
+
+const queue = new PQueue({ interval: 1000, intervalCap: 4 })
+
+export const fetchNotion = (...args: Parameters<typeof fetch>) => {
+  return queue.add(() =>
+    pRetry(
+      () => {
+        console.log(`Fetching ${args[0]}`)
+        return fetch(...args)
+      },
+      {
+        retries: 5,
+        onFailedAttempt: async (r) => {
+          console.warn(`WARN: Error while querying Notion, retry #${r.attemptNumber}`)
+          await new Promise((res) => setTimeout(res, 1000))
+        },
+      },
+    ),
+  ) as Promise<Response>
+}

packages/@contentlayer/source-notion/src/types.ts

@@ -5,8 +5,8 @@ import type * as notion from '@notionhq/client'
 import type { DatabaseTypes } from './schema/types/database.js'
 
 export type PluginOptions = {
-  client?: notion.Client
-  renderer?: NotionRenderer
+  clientOptions?: ConstructorParameters<typeof notion.Client>[0]
+  rendererOptions?: ConstructorParameters<typeof NotionRenderer>[0]
   databaseTypes: DatabaseTypes
 }
 

yarn.lock

@@ -1792,6 +1792,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/retry@npm:0.12.1":
+  version: 0.12.1
+  resolution: "@types/retry@npm:0.12.1"
+  checksum: 5f46b2556053655f78262bb33040dc58417c900457cc63ff37d6c35349814471453ef511af0cec76a540c601296cd2b22f64bab1ab649c0dacc0223765ba876c
+  languageName: node
+  linkType: hard
+
 "@types/scheduler@npm:*":
   version: 0.16.2
   resolution: "@types/scheduler@npm:0.16.2"
@@ -3032,6 +3039,8 @@ __metadata:
     "@notionhq/client": ^2.2.3
     "@types/node": ^18.13.0
     contentful-management: 7.22.4
+    p-queue: ^7.3.4
+    p-retry: ^5.1.2
     slugify: ^1.6.5
   languageName: unknown
   linkType: soft
@@ -3898,6 +3907,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"eventemitter3@npm:^4.0.7":
+  version: 4.0.7
+  resolution: "eventemitter3@npm:4.0.7"
+  checksum: 1875311c42fcfe9c707b2712c32664a245629b42bb0a5a84439762dd0fd637fc54d078155ea83c2af9e0323c9ac13687e03cfba79b03af9f40c89b4960099374
+  languageName: node
+  linkType: hard
+
 "execa@npm:^0.7.0":
   version: 0.7.0
   resolution: "execa@npm:0.7.0"
@@ -6888,13 +6904,40 @@ __metadata:
   languageName: node
   linkType: hard
 
+"p-queue@npm:^7.3.4":
+  version: 7.3.4
+  resolution: "p-queue@npm:7.3.4"
+  dependencies:
+    eventemitter3: ^4.0.7
+    p-timeout: ^5.0.2
+  checksum: a21b8a4dd75f64a4988e4468cc344d1b45132506ddd2c771932d3de446d108ee68713b629e0d3f0809c227bc10eafc613edde6ae741d9f60db89b6031e40921c
+  languageName: node
+  linkType: hard
+
+"p-retry@npm:^5.1.2":
+  version: 5.1.2
+  resolution: "p-retry@npm:5.1.2"
+  dependencies:
+    "@types/retry": 0.12.1
+    retry: ^0.13.1
+  checksum: f063c08b1adc3cf7c01de01eb2dbda841970229f9f229c5167ebf4e2080d8a38b1f4e6eccefac74bca97cfaf4436d0a0eeb0b551175b26bc8b3116195f61bba8
+  languageName: node
+  linkType: hard
+
 "p-throttle@npm:^4.1.1":
   version: 4.1.1
   resolution: "p-throttle@npm:4.1.1"
   checksum: fe8709f3c3b1da7c033479375c2c302e80c1a5d86449013afa7cd46d1dc210bc824a7e4a9d088e66d31987d00878c2b5491bb2fe76246d4d2fc9a1636f5f8298
   languageName: node
   linkType: hard
 
+"p-timeout@npm:^5.0.2":
+  version: 5.1.0
+  resolution: "p-timeout@npm:5.1.0"
+  checksum: f5cd4e17301ff1ff1d8dbf2817df0ad88c6bba99349fc24d8d181827176ad4f8aca649190b8a5b1a428dfd6ddc091af4606835d3e0cb0656e04045da5c9e270c
+  languageName: node
+  linkType: hard
+
 "p-try@npm:^2.0.0":
   version: 2.2.0
   resolution: "p-try@npm:2.2.0"
@@ -7557,6 +7600,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"retry@npm:^0.13.1":
+  version: 0.13.1
+  resolution: "retry@npm:0.13.1"
+  checksum: 47c4d5be674f7c13eee4cfe927345023972197dbbdfba5d3af7e461d13b44de1bfd663bfc80d2f601f8ef3fc8164c16dd99655a221921954a65d044a2fc1233b
+  languageName: node
+  linkType: hard
+
 "reusify@npm:^1.0.4":
   version: 1.0.4
   resolution: "reusify@npm:1.0.4"