feat(notion): add cache system

Author: kerwanp

packages/@contentlayer/source-notion/src/fetchData/fetchAllDocuments.ts

@@ -10,11 +10,12 @@ import { makeCacheItem } from './makeCacheItem.js'
 
 export type FetchAllDocumentsArgs = {
   databaseTypeDefs: DatabaseTypeDef[]
+  previousCache: core.DataCache.Cache | undefined
   schemaDef: core.SchemaDef
   options: core.PluginOptions
 }
 
-export const fetchAllDocuments = ({ databaseTypeDefs, schemaDef, options }: FetchAllDocumentsArgs) =>
+export const fetchAllDocuments = ({ databaseTypeDefs, previousCache, schemaDef, options }: FetchAllDocumentsArgs) =>
   pipe(
     T.forEachPar_(databaseTypeDefs, (databaseTypeDef) =>
       pipe(
@@ -27,6 +28,7 @@ export const fetchAllDocuments = ({ databaseTypeDefs, schemaDef, options }: Fetc
                   page,
                   documentTypeDef: schemaDef.documentTypeDefMap[databaseTypeDef.name]!,
                   databaseTypeDef,
+                  previousCache,
                   options,
                 }),
               ),

packages/@contentlayer/source-notion/src/fetchData/fetchData.ts

@@ -0,0 +1,27 @@
+import * as core from '@contentlayer/core'
+import { OT, pipe, T } from '@contentlayer/utils/effect'
+
+import type { DatabaseTypeDef } from '../schema/types/database.js'
+import { fetchAllDocuments } from './fetchAllDocuments.js'
+
+export type FetchDataArgs = {
+  databaseTypeDefs: DatabaseTypeDef[]
+  schemaDef: core.SchemaDef
+  options: core.PluginOptions
+}
+
+export const fetchData = ({ schemaDef, databaseTypeDefs, options }: FetchDataArgs) => {
+  const resolveParams = pipe(core.DataCache.loadPreviousCacheFromDisk({ schemaHash: schemaDef.hash }), T.either)
+  return pipe(
+    T.rightOrFail(resolveParams),
+    T.chain((cache) =>
+      pipe(
+        fetchAllDocuments({ schemaDef, databaseTypeDefs, previousCache: cache, options }),
+        T.tap((cache_) => T.succeedWith(() => (cache = cache_))),
+        T.tap((cache_) => core.DataCache.writeCacheToDisk({ cache: cache_, schemaHash: schemaDef.hash })),
+      ),
+    ),
+    OT.withSpan('@contentlayer/source-notion/fetchData:fetchData'),
+    T.mapError((error) => new core.SourceFetchDataError({ error, alreadyHandled: false })),
+  )
+}

packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts

@@ -11,32 +11,38 @@ export type MakeCacheItemArgs = {
   databaseTypeDef: DatabaseTypeDef
   documentTypeDef: core.DocumentTypeDef
   page: PageObjectResponse
+  previousCache: core.DataCache.Cache | undefined
   options: core.PluginOptions
 }
 
-export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, page, options }: MakeCacheItemArgs) =>
+export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache, page, options }: MakeCacheItemArgs) =>
   pipe(
     T.gen(function* ($) {
-      const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }))
+      const documentHash = new Date(page.last_edited_time).getTime().toString()
+
+      if (
+        previousCache &&
+        previousCache.cacheItemsMap[page.id] &&
+        previousCache.cacheItemsMap[page.id]!.documentHash === documentHash &&
+        previousCache.cacheItemsMap[page.id]!.hasWarnings === false
+      ) {
+        const cacheItem = previousCache.cacheItemsMap[page.id]!
+        return cacheItem
+      }
 
+      const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }))
       const computedValues = yield* $(getComputedValues({ document, documentTypeDef }))
 
       Object.entries(computedValues).forEach(([fieldName, value]) => {
         document[fieldName] = value
       })
 
-      return document
+      return {
+        document,
+        documentHash,
+        hasWarnings: false,
+        documentTypeName: documentTypeDef.name,
+      }
     }),
-    T.chain((document) =>
-      pipe(
-        hashObject(document),
-        T.map((hash) => ({
-          document,
-          documentHash: hash,
-          hasWarnings: false,
-          documentTypeName: documentTypeDef.name,
-        })),
-      ),
-    ),
     OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem'),
   )

packages/@contentlayer/source-notion/src/index.ts

@@ -4,7 +4,7 @@ import { pipe, S, T } from '@contentlayer/utils/effect'
 import { NotionRenderer } from '@notion-render/client'
 import * as notion from '@notionhq/client'
 
-import { fetchAllDocuments } from './fetchData/fetchAllDocuments.js'
+import { fetchData } from './fetchData/fetchData.js'
 import { fetchNotion } from './notion/fetchNotion.js'
 import { provideSchema } from './schema/provideSchema.js'
 import { flattendDatabaseTypeDef } from './schema/utils/flattenDatabaseTypeDef.js'
@@ -49,7 +49,7 @@ export const makeSource: core.MakeSourcePlugin<PluginOptions & core.PartialArgs>
       pipe(
         S.fromEffect(
           pipe(
-            fetchAllDocuments({
+            fetchData({
               databaseTypeDefs: databaseTypeDefs.map((databaseTypeDef) => flattendDatabaseTypeDef(databaseTypeDef)),
               schemaDef,
               options,