Merge pull request #81 from sillsdev/moreTimeoutResilience

fix: Increase timeout resilience (#81)

Author: andrew-polk

package.json

@@ -11,7 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
-    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr   --log-level debug",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",

src/pull.ts

@@ -284,10 +284,12 @@ export async function executeWithRateLimitAndRetries<T>(
         e.message.includes("timeout") ||
         e.message.includes("Timeout") ||
         e.message.includes("limit") ||
-        e.message.includes("Limit")
+        e.message.includes("Limit") ||
+        e?.code === "notionhq_client_response_error" ||
+        e?.code === "service_unavailable"
       ) {
         const secondsToWait = i + 1;
-        info(
+        warning(
           `While doing "${label}", got error "${
             e.message as string
           }". Will retry after  ${secondsToWait}s...`

src/transform.ts

@@ -7,6 +7,7 @@ import { error, info, logDebug, logDebugFn, verbose, warning } from "./log";
 import { NotionPage } from "./NotionPage";
 import { IDocuNotionConfig } from "./config/configuration";
 import { NotionBlock } from "./types";
+import { executeWithRateLimitAndRetries } from "./pull";
 
 export async function getMarkdownForPage(
   config: IDocuNotionConfig,
@@ -157,8 +158,14 @@ async function doNotionToMarkdown(
   docunotionContext: IDocuNotionContext,
   blocks: Array<NotionBlock>
 ) {
-  const mdBlocks = await docunotionContext.notionToMarkdown.blocksToMarkdown(
-    blocks
+  let mdBlocks: any;
+  await executeWithRateLimitAndRetries(
+    "notionToMarkdown.blocksToMarkdown",
+    async () => {
+      mdBlocks = await docunotionContext.notionToMarkdown.blocksToMarkdown(
+        blocks
+      );
+    }
   );
 
   const markdown =