feat: add both full & partial sync

lcjnil · Jul 22, 2021 · 238fe20 · 238fe20
1 parent dc0067b
commit 238fe20
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 17 deletions.
diff --git a/.github/workflows/full-sync.yml b/.github/workflows/full-sync.yml
@@ -0,0 +1,30 @@
+name: FullSync Notion Star
+
+on:
+  workflow_dispatch:
+
+jobs:
+  full-sync:
+    runs-on: ubuntu-latest
+    environment: notion-sync
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup environment
+        uses: actions/setup-node@v2.2.0
+        with:
+          always-auth: false
+          node-version: 16.x
+          cache: yarn
+
+      - run: yarn
+
+      - name: Run Full Sync
+        env:
+          NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}
+          NOTION_DATABASE_ID: ${{ secrets.NOTION_DATABASE_ID }}
+          TOKEN_OF_GITHUB: ${{ secrets.TOKEN_OF_GITHUB }}
+          FULLSYNC_LIMIT: 2000
+          PARTIALSYNC_LIMIT: 10
+          SYNC_NOTION_FIRST: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: npm start
diff --git a/.github/workflows/blank.yml → .github/workflows/partial-sync.yml b/.github/workflows/blank.yml → .github/workflows/partial-sync.yml
@@ -1,29 +1,25 @@
-# This is a basic workflow to help you get started with Actions
-name: Sync Notion Star
+name: Partial Sync Notion Star
 
-# Controls when the workflow will run
 on:
-  workflow_dispatch:
-
+  schedule:
+    - cron: '*/10 * * * *'
 jobs:
-  full-sync:
+  partial-sync:
     runs-on: ubuntu-latest
     environment: notion-sync
     steps:
       - uses: actions/checkout@v2
-      
-      - name: Download artifact of last build
+
+      - name: Download cache storage of last build
         uses: dawidd6/action-download-artifact@v2
         continue-on-error: true
         with:
-          workflow: blank.yml
+          workflow: partial-sync.yml
           workflow_conclusion: success
           name: notion-sync-storage
           path: .cache
-
-      - run: pwd && ls -al && ls .cache && cat .cache/notion-page.json
 
-      - name: Setup Node.js environment
+      - name: Setup environment
         uses: actions/setup-node@v2.2.0
         with:
           always-auth: false
@@ -32,7 +28,7 @@ jobs:
 
       - run: yarn
 
-      - name: Run Sync
+      - name: Run Partial Sync
         env:
           NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}
           NOTION_DATABASE_ID: ${{ secrets.NOTION_DATABASE_ID }}
@@ -41,7 +37,7 @@ jobs:
           PARTIALSYNC_LIMIT: 10
           SYNC_NOTION_FIRST: ${{ steps.cache.outputs.cache-hit != 'true' }}
         run: npm start
-        
+
       - uses: actions/upload-artifact@v2
         with:
           name: notion-sync-storage

diff --git a/libs/notion.ts b/libs/notion.ts
@@ -34,7 +34,12 @@ export class Notion {
     /**
      * full-sync pages in database
      */
-    async fullSync() {
+    async fullSyncIfNeeded() {
+        if (Object.keys(this.pages).length) {
+            console.log(`Notion: skipped sync due to cache`);
+            return;
+        }
+
         console.log('Notion: Start to get all pages');
 
         let hasNext = true;

diff --git a/main.ts b/main.ts
@@ -3,7 +3,7 @@ import { notion } from './libs/notion';
 import assert from 'assert';
 
 async function fullSync() {
-    await Promise.all([github.fullSync(), notion.fullSync()]);
+    await Promise.all([github.fullSync(), notion.fullSyncIfNeeded()]);
 
     for (const repo of github.repoList) {
         if (!notion.hasPage(repo.nameWithOwner)) {
@@ -13,13 +13,14 @@ async function fullSync() {
 }
 
 async function partialSync() {
-    await github.getList();
+    await Promise.all([github.getList(), notion.fullSyncIfNeeded()]);
 
     for (const repo of github.repoList) {
         if (notion.hasPage(repo.nameWithOwner)) {
             console.log(`Skip saved page ${repo.nameWithOwner}`);
             continue;
         }
+
         await notion.insertPage(repo);
     }
 }