feat: support JSON export and export All

closes #39 #71

packages/userscript/package.json

@@ -24,6 +24,7 @@
     "@radix-ui/react-dialog": "^1.0.2",
     "hast-util-to-html": "^8.0.4",
     "html2canvas": "^1.4.1",
+    "jszip": "^3.10.1",
     "mdast": "^3.0.0",
     "mdast-util-from-markdown": "^1.3.0",
     "mdast-util-frontmatter": "^1.0.1",

packages/userscript/src/api.ts

@@ -1,5 +1,5 @@
 import urlcat from 'urlcat'
-import { getConversationChoice, getPageAccessToken } from './page'
+import { getPageAccessToken } from './page'
 
 interface ApiSession {
     accessToken: string
@@ -37,7 +37,7 @@ interface ConversationNode {
     parent?: string
 }
 
-interface ApiConversation {
+export interface ApiConversation {
     create_time: number
     current_node: string
     mapping: {
@@ -47,12 +47,18 @@ interface ApiConversation {
     title: string
 }
 
-interface ApiConversations {
-    items: {
-        id: string
-        title: string
-        create_time: number
-    }[]
+export type ApiConversationWithId = ApiConversation & {
+    id: string
+}
+
+export interface ApiConversationItem {
+    id: string
+    title: string
+    create_time: number
+}
+
+export interface ApiConversations {
+    items: ApiConversationItem[]
     limit: number
     offset: number
     total: number
@@ -65,7 +71,7 @@ const sessionApi = urlcat(baseUrl, '/api/auth/session')
 const conversationApi = (id: string) => urlcat(apiUrl, '/conversation/:id', { id })
 const conversationsApi = (offset: number, limit: number) => urlcat(apiUrl, '/conversations', { offset, limit })
 
-async function getCurrentChatId(): Promise<string> {
+export async function getCurrentChatId(): Promise<string> {
     const match = location.pathname.match(/^\/chat\/([a-z0-9-]+)$/i)
     if (match) return match[1]
 
@@ -77,8 +83,7 @@ async function getCurrentChatId(): Promise<string> {
     throw new Error('No chat id found.')
 }
 
-export async function fetchConversation(): Promise<ApiConversation & { id: string }> {
-    const chatId = await getCurrentChatId()
+export async function fetchConversation(chatId: string): Promise<ApiConversationWithId> {
     const url = conversationApi(chatId)
     const conversation = await fetchApi<ApiConversation>(url)
     return {
@@ -87,11 +92,24 @@ export async function fetchConversation(): Promise<ApiConversation & { id: strin
     }
 }
 
-async function fetchConversations(offset = 0, limit = 20): Promise<ApiConversations> {
+export async function fetchConversations(offset = 0, limit = 20): Promise<ApiConversations> {
     const url = conversationsApi(offset, limit)
     return fetchApi(url)
 }
 
+export async function fetchAllConversations(): Promise<ApiConversationItem[]> {
+    const conversations: ApiConversationItem[] = []
+    let offset = 0
+    const limit = 20
+    while (true) {
+        const result = await fetchConversations(offset, limit)
+        conversations.push(...result.items)
+        if (result.items.length < limit) break
+        offset += limit
+    }
+    return conversations
+}
+
 async function fetchApi<T>(url: string): Promise<T> {
     const accessToken = await getAccessToken()
 
@@ -130,16 +148,14 @@ class LinkedListItem<T> {
     }
 }
 
-interface ConversationResult {
+export interface ConversationResult {
     id: string
     title: string
     createTime: number
-    conversations: ConversationNode[]
+    conversationNodes: ConversationNode[]
 }
 
-export async function getConversations(): Promise<ConversationResult> {
-    const conversation = await fetchConversation()
-
+export function processConversation(conversation: ApiConversationWithId, conversationChoices: (number | null)[] = []): ConversationResult {
     const title = conversation.title || 'ChatGPT Conversation'
     const createTime = conversation.create_time
 
@@ -148,9 +164,7 @@ export async function getConversations(): Promise<ConversationResult> {
     const root = nodes.find(node => !node.parent)
     if (!root) throw new Error('No root node found.')
 
-    const conversationChoices = getConversationChoice()
     const nodeMap = new Map(Object.entries(conversation.mapping))
-
     const tail = new LinkedListItem(root)
     const queue = [tail]
     let index = -1
@@ -168,7 +182,7 @@ export async function getConversations(): Promise<ConversationResult> {
 
         const _last = node.children.length - 1
         const choice = conversationChoices[index++] ?? _last
-        const childId = node.children[choice]
+        const childId = node.children[choice] ?? node.children[_last]
         if (!childId) throw new Error('No child node found.')
 
         const child = nodeMap.get(childId)
@@ -183,6 +197,6 @@ export async function getConversations(): Promise<ConversationResult> {
         id: conversation.id,
         title,
         createTime,
-        conversations: result,
+        conversationNodes: result,
     }
 }

packages/userscript/src/exporter/html.ts

@@ -1,22 +1,59 @@
+import JSZip from 'jszip'
 import templateHtml from '../template.html?raw'
 import { downloadFile, getFileNameWithFormat } from '../utils/download'
 import { dateStr, getColorScheme } from '../utils/utils'
 import { standardizeLineBreaks } from '../utils/text'
-import { baseUrl, getConversations } from '../api'
+import { type ConversationResult, baseUrl, fetchConversation, getCurrentChatId, processConversation } from '../api'
 import { fromMarkdown, toHtml } from '../utils/markdown'
-import { checkIfConversationStarted, getUserAvatar } from '../page'
+import { checkIfConversationStarted, getConversationChoice, getUserAvatar } from '../page'
 
 export async function exportToHtml(fileNameFormat: string) {
     if (!checkIfConversationStarted()) {
         alert('Please start a conversation first.')
         return false
     }
 
-    const { id, title, conversations } = await getConversations()
+    const userAvatar = await getUserAvatar()
+
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId)
+    const conversationChoices = getConversationChoice()
+    const conversation = processConversation(rawConversation, conversationChoices)
+    const html = conversationToHtml(conversation, userAvatar)
+
+    const fileName = getFileNameWithFormat(fileNameFormat, 'html', { title: conversation.title })
+    downloadFile(fileName, 'text/html', standardizeLineBreaks(html))
+
+    return true
+}
+
+export async function exportAllToHtml(fileNameFormat: string, conversationIds: string[]) {
+    const conversations = await Promise.all(
+        conversationIds.map(async (id) => {
+            const rawConversation = await fetchConversation(id)
+            return processConversation(rawConversation)
+        }),
+    )
 
     const userAvatar = await getUserAvatar()
 
-    const conversationHtml = conversations.map((item) => {
+    const zip = new JSZip()
+    conversations.forEach((conversation) => {
+        const fileName = getFileNameWithFormat(fileNameFormat, 'html', { title: conversation.title })
+        const content = conversationToHtml(conversation, userAvatar)
+        zip.file(fileName, content)
+    })
+
+    const blob = await zip.generateAsync({ type: 'blob' })
+    downloadFile('chatgpt-export.zip', 'application/zip', blob)
+
+    return true
+}
+
+function conversationToHtml(conversation: ConversationResult, avatar: string) {
+    const { id, title, conversationNodes } = conversation
+
+    const conversationHtml = conversationNodes.map((item) => {
         const author = item.message?.author.role === 'assistant' ? 'ChatGPT' : 'You'
         const avatarEl = author === 'ChatGPT'
             ? '<svg width="41" height="41"><use xlink:href="#chatgpt" /></svg>'
@@ -70,11 +107,7 @@ export async function exportToHtml(fileNameFormat: string) {
         .replaceAll('{{source}}', source)
         .replaceAll('{{lang}}', lang)
         .replaceAll('{{theme}}', theme)
-        .replaceAll('{{avatar}}', userAvatar)
+        .replaceAll('{{avatar}}', avatar)
         .replaceAll('{{content}}', conversationHtml)
-
-    const fileName = getFileNameWithFormat(fileNameFormat, 'html', { title })
-    downloadFile(fileName, 'text/html', standardizeLineBreaks(html))
-
-    return true
+    return html
 }

packages/userscript/src/exporter/json.ts

@@ -0,0 +1,48 @@
+import JSZip from 'jszip'
+import { type ApiConversationWithId, fetchConversation, getCurrentChatId, processConversation } from '../api'
+import { downloadFile, getFileNameWithFormat } from '../utils/download'
+import { checkIfConversationStarted, getConversationChoice } from '../page'
+
+export async function exportToJson(fileNameFormat: string) {
+    if (!checkIfConversationStarted()) {
+        alert('Please start a conversation first.')
+        return false
+    }
+
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId)
+    const conversationChoices = getConversationChoice()
+    const conversation = processConversation(rawConversation, conversationChoices)
+
+    const fileName = getFileNameWithFormat(fileNameFormat, 'json', { title: conversation.title })
+    const content = conversationToJson(rawConversation)
+    downloadFile(fileName, 'application/json', content)
+
+    return true
+}
+
+export async function exportAllToJson(fileNameFormat: string, conversationIds: string[]) {
+    const conversations = await Promise.all(
+        conversationIds.map(async (id) => {
+            const rawConversation = await fetchConversation(id)
+            const conversation = processConversation(rawConversation)
+            return { conversation, rawConversation }
+        }),
+    )
+
+    const zip = new JSZip()
+    conversations.forEach(({ conversation, rawConversation }) => {
+        const fileName = getFileNameWithFormat(fileNameFormat, 'json', { title: conversation.title })
+        const content = conversationToJson(rawConversation)
+        zip.file(fileName, content)
+    })
+
+    const blob = await zip.generateAsync({ type: 'blob' })
+    downloadFile('chatgpt-export.zip', 'application/zip', blob)
+
+    return true
+}
+
+function conversationToJson(conversation: ApiConversationWithId) {
+    return JSON.stringify(conversation)
+}

packages/userscript/src/exporter/markdown.ts

@@ -1,8 +1,9 @@
-import { getConversations } from '../api'
+import JSZip from 'jszip'
+import { type ConversationResult, fetchConversation, getCurrentChatId, processConversation } from '../api'
 import { fromMarkdown, toMarkdown } from '../utils/markdown'
 import { downloadFile, getFileNameWithFormat } from '../utils/download'
 import { standardizeLineBreaks } from '../utils/text'
-import { checkIfConversationStarted } from '../page'
+import { checkIfConversationStarted, getConversationChoice } from '../page'
 // import { dateStr } from '../utils/utils'
 
 export async function exportToMarkdown(fileNameFormat: string) {
@@ -11,8 +12,41 @@ export async function exportToMarkdown(fileNameFormat: string) {
         return false
     }
 
-    // const { id, title, createTime, conversations } = await getConversations()
-    const { title, conversations } = await getConversations()
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId)
+    const conversationChoices = getConversationChoice()
+    const conversation = processConversation(rawConversation, conversationChoices)
+    const markdown = conversationToMarkdown(conversation)
+
+    const fileName = getFileNameWithFormat(fileNameFormat, 'md', { title: conversation.title })
+    downloadFile(fileName, 'text/markdown', standardizeLineBreaks(markdown))
+
+    return true
+}
+
+export async function exportAllToMarkdown(fileNameFormat: string, conversationIds: string[]) {
+    const conversations = await Promise.all(
+        conversationIds.map(async (id) => {
+            const rawConversation = await fetchConversation(id)
+            return processConversation(rawConversation)
+        }),
+    )
+
+    const zip = new JSZip()
+    conversations.forEach((conversation) => {
+        const fileName = getFileNameWithFormat(fileNameFormat, 'md', { title: conversation.title })
+        const content = conversationToMarkdown(conversation)
+        zip.file(fileName, content)
+    })
+
+    const blob = await zip.generateAsync({ type: 'blob' })
+    downloadFile('chatgpt-export.zip', 'application/zip', blob)
+
+    return true
+}
+
+function conversationToMarkdown(conversation: ConversationResult) {
+    const { conversationNodes } = conversation
 
     // const date = dateStr()
     // const source = `${baseUrl}/chat/${id}`
@@ -22,7 +56,7 @@ export async function exportToMarkdown(fileNameFormat: string) {
     // source: ${source}
     // ---`
 
-    const content = conversations.map((item) => {
+    const content = conversationNodes.map((item) => {
         const author = item.message?.author.role === 'assistant' ? 'ChatGPT' : 'You'
         const content = item.message?.content.parts.join('\n') ?? ''
         let message = content
@@ -38,8 +72,5 @@ export async function exportToMarkdown(fileNameFormat: string) {
     // const markdown = `${frontMatter}\n\n${content}`
     const markdown = content
 
-    const fileName = getFileNameWithFormat(fileNameFormat, 'md', { title })
-    downloadFile(fileName, 'text/markdown', standardizeLineBreaks(markdown))
-
-    return true
+    return markdown
 }

packages/userscript/src/exporter/text.ts

@@ -1,18 +1,21 @@
 import type { Emphasis, Strong } from 'mdast'
 import { copyToClipboard } from '../utils/clipboard'
 import { standardizeLineBreaks } from '../utils/text'
-import { getConversations } from '../api'
+import { fetchConversation, getCurrentChatId, processConversation } from '../api'
 import { flatMap, fromMarkdown, toMarkdown } from '../utils/markdown'
-import { checkIfConversationStarted } from '../page'
+import { checkIfConversationStarted, getConversationChoice } from '../page'
 
 export async function exportToText() {
     if (!checkIfConversationStarted()) {
         alert('Please start a conversation first.')
         return false
     }
 
-    const { conversations } = await getConversations()
-    const text = conversations.map((item) => {
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId)
+    const conversationChoices = getConversationChoice()
+    const { conversationNodes } = processConversation(rawConversation, conversationChoices)
+    const text = conversationNodes.map((item) => {
         const author = item.message?.author.role === 'assistant' ? 'ChatGPT' : 'You'
         const content = item.message?.content.parts.join('\n') ?? ''
         let message = content