media persistence workflow

.github/copilot-instructions.md

@@ -0,0 +1 @@
+When asked to write Convex functions, always follow [convex](./prompts/convex.prompt.md)

.gitignore

@@ -50,10 +50,14 @@ certificates
 
 # Env
 .env
+.env.local
 coursition-storage.json
 
 # Prisma
 generated
 
 # Aider
 .aider*
+
+# Convex
+_generated

apps/coursition/modern.config.ts

@@ -4,7 +4,7 @@ import { tailwindcssPlugin } from '@modern-js/plugin-tailwindcss'
 import { loadEnv } from '@rsbuild/core'
 import { pluginImageCompress } from '@rsbuild/plugin-image-compress'
 import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill'
-import { withZephyr } from 'zephyr-modernjs-plugin'
+// import { withZephyr } from 'zephyr-modernjs-plugin'
 
 const { publicVars } = loadEnv({ cwd: '../..' })
 
@@ -42,7 +42,7 @@ export default defineConfig({
       bundler: 'rspack',
     }),
     tailwindcssPlugin(),
-    withZephyr(),
+    //withZephyr(),
   ],
   tools: {
     rspack: {

apps/coursition/src/components/convex-provider.tsx

@@ -0,0 +1,12 @@
+import { ConvexProvider, ConvexReactClient } from 'convex/react'
+import type { ReactNode } from 'react'
+
+if (!process.env['PUBLIC_CONVEX_URL']) {
+  throw new Error('PUBLIC_CONVEX_URL is not defined')
+}
+
+const convex = new ConvexReactClient(process.env['PUBLIC_CONVEX_URL'])
+
+export default function ConvexProviderWithClient({ children }: { children: ReactNode }) {
+  return <ConvexProvider client={convex}>{children}</ConvexProvider>
+}

apps/coursition/src/routes/dashboard/page.tsx

@@ -0,0 +1,30 @@
+import { publicConfig } from '@nmit-coursition/env/typed.ts'
+import { api } from '@nmit-coursition/parse-engine/_generated/api'
+import { useQuery } from 'convex/react'
+import { Effect } from 'effect'
+
+const typedPublic = Effect.runSync(publicConfig)
+
+export default function MockDashboard() {
+  const { items } = useQuery(api.media.getMedia, { count: 10 }) ?? {}
+
+  console.log('Items:', items || 'none')
+
+  return (
+    <div className='flex justify-center h-screen'>
+      <div className='p-4 max-w-2xl w-full'>
+        <div>mock-dashboard</div>
+        <ul>
+          {!!items &&
+            items.map((item, index) => (
+              <li key={index}>
+                <a href={`${typedPublic.FRONTEND_URL.href}media/${item._id.toString()}`}>
+                  {item._id} <b>{item.status}</b>
+                </a>
+              </li>
+            ))}
+        </ul>
+      </div>
+    </div>
+  )
+}

apps/coursition/src/routes/layout.tsx

@@ -1,10 +1,13 @@
 import { Outlet } from '@modern-js/runtime/router'
 import './index.css'
+import ConvexProviderWithClient from '../components/convex-provider.tsx'
 
 export default function Layout() {
   return (
     <div>
-      <Outlet />
+      <ConvexProviderWithClient>
+        <Outlet />
+      </ConvexProviderWithClient>
     </div>
   )
 }

apps/coursition/src/routes/media/[id]/page.tsx

@@ -0,0 +1,68 @@
+import { TabPane, Tabs } from '@douyinfe/semi-ui'
+import { useParams } from '@modern-js/runtime/router'
+import { api } from '@nmit-coursition/parse-engine/_generated/api'
+import { convertSubtitlesToBlob, extractFileMetadata } from '@nmit-coursition/utils/media'
+import { type MediaMetadata } from '@nmit-coursition/utils/media'
+import { useQuery } from 'convex/react'
+import { useEffect, useState } from 'react'
+import { StatusDisplay } from '../../../components/status-display.tsx'
+import { TranscriptionResults } from '../../../components/transcription-results.tsx'
+import { VideoPlayer } from '../../../components/video-player.tsx'
+
+const statusStates = [
+  { key: 'upload', text: 'Uploading media' },
+  { key: 'parse', text: 'Transcribing' },
+]
+
+export default function MediaId() {
+  const { id: mediaId } = useParams()
+  const [status, setStatus] = useState<'upload' | 'parse' | 'done'>('parse')
+  const { media } = useQuery(api.media.getMediaById, { mediaId: mediaId ?? '' }) ?? {}
+  const videoUrl = useQuery(api.media.getFileUrl, { storageId: media?.fileId })
+  const [mediaMetadata, setMediaMetadata] = useState<MediaMetadata | null>(null)
+
+  useEffect(() => {
+    if (media?.status) {
+      setStatus(media.status)
+    }
+  }, [media])
+
+  useEffect(() => {
+    if (videoUrl) {
+      handleMetadata(videoUrl).then()
+    }
+  }, [videoUrl])
+
+  const handleMetadata = async (url: string) => {
+    const response = await fetch(url)
+    const blob = await response.blob()
+    const file = new File([blob], 'video', { type: blob.type })
+    setMediaMetadata(await extractFileMetadata(file))
+  }
+
+  return (
+    <div className='flex justify-center h-screen'>
+      <div className='p-4 max-w-2xl w-full'>
+        {status !== 'done' && <StatusDisplay states={statusStates} status={status} />}
+        {media && status === 'done' && (
+          <div>
+            <Tabs type='line' className='mt-4'>
+              {videoUrl && (
+                <TabPane tab='Video' itemKey='video'>
+                  <VideoPlayer
+                    source={videoUrl}
+                    subtitles={convertSubtitlesToBlob(media.vtt)}
+                    aspectRatio={`${mediaMetadata?.dimensions?.aspectRatio}` || '16/9'}
+                  />
+                </TabPane>
+              )}
+              <TabPane tab='Text' itemKey='text'>
+                <TranscriptionResults raw={media.text} srt={media.srt} vtt={media.vtt} />
+              </TabPane>
+            </Tabs>
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}

apps/coursition/src/routes/media/page.tsx

@@ -4,11 +4,14 @@ import type { BeforeUploadProps } from '@douyinfe/semi-ui/lib/es/upload/interfac
 import { allowedDeepgramLanguages, deepgramLanguageNames } from '@nmit-coursition/utils/languages'
 import {
   type MediaMetadata,
+  // allowedDeepgramLanguages,
   convertSubtitlesToBlob,
+  // deepgramLanguageNames,
   extractFileMetadata,
   extractUrlMetadata,
 } from '@nmit-coursition/utils/media'
 import { useState } from 'react'
+import { useNavigate } from 'react-router-dom'
 import { z } from 'zod'
 import { zfd } from 'zod-form-data'
 import { MediaFileDetails } from '../../components/media-file-details'
@@ -61,6 +64,7 @@ export default function Media() {
   const [isProcessingUrl, setIsProcessingUrl] = useState(false)
   const [currentUrl, setCurrentUrl] = useState('')
   const [modal, contextHolder] = Modal.useModal()
+  const navigate = useNavigate()
 
   const showFileDetails = () => {
     if (!state.mediaMetadata) return
@@ -218,7 +222,11 @@ export default function Media() {
             )
 
       if (error) throw new Error(error.value.description)
-      const { text = '', srt = '', vtt = '' } = data
+      const { text = '', srt = '', vtt = '', id } = data
+
+      if (id) {
+        navigate(`/media/${id}`)
+      }
       setStatus('done')
       setState((prev) => ({ ...prev, raw: text, srt, vtt, videoSource }))
     } catch (error) {

biome.json

@@ -5,7 +5,7 @@
     "clientKind": "git"
   },
   "files": {
-    "ignore": [".vercel", ".vinxi", ".nx", "apps/legacy_nmit/**"]
+    "ignore": [".vercel", ".vinxi", ".nx", "apps/legacy_nmit/**", "_generated"]
   },
   "formatter": {
     "enabled": true,

convex.json

@@ -0,0 +1,3 @@
+{
+  "functions": "libs/parse-engine/convex"
+}

libs/api/src/api-dev.ts

@@ -1,6 +1,14 @@
-import { Elysia } from 'elysia'
+import { publicConfig } from '@nmit-coursition/env/typed'
+import { api } from '@nmit-coursition/parse-engine/_generated/api'
+import { ConvexClient } from 'convex/browser'
+import { Effect } from 'effect'
+import { Elysia, t } from 'elysia'
+import { formatApiErrorResponse } from './utils/api'
 import { apiCommonGuard, computeUsage, reportSpend } from './utils/api-utils'
 
+const { CONVEX_URL } = await Effect.runPromise(publicConfig)
+const convexClient = new ConvexClient(CONVEX_URL.href)
+
 export const apiDev = new Elysia({ prefix: '/dev', tags: ['dev'] })
   .use(apiCommonGuard)
   .get('/ping', () => ({ status: 'ZEROPS' }), {
@@ -14,3 +22,35 @@ export const apiDev = new Elysia({ prefix: '/dev', tags: ['dev'] })
   .get('/report-usage', async () => await computeUsage({ organisationId: 1 }), {
     afterResponse: ({ request }) => reportSpend({ request }),
   })
+  .group('/convex', (convexApp) =>
+    convexApp.get(
+      '/tasks',
+      async ({ error: errorFn, request }) => {
+        try {
+          const mediaItems = await convexClient.query(api.media.getMedia, { count: 10 })
+          return {
+            tasks: mediaItems.items.map((item) => ({
+              text: item.text || '',
+              isCompleted: true,
+              _id: item._id,
+            })),
+          }
+        } catch (error) {
+          return errorFn(500, formatApiErrorResponse(request, `Failed to fetch tasks: ${error}`))
+        }
+      },
+      {
+        response: {
+          200: t.Object({
+            tasks: t.Array(
+              t.Object({
+                text: t.String(),
+                isCompleted: t.Boolean(),
+                _id: t.Any(),
+              }),
+            ),
+          }),
+        },
+      },
+    ),
+  )

libs/api/src/api-v1.ts

@@ -1,8 +1,6 @@
-import { unlink } from 'node:fs/promises'
 import FirecrawlApp from '@mendable/firecrawl-js'
 import { getResult, uploadFile, waitUntilJobIsDone } from '@nmit-coursition/ai/document'
 import { generateQuiz } from '@nmit-coursition/ai/generate'
-import { getTranscript } from '@nmit-coursition/ai/media'
 import { AUTH_BRJ_COOKIES_NAME } from '@nmit-coursition/auth/constants'
 import { secretsEffect } from '@nmit-coursition/env/secrets'
 import {
@@ -16,7 +14,7 @@ import { Redacted } from 'effect'
 import { Elysia, t } from 'elysia'
 import { formatApiErrorResponse } from './utils/api'
 import { apiCommonGuard, reportUsage } from './utils/api-utils'
-import { downloadPublicMedia } from './utils/download-media'
+import { persistNewMedia } from './utils/convex-client.ts'
 
 const secretsEnv = await Effect.runPromise(secretsEffect)
 export const apiV1 = new Elysia({ prefix: '/v1', tags: ['v1'] })
@@ -26,17 +24,22 @@ export const apiV1 = new Elysia({ prefix: '/v1', tags: ['v1'] })
       .post(
         '/media',
         async ({ body: { output, keywords, file, language }, error: errorFn, request }) => {
-          const transcript = await getTranscript(file, keywords, language)
+          const id = await persistNewMedia({
+            identityId: 'undefined_user',
+            output,
+            keywords,
+            file,
+            language,
+          })
+          // const transcript = await getTranscript(file, keywords, language)
+          const transcript = {}
 
           if ('error' in transcript) {
             return errorFn(500, formatApiErrorResponse(request, `Failed to process media: ${transcript.error}`))
           }
 
           return {
-            ...(output.includes('vtt') ? { vtt: transcript.vtt } : {}),
-            ...(output.includes('srt') ? { srt: transcript.srt } : {}),
-            ...(output.includes('text') ? { text: transcript.raw } : {}),
-            duration: transcript.metadata?.duration,
+            id,
           }
         },
         {
@@ -58,6 +61,7 @@ export const apiV1 = new Elysia({ prefix: '/v1', tags: ['v1'] })
               srt: t.Optional(t.String()),
               text: t.Optional(t.String()),
               duration: t.Optional(t.Number()),
+              id: t.String(),
             }),
           },
           afterResponse({ response, cookie }) {
@@ -72,23 +76,24 @@ export const apiV1 = new Elysia({ prefix: '/v1', tags: ['v1'] })
         '/public-media',
         async ({ body: { url, output, keywords, language }, error: errorFn, request }) => {
           try {
-            const { path } = await downloadPublicMedia(url)
-            const audioFile = Buffer.from(await Bun.file(path).arrayBuffer())
-            const transcript = await getTranscript(audioFile, keywords, language)
-            await unlink(path)
-
-            if ('error' in transcript) {
-              return errorFn(
-                500,
-                formatApiErrorResponse(request, `Failed to process public media: ${transcript.error}`),
-              )
-            }
+            const id = await persistNewMedia({
+              identityId: 'undefined_user',
+              output,
+              keywords,
+              publicMediaUrl: url,
+              language,
+            })
+            // const { path } = await downloadPublicMedia(url)
+            // const audioFile = Buffer.from(await Bun.file(path).arrayBuffer())
+            // const transcript = await getTranscript(audioFile, keywords, language)
+            // await unlink(path)
 
             return {
-              ...(output.includes('vtt') ? { vtt: transcript.vtt } : {}),
-              ...(output.includes('srt') ? { srt: transcript.srt } : {}),
-              ...(output.includes('text') ? { text: transcript.raw } : {}),
-              duration: transcript.metadata?.duration,
+              // ...(output.includes('vtt') ? { vtt: transcript.vtt } : {}),
+              // ...(output.includes('srt') ? { srt: transcript.srt } : {}),
+              // ...(output.includes('text') ? { text: transcript.raw } : {}),
+              // duration: transcript.metadata?.duration,
+              id,
             }
           } catch (error) {
             return errorFn(500, formatApiErrorResponse(request, `Failed to process public media: ${error}`))
@@ -113,6 +118,7 @@ export const apiV1 = new Elysia({ prefix: '/v1', tags: ['v1'] })
               srt: t.Optional(t.String()),
               text: t.Optional(t.String()),
               duration: t.Optional(t.Number()),
+              id: t.Optional(t.String()),
             }),
           },
           afterResponse({ response, cookie }) {

libs/api/src/utils/api-utils.ts

@@ -34,6 +34,9 @@ export const apiCommonGuard = new Elysia().guard({
   },
   headers: headersModel,
   beforeHandle: async ({ headers, request: r, error, set, cookie }) => {
+    if (process.env['NODE_ENV'] === 'development' && r.url.includes('localhost')) {
+      return
+    }
     const session = cookie[AUTH_COOKIES_NAME]?.toString() || ''
     const identityId = cookie[AUTH_BRJ_COOKIES_NAME]?.toString() || ''
     const apiKeyRaw = headers['authorization'] || ''