画面共有の内容を回答できる機能を追加

src/components/capture.tsx

@@ -0,0 +1,152 @@
+import { useRef, useState, useEffect, useCallback } from 'react'
+import homeStore from '@/features/stores/home'
+import { IconButton } from './iconButton'
+
+const Capture = () => {
+  const triggerShutter = homeStore((s) => s.triggerShutter)
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const mediaStreamRef = useRef<MediaStream | null>(null)
+  const captureStartedRef = useRef<boolean>(false)
+
+  const [permissionGranted, setPermissionGranted] = useState<boolean>(false)
+  const [showPermissionModal, setShowPermissionModal] = useState<boolean>(true)
+
+  // 初回のみ許可を要求するために useRef で状態を保持
+  const requestCapturePermissionAttempted = useRef<boolean>(false)
+
+  // Capture permission request
+  const requestCapturePermission = async () => {
+    try {
+      if (!navigator.mediaDevices) {
+        throw new Error('Media Devices API non supported.')
+      }
+      const stream = await navigator.mediaDevices.getDisplayMedia({
+        video: true,
+      })
+      mediaStreamRef.current = stream
+      console.log('MediaStream obtained:', mediaStreamRef.current)
+      if (videoRef.current) {
+        videoRef.current.srcObject = stream
+        await videoRef.current.play()
+      }
+      setPermissionGranted(true)
+      // menuStore.setState({ capturePermissionGranted: true })
+      setShowPermissionModal(false)
+    } catch (error) {
+      console.error('Error capturing display:', error)
+      setShowPermissionModal(true)
+    }
+  }
+
+  useEffect(() => {
+    // 初回のみ許可を要求
+    if (!requestCapturePermissionAttempted.current && !permissionGranted) {
+      requestCapturePermission()
+      requestCapturePermissionAttempted.current = true
+    }
+  }, [permissionGranted])
+
+  const startCapture = async () => {
+    // すでに画面共有中の場合は停止
+    if (captureStartedRef.current && mediaStreamRef.current) {
+      const tracks = mediaStreamRef.current.getTracks()
+      tracks.forEach((track) => track.stop())
+      mediaStreamRef.current = null
+      captureStartedRef.current = false
+      if (videoRef.current) {
+        videoRef.current.srcObject = null
+      }
+    }
+
+    // 新たに画面共有を開始
+    try {
+      const stream = await navigator.mediaDevices.getDisplayMedia({
+        video: true,
+      })
+      mediaStreamRef.current = stream
+
+      if (videoRef.current) {
+        videoRef.current.srcObject = stream
+        await videoRef.current.play()
+        captureStartedRef.current = true
+      }
+    } catch (error) {
+      console.error('Error capturing display:', error)
+    }
+  }
+
+  const handleCapture = useCallback(() => {
+    if (videoRef.current && mediaStreamRef.current) {
+      const canvas = document.createElement('canvas')
+      const context = canvas.getContext('2d')
+      const { videoWidth, videoHeight } = videoRef.current
+
+      canvas.width = videoWidth
+      canvas.height = videoHeight
+      context?.drawImage(videoRef.current, 0, 0)
+
+      const dataUrl = canvas.toDataURL('image/png')
+
+      if (dataUrl !== '') {
+        console.log('capture')
+        homeStore.setState({
+          modalImage: dataUrl,
+          triggerShutter: false, // シャッターをリセット
+        })
+      } else {
+        homeStore.setState({ modalImage: '' })
+      }
+    } else {
+      console.error('Video or media stream is not available')
+    }
+  }, [])
+
+  useEffect(() => {
+    if (triggerShutter) {
+      handleCapture()
+    }
+  }, [triggerShutter, handleCapture])
+
+  useEffect(() => {
+    const videoElement = videoRef.current
+
+    return () => {
+      if (mediaStreamRef.current) {
+        const tracks = mediaStreamRef.current.getTracks()
+        tracks.forEach((track) => track.stop())
+        mediaStreamRef.current = null
+      }
+      captureStartedRef.current = false
+      if (videoElement) {
+        videoElement.srcObject = null
+      }
+    }
+  }, [])
+
+  return (
+    <div className="absolute row-span-1 flex right-0 max-h-[40vh] z-10">
+      <div className="relative w-full md:max-w-[512px] max-w-[50%] m-16">
+        <video ref={videoRef} autoPlay playsInline width={512} height={512} />
+
+        <div className="md:block hidden absolute top-4 right-4">
+          <IconButton
+            iconName="24/Reload"
+            className="bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled m-8"
+            isProcessing={false}
+            onClick={startCapture}
+          />
+        </div>
+        <div className="block absolute bottom-4 right-4">
+          <IconButton
+            iconName="24/Shutter"
+            className="z-30 bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled m-8"
+            isProcessing={false}
+            onClick={handleCapture}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default Capture

src/components/form.tsx

@@ -17,6 +17,7 @@ import { SlideText } from './slideText'
 export const Form = () => {
   const modalImage = homeStore((s) => s.modalImage)
   const webcamStatus = homeStore((s) => s.webcamStatus)
+  const captureStatus = homeStore((s) => s.captureStatus)
   const slideMode = settingsStore((s) => s.slideMode)
   const slideVisible = menuStore((s) => s.slideVisible)
   const slidePlaying = slideStore((s) => s.isPlaying)
@@ -47,14 +48,14 @@ export const Form = () => {
       homeStore.setState({ triggerShutter: true })
 
       // MENUの中でshowCameraがtrueの場合、画像が取得されるまで待機
-      if (webcamStatus) {
+      if (webcamStatus || captureStatus) {
         // Webcamが開いている場合
         setDelayedText(text) // 画像が取得されるまで遅延させる
       } else {
         handleSendChat(text)
       }
     },
-    [handleSendChat, webcamStatus, setDelayedText]
+    [handleSendChat, webcamStatus, captureStatus, setDelayedText]
   )
 
   useEffect(() => {

src/components/menu.tsx

@@ -3,14 +3,18 @@ import { useTranslation } from 'react-i18next'
 
 import homeStore from '@/features/stores/home'
 import menuStore from '@/features/stores/menu'
-import settingsStore from '@/features/stores/settings'
+import settingsStore, {
+  multiModalAIServiceKey,
+} from '@/features/stores/settings'
 import slideStore from '@/features/stores/slide'
 import { AssistantText } from './assistantText'
 import { ChatLog } from './chatLog'
 import { IconButton } from './iconButton'
 import Settings from './settings'
 import { Webcam } from './webcam'
 import Slides from './slides'
+import Capture from './capture'
+import { multiModalAIServices } from '@/features/stores/settings'
 
 export const Menu = () => {
   const selectAIService = settingsStore((s) => s.selectAIService)
@@ -22,6 +26,7 @@ export const Menu = () => {
   const chatLog = homeStore((s) => s.chatLog)
   const assistantMessage = homeStore((s) => s.assistantMessage)
   const showWebcam = menuStore((s) => s.showWebcam)
+  const showCapture = menuStore((s) => s.showCapture)
   const showControlPanel = menuStore((s) => s.showControlPanel)
   const slidePlaying = slideStore((s) => s.isPlaying)
   const showAssistantText = settingsStore((s) => s.showAssistantText)
@@ -100,6 +105,11 @@ export const Menu = () => {
     }
   }, [showWebcam])
 
+  useEffect(() => {
+    console.log('onChangeCaptureStatus')
+    homeStore.setState({ captureStatus: showCapture })
+  }, [showCapture])
+
   useEffect(() => {
     if (!youtubePlaying) {
       settingsStore.setState({
@@ -110,6 +120,16 @@ export const Menu = () => {
     }
   }, [youtubePlaying])
 
+  const toggleCapture = useCallback(() => {
+    menuStore.setState(({ showCapture }) => ({ showCapture: !showCapture }))
+    menuStore.setState({ showWebcam: false }) // Captureを表示するときWebcamを非表示にする
+  }, [])
+
+  const toggleWebcam = useCallback(() => {
+    menuStore.setState(({ showWebcam }) => ({ showWebcam: !showWebcam }))
+    menuStore.setState({ showCapture: false }) // Webcamを表示するときCaptureを非表示にする
+  }, [])
+
   return (
     <>
       <div className="absolute z-15 m-24">
@@ -144,45 +164,51 @@ export const Menu = () => {
                   />
                 )}
               </div>
-              {!youtubeMode && (
-                <>
-                  <div className="order-3">
-                    <IconButton
-                      iconName="24/Camera"
-                      isProcessing={false}
-                      onClick={() =>
-                        menuStore.setState(({ showWebcam }) => ({
-                          showWebcam: !showWebcam,
-                        }))
-                      }
-                    />
-                  </div>
-                  <div className="order-4">
-                    <IconButton
-                      iconName="24/AddImage"
-                      isProcessing={false}
-                      onClick={() => imageFileInputRef.current?.click()}
-                    />
-                    <input
-                      type="file"
-                      className="hidden"
-                      accept="image/*"
-                      ref={imageFileInputRef}
-                      onChange={(e) => {
-                        const file = e.target.files?.[0]
-                        if (file) {
-                          const reader = new FileReader()
-                          reader.onload = (e) => {
-                            const imageUrl = e.target?.result as string
-                            homeStore.setState({ modalImage: imageUrl })
+              {!youtubeMode &&
+                multiModalAIServices.includes(
+                  selectAIService as multiModalAIServiceKey
+                ) && (
+                  <>
+                    <div className="order-3">
+                      <IconButton
+                        iconName="24/ShareIos"
+                        isProcessing={false}
+                        onClick={toggleCapture}
+                      />
+                    </div>
+                    <div className="order-4">
+                      <IconButton
+                        iconName="24/Camera"
+                        isProcessing={false}
+                        onClick={toggleWebcam}
+                      />
+                    </div>
+                    <div className="order-4">
+                      <IconButton
+                        iconName="24/AddImage"
+                        isProcessing={false}
+                        onClick={() => imageFileInputRef.current?.click()}
+                      />
+                      <input
+                        type="file"
+                        className="hidden"
+                        accept="image/*"
+                        ref={imageFileInputRef}
+                        onChange={(e) => {
+                          const file = e.target.files?.[0]
+                          if (file) {
+                            const reader = new FileReader()
+                            reader.onload = (e) => {
+                              const imageUrl = e.target?.result as string
+                              homeStore.setState({ modalImage: imageUrl })
+                            }
+                            reader.readAsDataURL(file)
                           }
-                          reader.readAsDataURL(file)
-                        }
-                      }}
-                    />
-                  </div>
-                </>
-              )}
+                        }}
+                      />
+                    </div>
+                  </>
+                )}
               {youtubeMode && (
                 <div className="order-5">
                   <IconButton
@@ -222,6 +248,7 @@ export const Menu = () => {
         (!slideMode || !slideVisible) &&
         showAssistantText && <AssistantText message={assistantMessage} />}
       {showWebcam && navigator.mediaDevices && <Webcam />}
+      {showCapture && <Capture />}
       {showPermissionModal && (
         <div className="modal">
           <div className="modal-content">

src/features/stores/home.ts

@@ -22,6 +22,7 @@ export interface TransientState {
   modalImage: string
   triggerShutter: boolean
   webcamStatus: boolean
+  captureStatus: boolean
   ws: WebSocket | null
   wsStreaming: boolean
 }
@@ -57,16 +58,24 @@ const homeStore = create<HomeState>()(
       modalImage: '',
       triggerShutter: false,
       webcamStatus: false,
+      captureStatus: false,
       ws: null,
       wsStreaming: false,
     }),
     {
       name: 'aitube-kit-home',
       partialize: ({ chatLog, dontShowIntroduction }) => ({
-        chatLog,
+        chatLog: chatLog.map((message: Message) => ({
+          ...message,
+          content:
+            typeof message.content === 'string'
+              ? message.content
+              : message.content[0].text,
+        })),
         dontShowIntroduction,
       }),
     }
   )
 )
+
 export default homeStore

src/features/stores/menu.ts

@@ -2,6 +2,7 @@ import { create } from 'zustand'
 
 interface MenuState {
   showWebcam: boolean
+  showCapture: boolean
   showControlPanel: boolean
   fileInput: HTMLInputElement | null
   bgFileInput: HTMLInputElement | null
@@ -10,6 +11,7 @@ interface MenuState {
 
 const menuStore = create<MenuState>((set, get) => ({
   showWebcam: false,
+  showCapture: false,
   showControlPanel: true,
   fileInput: null,
   bgFileInput: null,