add youtube video summary

app.go

@@ -479,3 +479,49 @@ func (a *App) ShareWorkspace(id int) error {
 	err = util.OpenURL("https://sharegpt.com/c/" + response.ID)
 	return nil
 }
+
+type YoutubeVideoResult struct {
+	Details  YoutubeVideoDetails    `json:"details"`
+	Captions []util.YtCustomCaption `json:"captions"`
+}
+type YoutubeVideoDetails struct {
+	Title         string   `json:"title"`
+	LengthSeconds string   `json:"length_seconds"`
+	Description   string   `json:"description"`
+	Keywords      []string `json:"keywords"`
+	PicURL        string   `json:"pic_url"`
+	Author        string   `json:"author"`
+}
+
+func (a *App) GetYoutubeVideo(url string) (YoutubeVideoResult, error) {
+	var result YoutubeVideoResult
+	yt, err := util.NewYoutube(url, a.settings.config.Proxy)
+	if err != nil {
+		return result, err
+	}
+	vd, err := yt.GetVideoDetails()
+	if err != nil {
+		return result, err
+	}
+	cp, err := yt.GetCaptions()
+	if err != nil {
+		return result, err
+	}
+	th, _ := lo.Last(vd.Thumbnail.Thumbnails)
+	result = YoutubeVideoResult{
+		Details: YoutubeVideoDetails{
+			Title:         vd.Title,
+			LengthSeconds: vd.LengthSeconds,
+			Description:   vd.ShortDescription,
+			Keywords:      vd.Keywords,
+			PicURL:        th.Url,
+			Author:        vd.Author,
+		},
+		Captions: cp,
+	}
+	return result, nil
+}
+
+func (a *App) GetYoutubeTranscript(caption util.YtCustomCaption) ([]util.YtTranscriptText, error) {
+	return caption.GetTranscript(a.settings.config.Proxy)
+}

config.go

@@ -81,6 +81,7 @@ type Migration struct {
 	SydneyPreset20240304 bool `json:"sydney_preset_20240304"`
 	ThemeColor20240304   bool `json:"theme_color_20240304"`
 	Quick20240326        bool `json:"quick_20240326"`
+	Quick20240405        bool `json:"quick_20240405"`
 }
 
 func fillDefault[T comparable](pointer *T, defaultValue T) {
@@ -126,6 +127,15 @@ func (o *Config) DoMigration() {
 		}
 		o.Migration.Quick20240326 = true
 	}
+	if !o.Migration.Quick20240405 {
+		_, ok := lo.Find(o.Quick, func(item string) bool {
+			return item == "Summarize the video from Youtube according to the given information."
+		})
+		if !ok {
+			o.Quick = append(o.Quick, "Summarize the video from Youtube according to the given information.")
+		}
+		o.Migration.Quick20240405 = true
+	}
 }
 func (o *Config) FillDefault() {
 	if len(o.Presets) == 0 {
@@ -151,6 +161,7 @@ func (o *Config) FillDefault() {
 			"Fix grammar errors and polish the writing of the text above.",
 			"Translate the text above into Chinese in a fluent way.",
 			"Continue the conversation in context. Assistant:",
+			"Summarize the video from Youtube according to the given information.",
 			"[user](#message)"}
 	}
 	if len(o.OpenAIBackends) == 0 {

frontend/src/components/index/InsertVideoButton.vue

@@ -0,0 +1,103 @@
+<script setup lang="ts">
+import UserInputToolButton from "./UserInputToolButton.vue"
+import {ref} from "vue"
+import {GetYoutubeTranscript, GetYoutubeVideo} from "../../../wailsjs/go/main/App"
+import {main, util} from "../../../wailsjs/go/models"
+import {swal} from "../../helper"
+import YoutubeVideoResult = main.YoutubeVideoResult
+import YtCustomCaption = util.YtCustomCaption
+
+let props = defineProps<{
+  isAsking: boolean,
+}>()
+let emit = defineEmits<{
+  (e: 'append', text: string): void
+}>()
+let dialog = ref(false)
+let youtubeLink = ref('')
+let youtubeResult = ref<undefined | YoutubeVideoResult>(undefined)
+let loading = ref(false)
+
+function handleInsert(caption: YtCustomCaption) {
+  loading.value = true
+  GetYoutubeTranscript(caption).then(res => {
+    let obj: any = {
+      ...youtubeResult.value?.details,
+      'transcript': res,
+      'transcript_lang': caption.language_code,
+    }
+    emit('append', '[user](#youtube_video_info)\n' + JSON.stringify(obj) + '\n\n')
+    dialog.value = false
+    youtubeLink.value = ''
+    youtubeResult.value = undefined
+  }).catch(err => {
+    swal.error(err)
+  }).finally(() => {
+    loading.value = false
+  })
+}
+
+function handleFetch() {
+  loading.value = true
+  youtubeResult.value = undefined
+  GetYoutubeVideo(youtubeLink.value).then(res => {
+    youtubeResult.value = res
+  }).catch(err => {
+    swal.error(err)
+  }).finally(() => {
+    loading.value = false
+  })
+}
+</script>
+
+<template>
+  <div>
+    <user-input-tool-button icon="mdi-video" :disabled="isAsking" @click="dialog=true"
+                            tooltip="Insert a Youtube Video to chat context"></user-input-tool-button>
+    <v-dialog max-width="500" v-model="dialog" :scrollable="true">
+      <v-card title="Insert a Youtube Video">
+        <v-card-text>
+          <div class="d-flex align-center">
+            <v-text-field @keydown.enter="handleFetch" density="compact" v-model="youtubeLink" label="Youtube Link"
+                          color="primary"></v-text-field>
+            <v-btn color="primary" class="ml-3 mb-4" :loading="loading" variant="tonal" @click="handleFetch">Fetch
+            </v-btn>
+          </div>
+          <div v-if="youtubeResult">
+            <div style="font-size: 17px;font-weight: bold">Video Details</div>
+            <p>Title: <i>{{ youtubeResult.details.title }}</i></p>
+            <p>Author: <i>{{ youtubeResult.details.author }}</i></p>
+            <p>Cover:</p>
+            <v-img :src="youtubeResult.details.pic_url"></v-img>
+            <div style="font-size: 17px;font-weight: bold" class="mt-3">Select a Caption</div>
+            <v-list density="compact">
+              <v-list-item v-for="caption in youtubeResult.captions">
+                <template #title>
+                  <div class="d-flex align-center">
+                    <p>{{ caption.name }}</p>
+                    <v-chip class="ml-3" size="small" color="#5c5c5c" v-if="caption.is_asr">Auto-generated</v-chip>
+                    <v-chip class="ml-3" size="small" color="#2b28ec" v-else-if="caption.is_translated">Translated
+                    </v-chip>
+                    <v-chip class="ml-3" size="small" color="green" v-else>Manually-inputted</v-chip>
+                    <v-spacer></v-spacer>
+                    <v-btn density="compact" variant="tonal" color="primary" @click="handleInsert(caption)"
+                           :loading="loading">Insert
+                    </v-btn>
+                  </div>
+                </template>
+              </v-list-item>
+            </v-list>
+          </div>
+        </v-card-text>
+        <v-card-actions>
+          <v-spacer></v-spacer>
+          <v-btn variant="text" color="primary" @click="dialog=false">Cancel</v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
+  </div>
+</template>
+
+<style scoped>
+
+</style>

frontend/src/pages/IndexPage.vue

@@ -17,6 +17,7 @@ import UploadPanelButton from "../components/index/UploadPanelButton.vue"
 import UploadDocumentButton from "../components/index/UploadDocumentButton.vue"
 import FetchWebpageButton from "../components/index/FetchWebpageButton.vue"
 import RevokeButton from "../components/index/RevokeButton.vue"
+import InsertVideoButton from "../components/index/InsertVideoButton.vue"
 import AskOptions = main.AskOptions
 import Workspace = main.Workspace
 import ChatFinishResult = main.ChatFinishResult
@@ -620,6 +621,7 @@ function generateTitle() {
           ></upload-document-button>
           <fetch-webpage-button :is-asking="isAsking"
                                 @append-block-to-current-workspace="appendBlockToCurrentWorkspace"></fetch-webpage-button>
+          <insert-video-button :is-asking="isAsking" @append="appendBlockToCurrentWorkspace"></insert-video-button>
           <revoke-button :is-asking="isAsking" :current-workspace="currentWorkspace"></revoke-button>
           <v-menu>
             <template #activator="{props}">

frontend/wailsjs/go/main/App.d.ts

@@ -2,6 +2,7 @@
 // This file is automatically generated. DO NOT EDIT
 import {main} from '../models';
 import {sydney} from '../models';
+import {util} from '../models';
 
 export function AskAI(arg1:main.AskOptions):Promise<void>;
 
@@ -23,6 +24,10 @@ export function GetConciseAnswer(arg1:main.ConciseAnswerReq):Promise<string>;
 
 export function GetUser():Promise<string>;
 
+export function GetYoutubeTranscript(arg1:util.YtCustomCaption):Promise<Array<util.YtTranscriptText>>;
+
+export function GetYoutubeVideo(arg1:string):Promise<main.YoutubeVideoResult>;
+
 export function SaveRemoteFile(arg1:string,arg2:string,arg3:string):Promise<void>;
 
 export function SaveRemoteJPEGImage(arg1:string):Promise<void>;

frontend/wailsjs/go/main/App.js

@@ -42,6 +42,14 @@ export function GetUser() {
   return window['go']['main']['App']['GetUser']();
 }
 
+export function GetYoutubeTranscript(arg1) {
+  return window['go']['main']['App']['GetYoutubeTranscript'](arg1);
+}
+
+export function GetYoutubeVideo(arg1) {
+  return window['go']['main']['App']['GetYoutubeVideo'](arg1);
+}
+
 export function SaveRemoteFile(arg1, arg2, arg3) {
   return window['go']['main']['App']['SaveRemoteFile'](arg1, arg2, arg3);
 }

frontend/wailsjs/go/models.ts

@@ -78,6 +78,7 @@ export namespace main {
 	    sydney_preset_20240304: boolean;
 	    theme_color_20240304: boolean;
 	    quick_20240326: boolean;
+	    quick_20240405: boolean;
 
 	    static createFrom(source: any = {}) {
 	        return new Migration(source);
@@ -88,6 +89,7 @@ export namespace main {
 	        this.sydney_preset_20240304 = source["sydney_preset_20240304"];
 	        this.theme_color_20240304 = source["theme_color_20240304"];
 	        this.quick_20240326 = source["quick_20240326"];
+	        this.quick_20240405 = source["quick_20240405"];
 	    }
 	}
 	export class OpenAIBackend {
@@ -337,6 +339,61 @@ export namespace main {
 	        this.canceled = source["canceled"];
 	    }
 	}
+
+	export class YoutubeVideoDetails {
+	    title: string;
+	    length_seconds: string;
+	    description: string;
+	    keywords: string[];
+	    pic_url: string;
+	    author: string;
+
+	    static createFrom(source: any = {}) {
+	        return new YoutubeVideoDetails(source);
+	    }
+
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.title = source["title"];
+	        this.length_seconds = source["length_seconds"];
+	        this.description = source["description"];
+	        this.keywords = source["keywords"];
+	        this.pic_url = source["pic_url"];
+	        this.author = source["author"];
+	    }
+	}
+	export class YoutubeVideoResult {
+	    details: YoutubeVideoDetails;
+	    captions: util.YtCustomCaption[];
+
+	    static createFrom(source: any = {}) {
+	        return new YoutubeVideoResult(source);
+	    }
+
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.details = this.convertValues(source["details"], YoutubeVideoDetails);
+	        this.captions = this.convertValues(source["captions"], util.YtCustomCaption);
+	    }
+
+		convertValues(a: any, classs: any, asMap: boolean = false): any {
+		    if (!a) {
+		        return a;
+		    }
+		    if (a.slice) {
+		        return (a as any[]).map(elem => this.convertValues(elem, classs));
+		    } else if ("object" === typeof a) {
+		        if (asMap) {
+		            for (const key of Object.keys(a)) {
+		                a[key] = new classs(a[key]);
+		            }
+		            return a;
+		        }
+		        return new classs(a);
+		    }
+		    return a;
+		}
+	}
 
 }
 
@@ -425,3 +482,44 @@ export namespace sydney {
 
 }
 
+export namespace util {
+
+	export class YtCustomCaption {
+	    name: string;
+	    language_code: string;
+	    url: string;
+	    is_asr: boolean;
+	    is_translated: boolean;
+
+	    static createFrom(source: any = {}) {
+	        return new YtCustomCaption(source);
+	    }
+
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.name = source["name"];
+	        this.language_code = source["language_code"];
+	        this.url = source["url"];
+	        this.is_asr = source["is_asr"];
+	        this.is_translated = source["is_translated"];
+	    }
+	}
+	export class YtTranscriptText {
+	    start: number;
+	    dur: number;
+	    value: string;
+
+	    static createFrom(source: any = {}) {
+	        return new YtTranscriptText(source);
+	    }
+
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.start = source["start"];
+	        this.dur = source["dur"];
+	        this.value = source["value"];
+	    }
+	}
+
+}
+

sydney/captcha.go

@@ -137,6 +137,7 @@ func (o *Sydney) postprocessCaptchaCookies(modifiedCookies map[string]string) er
 	if _, ok := modifiedCookies["cct"]; !ok {
 		return errors.New("captcha cookies not valid: no cookie named cct found")
 	}
+	slog.Info("postprocessCaptchaCookies", "cookies", modifiedCookies)
 	o.UpdateModifiedCookies(modifiedCookies)
 	return nil
 }

util/youtube.go

@@ -190,7 +190,7 @@ type YtTranscript struct {
 	Texts   []YtTranscriptText `xml:"text"`
 }
 type YtTranscriptText struct {
-	Start float64 `xml:"start,attr"`
-	Dur   float64 `xml:"dur,attr"`
-	Value string  `xml:",chardata"`
+	Start float64 `xml:"start,attr" json:"start"`
+	Dur   float64 `xml:"dur,attr" json:"dur"`
+	Value string  `xml:",chardata" json:"value"`
 }