Audio video sync perfection

apps/desktop/src-tauri/src/export.rs

@@ -225,7 +225,7 @@ pub async fn generate_export_preview(
         RenderVideoConstants::new(
             &recordings.segments,
             recording_meta.clone(),
-            studio_meta.clone(),
+            (**studio_meta).clone(),
         )
         .await
         .map_err(|e| format!("Failed to create render constants: {e}"))?,

apps/desktop/src-tauri/src/lib.rs

@@ -1456,7 +1456,7 @@ async fn get_video_metadata(path: PathBuf) -> Result<VideoRecordingMetadata, Str
                 return Err("Unable to get metadata on in-progress recording".to_string());
             }
 
-            match meta {
+            match &**meta {
                 StudioRecordingMeta::SingleSegment { segment } => {
                     vec![recording_meta.path(&segment.display.path)]
                 }
@@ -1888,15 +1888,13 @@ impl RecordingMetaWithMetadata {
                 RecordingMetaInner::Instant(_) => RecordingMode::Instant,
             },
             status: match &inner.inner {
-                RecordingMetaInner::Studio(StudioRecordingMeta::MultipleSegments { inner }) => {
-                    inner
+                RecordingMetaInner::Studio(meta) => match &**meta {
+                    StudioRecordingMeta::MultipleSegments { inner } => inner
                         .status
                         .clone()
-                        .unwrap_or(StudioRecordingStatus::Complete)
-                }
-                RecordingMetaInner::Studio(StudioRecordingMeta::SingleSegment { .. }) => {
-                    StudioRecordingStatus::Complete
-                }
+                        .unwrap_or(StudioRecordingStatus::Complete),
+                    StudioRecordingMeta::SingleSegment { .. } => StudioRecordingStatus::Complete,
+                },
                 RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => {
                     StudioRecordingStatus::InProgress
                 }
@@ -3072,8 +3070,10 @@ async fn resume_uploads(app: AppHandle) -> Result<(), String> {
                 // Check if recording is still marked as in-progress and if so mark as failed
                 // This should only happen if the application crashes while recording
                 match &mut meta.inner {
-                    RecordingMetaInner::Studio(StudioRecordingMeta::MultipleSegments { inner }) => {
-                        if let Some(StudioRecordingStatus::InProgress) = &inner.status {
+                    RecordingMetaInner::Studio(meta_box) => {
+                        if let StudioRecordingMeta::MultipleSegments { inner } = &mut **meta_box
+                            && let Some(StudioRecordingStatus::InProgress) = &inner.status
+                        {
                             inner.status = Some(StudioRecordingStatus::Failed {
                                 error: "Recording crashed".to_string(),
                             });

apps/desktop/src-tauri/src/recording.rs

@@ -543,13 +543,13 @@ pub async fn start_recording(
         pretty_name: project_name.clone(),
         inner: match inputs.mode {
             RecordingMode::Studio => {
-                RecordingMetaInner::Studio(StudioRecordingMeta::MultipleSegments {
+                RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments {
                     inner: MultipleSegments {
                         segments: Default::default(),
                         cursors: Default::default(),
                         status: Some(StudioRecordingStatus::InProgress),
                     },
-                })
+                }))
             }
             RecordingMode::Instant => {
                 RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { recording: true })
@@ -1213,6 +1213,7 @@ pub async fn take_screenshot(
         path: relative_path,
         fps: 0,
         start_time: Some(0.0),
+        device_id: None,
     };
 
     let segment = cap_project::SingleSegment {
@@ -1227,9 +1228,9 @@ pub async fn take_screenshot(
         project_path: project_file_path.clone(),
         pretty_name: project_name,
         sharing: None,
-        inner: cap_project::RecordingMetaInner::Studio(
+        inner: cap_project::RecordingMetaInner::Studio(Box::new(
             cap_project::StudioRecordingMeta::SingleSegment { segment },
-        ),
+        )),
         upload: None,
     };
 
@@ -1331,7 +1332,7 @@ async fn handle_recording_end(
             {
                 match &mut project_meta.inner {
                     RecordingMetaInner::Studio(meta) => {
-                        if let StudioRecordingMeta::MultipleSegments { inner } = meta {
+                        if let StudioRecordingMeta::MultipleSegments { inner } = &mut **meta {
                             inner.status = Some(StudioRecordingStatus::Failed { error });
                         }
                     }
@@ -1396,7 +1397,7 @@ async fn handle_recording_finish(
 
     let (meta_inner, sharing) = match completed_recording {
         CompletedRecording::Studio { recording, .. } => {
-            let meta_inner = RecordingMetaInner::Studio(recording.meta.clone());
+            let meta_inner = RecordingMetaInner::Studio(Box::new(recording.meta.clone()));
 
             if let Ok(mut meta) = RecordingMeta::load_for_project(&recording_dir).map_err(|err| {
                 error!("Failed to load recording meta while saving finished recording: {err}")
@@ -1508,7 +1509,10 @@ async fn handle_recording_finish(
 
             config.write(&recording_dir).map_err(|e| e.to_string())?;
 
-            (RecordingMetaInner::Studio(updated_studio_meta), None)
+            (
+                RecordingMetaInner::Studio(Box::new(updated_studio_meta)),
+                None,
+            )
         }
         CompletedRecording::Instant {
             recording,
@@ -1910,7 +1914,7 @@ pub fn generate_zoom_segments_from_clicks(
         project_path: recording.project_path.clone(),
         pretty_name: String::new(),
         sharing: None,
-        inner: RecordingMetaInner::Studio(recording.meta.clone()),
+        inner: RecordingMetaInner::Studio(Box::new(recording.meta.clone())),
         upload: None,
     };
 
@@ -1930,7 +1934,7 @@ pub fn generate_zoom_segments_for_project(
     let mut all_clicks = Vec::new();
     let mut all_moves = Vec::new();
 
-    match studio_meta {
+    match &**studio_meta {
         StudioRecordingMeta::SingleSegment { segment } => {
             if let Some(cursor_path) = &segment.cursor {
                 let mut events = CursorEvents::load_from_file(&recording_meta.path(cursor_path))

apps/desktop/src-tauri/src/screenshot_editor.rs

@@ -214,6 +214,7 @@ impl ScreenshotEditorInstances {
                         path: relative_path.clone(),
                         fps: 30,
                         start_time: Some(0.0),
+                        device_id: None,
                     };
                     let segment = SingleSegment {
                         display: video_meta.clone(),
@@ -227,7 +228,7 @@ impl ScreenshotEditorInstances {
                         project_path: path.parent().unwrap().to_path_buf(),
                         pretty_name: "Screenshot".to_string(),
                         sharing: None,
-                        inner: RecordingMetaInner::Studio(studio_meta.clone()),
+                        inner: RecordingMetaInner::Studio(Box::new(studio_meta.clone())),
                         upload: None,
                     }
                 };
@@ -283,7 +284,7 @@ impl ScreenshotEditorInstances {
                     queue: (*queue).clone(),
                     device: (*device).clone(),
                     options,
-                    meta: studio_meta,
+                    meta: *studio_meta,
                     recording_meta: recording_meta.clone(),
                     background_textures: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
                     is_software_adapter,

apps/desktop/src/utils/tauri.ts

@@ -374,7 +374,7 @@ export type AspectRatio = "wide" | "vertical" | "square" | "classic" | "tall"
 export type Audio = { duration: number; sample_rate: number; channels: number; start_time: number }
 export type AudioConfiguration = { mute: boolean; improve: boolean; micVolumeDb?: number; micStereoMode?: StereoMode; systemVolumeDb?: number }
 export type AudioInputLevelChange = number
-export type AudioMeta = { path: string; start_time?: number | null }
+export type AudioMeta = { path: string; start_time?: number | null; device_id?: string | null }
 export type AuthSecret = { api_key: string } | { token: string; expires: number }
 export type AuthStore = { secret: AuthSecret; user_id: string | null; plan: Plan | null; intercom_hash: string | null; organizations?: Organization[] }
 export type BackgroundConfiguration = { source: BackgroundSource; blur: number; padding: number; rounding: number; roundingType?: CornerStyle; inset: number; crop: Crop | null; shadow?: number; advancedShadow?: ShadowConfiguration | null; border?: BorderConfiguration | null }
@@ -520,7 +520,7 @@ export type UploadProgress = { progress: number }
 export type UploadProgressEvent = { video_id: string; uploaded: string; total: string }
 export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired"
 export type Video = { duration: number; width: number; height: number; fps: number; start_time: number }
-export type VideoMeta = { path: string; fps?: number; start_time?: number | null }
+export type VideoMeta = { path: string; fps?: number; start_time?: number | null; device_id?: string | null }
 export type VideoRecordingMetadata = { duration: number; size: number }
 export type VideoUploadInfo = { id: string; link: string; config: S3UploadMeta }
 export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: string | null; windowTitle?: string | null }

crates/audio/Cargo.toml

@@ -9,10 +9,15 @@ ffmpeg = { workspace = true }
 cpal = { workspace = true }
 tokio.workspace = true
 tracing = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 cidre = { workspace = true }
 
+[dev-dependencies]
+tempfile = "3"
+
 [lints]
 workspace = true