Refine auto-zoom segment generation and cursor stability (#1106)

* feat: refine auto-zoom segments and cursor stability

* fix: Coderabbit suggestion

* fix: Cargo fmt

Author: richiemcilroy

apps/desktop/src-tauri/src/recording.rs

@@ -1,8 +1,9 @@
 use cap_fail::fail;
+use cap_project::cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS;
 use cap_project::{
-    CursorClickEvent, Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner,
-    SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, ZoomMode,
-    ZoomSegment, cursor::CursorEvents,
+    CursorClickEvent, CursorMoveEvent, Platform, ProjectConfiguration, RecordingMeta,
+    RecordingMetaInner, SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment,
+    ZoomMode, ZoomSegment, cursor::CursorEvents,
 };
 use cap_recording::{
     RecordingError, RecordingMode,
@@ -15,7 +16,13 @@ use cap_rendering::ProjectRecordingsMeta;
 use cap_utils::{ensure_dir, spawn_actor};
 use serde::{Deserialize, Serialize};
 use specta::Type;
-use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration};
+use std::{
+    collections::{HashMap, VecDeque},
+    path::PathBuf,
+    str::FromStr,
+    sync::Arc,
+    time::Duration,
+};
 use tauri::{AppHandle, Manager};
 use tauri_plugin_dialog::{DialogExt, MessageDialogBuilder};
 use tauri_specta::Event;
@@ -1718,56 +1725,161 @@ async fn handle_recording_finish(
 /// around user interactions to highlight important moments.
 fn generate_zoom_segments_from_clicks_impl(
     mut clicks: Vec<CursorClickEvent>,
-    recordings: &ProjectRecordingsMeta,
+    mut moves: Vec<CursorMoveEvent>,
+    max_duration: f64,
 ) -> Vec<ZoomSegment> {
-    const ZOOM_SEGMENT_AFTER_CLICK_PADDING: f64 = 1.5;
-    const ZOOM_SEGMENT_BEFORE_CLICK_PADDING: f64 = 0.8;
-    const ZOOM_DURATION: f64 = 1.0;
-    const CLICK_GROUP_THRESHOLD: f64 = 0.6; // seconds
-    const MIN_SEGMENT_PADDING: f64 = 2.0; // minimum gap between segments
+    const STOP_PADDING_SECONDS: f64 = 0.8;
+    const CLICK_PRE_PADDING: f64 = 0.6;
+    const CLICK_POST_PADDING: f64 = 1.6;
+    const MOVEMENT_PRE_PADDING: f64 = 0.4;
+    const MOVEMENT_POST_PADDING: f64 = 1.2;
+    const MERGE_GAP_THRESHOLD: f64 = 0.6;
+    const MIN_SEGMENT_DURATION: f64 = 1.3;
+    const MOVEMENT_WINDOW_SECONDS: f64 = 1.2;
+    const MOVEMENT_EVENT_DISTANCE_THRESHOLD: f64 = 0.025;
+    const MOVEMENT_WINDOW_DISTANCE_THRESHOLD: f64 = 0.1;
+
+    if max_duration <= 0.0 {
+        return Vec::new();
+    }
 
-    let max_duration = recordings.duration();
+    // We trim the tail of the recording to avoid using the final
+    // "stop recording" click as a zoom target.
+    let activity_end_limit = if max_duration > STOP_PADDING_SECONDS {
+        max_duration - STOP_PADDING_SECONDS
+    } else {
+        max_duration
+    };
+
+    if activity_end_limit <= f64::EPSILON {
+        return Vec::new();
+    }
 
     clicks.sort_by(|a, b| {
         a.time_ms
             .partial_cmp(&b.time_ms)
             .unwrap_or(std::cmp::Ordering::Equal)
     });
+    moves.sort_by(|a, b| {
+        a.time_ms
+            .partial_cmp(&b.time_ms)
+            .unwrap_or(std::cmp::Ordering::Equal)
+    });
 
-    let mut segments = Vec::<ZoomSegment>::new();
+    // Remove trailing click-down events that are too close to the end.
+    while let Some(index) = clicks.iter().rposition(|c| c.down) {
+        let time_secs = clicks[index].time_ms / 1000.0;
+        if time_secs > activity_end_limit {
+            clicks.remove(index);
+        } else {
+            break;
+        }
+    }
 
-    // Generate segments around mouse clicks
-    for click in &clicks {
-        if !click.down {
+    let mut intervals: Vec<(f64, f64)> = Vec::new();
+
+    for click in clicks.into_iter().filter(|c| c.down) {
+        let time = click.time_ms / 1000.0;
+        if time >= activity_end_limit {
             continue;
         }
 
-        let time = click.time_ms / 1000.0;
+        let start = (time - CLICK_PRE_PADDING).max(0.0);
+        let end = (time + CLICK_POST_PADDING).min(activity_end_limit);
 
-        let proposed_start = (time - ZOOM_SEGMENT_BEFORE_CLICK_PADDING).max(0.0);
-        let proposed_end = (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(max_duration);
+        if end > start {
+            intervals.push((start, end));
+        }
+    }
 
-        if let Some(last) = segments.last_mut() {
-            // Merge if within group threshold OR if segments would be too close together
-            if time <= last.end + CLICK_GROUP_THRESHOLD
-                || proposed_start <= last.end + MIN_SEGMENT_PADDING
-            {
-                last.end = proposed_end;
-                continue;
+    let mut last_move_by_cursor: HashMap<String, (f64, f64, f64)> = HashMap::new();
+    let mut distance_window: VecDeque<(f64, f64)> = VecDeque::new();
+    let mut window_distance = 0.0_f64;
+
+    for mv in moves.iter() {
+        let time = mv.time_ms / 1000.0;
+        if time >= activity_end_limit {
+            break;
+        }
+
+        let distance = if let Some((_, last_x, last_y)) = last_move_by_cursor.get(&mv.cursor_id) {
+            let dx = mv.x - last_x;
+            let dy = mv.y - last_y;
+            (dx * dx + dy * dy).sqrt()
+        } else {
+            0.0
+        };
+
+        last_move_by_cursor.insert(mv.cursor_id.clone(), (time, mv.x, mv.y));
+
+        if distance <= f64::EPSILON {
+            continue;
+        }
+
+        distance_window.push_back((time, distance));
+        window_distance += distance;
+
+        while let Some(&(old_time, old_distance)) = distance_window.front() {
+            if time - old_time > MOVEMENT_WINDOW_SECONDS {
+                distance_window.pop_front();
+                window_distance -= old_distance;
+            } else {
+                break;
             }
         }
 
-        if time < max_duration - ZOOM_DURATION {
-            segments.push(ZoomSegment {
-                start: proposed_start,
-                end: proposed_end,
-                amount: 2.0,
-                mode: ZoomMode::Auto,
-            });
+        if window_distance < 0.0 {
+            window_distance = 0.0;
+        }
+
+        let significant_movement = distance >= MOVEMENT_EVENT_DISTANCE_THRESHOLD
+            || window_distance >= MOVEMENT_WINDOW_DISTANCE_THRESHOLD;
+
+        if !significant_movement {
+            continue;
+        }
+
+        let start = (time - MOVEMENT_PRE_PADDING).max(0.0);
+        let end = (time + MOVEMENT_POST_PADDING).min(activity_end_limit);
+
+        if end > start {
+            intervals.push((start, end));
+        }
+    }
+
+    if intervals.is_empty() {
+        return Vec::new();
+    }
+
+    intervals.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));
+
+    let mut merged: Vec<(f64, f64)> = Vec::new();
+    for interval in intervals {
+        if let Some(last) = merged.last_mut() {
+            if interval.0 <= last.1 + MERGE_GAP_THRESHOLD {
+                last.1 = last.1.max(interval.1);
+                continue;
+            }
         }
+        merged.push(interval);
     }
 
-    segments
+    merged
+        .into_iter()
+        .filter_map(|(start, end)| {
+            let duration = end - start;
+            if duration < MIN_SEGMENT_DURATION {
+                return None;
+            }
+
+            Some(ZoomSegment {
+                start,
+                end,
+                amount: 2.0,
+                mode: ZoomMode::Auto,
+            })
+        })
+        .collect()
 }
 
 /// Generates zoom segments based on mouse click events during recording.
@@ -1798,24 +1910,34 @@ pub fn generate_zoom_segments_for_project(
         return Vec::new();
     };
 
-    let all_events = match studio_meta {
+    let mut all_clicks = Vec::new();
+    let mut all_moves = Vec::new();
+
+    match studio_meta {
         StudioRecordingMeta::SingleSegment { segment } => {
             if let Some(cursor_path) = &segment.cursor {
-                CursorEvents::load_from_file(&recording_meta.path(cursor_path))
-                    .unwrap_or_default()
-                    .clicks
-            } else {
-                vec![]
+                let mut events = CursorEvents::load_from_file(&recording_meta.path(cursor_path))
+                    .unwrap_or_default();
+                let pointer_ids = studio_meta.pointer_cursor_ids();
+                let pointer_ids_ref = (!pointer_ids.is_empty()).then_some(&pointer_ids);
+                events.stabilize_short_lived_cursor_shapes(
+                    pointer_ids_ref,
+                    SHORT_CURSOR_SHAPE_DEBOUNCE_MS,
+                );
+                all_clicks = events.clicks;
+                all_moves = events.moves;
             }
         }
-        StudioRecordingMeta::MultipleSegments { inner, .. } => inner
-            .segments
-            .iter()
-            .flat_map(|s| s.cursor_events(recording_meta).clicks)
-            .collect(),
-    };
+        StudioRecordingMeta::MultipleSegments { inner, .. } => {
+            for segment in inner.segments.iter() {
+                let events = segment.cursor_events(recording_meta);
+                all_clicks.extend(events.clicks);
+                all_moves.extend(events.moves);
+            }
+        }
+    }
 
-    generate_zoom_segments_from_clicks_impl(all_events, recordings)
+    generate_zoom_segments_from_clicks_impl(all_clicks, all_moves, recordings.duration())
 }
 
 fn project_config_from_recording(
@@ -1828,26 +1950,110 @@ fn project_config_from_recording(
         .unwrap_or(None)
         .unwrap_or_default();
 
-    ProjectConfiguration {
-        timeline: Some(TimelineConfiguration {
-            segments: recordings
-                .segments
-                .iter()
-                .enumerate()
-                .map(|(i, segment)| TimelineSegment {
-                    recording_segment: i as u32,
-                    start: 0.0,
-                    end: segment.duration(),
-                    timescale: 1.0,
-                })
-                .collect(),
-            zoom_segments: if settings.auto_zoom_on_clicks {
-                generate_zoom_segments_from_clicks(completed_recording, recordings)
-            } else {
-                Vec::new()
-            },
-            scene_segments: Vec::new(),
-        }),
-        ..default_config.unwrap_or_default()
+    let mut config = default_config.unwrap_or_default();
+
+    let timeline_segments = recordings
+        .segments
+        .iter()
+        .enumerate()
+        .map(|(i, segment)| TimelineSegment {
+            recording_segment: i as u32,
+            start: 0.0,
+            end: segment.duration(),
+            timescale: 1.0,
+        })
+        .collect::<Vec<_>>();
+
+    let zoom_segments = if settings.auto_zoom_on_clicks {
+        generate_zoom_segments_from_clicks(completed_recording, recordings)
+    } else {
+        Vec::new()
+    };
+
+    if !zoom_segments.is_empty() {
+        config.cursor.size = 200;
+    }
+
+    config.timeline = Some(TimelineConfiguration {
+        segments: timeline_segments,
+        zoom_segments,
+        scene_segments: Vec::new(),
+    });
+
+    config
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn click_event(time_ms: f64) -> CursorClickEvent {
+        CursorClickEvent {
+            active_modifiers: vec![],
+            cursor_num: 0,
+            cursor_id: "default".to_string(),
+            time_ms,
+            down: true,
+        }
+    }
+
+    fn move_event(time_ms: f64, x: f64, y: f64) -> CursorMoveEvent {
+        CursorMoveEvent {
+            active_modifiers: vec![],
+            cursor_id: "default".to_string(),
+            time_ms,
+            x,
+            y,
+        }
+    }
+
+    #[test]
+    fn skips_trailing_stop_click() {
+        let segments =
+            generate_zoom_segments_from_clicks_impl(vec![click_event(11_900.0)], vec![], 12.0);
+
+        assert!(
+            segments.is_empty(),
+            "expected trailing stop click to be ignored"
+        );
+    }
+
+    #[test]
+    fn generates_segment_for_sustained_activity() {
+        let clicks = vec![click_event(1_200.0), click_event(4_200.0)];
+        let moves = vec![
+            move_event(1_500.0, 0.10, 0.12),
+            move_event(1_720.0, 0.42, 0.45),
+            move_event(1_940.0, 0.74, 0.78),
+        ];
+
+        let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0);
+
+        assert!(
+            !segments.is_empty(),
+            "expected activity to produce zoom segments"
+        );
+        let first = &segments[0];
+        assert!(first.start < first.end);
+        assert!(first.end - first.start >= 1.3);
+        assert!(first.end <= 19.5);
+    }
+
+    #[test]
+    fn ignores_cursor_jitter() {
+        let jitter_moves = (0..30)
+            .map(|i| {
+                let t = 1_000.0 + (i as f64) * 30.0;
+                let delta = (i as f64) * 0.0004;
+                move_event(t, 0.5 + delta, 0.5)
+            })
+            .collect::<Vec<_>>();
+
+        let segments = generate_zoom_segments_from_clicks_impl(Vec::new(), jitter_moves, 15.0);
+
+        assert!(
+            segments.is_empty(),
+            "small jitter should not generate segments"
+        );
     }
 }