Merge pull request #1471 from CapSoftware/recording-benchmark

Add synthetic recording test framework and improve frame converter reliability

Author: richiemcilroy

.claude/settings.local.json

@@ -54,7 +54,25 @@
       "Bash(./target/release/examples/decode-benchmark:*)",
       "Bash(RUST_LOG=warn ./target/release/examples/decode-benchmark:*)",
       "Bash(git mv:*)",
-      "Bash(xargs cat:*)"
+      "Bash(xargs cat:*)",
+      "Bash(cargo test:*)",
+      "Bash(/opt/homebrew/bin/ffmpeg:*)",
+      "Bash(RUST_LOG=info cargo run:*)",
+      "Bash(RUST_LOG=warn cargo run:*)",
+      "Bash(CAP_MUXER_BUFFER_SIZE=240 RUST_LOG=info cargo run:*)",
+      "Bash(CAP_MUXER_BUFFER_SIZE=480 RUST_LOG=debug cargo run:*)",
+      "Bash(CAP_MUXER_BUFFER_SIZE=480 RUST_LOG=info cargo run:*)",
+      "Bash(RUST_LOG=debug cargo run:*)",
+      "Bash(RUST_LOG=info ./target/release/examples/synthetic-test-runner:*)",
+      "Bash(CAP_MUXER_BUFFER_SIZE=960 RUST_LOG=info cargo run:*)",
+      "Bash(CAP_MUXER_BUFFER_SIZE=1200 RUST_LOG=warn ./target/release/examples/synthetic-test-runner:*)",
+      "Bash(./target/release/examples/synthetic-test-runner:*)",
+      "Bash(CAP_VIDEO_SOURCE_BUFFER_SIZE=2400 CAP_MUXER_BUFFER_SIZE=2400 RUST_LOG=warn ./target/release/examples/synthetic-test-runner:*)",
+      "Bash(CAP_VIDEO_SOURCE_BUFFER_SIZE=2400 CAP_MUXER_BUFFER_SIZE=2400 RUST_LOG=info cargo run:*)",
+      "Bash(sysctl:*)",
+      "Bash(CAP_VIDEO_SOURCE_BUFFER_SIZE=4800 CAP_MUXER_BUFFER_SIZE=4800 RUST_LOG=warn cargo run:*)",
+      "Bash(RUST_LOG=cap_enc_ffmpeg::video::h264=debug cargo run:*)",
+      "Bash(CAP_VIDEO_SOURCE_BUFFER_SIZE=5400 CAP_MUXER_BUFFER_SIZE=5400 RUST_LOG=info cargo run:*)"
     ],
     "deny": [],
     "ask": []

Cargo.lock

@@ -347,25 +347,12 @@ pub enum RecordingInputKind {
 #[derive(tauri_specta::Event, specta::Type, Clone, Debug, serde::Serialize)]
 #[serde(tag = "variant")]
 pub enum RecordingEvent {
-    Countdown {
-        value: u32,
-    },
+    Countdown { value: u32 },
     Started,
     Stopped,
-    Failed {
-        error: String,
-    },
-    InputLost {
-        input: RecordingInputKind,
-    },
-    InputRestored {
-        input: RecordingInputKind,
-    },
-    DiskSpaceLow {
-        available_mb: u32,
-        threshold_mb: u32,
-        path: String,
-    },
+    Failed { error: String },
+    InputLost { input: RecordingInputKind },
+    InputRestored { input: RecordingInputKind },
 }
 
 #[derive(Serialize, Type)]

apps/desktop/src-tauri/src/recording.rs

@@ -261,7 +261,11 @@ export function createImageDataWS(
 						directCanvas.height = height;
 					}
 
-					const imageData = new ImageData(frameData, width, height);
+					const imageData = new ImageData(
+						new Uint8ClampedArray(frameData.buffer),
+						width,
+						height,
+					);
 					directCtx.putImageData(imageData, 0, 0);
 
 					if (!hasRenderedFrame()) {

apps/desktop/src/utils/socket.ts

@@ -477,7 +477,7 @@ export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background
 export type ProjectRecordingsMeta = { segments: SegmentRecordings[] }
 export type RecordingAction = "Started" | "InvalidAuthentication" | "UpgradeRequired"
 export type RecordingDeleted = { path: string }
-export type RecordingEvent = { variant: "Countdown"; value: number } | { variant: "Started" } | { variant: "Stopped" } | { variant: "Failed"; error: string } | { variant: "InputLost"; input: RecordingInputKind } | { variant: "InputRestored"; input: RecordingInputKind } | { variant: "DiskSpaceLow"; available_mb: number; threshold_mb: number; path: string }
+export type RecordingEvent = { variant: "Countdown"; value: number } | { variant: "Started" } | { variant: "Stopped" } | { variant: "Failed"; error: string } | { variant: "InputLost"; input: RecordingInputKind } | { variant: "InputRestored"; input: RecordingInputKind }
 export type RecordingInputKind = "microphone" | "camera"
 export type RecordingMeta = (StudioRecordingMeta | InstantRecordingMeta) & { platform?: Platform | null; pretty_name: string; sharing?: SharingMeta | null; upload?: UploadMeta | null }
 export type RecordingMetaWithMetadata = ((StudioRecordingMeta | InstantRecordingMeta) & { platform?: Platform | null; pretty_name: string; sharing?: SharingMeta | null; upload?: UploadMeta | null }) & { mode: RecordingMode; status: StudioRecordingStatus }

apps/desktop/src/utils/tauri.ts

@@ -197,7 +197,7 @@ export function renderFrameWebGPU(
 
 	device.queue.writeTexture(
 		{ texture: renderer.frameTexture },
-		textureData,
+		textureData.buffer as unknown as GPUAllowSharedBufferSource,
 		{ bytesPerRow: width * 4, rowsPerImage: height },
 		{ width, height },
 	);

apps/desktop/src/utils/webgpu-renderer.ts

@@ -10,7 +10,9 @@ use std::{
 };
 use sysinfo::Disks;
 
-use crate::video::h264::{H264Encoder, H264EncoderBuilder, H264EncoderError, H264Preset};
+use crate::video::h264::{
+    DEFAULT_KEYFRAME_INTERVAL_SECS, H264Encoder, H264EncoderBuilder, H264EncoderError, H264Preset,
+};
 
 const INIT_SEGMENT_NAME: &str = "init.mp4";
 const DISK_SPACE_WARNING_THRESHOLD_MB: u64 = 500;
@@ -142,7 +144,7 @@ pub enum QueueFrameError {
     FFmpeg(#[from] ffmpeg::Error),
     #[error("Init: {0}")]
     Init(#[from] InitError),
-    #[error("Encode: {0}")]
+    #[error(transparent)]
     Encode(#[from] crate::video::h264::QueueFrameError),
     #[error("Init segment validation failed: {0}")]
     InitSegmentInvalid(String),
@@ -164,7 +166,7 @@ pub struct SegmentedVideoEncoderConfig {
 impl Default for SegmentedVideoEncoderConfig {
     fn default() -> Self {
         Self {
-            segment_duration: Duration::from_secs(3),
+            segment_duration: Duration::from_secs(DEFAULT_KEYFRAME_INTERVAL_SECS as u64),
             preset: H264Preset::Ultrafast,
             bpp: H264EncoderBuilder::QUALITY_BPP,
             output_size: None,
@@ -210,6 +212,7 @@ impl SegmentedVideoEncoder {
             set_opt("use_timeline", "0");
             set_opt("use_template", "1");
             set_opt("single_file", "0");
+            set_opt("hls_playlist", "1");
         }
 
         let mut builder = H264EncoderBuilder::new(video_config)