feat: Improve Media Foundation initialization and error handling

crates/mediafoundation-utils/Cargo.toml

@@ -11,6 +11,7 @@ windows = { workspace = true, features = [
     "Win32_Media_MediaFoundation",
     "Win32_Media_DirectShow",
     "Win32_System_Com",
+    "Win32_System_LibraryLoader",
 ] }
 
 [lints]

crates/mediafoundation-utils/src/lib.rs

@@ -7,19 +7,85 @@ use std::{
 use windows::{
     Win32::{
         Media::MediaFoundation::{IMFMediaBuffer, MFSTARTUP_FULL, MFStartup},
-        System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize},
+        System::{
+            LibraryLoader::GetModuleHandleW,
+            WinRT::{RO_INIT_MULTITHREADED, RoInitialize},
+        },
     },
-    core::Result,
+    core::{Result, PCWSTR},
 };
 
 // This is the value for Win7+
 pub const MF_VERSION: u32 = 131184;
 
+/// Custom error type for initialization failures
+#[derive(Debug, thiserror::Error)]
+pub enum InitError {
+    #[error("Third-party software conflict detected: {software}. {message}")]
+    ThirdPartyConflict { software: String, message: String },
+    #[error("Failed to initialize Windows Runtime: {0}")]
+    RoInitializeFailed(windows::core::Error),
+    #[error("Failed to initialize Media Foundation: {0}")]
+    MFStartupFailed(windows::core::Error),
+}
+
+/// Detects if Wondershare DemoCreator's hook DLL is loaded in the process.
+/// This DLL is known to cause crashes when interfering with WinRT initialization.
+pub fn detect_wondershare_democreator() -> bool {
+    unsafe {
+        // Check for CaptureGameHook_64.dll (64-bit version)
+        let dll_name = windows::core::w!("CaptureGameHook_64.dll");
+        let handle = GetModuleHandleW(PCWSTR(dll_name.as_ptr())).ok();
+
+        if handle.is_some() {
+            return true;
+        }
+
+        // Also check for the 32-bit version just in case
+        let dll_name_32 = windows::core::w!("CaptureGameHook.dll");
+        let handle_32 = GetModuleHandleW(PCWSTR(dll_name_32.as_ptr())).ok();
+
+        handle_32.is_some()
+    }
+}
+
+/// Checks for known incompatible third-party software before initializing.
+/// Returns an error if incompatible software is detected.
+pub fn check_compatibility() -> Result<(), InitError> {
+    if detect_wondershare_democreator() {
+        return Err(InitError::ThirdPartyConflict {
+            software: "Wondershare DemoCreator".to_string(),
+            message: "The Wondershare DemoCreator hook DLL (CaptureGameHook_64.dll) is interfering with Cap's screen capture system. Please close Wondershare DemoCreator and try again.".to_string(),
+        });
+    }
+
+    Ok(())
+}
+
+/// Initializes Windows Runtime and Media Foundation for the current thread.
+/// Should be called on threads that use WinRT or Media Foundation APIs.
 pub fn thread_init() {
     let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) };
     let _ = unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) };
 }
 
+/// Initializes Windows Runtime and Media Foundation for the current thread with error handling.
+/// Returns an error if initialization fails or if incompatible software is detected.
+pub fn thread_init_checked() -> Result<(), InitError> {
+    // First check for known incompatible software
+    check_compatibility()?;
+
+    // Initialize Windows Runtime
+    unsafe { RoInitialize(RO_INIT_MULTITHREADED) }
+        .map_err(InitError::RoInitializeFailed)?;
+
+    // Initialize Media Foundation
+    unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }
+        .map_err(InitError::MFStartupFailed)?;
+
+    Ok(())
+}
+
 pub trait IMFMediaBufferExt {
     fn lock(&self) -> Result<IMFMediaBufferLock<'_>>;
 }

crates/recording/src/instant_recording.rs

@@ -288,7 +288,13 @@ pub async fn spawn_instant_recording_actor(
     let content_dir = ensure_dir(&recording_dir.join("content"))?;
 
     #[cfg(windows)]
-    cap_mediafoundation_utils::thread_init();
+    {
+        // Check for incompatible third-party software before initializing
+        cap_mediafoundation_utils::check_compatibility()?;
+
+        // Initialize Windows Runtime and Media Foundation with error handling
+        cap_mediafoundation_utils::thread_init_checked()?;
+    }
 
     #[cfg(windows)]
     let d3d_device = crate::capture_pipeline::create_d3d_device()?;

crates/recording/src/output_pipeline/win.rs

@@ -64,7 +64,19 @@ impl Muxer for WindowsMuxer {
             let output = output.clone();
 
             tasks.spawn_thread("windows-encoder", move || {
-                cap_mediafoundation_utils::thread_init();
+                // Check for incompatible third-party software before initializing
+                if let Err(e) = cap_mediafoundation_utils::check_compatibility() {
+                    error!("Third-party software conflict detected: {}", e);
+                    let _ = ready_tx.send(Err(anyhow!("{}", e)));
+                    return;
+                }
+
+                // Initialize Windows Runtime and Media Foundation with error handling
+                if let Err(e) = cap_mediafoundation_utils::thread_init_checked() {
+                    error!("Failed to initialize encoder thread: {}", e);
+                    let _ = ready_tx.send(Err(anyhow!("{}", e)));
+                    return;
+                }
 
                 let encoder = (|| {
                     let mut output = output.lock().unwrap();

crates/recording/src/sources/screen_capture/windows.rs

@@ -185,7 +185,19 @@ impl output_pipeline::VideoSource for VideoSource {
         let tokio_rt = tokio::runtime::Handle::current();
 
         ctx.tasks().spawn_thread("d3d-capture-thread", move || {
-            cap_mediafoundation_utils::thread_init();
+            // Check for incompatible third-party software before initializing
+            if let Err(e) = cap_mediafoundation_utils::check_compatibility() {
+                error!("Third-party software conflict detected: {}", e);
+                let _ = error_tx.send(anyhow!("{}", e));
+                return;
+            }
+
+            // Initialize Windows Runtime and Media Foundation with error handling
+            if let Err(e) = cap_mediafoundation_utils::thread_init_checked() {
+                error!("Failed to initialize capture thread: {}", e);
+                let _ = error_tx.send(anyhow!("{}", e));
+                return;
+            }
 
             // Look up the display and create the GraphicsCaptureItem on this thread to avoid COM threading issues
             let capture_item = match Display::from_id(&display_id) {