Skip to content

Commit 352c0cd

Browse files
richiemcilroyrichiemcilroy
committed
Add support for more pixel formats and GPU converters
1 parent db01c9d commit 352c0cd

File tree

18 files changed

+984
-9
lines changed

18 files changed

+984
-9
lines changed

‎.claude/settings.local.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"Bash(rustfmt:*)",
4242
"Bash(cargo tree:*)",
4343
"WebFetch(domain:github.com)",
44-
"WebFetch(domain:docs.rs)"
44+
"WebFetch(domain:docs.rs)",
45+
"WebFetch(domain:gix.github.io)"
4546
],
4647
"deny": [],
4748
"ask": []

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/camera-ffmpeg/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ license = "MIT"
77
[dependencies]
88
ffmpeg = { workspace = true }
99
thiserror.workspace = true
10+
tracing.workspace = true
1011
cap-camera = { path = "../camera" }
1112
workspace-hack = { version = "0.1", path = "../workspace-hack" }
1213

‎crates/camera-ffmpeg/src/macos.rs‎

Lines changed: 154 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
use cap_camera::CapturedFrame;
22
use cap_camera_avfoundation::ImageBufExt;
33
use cidre::*;
4+
use ffmpeg::{format::Pixel, software::scaling};
5+
use std::sync::atomic::{AtomicBool, Ordering};
46

57
use crate::CapturedFrameExt;
68

@@ -14,10 +16,41 @@ pub enum AsFFmpegError {
1416
expected: usize,
1517
found: usize,
1618
},
19+
#[error("Swscale fallback failed for format '{format}': {reason}")]
20+
SwscaleFallbackFailed { format: String, reason: String },
1721
#[error("{0}")]
1822
Native(#[from] cidre::os::Error),
1923
}
2024

25+
struct FourccInfo {
26+
pixel: Pixel,
27+
bytes_per_pixel: usize,
28+
}
29+
30+
fn fourcc_to_pixel_format(fourcc: &str) -> Option<FourccInfo> {
31+
match fourcc {
32+
"ABGR" => Some(FourccInfo {
33+
pixel: Pixel::ABGR,
34+
bytes_per_pixel: 4,
35+
}),
36+
"b64a" => Some(FourccInfo {
37+
pixel: Pixel::RGBA64BE,
38+
bytes_per_pixel: 8,
39+
}),
40+
"b48r" => Some(FourccInfo {
41+
pixel: Pixel::RGB48BE,
42+
bytes_per_pixel: 6,
43+
}),
44+
"L016" => Some(FourccInfo {
45+
pixel: Pixel::GRAY16LE,
46+
bytes_per_pixel: 2,
47+
}),
48+
_ => None,
49+
}
50+
}
51+
52+
static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false);
53+
2154
impl CapturedFrameExt for CapturedFrame {
2255
fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
2356
let native = self.native().clone();
@@ -162,6 +195,29 @@ impl CapturedFrameExt for CapturedFrame {
162195

163196
ff_frame
164197
}
198+
"RGBA" => {
199+
let mut ff_frame = ffmpeg::frame::Video::new(
200+
ffmpeg::format::Pixel::RGBA,
201+
width as u32,
202+
height as u32,
203+
);
204+
205+
let src_stride = native.image_buf().plane_bytes_per_row(0);
206+
let dest_stride = ff_frame.stride(0);
207+
208+
let src_bytes = bytes_lock.plane_data(0);
209+
let dest_bytes = &mut ff_frame.data_mut(0);
210+
211+
for y in 0..height {
212+
let row_width = width * 4;
213+
let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
214+
let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
215+
216+
dest_row.copy_from_slice(src_row);
217+
}
218+
219+
ff_frame
220+
}
165221
"24BG" => {
166222
let mut ff_frame = ffmpeg::frame::Video::new(
167223
ffmpeg::format::Pixel::BGR24,
@@ -185,6 +241,29 @@ impl CapturedFrameExt for CapturedFrame {
185241

186242
ff_frame
187243
}
244+
"24RG" => {
245+
let mut ff_frame = ffmpeg::frame::Video::new(
246+
ffmpeg::format::Pixel::RGB24,
247+
width as u32,
248+
height as u32,
249+
);
250+
251+
let src_stride = native.image_buf().plane_bytes_per_row(0);
252+
let dest_stride = ff_frame.stride(0);
253+
254+
let src_bytes = bytes_lock.plane_data(0);
255+
let dest_bytes = &mut ff_frame.data_mut(0);
256+
257+
for y in 0..height {
258+
let row_width = width * 3;
259+
let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
260+
let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
261+
262+
dest_row.copy_from_slice(src_row);
263+
}
264+
265+
ff_frame
266+
}
188267
"y420" => {
189268
let plane_count = native.image_buf().plane_count();
190269
if plane_count < 3 {
@@ -220,8 +299,82 @@ impl CapturedFrameExt for CapturedFrame {
220299

221300
ff_frame
222301
}
302+
"L008" | "GRAY" => {
303+
let mut ff_frame = ffmpeg::frame::Video::new(
304+
ffmpeg::format::Pixel::GRAY8,
305+
width as u32,
306+
height as u32,
307+
);
308+
309+
let src_stride = native.image_buf().plane_bytes_per_row(0);
310+
let dest_stride = ff_frame.stride(0);
311+
312+
let src_bytes = bytes_lock.plane_data(0);
313+
let dest_bytes = &mut ff_frame.data_mut(0);
314+
315+
for y in 0..height {
316+
let row_width = width;
317+
let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
318+
let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
319+
320+
dest_row.copy_from_slice(src_row);
321+
}
322+
323+
ff_frame
324+
}
223325
format => {
224-
return Err(AsFFmpegError::UnsupportedSubType(format.to_string()));
326+
if let Some(info) = fourcc_to_pixel_format(format) {
327+
if !FALLBACK_WARNING_LOGGED.swap(true, Ordering::Relaxed) {
328+
tracing::warn!(
329+
"Using swscale fallback for camera format '{}' - this may impact performance",
330+
format
331+
);
332+
}
333+
334+
let mut src_frame =
335+
ffmpeg::frame::Video::new(info.pixel, width as u32, height as u32);
336+
337+
let src_stride = native.image_buf().plane_bytes_per_row(0);
338+
let dest_stride = src_frame.stride(0);
339+
let src_bytes = bytes_lock.plane_data(0);
340+
let dest_bytes = &mut src_frame.data_mut(0);
341+
342+
let row_width = width * info.bytes_per_pixel;
343+
for y in 0..height {
344+
let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
345+
let dest_row =
346+
&mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
347+
dest_row.copy_from_slice(src_row);
348+
}
349+
350+
let mut scaler = scaling::Context::get(
351+
info.pixel,
352+
width as u32,
353+
height as u32,
354+
Pixel::RGBA,
355+
width as u32,
356+
height as u32,
357+
scaling::flag::Flags::FAST_BILINEAR,
358+
)
359+
.map_err(|e| AsFFmpegError::SwscaleFallbackFailed {
360+
format: format.to_string(),
361+
reason: format!("Failed to create scaler: {e}"),
362+
})?;
363+
364+
let mut output_frame =
365+
ffmpeg::frame::Video::new(Pixel::RGBA, width as u32, height as u32);
366+
367+
scaler.run(&src_frame, &mut output_frame).map_err(|e| {
368+
AsFFmpegError::SwscaleFallbackFailed {
369+
format: format.to_string(),
370+
reason: format!("Conversion failed: {e}"),
371+
}
372+
})?;
373+
374+
output_frame
375+
} else {
376+
return Err(AsFFmpegError::UnsupportedSubType(format.to_string()));
377+
}
225378
}
226379
};
227380

‎crates/camera-ffmpeg/src/windows.rs‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ pub enum AsFFmpegError {
1212
Empty,
1313
#[error("MJPEG decode error: {0}")]
1414
MjpegDecodeError(String),
15+
#[error("H264 decode error: {0}")]
16+
H264DecodeError(String),
1517
}
1618

1719
fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
@@ -38,6 +40,30 @@ fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
3840
Ok(decoded_frame)
3941
}
4042

43+
fn decode_h264(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
44+
let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264)
45+
.ok_or_else(|| AsFFmpegError::H264DecodeError("H264 codec not found".to_string()))?;
46+
47+
let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);
48+
49+
let mut decoder = decoder_context
50+
.decoder()
51+
.video()
52+
.map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to create decoder: {e}")))?;
53+
54+
let packet = Packet::copy(bytes);
55+
decoder
56+
.send_packet(&packet)
57+
.map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to send packet: {e}")))?;
58+
59+
let mut decoded_frame = FFVideo::empty();
60+
decoder
61+
.receive_frame(&mut decoded_frame)
62+
.map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to receive frame: {e}")))?;
63+
64+
Ok(decoded_frame)
65+
}
66+
4167
impl CapturedFrameExt for CapturedFrame {
4268
fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
4369
let native = self.native();
@@ -232,6 +258,98 @@ impl CapturedFrameExt for CapturedFrame {
232258

233259
ff_frame
234260
}
261+
PixelFormat::GRAY8 => {
262+
let mut ff_frame = FFVideo::new(Pixel::GRAY8, width as u32, height as u32);
263+
264+
let stride = ff_frame.stride(0);
265+
266+
for y in 0..height {
267+
let row_width = width;
268+
let src_row = &bytes[y * row_width..];
269+
let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
270+
dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
271+
}
272+
273+
ff_frame
274+
}
275+
PixelFormat::GRAY16 => {
276+
let mut ff_frame = FFVideo::new(Pixel::GRAY16LE, width as u32, height as u32);
277+
278+
let stride = ff_frame.stride(0);
279+
let src_stride = width * 2;
280+
281+
for y in 0..height {
282+
let src_row = &bytes[y * src_stride..];
283+
let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
284+
dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
285+
}
286+
287+
ff_frame
288+
}
289+
PixelFormat::NV21 => {
290+
let mut ff_frame = FFVideo::new(Pixel::NV12, width as u32, height as u32);
291+
292+
let stride = ff_frame.stride(0);
293+
for y in 0..height {
294+
let src_row = &bytes[y * width..];
295+
let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
296+
dest_row[0..width].copy_from_slice(&src_row[0..width]);
297+
}
298+
299+
let stride = ff_frame.stride(1);
300+
let src_uv = &bytes[width * height..];
301+
302+
for y in 0..height / 2 {
303+
let row_width = width;
304+
let src_row = &src_uv[y * row_width..];
305+
let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
306+
for x in 0..width / 2 {
307+
dest_row[x * 2] = src_row[x * 2 + 1];
308+
dest_row[x * 2 + 1] = src_row[x * 2];
309+
}
310+
}
311+
312+
ff_frame
313+
}
314+
PixelFormat::RGB565 => {
315+
let mut ff_frame = FFVideo::new(Pixel::RGB565LE, width as u32, height as u32);
316+
317+
let stride = ff_frame.stride(0);
318+
let src_stride = width * 2;
319+
320+
for y in 0..height {
321+
let src_row = &bytes[(height - y - 1) * src_stride..];
322+
let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
323+
dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
324+
}
325+
326+
ff_frame
327+
}
328+
PixelFormat::P010 => {
329+
let mut ff_frame = FFVideo::new(Pixel::P010LE, width as u32, height as u32);
330+
331+
let stride = ff_frame.stride(0);
332+
let src_stride = width * 2;
333+
334+
for y in 0..height {
335+
let src_row = &bytes[y * src_stride..];
336+
let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
337+
dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
338+
}
339+
340+
let stride = ff_frame.stride(1);
341+
let uv_offset = width * height * 2;
342+
let src_stride = width * 2;
343+
344+
for y in 0..height / 2 {
345+
let src_row = &bytes[uv_offset + y * src_stride..];
346+
let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
347+
dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
348+
}
349+
350+
ff_frame
351+
}
352+
PixelFormat::H264 => decode_h264(&bytes)?,
235353
})
236354
}
237355
}

0 commit comments

Comments
 (0)