feat(camera): direct V4L2 capture via v4l crate — eliminates ffmpeg orphans
Replaces ffmpeg subprocess with direct V4L2 mmap capture using the `v4l` Rust crate. Supports MJPG (decoded via jpeg-decoder) and YUYV formats. Key changes: - Primary backend: v4l::io::mmap::Stream (no subprocess, no orphans) - Fallback: ffmpeg with 10-second timeout + kill on hang - MJPG → RGB via jpeg-decoder, YUYV → RGB inline conversion - Device released cleanly on drop (no zombie processes) Fixes the recurring stale ffmpeg issue (killed ~8 times in 61 hours of continuous monitoring). The ffmpeg subprocess would hang on V4L2 device access and become an orphan consuming 99%+ CPU. Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
parent
0824de7665
commit
e20bed197b
|
|
@ -18,3 +18,7 @@ clap = { version = "4", features = ["derive"] }
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
dirs = "5"
|
dirs = "5"
|
||||||
reqwest = { version = "0.12", features = ["json"], default-features = false }
|
reqwest = { version = "0.12", features = ["json"], default-features = false }
|
||||||
|
|
||||||
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
|
v4l = "0.14"
|
||||||
|
jpeg-decoder = "0.3"
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,20 @@
|
||||||
//! Camera capture — cross-platform frame grabber.
|
//! Camera capture — cross-platform frame grabber.
|
||||||
//!
|
//!
|
||||||
//! macOS: uses `screencapture` or `ffmpeg -f avfoundation` for camera frames
|
//! Linux: direct V4L2 via `v4l` crate (no subprocess, no orphans)
|
||||||
//! Linux: uses `v4l2-ctl` or `ffmpeg -f v4l2` for camera frames
|
//! macOS: ffmpeg -f avfoundation (subprocess)
|
||||||
//! Both: capture to JPEG, decode to RGB, return raw pixel data
|
//! Fallback: ffmpeg subprocess on all platforms
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{bail, Result};
|
||||||
use std::process::Command;
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
/// Captured frame with raw RGB data.
|
/// Captured frame with raw RGB data.
|
||||||
pub struct Frame {
|
pub struct Frame {
|
||||||
pub width: u32,
|
pub width: u32,
|
||||||
pub height: u32,
|
pub height: u32,
|
||||||
pub rgb: Vec<u8>, // row-major [height * width * 3]
|
pub rgb: Vec<u8>,
|
||||||
|
pub timestamp_ms: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Camera source configuration.
|
/// Camera source configuration.
|
||||||
|
|
@ -31,41 +33,139 @@ impl Default for CameraConfig {
|
||||||
|
|
||||||
/// Capture a single frame from the camera.
|
/// Capture a single frame from the camera.
|
||||||
///
|
///
|
||||||
/// Tries multiple backends in order: ffmpeg, v4l2, imagesnap (macOS).
|
/// On Linux: uses direct V4L2 (no subprocess, no orphans).
|
||||||
|
/// On macOS: uses ffmpeg subprocess.
|
||||||
pub fn capture_frame(config: &CameraConfig) -> Result<Frame> {
|
pub fn capture_frame(config: &CameraConfig) -> Result<Frame> {
|
||||||
let tmp = tmp_path();
|
// Linux: direct V4L2 (preferred — no subprocess)
|
||||||
|
|
||||||
// Try ffmpeg first (cross-platform)
|
|
||||||
if let Ok(frame) = capture_ffmpeg(config, &tmp) {
|
|
||||||
return Ok(frame);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Linux: try v4l2
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
if let Ok(frame) = capture_v4l2(config, &tmp) {
|
{
|
||||||
|
match capture_v4l2_direct(config) {
|
||||||
|
Ok(frame) => return Ok(frame),
|
||||||
|
Err(e) => eprintln!("[camera] V4L2 direct failed: {e}, falling back to ffmpeg"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: ffmpeg subprocess (with timeout to prevent orphans)
|
||||||
|
let tmp = tmp_path();
|
||||||
|
if let Ok(frame) = capture_ffmpeg_safe(config, &tmp) {
|
||||||
return Ok(frame);
|
return Ok(frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
// macOS: try screencapture (camera mode)
|
// macOS: screencapture
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
if let Ok(frame) = capture_macos(config, &tmp) {
|
if let Ok(frame) = capture_macos(config, &tmp) {
|
||||||
return Ok(frame);
|
return Ok(frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
bail!("No camera backend available. Install ffmpeg or run on a machine with a camera.")
|
bail!("No camera backend available")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Capture via ffmpeg (works on Linux + macOS).
|
// ============================================================
|
||||||
fn capture_ffmpeg(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
// Linux: Direct V4L2 capture (no subprocess, no orphans)
|
||||||
let input = if cfg!(target_os = "macos") {
|
// ============================================================
|
||||||
format!("{}:none", config.device_index) // avfoundation: video:audio
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn capture_v4l2_direct(config: &CameraConfig) -> Result<Frame> {
|
||||||
|
use v4l::buffer::Type;
|
||||||
|
use v4l::io::mmap::Stream;
|
||||||
|
use v4l::io::traits::CaptureStream;
|
||||||
|
use v4l::video::Capture;
|
||||||
|
use v4l::{Device, FourCC};
|
||||||
|
|
||||||
|
let device_path = format!("/dev/video{}", config.device_index);
|
||||||
|
if !std::path::Path::new(&device_path).exists() {
|
||||||
|
bail!("no camera at {device_path}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let dev = Device::with_path(&device_path)?;
|
||||||
|
|
||||||
|
// Try MJPG first (most webcams support it), fall back to YUYV
|
||||||
|
let mut fmt = dev.format()?;
|
||||||
|
fmt.width = config.width;
|
||||||
|
fmt.height = config.height;
|
||||||
|
fmt.fourcc = FourCC::new(b"MJPG");
|
||||||
|
let use_mjpg = dev.set_format(&fmt).is_ok();
|
||||||
|
|
||||||
|
if !use_mjpg {
|
||||||
|
fmt.fourcc = FourCC::new(b"YUYV");
|
||||||
|
dev.set_format(&fmt)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fmt = dev.format()?;
|
||||||
|
let actual_w = fmt.width;
|
||||||
|
let actual_h = fmt.height;
|
||||||
|
|
||||||
|
// Stream one frame via mmap
|
||||||
|
let mut stream = Stream::with_buffers(&dev, Type::VideoCapture, 2)?;
|
||||||
|
let (buf, _meta) = stream.next()?;
|
||||||
|
|
||||||
|
let rgb = if use_mjpg {
|
||||||
|
decode_mjpeg_to_rgb(buf, actual_w, actual_h)?
|
||||||
} else {
|
} else {
|
||||||
format!("/dev/video{}", config.device_index) // v4l2
|
yuyv_to_rgb(buf, actual_w, actual_h)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Stream is dropped here — device released cleanly, no orphan process
|
||||||
|
|
||||||
|
Ok(Frame {
|
||||||
|
width: actual_w,
|
||||||
|
height: actual_h,
|
||||||
|
rgb,
|
||||||
|
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn decode_mjpeg_to_rgb(data: &[u8], _w: u32, _h: u32) -> Result<Vec<u8>> {
|
||||||
|
// Use a minimal JPEG decoder
|
||||||
|
let mut decoder = jpeg_decoder::Decoder::new(std::io::Cursor::new(data));
|
||||||
|
let pixels = decoder.decode()?;
|
||||||
|
let info = decoder.info().ok_or_else(|| anyhow::anyhow!("no JPEG info"))?;
|
||||||
|
|
||||||
|
if info.pixel_format == jpeg_decoder::PixelFormat::RGB24 {
|
||||||
|
Ok(pixels)
|
||||||
|
} else if info.pixel_format == jpeg_decoder::PixelFormat::L8 {
|
||||||
|
// Grayscale → RGB
|
||||||
|
Ok(pixels.iter().flat_map(|&g| [g, g, g]).collect())
|
||||||
|
} else {
|
||||||
|
bail!("unsupported JPEG pixel format: {:?}", info.pixel_format)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn yuyv_to_rgb(data: &[u8], w: u32, h: u32) -> Vec<u8> {
|
||||||
|
let pixel_count = (w * h) as usize;
|
||||||
|
let mut rgb = Vec::with_capacity(pixel_count * 3);
|
||||||
|
|
||||||
|
for chunk in data.chunks(4) {
|
||||||
|
if chunk.len() < 4 { break; }
|
||||||
|
let (y0, u, y1, v) = (chunk[0] as f32, chunk[1] as f32, chunk[2] as f32, chunk[3] as f32);
|
||||||
|
|
||||||
|
for y in [y0, y1] {
|
||||||
|
let r = (y + 1.402 * (v - 128.0)).clamp(0.0, 255.0) as u8;
|
||||||
|
let g = (y - 0.344136 * (u - 128.0) - 0.714136 * (v - 128.0)).clamp(0.0, 255.0) as u8;
|
||||||
|
let b = (y + 1.772 * (u - 128.0)).clamp(0.0, 255.0) as u8;
|
||||||
|
rgb.extend_from_slice(&[r, g, b]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rgb.truncate(pixel_count * 3);
|
||||||
|
rgb
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Fallback: ffmpeg subprocess (with timeout + cleanup)
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
fn capture_ffmpeg_safe(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
||||||
|
let input = if cfg!(target_os = "macos") {
|
||||||
|
format!("{}:none", config.device_index)
|
||||||
|
} else {
|
||||||
|
format!("/dev/video{}", config.device_index)
|
||||||
|
};
|
||||||
let format = if cfg!(target_os = "macos") { "avfoundation" } else { "v4l2" };
|
let format = if cfg!(target_os = "macos") { "avfoundation" } else { "v4l2" };
|
||||||
|
|
||||||
let status = Command::new("ffmpeg")
|
// Spawn with timeout to prevent orphans
|
||||||
|
let mut child = Command::new("ffmpeg")
|
||||||
.args([
|
.args([
|
||||||
"-y", "-f", format,
|
"-y", "-f", format,
|
||||||
"-video_size", &format!("{}x{}", config.width, config.height),
|
"-video_size", &format!("{}x{}", config.width, config.height),
|
||||||
|
|
@ -76,59 +176,54 @@ fn capture_ffmpeg(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
||||||
"-pix_fmt", "rgb24",
|
"-pix_fmt", "rgb24",
|
||||||
tmp.to_str().unwrap_or("/tmp/ruview-frame.raw"),
|
tmp.to_str().unwrap_or("/tmp/ruview-frame.raw"),
|
||||||
])
|
])
|
||||||
.output()?;
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::piped())
|
||||||
|
.spawn()?;
|
||||||
|
|
||||||
if !status.status.success() {
|
// Wait with 10-second timeout
|
||||||
bail!("ffmpeg capture failed: {}", String::from_utf8_lossy(&status.stderr));
|
let timeout = std::time::Duration::from_secs(10);
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match child.try_wait()? {
|
||||||
|
Some(status) => {
|
||||||
|
if !status.success() {
|
||||||
|
bail!("ffmpeg capture failed (exit {})", status.code().unwrap_or(-1));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
if start.elapsed() > timeout {
|
||||||
|
// Kill the stuck process — this is the orphan prevention
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
bail!("ffmpeg capture timed out after 10s — killed");
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let rgb = std::fs::read(tmp)?;
|
let rgb = std::fs::read(tmp)?;
|
||||||
let expected = (config.width * config.height * 3) as usize;
|
let expected = (config.width * config.height * 3) as usize;
|
||||||
|
let _ = std::fs::remove_file(tmp);
|
||||||
|
|
||||||
if rgb.len() < expected {
|
if rgb.len() < expected {
|
||||||
bail!("frame too small: {} bytes, expected {}", rgb.len(), expected);
|
bail!("frame too small: {} bytes, expected {}", rgb.len(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = std::fs::remove_file(tmp);
|
|
||||||
|
|
||||||
Ok(Frame {
|
Ok(Frame {
|
||||||
width: config.width,
|
width: config.width,
|
||||||
height: config.height,
|
height: config.height,
|
||||||
rgb: rgb[..expected].to_vec(),
|
rgb: rgb[..expected].to_vec(),
|
||||||
|
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Linux: capture via v4l2-ctl.
|
/// macOS: capture via swift/screencapture.
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn capture_v4l2(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
|
||||||
let device = format!("/dev/video{}", config.device_index);
|
|
||||||
if !std::path::Path::new(&device).exists() {
|
|
||||||
bail!("no camera at {device}");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use v4l2-ctl to grab a frame
|
|
||||||
let status = Command::new("v4l2-ctl")
|
|
||||||
.args([
|
|
||||||
"--device", &device,
|
|
||||||
"--set-fmt-video", &format!("width={},height={},pixelformat=MJPG", config.width, config.height),
|
|
||||||
"--stream-mmap", "--stream-count=1",
|
|
||||||
"--stream-to", tmp.to_str().unwrap_or("/tmp/frame.mjpg"),
|
|
||||||
])
|
|
||||||
.output()?;
|
|
||||||
|
|
||||||
if !status.status.success() {
|
|
||||||
bail!("v4l2-ctl failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decode MJPEG to RGB
|
|
||||||
decode_jpeg_to_rgb(tmp, config.width, config.height)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// macOS: capture via screencapture or swift.
|
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
fn capture_macos(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
fn capture_macos(config: &CameraConfig, tmp: &PathBuf) -> Result<Frame> {
|
||||||
let jpg_path = tmp.with_extension("jpg");
|
let jpg_path = tmp.with_extension("jpg");
|
||||||
|
|
||||||
// Try swift-based capture (requires camera permission)
|
|
||||||
let swift = format!(
|
let swift = format!(
|
||||||
r#"import AVFoundation; import AppKit
|
r#"import AVFoundation; import AppKit
|
||||||
let sem = DispatchSemaphore(value: 0)
|
let sem = DispatchSemaphore(value: 0)
|
||||||
|
|
@ -147,34 +242,25 @@ o.capturePhoto(with: AVCapturePhotoSettings(), delegate: dl)
|
||||||
Thread.sleep(forTimeInterval: 3)"#,
|
Thread.sleep(forTimeInterval: 3)"#,
|
||||||
path = jpg_path.display()
|
path = jpg_path.display()
|
||||||
);
|
);
|
||||||
|
|
||||||
let _ = Command::new("swift").args(["-e", &swift]).output();
|
let _ = Command::new("swift").args(["-e", &swift]).output();
|
||||||
|
|
||||||
if jpg_path.exists() {
|
if jpg_path.exists() {
|
||||||
return decode_jpeg_to_rgb(&jpg_path, config.width, config.height);
|
let data = std::fs::read(&jpg_path)?;
|
||||||
|
let _ = std::fs::remove_file(&jpg_path);
|
||||||
|
return Ok(Frame {
|
||||||
|
width: config.width,
|
||||||
|
height: config.height,
|
||||||
|
rgb: data,
|
||||||
|
timestamp_ms: chrono::Utc::now().timestamp_millis(),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
bail!("macOS camera capture requires GUI session with camera permission")
|
bail!("macOS camera capture requires GUI session with camera permission")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode_jpeg_to_rgb(path: &PathBuf, _width: u32, _height: u32) -> Result<Frame> {
|
|
||||||
let data = std::fs::read(path)?;
|
|
||||||
let _ = std::fs::remove_file(path);
|
|
||||||
|
|
||||||
// Simple JPEG decode — use the image crate if available, otherwise raw
|
|
||||||
// For now, return the raw data and let the caller handle format
|
|
||||||
Ok(Frame {
|
|
||||||
width: _width,
|
|
||||||
height: _height,
|
|
||||||
rgb: data,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tmp_path() -> PathBuf {
|
fn tmp_path() -> PathBuf {
|
||||||
std::env::temp_dir().join(format!("ruview-frame-{}.raw", std::process::id()))
|
std::env::temp_dir().join(format!("ruview-frame-{}.raw", std::process::id()))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a camera is available on this system.
|
/// Check if a camera is available.
|
||||||
pub fn camera_available() -> bool {
|
pub fn camera_available() -> bool {
|
||||||
if cfg!(target_os = "macos") {
|
if cfg!(target_os = "macos") {
|
||||||
Command::new("system_profiler")
|
Command::new("system_profiler")
|
||||||
|
|
@ -190,7 +276,6 @@ pub fn camera_available() -> bool {
|
||||||
/// List available cameras.
|
/// List available cameras.
|
||||||
pub fn list_cameras() -> Vec<String> {
|
pub fn list_cameras() -> Vec<String> {
|
||||||
let mut cameras = Vec::new();
|
let mut cameras = Vec::new();
|
||||||
|
|
||||||
if cfg!(target_os = "macos") {
|
if cfg!(target_os = "macos") {
|
||||||
if let Ok(output) = Command::new("system_profiler").args(["SPCameraDataType"]).output() {
|
if let Ok(output) = Command::new("system_profiler").args(["SPCameraDataType"]).output() {
|
||||||
let text = String::from_utf8_lossy(&output.stdout);
|
let text = String::from_utf8_lossy(&output.stdout);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue