Add native self-hosted instance connection to fluxer_desktop

Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 +
tools/ci) with a "Connect to a Different Server" menu item and popout
that lets the desktop app switch to any self-hosted Fluxer instance,
plus fixes for well-known discovery on single-domain self-hosted
deployments and a false-positive ERR_ABORTED on same-origin client
redirects during the switch. Defaults to chat.fluxr.chat and uses an
isolated userData directory from the official build.
This commit is contained in:
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
@@ -0,0 +1,364 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::collections::HashMap;
use std::ffi::c_void;
use std::ptr;
use std::sync::Arc;
use libloading::{Library, Symbol};
use windows::Win32::Graphics::Direct3D11::ID3D11Device;
use windows::core::Interface;
use crate::encoder_handoff::{
AmfHandoff, EncodedBitstream, EncoderCompletionCallback, EncoderDims, EncoderError,
EncoderFrameRate, EncoderSubmission, HandoffSlot, PicParams, apply_dts_offset,
compute_dts_offset_us,
};
use crate::ring::RingError;
pub const AMF_DLL_NAME: &str = "amfrt64.dll";
const AMF_OK: i32 = 0;
#[allow(
dead_code,
reason = "documented as a possible AMF QueryOutput status code per the SDK"
)]
const AMF_REPEAT: i32 = 5;
#[allow(
dead_code,
reason = "documented as a possible AMF status code per the SDK"
)]
const AMF_NOT_READY: i32 = 1;
type AmfStatus = i32;
type AmfInitFn = unsafe extern "C" fn(version: u64, factory: *mut *mut c_void) -> AmfStatus;
#[repr(C)]
struct AmfFactoryVtbl {
query_interface:
unsafe extern "system" fn(*mut c_void, *const u128, *mut *mut c_void) -> AmfStatus,
acquire: unsafe extern "system" fn(*mut c_void) -> u32,
release: unsafe extern "system" fn(*mut c_void) -> u32,
create_context: unsafe extern "system" fn(*mut c_void, *mut *mut c_void) -> AmfStatus,
create_component: unsafe extern "system" fn(
*mut c_void,
*mut c_void,
*const u16,
*mut *mut c_void,
) -> AmfStatus,
set_cache_folder: unsafe extern "system" fn(*mut c_void, *const u16) -> AmfStatus,
get_cache_folder: unsafe extern "system" fn(*mut c_void) -> *const u16,
get_debug: unsafe extern "system" fn(*mut c_void, *mut *mut c_void) -> AmfStatus,
get_trace: unsafe extern "system" fn(*mut c_void, *mut *mut c_void) -> AmfStatus,
get_program_versions:
unsafe extern "system" fn(*mut c_void, *mut u32, *mut u32, *mut u32, *mut u32) -> AmfStatus,
}
#[repr(C)]
struct AmfFactoryObject {
vtbl: *const AmfFactoryVtbl,
}
struct SlotState {
pending_pts_us: u64,
pending_force_keyframe: bool,
in_flight: bool,
}
pub struct AmfD3D11Handoff {
_library: Arc<Library>,
factory: *mut c_void,
context: *mut c_void,
encoder: *mut c_void,
slots: HashMap<u32, SlotState>,
next_slot_index: u32,
dts_offset_us: i64,
completed_count: u64,
frame_interval_us: u64,
}
unsafe impl Send for AmfD3D11Handoff {}
impl AmfD3D11Handoff {
pub fn new(
device: ID3D11Device,
dims: EncoderDims,
bitrate_bps: u32,
) -> Result<Self, EncoderError> {
Self::new_with_frame_rate(device, dims, bitrate_bps, EncoderFrameRate::default())
}
pub fn new_with_frame_rate(
device: ID3D11Device,
dims: EncoderDims,
bitrate_bps: u32,
frame_rate: EncoderFrameRate,
) -> Result<Self, EncoderError> {
assert!(dims.width > 0, "width positive");
assert!(dims.height > 0, "height positive");
assert!(frame_rate.numerator > 0, "frame rate numerator positive");
assert!(
frame_rate.denominator > 0,
"frame rate denominator positive"
);
if dims.width > 7680 || dims.height > 4320 {
return Err(EncoderError::DimensionsOutOfRange {
width: dims.width,
height: dims.height,
});
}
let library = load_runtime()?;
let factory = init_factory(&library)?;
let context = create_context(factory)?;
init_dx11_context(context, &device)?;
let encoder = create_video_encoder(factory, context, dims, bitrate_bps)?;
let frame_interval_us = frame_rate.frame_interval_us();
let dts_offset_us = compute_dts_offset_us(0, 0, frame_interval_us);
let handoff = Self {
_library: Arc::new(library),
factory,
context,
encoder,
slots: HashMap::new(),
next_slot_index: 0,
dts_offset_us,
completed_count: 0,
frame_interval_us,
};
assert!(!handoff.factory.is_null(), "factory non-null");
assert!(handoff.completed_count == 0, "fresh state");
Ok(handoff)
}
}
fn load_runtime() -> Result<Library, EncoderError> {
let library = unsafe { Library::new(AMF_DLL_NAME) }.map_err(|_| EncoderError::SdkNotFound {
vendor: "amf",
dll: AMF_DLL_NAME,
})?;
Ok(library)
}
fn init_factory(library: &Library) -> Result<*mut c_void, EncoderError> {
let init: Symbol<'_, AmfInitFn> =
unsafe { library.get(b"AMFInit\0") }.map_err(|_| EncoderError::SymbolMissing {
vendor: "amf",
symbol: "AMFInit",
})?;
let mut factory: *mut c_void = ptr::null_mut();
const AMF_FULL_VERSION: u64 = (1_u64 << 48) | (4_u64 << 32) | (30_u64 << 16);
let status = unsafe { init(AMF_FULL_VERSION, &mut factory) };
if status != AMF_OK {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: status as i64,
});
}
if factory.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: -1,
});
}
Ok(factory)
}
fn create_context(factory: *mut c_void) -> Result<*mut c_void, EncoderError> {
assert!(!factory.is_null(), "factory ptr non-null");
let object = factory as *mut AmfFactoryObject;
let vtbl = unsafe { (*object).vtbl };
let mut context: *mut c_void = ptr::null_mut();
let status = unsafe { ((*vtbl).create_context)(factory, &mut context) };
if status != AMF_OK {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: status as i64,
});
}
if context.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: -2,
});
}
Ok(context)
}
fn init_dx11_context(context: *mut c_void, device: &ID3D11Device) -> Result<(), EncoderError> {
assert!(!context.is_null(), "context non-null");
let _ = device.as_raw();
Ok(())
}
fn create_video_encoder(
factory: *mut c_void,
context: *mut c_void,
dims: EncoderDims,
bitrate_bps: u32,
) -> Result<*mut c_void, EncoderError> {
assert!(!factory.is_null(), "factory non-null");
assert!(!context.is_null(), "context non-null");
assert!(dims.width > 0, "width positive");
let _ = bitrate_bps;
let component_id: Vec<u16> = "AMFVideoEncoderVCE_AVC\0".encode_utf16().collect();
let object = factory as *mut AmfFactoryObject;
let vtbl = unsafe { (*object).vtbl };
let mut encoder: *mut c_void = ptr::null_mut();
let status = unsafe {
((*vtbl).create_component)(factory, context, component_id.as_ptr(), &mut encoder)
};
if status != AMF_OK {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: status as i64,
});
}
if encoder.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "amf",
status: -3,
});
}
Ok(encoder)
}
impl Drop for AmfD3D11Handoff {
fn drop(&mut self) {
self.slots.clear();
if !self.factory.is_null() {
let object = self.factory as *mut AmfFactoryObject;
unsafe {
let vtbl = (*object).vtbl;
if !self.encoder.is_null() {
let _ = ((*vtbl).release)(self.encoder);
self.encoder = ptr::null_mut();
}
if !self.context.is_null() {
let _ = ((*vtbl).release)(self.context);
self.context = ptr::null_mut();
}
let _ = ((*vtbl).release)(self.factory);
}
self.factory = ptr::null_mut();
}
}
}
impl AmfHandoff for AmfD3D11Handoff {
fn register_slot(
&mut self,
shared_handle: u64,
_key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError> {
assert!(shared_handle != 0, "shared_handle non-zero");
assert!(dims.width > 0, "width positive");
let slot_index = self.next_slot_index;
self.next_slot_index = self.next_slot_index.saturating_add(1);
let slot = HandoffSlot::new(slot_index, shared_handle);
self.slots.insert(
slot_index,
SlotState {
pending_pts_us: 0,
pending_force_keyframe: false,
in_flight: false,
},
);
assert!(self.slots.contains_key(&slot_index), "slot stored");
Ok(slot)
}
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
_key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError> {
assert!(slot.shared_handle != 0, "slot handle non-zero");
assert!(dims.width > 0, "width positive");
let state = self
.slots
.get_mut(&slot.slot_index)
.ok_or(EncoderError::SlotUnknown {
slot_index: slot.slot_index,
})?;
state.pending_pts_us = pic_params.pts_us;
state.pending_force_keyframe = pic_params.force_keyframe;
state.in_flight = true;
Ok(())
}
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream> {
let state = self.slots.get_mut(&slot.slot_index)?;
if !state.in_flight {
return None;
}
state.in_flight = false;
let pts = state.pending_pts_us;
let dts = apply_dts_offset(pts, self.dts_offset_us);
let _ = state.pending_force_keyframe || self.completed_count == 0;
self.completed_count = self.completed_count.saturating_add(1);
let _ = (pts, dts);
None
}
fn unregister_slot(&mut self, slot: HandoffSlot) {
self.slots.remove(&slot.slot_index);
}
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError> {
assert!(submission.shared_handle != 0, "submission handle non-zero");
assert!(submission.dims.width > 0, "submission width positive");
let slot = self
.register_slot(
submission.shared_handle,
submission.keyed_mutex_key,
submission.dims,
)
.map_err(|_| RingError::NotImplemented {
what: "amf::register_slot in encode_shared",
})?;
let pts_us = submission
.capture_pts_us
.unwrap_or_else(|| submission.sequence.saturating_mul(self.frame_interval_us));
let pic = PicParams::new(pts_us, false);
AmfHandoff::encode_shared_async(
self,
slot,
submission.keyed_mutex_key,
submission.dims,
pic,
)
.map_err(|_| RingError::NotImplemented {
what: "amf::encode_shared_async",
})?;
if let Some(bs) = AmfHandoff::poll_completed(self, slot) {
callback.on_complete(submission.sequence, bs.data.len() as u32);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sdk_not_found_when_dll_missing() {
let dummy_path = "/this/path/does/not/exist/fake-amfrt64.dll";
let result = unsafe { Library::new(dummy_path) };
assert!(result.is_err());
}
#[test]
fn amf_status_constants_match_spec() {
assert_eq!(AMF_OK, 0);
assert_eq!(AMF_REPEAT, 5);
assert_eq!(AMF_NOT_READY, 1);
}
}
@@ -0,0 +1,366 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use crate::{MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH, nv12_byte_size};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TextureFormat {
Nv12,
P010,
Bgra8,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackendError {
DimensionsOutOfRange { width: u32, height: u32 },
UnsupportedFormat { format: TextureFormat },
PlatformUnsupported { reason: &'static str },
KeyMismatch { expected: u64, observed: u64 },
AcquireWhileWriting { slot_index: u32 },
ReleaseWithoutAcquire { slot_index: u32 },
WouldBlock { slot_index: u32 },
}
impl std::fmt::Display for BackendError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::DimensionsOutOfRange { width, height } => {
write!(f, "dimensions out of range: {width}x{height}")
}
Self::UnsupportedFormat { format } => write!(f, "unsupported format: {format:?}"),
Self::PlatformUnsupported { reason } => write!(f, "platform unsupported: {reason}"),
Self::KeyMismatch { expected, observed } => {
write!(
f,
"keyed-mutex key mismatch: expected={expected} observed={observed}"
)
}
Self::AcquireWhileWriting { slot_index } => {
write!(f, "acquire_write while slot {slot_index} already acquired")
}
Self::ReleaseWithoutAcquire { slot_index } => {
write!(f, "release_write without acquire on slot {slot_index}")
}
Self::WouldBlock { slot_index } => {
write!(
f,
"keyed mutex busy on slot {slot_index}; skipped without blocking"
)
}
}
}
}
impl std::error::Error for BackendError {}
pub const NUM_SLOTS_DEFAULT: usize = 8;
pub trait KeyedMutexBackend: Send {
type SlotHandle: Send + Clone;
const NUM_SLOTS: usize;
fn create_slots(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<Vec<Self::SlotHandle>, BackendError>;
fn acquire_write(&mut self, slot: &Self::SlotHandle, key: u64) -> Result<(), BackendError>;
fn release_write(&mut self, slot: &Self::SlotHandle, next_key: u64)
-> Result<(), BackendError>;
fn poll_complete(&mut self, slot: &Self::SlotHandle) -> bool;
fn mark_consumed(&mut self, slot: &Self::SlotHandle);
fn fill_test_pattern(&mut self, _slot: &Self::SlotHandle, _value: u8) {}
}
#[derive(Clone)]
pub struct CpuSlotHandle {
inner: Arc<CpuSlotInner>,
}
impl CpuSlotHandle {
pub fn slot_index(&self) -> u32 {
let idx = self.inner.slot_index;
assert!((idx as usize) < NUM_SLOTS_DEFAULT, "slot_index in range");
assert!(
self.inner.buffer.len() == self.inner.byte_size,
"buffer matches byte_size"
);
idx
}
pub fn current_key(&self) -> u64 {
let key = self.inner.current_key.load(Ordering::Acquire);
assert!(
self.inner.buffer.len() == self.inner.byte_size,
"buffer intact"
);
assert!(
(self.inner.slot_index as usize) < NUM_SLOTS_DEFAULT,
"slot_index intact"
);
key
}
pub fn buffer_len(&self) -> usize {
let len = self.inner.byte_size;
assert!(len > 0, "byte_size positive");
assert!(self.inner.buffer.len() == len, "buffer matches byte_size");
len
}
pub fn write_byte(&self, offset: usize, value: u8) {
assert!(offset < self.inner.byte_size, "offset within buffer");
assert!(
self.inner.acquired.load(Ordering::Acquire),
"writes only while acquired",
);
unsafe {
let ptr = self.inner.buffer.as_ptr().add(offset) as *mut u8;
ptr.write_volatile(value);
}
}
}
struct CpuSlotInner {
slot_index: u32,
byte_size: usize,
buffer: Vec<u8>,
current_key: AtomicU64,
acquired: AtomicBool,
completed: AtomicBool,
}
pub struct CpuMemcpyBackend {
width: u32,
height: u32,
format: TextureFormat,
slots_created: bool,
slot_count: u32,
}
impl CpuMemcpyBackend {
pub fn new() -> Self {
let backend = Self {
width: 0,
height: 0,
format: TextureFormat::Nv12,
slots_created: false,
slot_count: 0,
};
assert!(!backend.slots_created, "fresh backend has no slots");
assert_eq!(backend.slot_count, 0, "fresh slot_count zero");
backend
}
pub fn width(&self) -> u32 {
assert!(self.width <= MAX_FRAME_WIDTH, "width within cap");
self.width
}
pub fn height(&self) -> u32 {
assert!(self.height <= MAX_FRAME_HEIGHT, "height within cap");
self.height
}
}
impl Default for CpuMemcpyBackend {
fn default() -> Self {
Self::new()
}
}
impl KeyedMutexBackend for CpuMemcpyBackend {
type SlotHandle = CpuSlotHandle;
const NUM_SLOTS: usize = NUM_SLOTS_DEFAULT;
fn create_slots(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<Vec<CpuSlotHandle>, BackendError> {
if width == 0 || height == 0 || width > MAX_FRAME_WIDTH || height > MAX_FRAME_HEIGHT {
return Err(BackendError::DimensionsOutOfRange { width, height });
}
if !matches!(format, TextureFormat::Nv12) {
return Err(BackendError::UnsupportedFormat { format });
}
assert!(!self.slots_created, "slots created once");
let byte_size = nv12_byte_size(width, height);
assert!(byte_size > 0, "byte_size positive");
let mut out: Vec<CpuSlotHandle> = Vec::with_capacity(Self::NUM_SLOTS);
for idx in 0..Self::NUM_SLOTS {
let inner = CpuSlotInner {
slot_index: idx as u32,
byte_size,
buffer: vec![0u8; byte_size],
current_key: AtomicU64::new(0),
acquired: AtomicBool::new(false),
completed: AtomicBool::new(false),
};
out.push(CpuSlotHandle {
inner: Arc::new(inner),
});
}
self.width = width;
self.height = height;
self.format = format;
self.slots_created = true;
self.slot_count = Self::NUM_SLOTS as u32;
assert_eq!(out.len(), Self::NUM_SLOTS, "slot vector length");
assert!(self.slots_created, "slots_created flipped");
Ok(out)
}
fn acquire_write(&mut self, slot: &CpuSlotHandle, key: u64) -> Result<(), BackendError> {
assert!(self.slots_created, "slots must exist before acquire");
let current = slot.inner.current_key.load(Ordering::Acquire);
if current != key {
return Err(BackendError::KeyMismatch {
expected: key,
observed: current,
});
}
let was_acquired = slot.inner.acquired.swap(true, Ordering::AcqRel);
if was_acquired {
return Err(BackendError::AcquireWhileWriting {
slot_index: slot.inner.slot_index,
});
}
slot.inner.completed.store(false, Ordering::Release);
assert!(
slot.inner.acquired.load(Ordering::Acquire),
"acquired flag set"
);
assert!(
!slot.inner.completed.load(Ordering::Acquire),
"completed cleared"
);
Ok(())
}
fn release_write(&mut self, slot: &CpuSlotHandle, next_key: u64) -> Result<(), BackendError> {
assert!(self.slots_created, "slots must exist before release");
let was_acquired = slot.inner.acquired.swap(false, Ordering::AcqRel);
if !was_acquired {
return Err(BackendError::ReleaseWithoutAcquire {
slot_index: slot.inner.slot_index,
});
}
slot.inner.current_key.store(next_key, Ordering::Release);
slot.inner.completed.store(true, Ordering::Release);
assert!(
!slot.inner.acquired.load(Ordering::Acquire),
"acquired cleared"
);
assert!(
slot.inner.completed.load(Ordering::Acquire),
"completed flag set"
);
Ok(())
}
fn poll_complete(&mut self, slot: &CpuSlotHandle) -> bool {
let done = slot.inner.completed.load(Ordering::Acquire);
assert!(self.slots_created, "slots exist for poll");
assert!(
(slot.inner.slot_index as usize) < Self::NUM_SLOTS,
"slot index in range"
);
done
}
fn mark_consumed(&mut self, slot: &CpuSlotHandle) {
assert!(self.slots_created, "slots exist for mark_consumed");
assert!(
(slot.inner.slot_index as usize) < Self::NUM_SLOTS,
"slot index in range"
);
slot.inner.completed.store(false, Ordering::Release);
}
fn fill_test_pattern(&mut self, slot: &CpuSlotHandle, value: u8) {
assert!(self.slots_created, "slots exist for fill");
assert!(
slot.inner.acquired.load(Ordering::Acquire),
"fill only while acquired"
);
let len = slot.inner.byte_size;
for offset in 0..len {
slot.write_byte(offset, value);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn create_slots_for_1080p_nv12_yields_eight_slots() {
let mut backend = CpuMemcpyBackend::new();
let slots = backend
.create_slots(1920, 1080, TextureFormat::Nv12)
.expect("creation succeeds");
assert_eq!(slots.len(), CpuMemcpyBackend::NUM_SLOTS);
assert_eq!(slots.len(), 8);
for (idx, slot) in slots.iter().enumerate() {
assert_eq!(slot.slot_index(), idx as u32);
assert_eq!(slot.buffer_len(), 1920 * 1080 * 3 / 2);
}
}
#[test]
fn create_slots_rejects_zero_dims() {
let mut backend = CpuMemcpyBackend::new();
let err = backend.create_slots(0, 1080, TextureFormat::Nv12).err();
assert!(matches!(
err,
Some(BackendError::DimensionsOutOfRange { .. })
));
}
#[test]
fn create_slots_rejects_unsupported_format() {
let mut backend = CpuMemcpyBackend::new();
let err = backend.create_slots(1920, 1080, TextureFormat::P010).err();
assert!(matches!(err, Some(BackendError::UnsupportedFormat { .. })));
}
#[test]
fn acquire_release_round_trip_marks_complete() {
let mut backend = CpuMemcpyBackend::new();
let slots = backend
.create_slots(64, 64, TextureFormat::Nv12)
.expect("create");
let slot = slots[0].clone();
backend.acquire_write(&slot, 0).expect("acquire");
backend.release_write(&slot, 1).expect("release");
assert!(backend.poll_complete(&slot));
backend.mark_consumed(&slot);
assert!(!backend.poll_complete(&slot));
}
#[test]
fn double_acquire_rejects() {
let mut backend = CpuMemcpyBackend::new();
let slots = backend
.create_slots(64, 64, TextureFormat::Nv12)
.expect("create");
let slot = slots[0].clone();
backend.acquire_write(&slot, 0).expect("first acquire");
let err = backend.acquire_write(&slot, 0).err();
assert!(matches!(
err,
Some(BackendError::AcquireWhileWriting { .. })
));
}
}
@@ -0,0 +1,473 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::backend::{BackendError, KeyedMutexBackend, TextureFormat};
#[derive(Clone)]
pub struct D3D11SharedHandle {
pub raw_handle: u64,
pub slot_index: u32,
pub width: u32,
pub height: u32,
}
#[cfg(target_os = "windows")]
pub struct D3D11KeyedMutexBackend {
device: Option<windows::Win32::Graphics::Direct3D11::ID3D11Device>,
#[allow(dead_code, reason = "held for RAII: device context lifetime")]
context: Option<windows::Win32::Graphics::Direct3D11::ID3D11DeviceContext>,
slots: Vec<D3D11SlotState>,
width: u32,
height: u32,
format: TextureFormat,
}
#[cfg(target_os = "windows")]
struct D3D11SlotState {
#[allow(
dead_code,
reason = "held for RAII: texture lifetime tied to keyed mutex"
)]
texture: windows::Win32::Graphics::Direct3D11::ID3D11Texture2D,
keyed_mutex: windows::Win32::Graphics::Dxgi::IDXGIKeyedMutex,
#[allow(dead_code, reason = "exposed via D3D11SharedHandle to clients")]
shared_handle: u64,
slot_index: u32,
expected_key: u64,
acquired: bool,
completed: bool,
}
#[cfg(target_os = "windows")]
impl D3D11KeyedMutexBackend {
pub fn new() -> Result<Self, BackendError> {
let (device, context) = unsafe { create_d3d11_device_windows()? };
let backend = Self {
device: Some(device),
context: Some(context),
slots: Vec::with_capacity(<Self as KeyedMutexBackend>::NUM_SLOTS),
width: 0,
height: 0,
format: TextureFormat::Nv12,
};
assert!(backend.slots.is_empty(), "fresh backend has no slots");
assert!(backend.device.is_some(), "device created");
Ok(backend)
}
fn find_slot(&mut self, slot: &D3D11SharedHandle) -> Option<usize> {
assert!((slot.slot_index as usize) < <Self as KeyedMutexBackend>::NUM_SLOTS);
self.slots
.iter()
.position(|s| s.slot_index == slot.slot_index)
}
pub fn texture_for_slot(
&self,
slot_index: u32,
) -> Option<windows::Win32::Graphics::Direct3D11::ID3D11Texture2D> {
assert!(
slot_index < (<Self as KeyedMutexBackend>::NUM_SLOTS as u32),
"slot_index in range"
);
assert!(!self.slots.is_empty(), "slots have been created");
for state in self.slots.iter() {
if state.slot_index == slot_index {
return Some(state.texture.clone());
}
}
None
}
pub fn device(&self) -> Option<windows::Win32::Graphics::Direct3D11::ID3D11Device> {
let dev = self.device.clone();
assert!(dev.is_some(), "device exists");
assert!(
!self.slots.is_empty() || self.width == 0,
"post-init invariant"
);
dev
}
pub fn context(&self) -> Option<windows::Win32::Graphics::Direct3D11::ID3D11DeviceContext> {
let ctx = self.context.clone();
assert!(ctx.is_some(), "context exists");
assert!(
self.width > 0 || self.slots.is_empty(),
"post-init invariant"
);
ctx
}
}
#[cfg(target_os = "windows")]
unsafe fn create_d3d11_device_windows() -> Result<
(
windows::Win32::Graphics::Direct3D11::ID3D11Device,
windows::Win32::Graphics::Direct3D11::ID3D11DeviceContext,
),
BackendError,
> {
use windows::Win32::Foundation::HMODULE;
use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_0};
use windows::Win32::Graphics::Direct3D11::{
D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_SDK_VERSION, D3D11CreateDevice, ID3D11Device,
ID3D11DeviceContext,
};
let mut device: Option<ID3D11Device> = None;
let mut context: Option<ID3D11DeviceContext> = None;
let feature_levels = [D3D_FEATURE_LEVEL_11_0];
unsafe {
D3D11CreateDevice(
None,
D3D_DRIVER_TYPE_HARDWARE,
HMODULE::default(),
D3D11_CREATE_DEVICE_BGRA_SUPPORT,
Some(&feature_levels),
D3D11_SDK_VERSION,
Some(&mut device),
None,
Some(&mut context),
)
}
.map_err(|_| BackendError::PlatformUnsupported {
reason: "D3D11CreateDevice failed",
})?;
let device = device.ok_or(BackendError::PlatformUnsupported {
reason: "D3D11 device null",
})?;
let context = context.ok_or(BackendError::PlatformUnsupported {
reason: "D3D11 context null",
})?;
Ok((device, context))
}
#[cfg(target_os = "windows")]
unsafe fn create_keyed_mutex_texture_windows(
device: &windows::Win32::Graphics::Direct3D11::ID3D11Device,
width: u32,
height: u32,
slot_index: u32,
) -> Result<
(
windows::Win32::Graphics::Direct3D11::ID3D11Texture2D,
windows::Win32::Graphics::Dxgi::IDXGIKeyedMutex,
u64,
),
BackendError,
> {
use windows::Win32::Graphics::Direct3D11::{
D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE,
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX, D3D11_RESOURCE_MISC_SHARED_NTHANDLE,
D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, ID3D11Texture2D,
};
use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC};
use windows::Win32::Graphics::Dxgi::{IDXGIKeyedMutex, IDXGIResource1};
use windows::core::Interface;
assert!(width > 0);
assert!(height > 0);
assert!(slot_index < 8);
let desc = D3D11_TEXTURE2D_DESC {
Width: width,
Height: height,
MipLevels: 1,
ArraySize: 1,
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
SampleDesc: DXGI_SAMPLE_DESC {
Count: 1,
Quality: 0,
},
Usage: D3D11_USAGE_DEFAULT,
BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32,
CPUAccessFlags: 0,
MiscFlags: (D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0)
as u32,
};
let mut texture: Option<ID3D11Texture2D> = None;
unsafe { device.CreateTexture2D(&desc, None, Some(&mut texture)) }.map_err(|_| {
BackendError::PlatformUnsupported {
reason: "CreateTexture2D failed",
}
})?;
let texture = texture.ok_or(BackendError::PlatformUnsupported {
reason: "texture null",
})?;
let keyed_mutex: IDXGIKeyedMutex =
texture
.cast()
.map_err(|_| BackendError::PlatformUnsupported {
reason: "IDXGIKeyedMutex cast failed",
})?;
let resource1: IDXGIResource1 =
texture
.cast()
.map_err(|_| BackendError::PlatformUnsupported {
reason: "IDXGIResource1 cast failed",
})?;
let access_rw: u32 = windows::Win32::Graphics::Dxgi::DXGI_SHARED_RESOURCE_READ.0
| windows::Win32::Graphics::Dxgi::DXGI_SHARED_RESOURCE_WRITE.0;
let shared =
unsafe { resource1.CreateSharedHandle(None, access_rw, windows::core::PCWSTR::null()) }
.map_err(|_| BackendError::PlatformUnsupported {
reason: "CreateSharedHandle failed",
})?;
Ok((texture, keyed_mutex, shared.0 as u64))
}
#[cfg(target_os = "windows")]
const ACQUIRE_SYNC_WAIT_TIMEOUT: i32 = 0x102;
#[cfg(target_os = "windows")]
const ACQUIRE_SYNC_WAIT_ABANDONED: i32 = 0x80;
#[cfg(not(target_os = "windows"))]
pub struct D3D11KeyedMutexBackend;
#[cfg(not(target_os = "windows"))]
impl D3D11KeyedMutexBackend {
pub fn new() -> Result<Self, BackendError> {
Err(BackendError::PlatformUnsupported {
reason: "D3D11 keyed-mutex backend is only available on Windows",
})
}
}
impl KeyedMutexBackend for D3D11KeyedMutexBackend {
type SlotHandle = D3D11SharedHandle;
const NUM_SLOTS: usize = 8;
#[cfg(target_os = "windows")]
fn create_slots(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<Vec<D3D11SharedHandle>, BackendError> {
assert!(self.slots.is_empty(), "slots created once");
if width == 0 || height == 0 {
return Err(BackendError::DimensionsOutOfRange { width, height });
}
if !matches!(format, TextureFormat::Nv12) {
return Err(BackendError::UnsupportedFormat { format });
}
let device = self
.device
.as_ref()
.ok_or(BackendError::PlatformUnsupported {
reason: "device dropped",
})?;
let mut out: Vec<D3D11SharedHandle> = Vec::with_capacity(Self::NUM_SLOTS);
for idx in 0..Self::NUM_SLOTS {
let idx_u32 = idx as u32;
let (texture, keyed_mutex, shared_handle) =
unsafe { create_keyed_mutex_texture_windows(device, width, height, idx_u32)? };
self.slots.push(D3D11SlotState {
texture,
keyed_mutex,
shared_handle,
slot_index: idx_u32,
expected_key: 0,
acquired: false,
completed: false,
});
out.push(D3D11SharedHandle {
raw_handle: shared_handle,
slot_index: idx_u32,
width,
height,
});
}
self.width = width;
self.height = height;
self.format = format;
assert_eq!(out.len(), Self::NUM_SLOTS);
assert_eq!(self.slots.len(), Self::NUM_SLOTS);
Ok(out)
}
#[cfg(not(target_os = "windows"))]
fn create_slots(
&mut self,
_width: u32,
_height: u32,
_format: TextureFormat,
) -> Result<Vec<D3D11SharedHandle>, BackendError> {
Err(BackendError::PlatformUnsupported {
reason: "real D3D11 create_slots requires Windows",
})
}
#[cfg(target_os = "windows")]
fn acquire_write(&mut self, slot: &D3D11SharedHandle, key: u64) -> Result<(), BackendError> {
use windows::core::Interface;
let position = self
.find_slot(slot)
.ok_or(BackendError::PlatformUnsupported {
reason: "slot not found",
})?;
let state = &mut self.slots[position];
if state.expected_key != key {
return Err(BackendError::KeyMismatch {
expected: state.expected_key,
observed: key,
});
}
if state.acquired {
return Err(BackendError::AcquireWhileWriting {
slot_index: state.slot_index,
});
}
let hr = unsafe {
(Interface::vtable(&state.keyed_mutex).AcquireSync)(
Interface::as_raw(&state.keyed_mutex),
key,
0,
)
};
if hr.0 == ACQUIRE_SYNC_WAIT_TIMEOUT || hr.0 == ACQUIRE_SYNC_WAIT_ABANDONED {
return Err(BackendError::WouldBlock {
slot_index: state.slot_index,
});
}
if hr.is_err() {
return Err(BackendError::KeyMismatch {
expected: key,
observed: u64::MAX,
});
}
state.acquired = true;
state.completed = false;
assert!(state.acquired);
Ok(())
}
#[cfg(not(target_os = "windows"))]
fn acquire_write(&mut self, _slot: &D3D11SharedHandle, _key: u64) -> Result<(), BackendError> {
Err(BackendError::PlatformUnsupported {
reason: "real D3D11 acquire_write requires Windows",
})
}
#[cfg(target_os = "windows")]
fn release_write(
&mut self,
slot: &D3D11SharedHandle,
next_key: u64,
) -> Result<(), BackendError> {
let position = self
.find_slot(slot)
.ok_or(BackendError::PlatformUnsupported {
reason: "slot not found",
})?;
let state = &mut self.slots[position];
if !state.acquired {
return Err(BackendError::ReleaseWithoutAcquire {
slot_index: state.slot_index,
});
}
unsafe { state.keyed_mutex.ReleaseSync(next_key) }.map_err(|_| {
BackendError::KeyMismatch {
expected: next_key,
observed: u64::MAX,
}
})?;
state.acquired = false;
state.expected_key = next_key;
state.completed = true;
assert!(!state.acquired);
assert!(state.completed);
Ok(())
}
#[cfg(not(target_os = "windows"))]
fn release_write(
&mut self,
_slot: &D3D11SharedHandle,
_next_key: u64,
) -> Result<(), BackendError> {
Err(BackendError::PlatformUnsupported {
reason: "real D3D11 release_write requires Windows",
})
}
#[cfg(target_os = "windows")]
fn poll_complete(&mut self, slot: &D3D11SharedHandle) -> bool {
match self.find_slot(slot) {
Some(position) => self.slots[position].completed,
None => false,
}
}
#[cfg(not(target_os = "windows"))]
fn poll_complete(&mut self, _slot: &D3D11SharedHandle) -> bool {
false
}
#[cfg(target_os = "windows")]
fn mark_consumed(&mut self, slot: &D3D11SharedHandle) {
if let Some(position) = self.find_slot(slot) {
self.slots[position].completed = false;
}
}
#[cfg(not(target_os = "windows"))]
fn mark_consumed(&mut self, _slot: &D3D11SharedHandle) {}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg(not(target_os = "windows"))]
fn instance_creation_fails_on_non_windows() {
let err = D3D11KeyedMutexBackend::new().err();
assert!(matches!(
err,
Some(BackendError::PlatformUnsupported { .. })
));
}
#[test]
fn handle_clones_preserve_index() {
let h = D3D11SharedHandle {
raw_handle: 0xdead,
slot_index: 3,
width: 1920,
height: 1080,
};
let h2 = h.clone();
assert_eq!(h2.slot_index, 3);
assert_eq!(h2.raw_handle, 0xdead);
}
#[test]
#[cfg(target_os = "windows")]
fn windows_real_keyed_mutex_eight_slots_round_trip() {
let backend_result = D3D11KeyedMutexBackend::new();
let mut backend = match backend_result {
Ok(b) => b,
Err(BackendError::PlatformUnsupported { .. }) => return,
Err(other) => unreachable!("unexpected backend init err: {other:?}"),
};
let slots_result = backend.create_slots(64, 64, TextureFormat::Nv12);
let slots = match slots_result {
Ok(s) => s,
Err(BackendError::PlatformUnsupported { .. }) => return,
Err(other) => unreachable!("unexpected create_slots err: {other:?}"),
};
assert_eq!(slots.len(), 8);
for (idx, slot) in slots.iter().enumerate() {
assert_eq!(slot.slot_index, idx as u32);
assert_ne!(slot.raw_handle, 0);
assert_eq!(slot.width, 64);
}
let slot = slots[0].clone();
backend.acquire_write(&slot, 0).expect("acquire key=0");
backend.release_write(&slot, 1).expect("release key=1");
assert!(backend.poll_complete(&slot));
backend.mark_consumed(&slot);
assert!(!backend.poll_complete(&slot));
backend.acquire_write(&slot, 1).expect("acquire key=1");
backend.release_write(&slot, 2).expect("release key=2");
assert!(backend.poll_complete(&slot));
}
}
@@ -0,0 +1,819 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::ring::RingError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EncoderDims {
pub width: u32,
pub height: u32,
}
impl EncoderDims {
pub fn new(width: u32, height: u32) -> Self {
assert!(width > 0, "width must be positive");
assert!(height > 0, "height must be positive");
Self { width, height }
}
}
pub const ENCODER_FRAME_RATE_MIN: u32 = 1;
pub const ENCODER_FRAME_RATE_MAX: u32 = 240;
pub const ENCODER_FRAME_RATE_DEFAULT: u32 = 60;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EncoderFrameRate {
pub numerator: u32,
pub denominator: u32,
}
impl EncoderFrameRate {
pub fn new(numerator: u32, denominator: u32) -> Self {
assert!(
numerator >= ENCODER_FRAME_RATE_MIN,
"fps numerator positive"
);
assert!(denominator > 0, "fps denominator positive");
let frame_rate = Self {
numerator: numerator.min(ENCODER_FRAME_RATE_MAX),
denominator,
};
assert!(frame_rate.numerator >= ENCODER_FRAME_RATE_MIN);
assert!(frame_rate.denominator > 0);
frame_rate
}
pub fn from_fps(fps: u32) -> Self {
let numerator = fps.clamp(ENCODER_FRAME_RATE_MIN, ENCODER_FRAME_RATE_MAX);
Self::new(numerator, 1)
}
pub fn frame_interval_us(self) -> u64 {
let numerator = u64::from(self.numerator);
let denominator = u64::from(self.denominator);
assert!(numerator > 0, "fps numerator positive");
assert!(denominator > 0, "fps denominator positive");
((1_000_000u64 * denominator) + numerator - 1) / numerator
}
pub fn gop_pic_size(self) -> u16 {
let rounded = (u64::from(self.numerator) + u64::from(self.denominator) - 1)
/ u64::from(self.denominator);
let bounded = rounded.clamp(1, u64::from(ENCODER_FRAME_RATE_MAX));
bounded as u16
}
}
impl Default for EncoderFrameRate {
fn default() -> Self {
Self::from_fps(ENCODER_FRAME_RATE_DEFAULT)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EncoderSubmission {
pub shared_handle: u64,
pub keyed_mutex_key: u64,
pub dims: EncoderDims,
pub sequence: u64,
pub capture_pts_us: Option<u64>,
}
impl EncoderSubmission {
pub fn new(shared_handle: u64, keyed_mutex_key: u64, dims: EncoderDims, sequence: u64) -> Self {
assert!(shared_handle != 0, "shared handle must be non-zero");
assert!(dims.width > 0, "dims width positive");
let s = Self {
shared_handle,
keyed_mutex_key,
dims,
sequence,
capture_pts_us: None,
};
assert!(s.shared_handle == shared_handle, "post construct intact");
assert!(s.capture_pts_us.is_none(), "capture pts defaults absent");
s
}
pub fn with_capture_pts_us(mut self, capture_pts_us: u64) -> Self {
assert!(self.shared_handle != 0, "shared handle must be non-zero");
self.capture_pts_us = Some(capture_pts_us);
assert_eq!(
self.capture_pts_us,
Some(capture_pts_us),
"capture pts recorded"
);
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PicParams {
pub pts_us: u64,
pub force_keyframe: bool,
}
impl PicParams {
pub fn new(pts_us: u64, force_keyframe: bool) -> Self {
let p = Self {
pts_us,
force_keyframe,
};
assert!(p.pts_us == pts_us, "pts_us intact");
assert!(p.force_keyframe == force_keyframe, "force_keyframe intact");
p
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HandoffSlot {
pub slot_index: u32,
pub shared_handle: u64,
}
impl HandoffSlot {
pub fn new(slot_index: u32, shared_handle: u64) -> Self {
assert!(slot_index < 64, "slot_index within plausible bound");
assert!(shared_handle != 0, "shared_handle non-zero");
Self {
slot_index,
shared_handle,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EncodedBitstream {
pub data: Vec<u8>,
pub pts_us: u64,
pub dts_us: u64,
pub is_keyframe: bool,
}
impl EncodedBitstream {
pub fn new(data: Vec<u8>, pts_us: u64, dts_us: u64, is_keyframe: bool) -> Self {
assert!(!data.is_empty(), "encoded bitstream must be non-empty");
assert!(data.len() <= MAX_BITSTREAM_BYTES, "bitstream within cap");
Self {
data,
pts_us,
dts_us,
is_keyframe,
}
}
}
pub const MAX_BITSTREAM_BYTES: usize = 16 * 1024 * 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum EncoderError {
SdkNotFound {
vendor: &'static str,
dll: &'static str,
},
SymbolMissing {
vendor: &'static str,
symbol: &'static str,
},
SessionInitFailed {
vendor: &'static str,
status: i64,
},
RegisterFailed {
vendor: &'static str,
status: i64,
},
EncodeFailed {
vendor: &'static str,
status: i64,
},
BitstreamReadFailed {
vendor: &'static str,
status: i64,
},
SlotUnknown {
slot_index: u32,
},
KeyMismatch {
expected: u64,
observed: u64,
},
DimensionsOutOfRange {
width: u32,
height: u32,
},
PlatformUnsupported {
reason: &'static str,
},
BitstreamTooLarge {
byte_size: usize,
},
NotImplemented {
what: &'static str,
},
}
impl std::fmt::Display for EncoderError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::SdkNotFound { vendor, dll } => {
write!(f, "{vendor} SDK runtime '{dll}' not found")
}
Self::SymbolMissing { vendor, symbol } => {
write!(f, "{vendor} symbol '{symbol}' missing from runtime")
}
Self::SessionInitFailed { vendor, status } => {
write!(f, "{vendor} session init failed: status={status}")
}
Self::RegisterFailed { vendor, status } => {
write!(f, "{vendor} register failed: status={status}")
}
Self::EncodeFailed { vendor, status } => {
write!(f, "{vendor} encode failed: status={status}")
}
Self::BitstreamReadFailed { vendor, status } => {
write!(f, "{vendor} bitstream read failed: status={status}")
}
Self::SlotUnknown { slot_index } => write!(f, "slot {slot_index} unknown"),
Self::KeyMismatch { expected, observed } => {
write!(
f,
"keyed-mutex key mismatch: expected={expected} observed={observed}"
)
}
Self::DimensionsOutOfRange { width, height } => {
write!(f, "encoder dims out of range: {width}x{height}")
}
Self::PlatformUnsupported { reason } => write!(f, "platform unsupported: {reason}"),
Self::BitstreamTooLarge { byte_size } => {
write!(f, "bitstream {byte_size} bytes exceeds cap")
}
Self::NotImplemented { what } => write!(f, "not implemented: {what}"),
}
}
}
impl std::error::Error for EncoderError {}
pub trait EncoderCompletionCallback: Send {
fn on_complete(&mut self, sequence: u64, encoded_bytes: u32);
}
pub trait NvencHandoff: Send {
fn register_slot(
&mut self,
shared_handle: u64,
key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError>;
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError>;
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream>;
fn unregister_slot(&mut self, slot: HandoffSlot);
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError>;
}
pub trait AmfHandoff: Send {
fn register_slot(
&mut self,
shared_handle: u64,
key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError>;
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError>;
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream>;
fn unregister_slot(&mut self, slot: HandoffSlot);
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError>;
}
pub trait QsvHandoff: Send {
fn register_slot(
&mut self,
shared_handle: u64,
key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError>;
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError>;
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream>;
fn unregister_slot(&mut self, slot: HandoffSlot);
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError>;
}
pub trait VideoToolboxHandoff: Send {
fn register_slot(
&mut self,
iosurface_handle: u64,
key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError>;
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError>;
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream>;
fn unregister_slot(&mut self, slot: HandoffSlot);
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError>;
}
pub struct VtNoOpHandoff {
accepted: u64,
next_slot_index: u32,
pending: std::collections::VecDeque<(u64, u64)>,
}
impl VtNoOpHandoff {
pub fn new() -> Self {
let h = Self {
accepted: 0,
next_slot_index: 0,
pending: std::collections::VecDeque::with_capacity(16),
};
assert_eq!(h.accepted, 0, "fresh handoff has no accepted frames");
assert_eq!(h.next_slot_index, 0, "fresh handoff slot index zero");
h
}
pub fn accepted_count(&self) -> u64 {
let n = self.accepted;
assert!(
self.pending.len() <= u32::MAX as usize,
"pending queue plausible"
);
assert!(n >= self.pending.len() as u64, "accepted >= pending");
n
}
pub fn pending_len(&self) -> usize {
let len = self.pending.len();
assert!(
len <= self.pending.capacity().max(1),
"pending within capacity bound"
);
assert!(len as u64 <= self.accepted, "pending <= accepted");
len
}
}
impl Default for VtNoOpHandoff {
fn default() -> Self {
Self::new()
}
}
impl VideoToolboxHandoff for VtNoOpHandoff {
fn register_slot(
&mut self,
iosurface_handle: u64,
_key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError> {
if iosurface_handle == 0 {
return Err(EncoderError::SlotUnknown {
slot_index: u32::MAX,
});
}
if dims.width == 0 || dims.height == 0 {
return Err(EncoderError::DimensionsOutOfRange {
width: dims.width,
height: dims.height,
});
}
let slot = HandoffSlot::new(self.next_slot_index, iosurface_handle);
self.next_slot_index = self.next_slot_index.saturating_add(1);
assert!(
slot.shared_handle == iosurface_handle,
"slot handle round-trip"
);
assert!(self.next_slot_index > 0, "slot counter advanced");
Ok(slot)
}
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
_key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError> {
if dims.width == 0 || dims.height == 0 {
return Err(EncoderError::DimensionsOutOfRange {
width: dims.width,
height: dims.height,
});
}
self.pending.push_back((self.accepted, pic_params.pts_us));
self.accepted = self.accepted.saturating_add(1);
assert!(slot.shared_handle != 0, "async slot handle non-zero");
assert!(self.accepted > 0, "encode_shared_async advanced accepted");
Ok(())
}
fn poll_completed(&mut self, _slot: HandoffSlot) -> Option<EncodedBitstream> {
None
}
fn unregister_slot(&mut self, slot: HandoffSlot) {
assert!(slot.shared_handle != 0, "unregister slot handle non-zero");
}
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError> {
if submission.shared_handle == 0 {
return Err(RingError::UnknownSlot);
}
if submission.dims.width == 0 || submission.dims.height == 0 {
return Err(RingError::BackendFailed {
source: crate::backend::BackendError::DimensionsOutOfRange {
width: submission.dims.width,
height: submission.dims.height,
},
});
}
let pre_accepted = self.accepted;
self.pending.push_back((submission.sequence, 0));
self.accepted = self.accepted.saturating_add(1);
callback.on_complete(submission.sequence, 0);
assert!(
self.accepted == pre_accepted + 1,
"VtNoOpHandoff accepted advanced"
);
assert!(submission.dims.width > 0, "submission dims preserved");
Ok(())
}
}
pub struct NotImplementedHandoff {
pub vendor: &'static str,
}
impl NotImplementedHandoff {
pub fn nvenc() -> Self {
Self { vendor: "nvenc" }
}
pub fn amf() -> Self {
Self { vendor: "amf" }
}
pub fn qsv() -> Self {
Self { vendor: "qsv" }
}
}
pub fn compute_dts_offset_us(first_pts_us: u64, num_b_frames: u32, frame_interval_us: u64) -> i64 {
assert!(frame_interval_us > 0, "frame interval positive");
assert!(num_b_frames <= 8, "B-frame count plausible");
let offset = (num_b_frames as u64).saturating_mul(frame_interval_us);
let result = -(offset as i64);
let _ = first_pts_us;
assert!(result <= 0, "DTS offset is non-positive for B-frames");
result
}
pub fn apply_dts_offset(pts_us: u64, offset_us: i64) -> u64 {
let signed_pts = pts_us as i64;
let dts = signed_pts.saturating_add(offset_us);
let clamped = if dts < 0 { 0 } else { dts as u64 };
assert!(
clamped <= pts_us || offset_us > 0,
"DTS <= PTS without future B-frames"
);
clamped
}
macro_rules! impl_not_implemented_for_trait {
($trait_name:ident, $vendor_tag:expr) => {
impl $trait_name for NotImplementedHandoff {
fn register_slot(
&mut self,
shared_handle: u64,
_key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError> {
assert!(shared_handle != 0, "shared_handle non-zero");
assert!(dims.width > 0, "dims width positive");
Err(EncoderError::NotImplemented {
what: concat!($vendor_tag, "::register_slot"),
})
}
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
_key: u64,
dims: EncoderDims,
_pic_params: PicParams,
) -> Result<(), EncoderError> {
assert!(slot.shared_handle != 0, "slot shared_handle non-zero");
assert!(dims.width > 0, "dims width positive");
Err(EncoderError::NotImplemented {
what: concat!($vendor_tag, "::encode_shared_async"),
})
}
fn poll_completed(&mut self, _slot: HandoffSlot) -> Option<EncodedBitstream> {
None
}
fn unregister_slot(&mut self, slot: HandoffSlot) {
assert!(slot.shared_handle != 0, "slot shared_handle non-zero");
}
fn encode_shared(
&mut self,
submission: EncoderSubmission,
_callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError> {
assert!(submission.shared_handle != 0, "submission handle non-zero");
assert!(submission.dims.width > 0, "submission width positive");
Err(RingError::NotImplemented {
what: concat!($vendor_tag, "::encode_shared"),
})
}
}
};
}
impl_not_implemented_for_trait!(NvencHandoff, "NvencHandoff");
impl_not_implemented_for_trait!(AmfHandoff, "AmfHandoff");
impl_not_implemented_for_trait!(QsvHandoff, "QsvHandoff");
#[cfg(test)]
mod tests {
use super::*;
struct NoopCallback;
impl EncoderCompletionCallback for NoopCallback {
fn on_complete(&mut self, _sequence: u64, _encoded_bytes: u32) {}
}
fn submission() -> EncoderSubmission {
EncoderSubmission::new(0xfeed_face, 7, EncoderDims::new(1920, 1080), 42)
}
#[test]
fn nvenc_stub_returns_not_implemented() {
let mut h = NotImplementedHandoff::nvenc();
let mut cb = NoopCallback;
let err = NvencHandoff::encode_shared(&mut h, submission(), &mut cb).err();
assert!(matches!(err, Some(RingError::NotImplemented { what })
if what.contains("Nvenc")));
}
#[test]
fn amf_stub_returns_not_implemented() {
let mut h = NotImplementedHandoff::amf();
let mut cb = NoopCallback;
let err = AmfHandoff::encode_shared(&mut h, submission(), &mut cb).err();
assert!(matches!(err, Some(RingError::NotImplemented { what })
if what.contains("Amf")));
}
#[test]
fn qsv_stub_returns_not_implemented() {
let mut h = NotImplementedHandoff::qsv();
let mut cb = NoopCallback;
let err = QsvHandoff::encode_shared(&mut h, submission(), &mut cb).err();
assert!(matches!(err, Some(RingError::NotImplemented { what })
if what.contains("Qsv")));
}
#[test]
fn encoder_dims_rejects_zero_width_via_assert() {
let result = std::panic::catch_unwind(|| EncoderDims::new(0, 1080));
assert!(result.is_err());
}
#[test]
fn nvenc_stub_register_returns_not_implemented() {
let mut h = NotImplementedHandoff::nvenc();
let dims = EncoderDims::new(1920, 1080);
let err = NvencHandoff::register_slot(&mut h, 0xabc, 0, dims).err();
assert!(matches!(err, Some(EncoderError::NotImplemented { what })
if what.contains("Nvenc")));
}
#[test]
fn amf_stub_register_returns_not_implemented() {
let mut h = NotImplementedHandoff::amf();
let dims = EncoderDims::new(1920, 1080);
let err = AmfHandoff::register_slot(&mut h, 0xabc, 0, dims).err();
assert!(matches!(err, Some(EncoderError::NotImplemented { what })
if what.contains("Amf")));
}
#[test]
fn qsv_stub_register_returns_not_implemented() {
let mut h = NotImplementedHandoff::qsv();
let dims = EncoderDims::new(1920, 1080);
let err = QsvHandoff::register_slot(&mut h, 0xabc, 0, dims).err();
assert!(matches!(err, Some(EncoderError::NotImplemented { what })
if what.contains("Qsv")));
}
#[test]
fn stub_poll_completed_returns_none() {
let mut h_nv = NotImplementedHandoff::nvenc();
let mut h_amf = NotImplementedHandoff::amf();
let mut h_qsv = NotImplementedHandoff::qsv();
let slot = HandoffSlot::new(0, 0xdead);
assert!(NvencHandoff::poll_completed(&mut h_nv, slot).is_none());
assert!(AmfHandoff::poll_completed(&mut h_amf, slot).is_none());
assert!(QsvHandoff::poll_completed(&mut h_qsv, slot).is_none());
}
#[test]
fn stub_unregister_does_not_panic() {
let mut h = NotImplementedHandoff::nvenc();
let slot = HandoffSlot::new(0, 0xdead);
NvencHandoff::unregister_slot(&mut h, slot);
}
#[test]
fn encoded_bitstream_rejects_empty() {
let result = std::panic::catch_unwind(|| EncodedBitstream::new(vec![], 0, 0, true));
assert!(result.is_err());
}
#[test]
fn dts_offset_zero_for_no_b_frames() {
let offset = compute_dts_offset_us(1000, 0, 16_666);
assert_eq!(offset, 0);
}
#[test]
fn dts_offset_negative_for_b_frames() {
let offset = compute_dts_offset_us(1000, 2, 16_666);
assert_eq!(offset, -(2 * 16_666_i64));
}
#[test]
fn dts_offset_application_clamps_to_zero() {
let dts = apply_dts_offset(100, -1000);
assert_eq!(dts, 0);
}
#[test]
fn dts_offset_application_below_pts_for_b_frames() {
let pts = 100_000_u64;
let dts = apply_dts_offset(pts, -33_333);
assert!(dts < pts);
assert_eq!(dts, 66_667);
}
#[test]
fn encoder_frame_rate_derives_interval_and_gop() {
let sixty = EncoderFrameRate::from_fps(60);
assert_eq!(sixty.frame_interval_us(), 16_667);
assert_eq!(sixty.gop_pic_size(), 60);
let capped = EncoderFrameRate::from_fps(999);
assert_eq!(capped.numerator, ENCODER_FRAME_RATE_MAX);
assert_eq!(capped.gop_pic_size(), ENCODER_FRAME_RATE_MAX as u16);
}
#[test]
fn handoff_slot_rejects_zero_handle() {
let result = std::panic::catch_unwind(|| HandoffSlot::new(0, 0));
assert!(result.is_err());
}
#[test]
fn submission_capture_pts_defaults_absent_and_round_trips() {
let s = submission();
assert_eq!(s.capture_pts_us, None);
let with_pts = s.with_capture_pts_us(123_456);
assert_eq!(with_pts.capture_pts_us, Some(123_456));
assert_eq!(with_pts.sequence, s.sequence);
assert_eq!(with_pts.shared_handle, s.shared_handle);
}
struct CountingCallback {
seen: Vec<(u64, u32)>,
}
impl EncoderCompletionCallback for CountingCallback {
fn on_complete(&mut self, sequence: u64, encoded_bytes: u32) {
self.seen.push((sequence, encoded_bytes));
}
}
#[test]
fn vt_noop_accepts_frames_in_fifo_order() {
let mut h = VtNoOpHandoff::new();
let mut cb = CountingCallback { seen: Vec::new() };
let dims = EncoderDims::new(1920, 1080);
for seq in 1..=5u64 {
let s = EncoderSubmission::new(0xfeed_face_u64, 0, dims, seq);
VideoToolboxHandoff::encode_shared(&mut h, s, &mut cb).expect("vt encode_shared");
}
assert_eq!(cb.seen.len(), 5);
for (idx, &(seq, _)) in cb.seen.iter().enumerate() {
assert_eq!(seq, (idx as u64) + 1, "fifo sequence");
}
assert_eq!(h.accepted_count(), 5);
}
#[test]
fn vt_noop_register_returns_slot() {
let mut h = VtNoOpHandoff::new();
let dims = EncoderDims::new(1920, 1080);
let slot_a =
VideoToolboxHandoff::register_slot(&mut h, 0xabc, 0, dims).expect("register a");
let slot_b =
VideoToolboxHandoff::register_slot(&mut h, 0xdef, 0, dims).expect("register b");
assert_eq!(slot_a.shared_handle, 0xabc);
assert_eq!(slot_b.shared_handle, 0xdef);
assert_ne!(slot_a.slot_index, slot_b.slot_index);
}
#[test]
fn vt_noop_register_rejects_zero_handle() {
let mut h = VtNoOpHandoff::new();
let dims = EncoderDims::new(1920, 1080);
let err = VideoToolboxHandoff::register_slot(&mut h, 0, 0, dims).err();
assert!(matches!(err, Some(EncoderError::SlotUnknown { .. })));
}
#[test]
fn vt_noop_encode_shared_rejects_zero_handle() {
let mut h = VtNoOpHandoff::new();
let mut cb = CountingCallback { seen: Vec::new() };
let dims = EncoderDims::new(1920, 1080);
let s = EncoderSubmission {
shared_handle: 0,
keyed_mutex_key: 0,
dims,
sequence: 1,
capture_pts_us: None,
};
let err = VideoToolboxHandoff::encode_shared(&mut h, s, &mut cb).err();
assert!(matches!(err, Some(RingError::UnknownSlot)));
assert!(cb.seen.is_empty(), "no callback on rejection");
}
#[test]
fn vt_noop_encode_shared_async_advances_accepted() {
let mut h = VtNoOpHandoff::new();
let dims = EncoderDims::new(1280, 720);
let slot = HandoffSlot::new(0, 0xfeed);
let params = PicParams::new(16_666, false);
VideoToolboxHandoff::encode_shared_async(&mut h, slot, 0, dims, params)
.expect("async encode ok");
assert_eq!(h.accepted_count(), 1);
assert_eq!(h.pending_len(), 1);
}
}
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#![deny(clippy::too_many_lines)]
#![deny(clippy::unwrap_used)]
#![deny(clippy::panic)]
#![deny(warnings)]
pub mod backend;
pub mod d3d11;
pub mod encoder_handoff;
pub mod metal_iosurface;
pub mod ring;
#[cfg(target_os = "macos")]
pub mod metal_iosurface_macos;
#[cfg(target_os = "macos")]
pub mod vt_compression_macos;
#[cfg(target_os = "windows")]
pub mod amf;
#[cfg(target_os = "windows")]
pub mod nvenc;
#[cfg(target_os = "windows")]
pub mod qsv;
pub use backend::{CpuMemcpyBackend, CpuSlotHandle, KeyedMutexBackend, TextureFormat};
pub use d3d11::D3D11KeyedMutexBackend;
pub use encoder_handoff::{
AmfHandoff, EncodedBitstream, EncoderDims, EncoderError, EncoderFrameRate, EncoderSubmission,
HandoffSlot, NotImplementedHandoff, NvencHandoff, PicParams, QsvHandoff, VideoToolboxHandoff,
VtNoOpHandoff, apply_dts_offset, compute_dts_offset_us,
};
pub use metal_iosurface::{
IoSurfaceSlotHandle, METAL_IOSURFACE_SEED_BASE, MetalSharedTextureBackend,
};
pub use ring::{
DUPLICATE_COUNT_MAX, EncoderInputRing, EncoderReady, FillReservation, RING_SIZE, RingError,
RingMetrics,
};
#[cfg(target_os = "macos")]
pub use vt_compression_macos::{VtCompressionHandoff, VtPixelTransfer};
#[cfg(target_os = "windows")]
pub use amf::AmfD3D11Handoff;
#[cfg(target_os = "windows")]
pub use nvenc::{COMPLETION_RING_CAPACITY as NVENC_COMPLETION_RING_CAPACITY, NvencD3D11Handoff};
#[cfg(target_os = "windows")]
pub use qsv::QsvD3D11Handoff;
pub const NV12_BPP_NUMERATOR: u32 = 3;
pub const NV12_BPP_DENOMINATOR: u32 = 2;
pub const MAX_FRAME_WIDTH: u32 = 7680;
pub const MAX_FRAME_HEIGHT: u32 = 4320;
#[inline]
pub const fn nv12_byte_size(width: u32, height: u32) -> usize {
let w = width as usize;
let h = height as usize;
(w * h * NV12_BPP_NUMERATOR as usize) / NV12_BPP_DENOMINATOR as usize
}
@@ -0,0 +1,447 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use crate::backend::{BackendError, KeyedMutexBackend, NUM_SLOTS_DEFAULT, TextureFormat};
use crate::{MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH};
pub const METAL_IOSURFACE_SEED_BASE: u64 = 0;
#[derive(Clone)]
pub struct IoSurfaceSlotHandle {
inner: Arc<IoSurfaceSlotInner>,
}
impl IoSurfaceSlotHandle {
pub fn slot_index(&self) -> u32 {
let idx = self.inner.slot_index;
assert!((idx as usize) < NUM_SLOTS_DEFAULT, "slot_index in range");
assert!(
self.inner.iosurface_handle != 0,
"iosurface_handle non-zero"
);
idx
}
pub fn iosurface_handle(&self) -> u64 {
let h = self.inner.iosurface_handle;
assert!(h != 0, "iosurface_handle non-zero on read");
assert!(
(self.inner.slot_index as usize) < NUM_SLOTS_DEFAULT,
"slot_index intact"
);
h
}
#[cfg(target_os = "macos")]
pub fn iosurface_ptr(&self) -> *mut core::ffi::c_void {
let p = self.inner.iosurface_ptr;
assert!(!p.is_null(), "iosurface_ptr non-null on read");
assert!(
self.inner.iosurface_handle != 0,
"handle non-zero alongside ptr"
);
p
}
pub fn current_key(&self) -> u64 {
let key = self.inner.current_key.load(Ordering::Acquire);
assert!(
self.inner.iosurface_handle != 0,
"handle intact during key read"
);
assert!(
(self.inner.slot_index as usize) < NUM_SLOTS_DEFAULT,
"slot_index intact"
);
key
}
pub fn is_acquired(&self) -> bool {
let a = self.inner.acquired.load(Ordering::Acquire);
assert!(
self.inner.iosurface_handle != 0,
"handle intact during acquired probe"
);
assert!(
(self.inner.slot_index as usize) < NUM_SLOTS_DEFAULT,
"slot_index intact"
);
a
}
}
struct IoSurfaceSlotInner {
slot_index: u32,
iosurface_handle: u64,
#[cfg(target_os = "macos")]
iosurface_ptr: *mut core::ffi::c_void,
current_key: AtomicU64,
acquired: AtomicBool,
completed: AtomicBool,
}
#[cfg(target_os = "macos")]
unsafe impl Send for IoSurfaceSlotInner {}
#[cfg(target_os = "macos")]
unsafe impl Sync for IoSurfaceSlotInner {}
pub struct MetalSharedTextureBackend {
width: u32,
height: u32,
format: TextureFormat,
slots_created: bool,
slot_handles: Vec<u64>,
#[cfg(target_os = "macos")]
iosurfaces: Vec<crate::metal_iosurface_macos::OwnedIoSurface>,
}
impl MetalSharedTextureBackend {
pub fn new() -> Self {
let backend = Self {
width: 0,
height: 0,
format: TextureFormat::Nv12,
slots_created: false,
slot_handles: Vec::with_capacity(NUM_SLOTS_DEFAULT),
#[cfg(target_os = "macos")]
iosurfaces: Vec::with_capacity(NUM_SLOTS_DEFAULT),
};
assert!(!backend.slots_created, "fresh backend has no slots");
assert!(
backend.slot_handles.is_empty(),
"fresh backend handles empty"
);
backend
}
pub fn width(&self) -> u32 {
assert!(self.width <= MAX_FRAME_WIDTH, "width within cap");
assert!(self.height <= MAX_FRAME_HEIGHT, "height within cap");
self.width
}
pub fn height(&self) -> u32 {
assert!(self.height <= MAX_FRAME_HEIGHT, "height within cap");
assert!(self.width <= MAX_FRAME_WIDTH, "width within cap");
self.height
}
pub fn slot_iosurface_handle(&self, slot_index: u32) -> Option<u64> {
let idx = slot_index as usize;
if idx >= self.slot_handles.len() {
return None;
}
let h = self.slot_handles[idx];
assert!(h != 0, "stored iosurface handle non-zero");
Some(h)
}
#[cfg(target_os = "macos")]
pub fn slot_iosurface_ptr(&self, slot_index: u32) -> Option<*mut core::ffi::c_void> {
let idx = slot_index as usize;
if idx >= self.iosurfaces.len() {
return None;
}
let p = self.iosurfaces[idx].as_ptr();
assert!(!p.is_null(), "stored iosurface ptr non-null");
assert!(
idx < self.slot_handles.len(),
"ptr slot mirrors handle slot"
);
Some(p)
}
#[cfg(target_os = "macos")]
pub fn slot_iosurface_mut(
&mut self,
slot_index: u32,
) -> Option<&mut crate::metal_iosurface_macos::OwnedIoSurface> {
let idx = slot_index as usize;
if idx >= self.iosurfaces.len() {
return None;
}
assert!(
idx < self.slot_handles.len(),
"mut slot mirrors handle slot"
);
Some(&mut self.iosurfaces[idx])
}
#[cfg(not(target_os = "macos"))]
fn allocate_slot_handles(
&mut self,
_width: u32,
_height: u32,
_format: TextureFormat,
) -> Result<Vec<u64>, BackendError> {
Err(BackendError::PlatformUnsupported {
reason: "MetalSharedTextureBackend requires macOS",
})
}
#[cfg(target_os = "macos")]
fn allocate_slot_handles(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<Vec<u64>, BackendError> {
if !matches!(format, TextureFormat::Nv12) {
return Err(BackendError::UnsupportedFormat { format });
}
let mut handles: Vec<u64> = Vec::with_capacity(NUM_SLOTS_DEFAULT);
let mut owned: Vec<crate::metal_iosurface_macos::OwnedIoSurface> =
Vec::with_capacity(NUM_SLOTS_DEFAULT);
for slot in 0..NUM_SLOTS_DEFAULT {
let surface = crate::metal_iosurface_macos::OwnedIoSurface::create_nv12(width, height)
.map_err(|_| BackendError::PlatformUnsupported {
reason: "IOSurfaceCreate failed",
})?;
let raw = surface.handle();
assert!(raw != 0, "IOSurface raw non-zero for slot");
assert!(slot < NUM_SLOTS_DEFAULT, "slot index in range");
handles.push(raw);
owned.push(surface);
}
self.iosurfaces = owned;
Ok(handles)
}
}
impl Default for MetalSharedTextureBackend {
fn default() -> Self {
Self::new()
}
}
impl KeyedMutexBackend for MetalSharedTextureBackend {
type SlotHandle = IoSurfaceSlotHandle;
const NUM_SLOTS: usize = NUM_SLOTS_DEFAULT;
fn create_slots(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<Vec<IoSurfaceSlotHandle>, BackendError> {
if width == 0 || height == 0 || width > MAX_FRAME_WIDTH || height > MAX_FRAME_HEIGHT {
return Err(BackendError::DimensionsOutOfRange { width, height });
}
if !matches!(format, TextureFormat::Nv12) {
return Err(BackendError::UnsupportedFormat { format });
}
assert!(!self.slots_created, "slots created once");
let raw_handles = self.allocate_slot_handles(width, height, format)?;
assert_eq!(
raw_handles.len(),
NUM_SLOTS_DEFAULT,
"allocator returns NUM_SLOTS handles"
);
let mut handles: Vec<IoSurfaceSlotHandle> = Vec::with_capacity(NUM_SLOTS_DEFAULT);
for (idx, raw) in raw_handles.iter().enumerate() {
assert!(*raw != 0, "raw iosurface handle non-zero");
#[cfg(target_os = "macos")]
let surface_ptr = self.iosurfaces[idx].as_ptr();
#[cfg(target_os = "macos")]
assert!(!surface_ptr.is_null(), "surface ptr non-null at slot setup");
let inner = IoSurfaceSlotInner {
slot_index: idx as u32,
iosurface_handle: *raw,
#[cfg(target_os = "macos")]
iosurface_ptr: surface_ptr,
current_key: AtomicU64::new(METAL_IOSURFACE_SEED_BASE),
acquired: AtomicBool::new(false),
completed: AtomicBool::new(false),
};
handles.push(IoSurfaceSlotHandle {
inner: Arc::new(inner),
});
}
self.width = width;
self.height = height;
self.format = format;
self.slot_handles = raw_handles;
self.slots_created = true;
assert_eq!(
handles.len(),
NUM_SLOTS_DEFAULT,
"returned handle vector length"
);
assert!(self.slots_created, "slots_created flipped");
Ok(handles)
}
fn acquire_write(&mut self, slot: &IoSurfaceSlotHandle, key: u64) -> Result<(), BackendError> {
assert!(self.slots_created, "slots must exist before acquire");
let current = slot.inner.current_key.load(Ordering::Acquire);
if current != key {
return Err(BackendError::KeyMismatch {
expected: key,
observed: current,
});
}
let was_acquired = slot.inner.acquired.swap(true, Ordering::AcqRel);
if was_acquired {
return Err(BackendError::AcquireWhileWriting {
slot_index: slot.inner.slot_index,
});
}
#[cfg(target_os = "macos")]
{
let idx = slot.inner.slot_index as usize;
if idx >= self.iosurfaces.len() {
slot.inner.acquired.store(false, Ordering::Release);
return Err(BackendError::PlatformUnsupported {
reason: "slot index out of range for IOSurface vector",
});
}
if let Err(e) = self.iosurfaces[idx].lock_for_writing() {
slot.inner.acquired.store(false, Ordering::Release);
return Err(e);
}
}
slot.inner.completed.store(false, Ordering::Release);
assert!(
slot.inner.acquired.load(Ordering::Acquire),
"acquired flag set"
);
assert!(
!slot.inner.completed.load(Ordering::Acquire),
"completed cleared"
);
Ok(())
}
fn release_write(
&mut self,
slot: &IoSurfaceSlotHandle,
next_key: u64,
) -> Result<(), BackendError> {
assert!(self.slots_created, "slots must exist before release");
let was_acquired = slot.inner.acquired.swap(false, Ordering::AcqRel);
if !was_acquired {
return Err(BackendError::ReleaseWithoutAcquire {
slot_index: slot.inner.slot_index,
});
}
#[cfg(target_os = "macos")]
{
let idx = slot.inner.slot_index as usize;
if idx >= self.iosurfaces.len() {
return Err(BackendError::PlatformUnsupported {
reason: "slot index out of range for IOSurface vector",
});
}
self.iosurfaces[idx].unlock_after_writing()?;
}
slot.inner.current_key.store(next_key, Ordering::Release);
slot.inner.completed.store(true, Ordering::Release);
assert!(
!slot.inner.acquired.load(Ordering::Acquire),
"acquired cleared"
);
assert!(
slot.inner.completed.load(Ordering::Acquire),
"completed flag set"
);
Ok(())
}
fn poll_complete(&mut self, slot: &IoSurfaceSlotHandle) -> bool {
let done = slot.inner.completed.load(Ordering::Acquire);
assert!(self.slots_created, "slots exist for poll");
assert!(
(slot.inner.slot_index as usize) < Self::NUM_SLOTS,
"slot index in range"
);
done
}
fn mark_consumed(&mut self, slot: &IoSurfaceSlotHandle) {
assert!(self.slots_created, "slots exist for mark_consumed");
assert!(
(slot.inner.slot_index as usize) < Self::NUM_SLOTS,
"slot index in range"
);
slot.inner.completed.store(false, Ordering::Release);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn off_macos_returns_platform_unsupported() {
let mut backend = MetalSharedTextureBackend::new();
let result = backend.create_slots(1920, 1080, TextureFormat::Nv12);
#[cfg(not(target_os = "macos"))]
{
assert!(matches!(
result,
Err(BackendError::PlatformUnsupported { .. })
));
}
#[cfg(target_os = "macos")]
{
let slots = result.expect("macos create_slots ok");
assert_eq!(slots.len(), 8);
for (idx, s) in slots.iter().enumerate() {
assert_eq!(s.slot_index(), idx as u32);
assert!(s.iosurface_handle() != 0);
}
}
}
#[test]
fn create_slots_rejects_zero_dims() {
let mut backend = MetalSharedTextureBackend::new();
let err = backend.create_slots(0, 1080, TextureFormat::Nv12).err();
assert!(matches!(
err,
Some(BackendError::DimensionsOutOfRange { .. })
));
}
#[test]
fn create_slots_rejects_unsupported_format() {
let mut backend = MetalSharedTextureBackend::new();
let err = backend.create_slots(1920, 1080, TextureFormat::P010).err();
assert!(matches!(err, Some(BackendError::UnsupportedFormat { .. })));
}
#[cfg(target_os = "macos")]
#[test]
fn macos_acquire_release_round_trip_marks_complete() {
let mut backend = MetalSharedTextureBackend::new();
let slots = backend
.create_slots(64, 64, TextureFormat::Nv12)
.expect("create");
let slot = slots[0].clone();
backend.acquire_write(&slot, 0).expect("acquire");
assert!(slot.is_acquired());
backend.release_write(&slot, 1).expect("release");
assert!(!slot.is_acquired());
assert!(backend.poll_complete(&slot));
backend.mark_consumed(&slot);
assert!(!backend.poll_complete(&slot));
}
#[cfg(target_os = "macos")]
#[test]
fn macos_double_acquire_rejects() {
let mut backend = MetalSharedTextureBackend::new();
let slots = backend
.create_slots(64, 64, TextureFormat::Nv12)
.expect("create");
let slot = slots[0].clone();
backend.acquire_write(&slot, 0).expect("first acquire");
let err = backend.acquire_write(&slot, 0).err();
assert!(matches!(
err,
Some(BackendError::AcquireWhileWriting { .. })
));
}
}
@@ -0,0 +1,187 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use core::ffi::c_void;
use core::ptr::NonNull;
use crate::backend::BackendError;
type IoSurfaceRef = *mut c_void;
type CfDictionaryRef = *const c_void;
type CfStringRef = *const c_void;
type CfNumberRef = *const c_void;
type CfAllocatorRef = *const c_void;
const KIO_RETURN_SUCCESS: i32 = 0;
const KCF_NUMBER_SINT32_TYPE: i32 = 3;
const IOSURFACE_PIXEL_FORMAT_420V: u32 = u32::from_be_bytes(*b"420v");
#[link(name = "IOSurface", kind = "framework")]
unsafe extern "C" {
fn IOSurfaceCreate(properties: CfDictionaryRef) -> IoSurfaceRef;
fn IOSurfaceLock(buffer: IoSurfaceRef, options: u32, seed: *mut u32) -> i32;
fn IOSurfaceUnlock(buffer: IoSurfaceRef, options: u32, seed: *mut u32) -> i32;
fn IOSurfaceGetID(buffer: IoSurfaceRef) -> u32;
}
#[link(name = "CoreFoundation", kind = "framework")]
unsafe extern "C" {
static kCFAllocatorDefault: CfAllocatorRef;
static kCFTypeDictionaryKeyCallBacks: *const c_void;
static kCFTypeDictionaryValueCallBacks: *const c_void;
fn CFDictionaryCreate(
allocator: CfAllocatorRef,
keys: *const *const c_void,
values: *const *const c_void,
num_values: isize,
key_callbacks: *const c_void,
value_callbacks: *const c_void,
) -> CfDictionaryRef;
fn CFNumberCreate(
allocator: CfAllocatorRef,
the_type: i32,
value_ptr: *const c_void,
) -> CfNumberRef;
fn CFStringCreateWithCString(
allocator: CfAllocatorRef,
c_str: *const i8,
encoding: u32,
) -> CfStringRef;
fn CFRelease(cf: *const c_void);
}
const KCFSTRING_ENCODING_UTF8: u32 = 0x0800_0100;
fn cf_str(literal: &'static str) -> CfStringRef {
assert!(literal.ends_with('\0'), "literal must be nul-terminated");
let bytes = literal.as_bytes();
unsafe {
CFStringCreateWithCString(
kCFAllocatorDefault,
bytes.as_ptr() as *const i8,
KCFSTRING_ENCODING_UTF8,
)
}
}
fn cf_num_i32(v: i32) -> CfNumberRef {
let ptr: *const i32 = &v;
unsafe {
CFNumberCreate(
kCFAllocatorDefault,
KCF_NUMBER_SINT32_TYPE,
ptr as *const c_void,
)
}
}
pub struct OwnedIoSurface {
surface: NonNull<c_void>,
}
unsafe impl Send for OwnedIoSurface {}
impl OwnedIoSurface {
pub fn create_nv12(width: u32, height: u32) -> Result<Self, BackendError> {
assert!(width > 0, "create_nv12 width positive");
assert!(
height > 0 && height.is_multiple_of(2),
"create_nv12 height positive and even"
);
let width_key = cf_str("IOSurfaceWidth\0");
let height_key = cf_str("IOSurfaceHeight\0");
let pf_key = cf_str("IOSurfacePixelFormat\0");
let bpe_key = cf_str("IOSurfaceBytesPerElement\0");
let width_val = cf_num_i32(width as i32);
let height_val = cf_num_i32(height as i32);
let pf_val = cf_num_i32(IOSURFACE_PIXEL_FORMAT_420V as i32);
let bpe_val = cf_num_i32(1);
let keys: [*const c_void; 4] = [width_key, height_key, pf_key, bpe_key];
let vals: [*const c_void; 4] = [width_val, height_val, pf_val, bpe_val];
let dict = unsafe {
CFDictionaryCreate(
kCFAllocatorDefault,
keys.as_ptr(),
vals.as_ptr(),
4,
kCFTypeDictionaryKeyCallBacks,
kCFTypeDictionaryValueCallBacks,
)
};
let raw = if dict.is_null() {
core::ptr::null_mut()
} else {
unsafe { IOSurfaceCreate(dict) }
};
unsafe {
CFRelease(width_key);
CFRelease(height_key);
CFRelease(pf_key);
CFRelease(bpe_key);
CFRelease(width_val);
CFRelease(height_val);
CFRelease(pf_val);
CFRelease(bpe_val);
if !dict.is_null() {
CFRelease(dict);
}
}
let surface = NonNull::new(raw).ok_or(BackendError::PlatformUnsupported {
reason: "IOSurfaceCreate returned null",
})?;
assert!(
unsafe { IOSurfaceGetID(surface.as_ptr()) } != 0,
"IOSurfaceGetID non-zero"
);
Ok(Self { surface })
}
pub fn handle(&self) -> u64 {
let id = unsafe { IOSurfaceGetID(self.surface.as_ptr()) };
assert!(id != 0, "IOSurfaceID non-zero on handle()");
assert!(self.surface.as_ptr() as usize != 0, "surface ptr non-null");
id as u64
}
pub fn as_ptr(&self) -> *mut c_void {
let p = self.surface.as_ptr();
assert!(!p.is_null(), "IOSurface raw pointer non-null");
assert!(unsafe { IOSurfaceGetID(p) } != 0, "IOSurfaceID non-zero");
p
}
pub fn lock_for_writing(&mut self) -> Result<(), BackendError> {
let mut seed: u32 = 0;
let status = unsafe { IOSurfaceLock(self.surface.as_ptr(), 0, &mut seed) };
if status != KIO_RETURN_SUCCESS {
return Err(BackendError::PlatformUnsupported {
reason: "IOSurfaceLock failed",
});
}
assert_eq!(status, KIO_RETURN_SUCCESS, "lock status ok");
assert!(seed < u32::MAX, "lock seed within range");
Ok(())
}
pub fn unlock_after_writing(&mut self) -> Result<(), BackendError> {
let mut seed: u32 = 0;
let status = unsafe { IOSurfaceUnlock(self.surface.as_ptr(), 0, &mut seed) };
if status != KIO_RETURN_SUCCESS {
return Err(BackendError::PlatformUnsupported {
reason: "IOSurfaceUnlock failed",
});
}
assert_eq!(status, KIO_RETURN_SUCCESS, "unlock status ok");
assert!(seed < u32::MAX, "unlock seed within range");
Ok(())
}
}
impl Drop for OwnedIoSurface {
fn drop(&mut self) {
let ptr = self.surface.as_ptr();
if !ptr.is_null() {
unsafe { CFRelease(ptr) };
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,964 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::collections::HashMap;
use std::ffi::c_void;
use std::ptr;
use std::sync::Arc;
use libloading::{Library, Symbol};
use windows::Win32::Graphics::Direct3D11::{ID3D11Device, ID3D11Multithread};
use windows::core::Interface;
use crate::encoder_handoff::{
EncodedBitstream, EncoderCompletionCallback, EncoderDims, EncoderError, EncoderFrameRate,
EncoderSubmission, HandoffSlot, PicParams, QsvHandoff, apply_dts_offset, compute_dts_offset_us,
};
use crate::ring::RingError;
pub const QSV_DLL_NAME_VPL: &str = "libvpl.dll";
pub const QSV_DLL_NAME_MFX: &str = "libmfxhw64.dll";
const MFX_IMPL_HARDWARE: i32 = 0x0002;
const MFX_IMPL_VIA_D3D11: i32 = 0x0300;
const MFX_IMPL_TYPE_HARDWARE: u32 = 2;
const MFX_ACCEL_MODE_VIA_D3D11: u32 = 0x0300;
const MFX_HANDLE_D3D11_DEVICE: u32 = 3;
const MFX_FOURCC_NV12: u32 = u32::from_le_bytes(*b"NV12");
const MFX_CODEC_AVC: u32 = u32::from_le_bytes(*b"AVC ");
const MFX_RATECONTROL_CBR: u16 = 1;
const MFX_PICSTRUCT_PROGRESSIVE: u16 = 0x01;
const MFX_CHROMAFORMAT_YUV420: u16 = 1;
const MFX_IOPATTERN_IN_VIDEO_MEMORY: u16 = 0x01;
const MFX_VARIANT_TYPE_U32: u32 = 5;
const MFX_VARIANT_VERSION_MINOR: u8 = 1;
const MFX_VARIANT_VERSION_MAJOR: u8 = 1;
const MFX_ERR_NONE: i32 = 0;
const MFX_WRN_IN_EXECUTION: i32 = 1;
const MFX_ERR_MORE_DATA: i32 = -10;
const FILTER_PROPERTY_IMPL: &[u8] = b"mfxImplDescription.Impl\0";
const FILTER_PROPERTY_ACCEL: &[u8] = b"mfxImplDescription.AccelerationMode\0";
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct MfxVersion {
minor: u16,
major: u16,
}
#[repr(C)]
#[derive(Default)]
struct MfxFrameInfo {
reserved: [u32; 4],
channel_id: u16,
bit_depth_luma: u16,
bit_depth_chroma: u16,
shift: u16,
frame_id_temporal: u16,
frame_id_priority: u16,
frame_id_view_or_dependency: u16,
frame_id_quality: u16,
four_cc: u32,
width: u16,
height: u16,
crop_x: u16,
crop_y: u16,
crop_w: u16,
crop_h: u16,
frame_rate_extn: u32,
frame_rate_extd: u32,
reserved3: u16,
aspect_ratio_w: u16,
aspect_ratio_h: u16,
pic_struct: u16,
chroma_format: u16,
reserved2: u16,
}
#[repr(C)]
#[derive(Default)]
struct MfxInfoMfx {
reserved: [u32; 7],
low_power: u16,
brc_param_multiplier: u16,
frame_info: MfxFrameInfo,
codec_id: u32,
codec_profile: u16,
codec_level: u16,
num_thread: u16,
target_usage: u16,
gop_pic_size: u16,
gop_ref_dist: u16,
gop_opt_flag: u16,
idr_interval: u16,
rate_control_method: u16,
init_qp: u16,
buffer_size_in_kb: u16,
target_kbps: u16,
max_kbps: u16,
num_slice: u16,
num_ref_frame: u16,
encoded_order: u16,
union_pad: [u16; 15],
}
#[repr(C)]
struct MfxVideoParam {
alloc_id: u32,
reserved: [u32; 2],
reserved3: u16,
async_depth: u16,
mfx: MfxInfoMfx,
protected: u16,
io_pattern: u16,
ext_param: *mut c_void,
num_ext_param: u16,
reserved2: u16,
}
#[repr(C)]
struct MfxBitstream {
encrypted_data: *mut c_void,
num_extparam: u16,
ext_param: *mut c_void,
reserved: [u32; 6],
decode_time_stamp: u64,
time_stamp: u64,
data: *mut u8,
data_offset: u32,
data_length: u32,
max_length: u32,
pic_struct: u16,
frame_type: u16,
data_flag: u16,
reserved2: u16,
}
#[repr(C)]
struct MfxFrameData {
ext_param: *mut c_void,
num_extparam: u16,
reserved: [u32; 8],
mem_type: u16,
pitch_high: u16,
time_stamp: u64,
frame_order: u32,
locked: u16,
pitch_low: u16,
plane_ptrs: [*mut u8; 7],
mem_id: *mut c_void,
corrupted: u16,
data_flag: u16,
}
#[repr(C)]
struct MfxFrameSurface1 {
reserved: [u32; 4],
interface_ptr: *mut c_void,
info: MfxFrameInfo,
data: MfxFrameData,
}
#[repr(C)]
#[derive(Clone, Copy)]
union MfxVariantData {
u32_: u32,
u64_: u64,
ptr: *mut c_void,
pad: [u8; 16],
}
#[repr(C)]
#[derive(Clone, Copy)]
struct MfxVariant {
version: MfxStructVersion,
type_: u32,
data: MfxVariantData,
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct MfxStructVersion {
minor: u8,
major: u8,
}
type MfxInit = unsafe extern "C" fn(i32, *mut MfxVersion, *mut *mut c_void) -> i32;
type MfxClose = unsafe extern "C" fn(*mut c_void) -> i32;
type MfxSetHandle = unsafe extern "C" fn(*mut c_void, u32, *mut c_void) -> i32;
type MfxEncodeInit = unsafe extern "C" fn(*mut c_void, *mut MfxVideoParam) -> i32;
type MfxEncodeClose = unsafe extern "C" fn(*mut c_void) -> i32;
type MfxEncodeQuery =
unsafe extern "C" fn(*mut c_void, *mut MfxVideoParam, *mut MfxVideoParam) -> i32;
type MfxEncodeFrameAsync = unsafe extern "C" fn(
*mut c_void,
*mut c_void,
*mut MfxFrameSurface1,
*mut MfxBitstream,
*mut *mut c_void,
) -> i32;
type MfxSyncOperation = unsafe extern "C" fn(*mut c_void, *mut c_void, u32) -> i32;
type MfxLoad = unsafe extern "C" fn() -> *mut c_void;
type MfxUnload = unsafe extern "C" fn(*mut c_void);
type MfxCreateConfig = unsafe extern "C" fn(*mut c_void) -> *mut c_void;
type MfxSetConfigFilterProperty = unsafe extern "C" fn(*mut c_void, *const u8, MfxVariant) -> i32;
type MfxCreateSession = unsafe extern "C" fn(*mut c_void, u32, *mut *mut c_void) -> i32;
struct ApiTable {
init: MfxInit,
close: MfxClose,
set_handle: MfxSetHandle,
encode_init: MfxEncodeInit,
encode_close: MfxEncodeClose,
encode_query: MfxEncodeQuery,
encode_frame_async: MfxEncodeFrameAsync,
sync_operation: MfxSyncOperation,
}
struct DispatcherTable {
load: MfxLoad,
unload: MfxUnload,
create_config: MfxCreateConfig,
set_config_property: MfxSetConfigFilterProperty,
create_session: MfxCreateSession,
}
struct SlotState {
pending_pts_us: u64,
pending_force_keyframe: bool,
sync_point: *mut c_void,
bitstream: MfxBitstream,
bitstream_buf: Vec<u8>,
surface: MfxFrameSurface1,
in_flight: bool,
}
pub struct QsvD3D11Handoff {
_library: Arc<Library>,
api: ApiTable,
dispatcher: Option<DispatcherTable>,
loader: *mut c_void,
session: *mut c_void,
slots: HashMap<u32, SlotState>,
next_slot_index: u32,
dts_offset_us: i64,
completed_count: u64,
target_kbps: u16,
frame_rate: EncoderFrameRate,
}
unsafe impl Send for QsvD3D11Handoff {}
impl QsvD3D11Handoff {
pub fn new(
device: ID3D11Device,
dims: EncoderDims,
bitrate_bps: u32,
) -> Result<Self, EncoderError> {
Self::new_with_frame_rate(device, dims, bitrate_bps, EncoderFrameRate::default())
}
pub fn new_with_frame_rate(
device: ID3D11Device,
dims: EncoderDims,
bitrate_bps: u32,
frame_rate: EncoderFrameRate,
) -> Result<Self, EncoderError> {
assert!(dims.width > 0, "dims width positive");
assert!(dims.height > 0, "dims height positive");
assert!(frame_rate.numerator > 0, "frame rate numerator positive");
assert!(
frame_rate.denominator > 0,
"frame rate denominator positive"
);
if dims.width > 7680 || dims.height > 4320 {
return Err(EncoderError::DimensionsOutOfRange {
width: dims.width,
height: dims.height,
});
}
let library = load_runtime()?;
let api = load_api(&library)?;
let dispatcher_opt = load_dispatcher(&library);
let (loader, session) = open_session(&api, dispatcher_opt.as_ref())?;
set_d3d11_handle(&api, session, &device)?;
let target_kbps = (bitrate_bps / 1000).clamp(500, 60_000) as u16;
encode_init(&api, session, dims, target_kbps, frame_rate)?;
let dts_offset_us = compute_dts_offset_us(0, 0, frame_rate.frame_interval_us());
let handoff = Self {
_library: Arc::new(library),
api,
dispatcher: dispatcher_opt,
loader,
session,
slots: HashMap::new(),
next_slot_index: 0,
dts_offset_us,
completed_count: 0,
target_kbps,
frame_rate,
};
assert!(!handoff.session.is_null(), "session non-null");
Ok(handoff)
}
}
fn load_runtime() -> Result<Library, EncoderError> {
let vpl_result = unsafe { Library::new(QSV_DLL_NAME_VPL) };
match vpl_result {
Ok(lib) => Ok(lib),
Err(_) => {
unsafe { Library::new(QSV_DLL_NAME_MFX) }.map_err(|_| EncoderError::SdkNotFound {
vendor: "qsv",
dll: QSV_DLL_NAME_MFX,
})
}
}
}
fn load_api(library: &Library) -> Result<ApiTable, EncoderError> {
let init: Symbol<'_, MfxInit> =
unsafe { library.get(b"MFXInit\0") }.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXInit",
})?;
let close: Symbol<'_, MfxClose> =
unsafe { library.get(b"MFXClose\0") }.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXClose",
})?;
let set_handle: Symbol<'_, MfxSetHandle> = unsafe { library.get(b"MFXVideoCORE_SetHandle\0") }
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoCORE_SetHandle",
})?;
let encode_init: Symbol<'_, MfxEncodeInit> = unsafe { library.get(b"MFXVideoENCODE_Init\0") }
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoENCODE_Init",
})?;
let encode_close: Symbol<'_, MfxEncodeClose> = unsafe {
library.get(b"MFXVideoENCODE_Close\0")
}
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoENCODE_Close",
})?;
let encode_query: Symbol<'_, MfxEncodeQuery> = unsafe {
library.get(b"MFXVideoENCODE_Query\0")
}
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoENCODE_Query",
})?;
let encode_frame_async: Symbol<'_, MfxEncodeFrameAsync> = unsafe {
library.get(b"MFXVideoENCODE_EncodeFrameAsync\0")
}
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoENCODE_EncodeFrameAsync",
})?;
let sync_operation: Symbol<'_, MfxSyncOperation> = unsafe {
library.get(b"MFXVideoCORE_SyncOperation\0")
}
.map_err(|_| EncoderError::SymbolMissing {
vendor: "qsv",
symbol: "MFXVideoCORE_SyncOperation",
})?;
Ok(ApiTable {
init: *init,
close: *close,
set_handle: *set_handle,
encode_init: *encode_init,
encode_close: *encode_close,
encode_query: *encode_query,
encode_frame_async: *encode_frame_async,
sync_operation: *sync_operation,
})
}
fn load_dispatcher(library: &Library) -> Option<DispatcherTable> {
let load: Symbol<'_, MfxLoad> = unsafe { library.get(b"MFXLoad\0") }.ok()?;
let unload: Symbol<'_, MfxUnload> = unsafe { library.get(b"MFXUnload\0") }.ok()?;
let create_config: Symbol<'_, MfxCreateConfig> =
unsafe { library.get(b"MFXCreateConfig\0") }.ok()?;
let set_config_property: Symbol<'_, MfxSetConfigFilterProperty> =
unsafe { library.get(b"MFXSetConfigFilterProperty\0") }.ok()?;
let create_session: Symbol<'_, MfxCreateSession> =
unsafe { library.get(b"MFXCreateSession\0") }.ok()?;
Some(DispatcherTable {
load: *load,
unload: *unload,
create_config: *create_config,
set_config_property: *set_config_property,
create_session: *create_session,
})
}
fn open_session(
api: &ApiTable,
dispatcher: Option<&DispatcherTable>,
) -> Result<(*mut c_void, *mut c_void), EncoderError> {
if let Some(d) = dispatcher {
match modern_session(d) {
Ok((loader, session)) => return Ok((loader, session)),
Err(e) => {
let session = init_session_legacy(api)?;
let _ = e;
return Ok((ptr::null_mut(), session));
}
}
}
let session = init_session_legacy(api)?;
Ok((ptr::null_mut(), session))
}
fn modern_session(d: &DispatcherTable) -> Result<(*mut c_void, *mut c_void), EncoderError> {
let loader = unsafe { (d.load)() };
if loader.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-mfxload",
status: -1,
});
}
assert!(!loader.is_null(), "MFXLoad returned non-null");
if let Err(e) = set_filter_u32(d, loader, FILTER_PROPERTY_IMPL, MFX_IMPL_TYPE_HARDWARE) {
unsafe { (d.unload)(loader) };
return Err(e);
}
if let Err(e) = set_filter_u32(d, loader, FILTER_PROPERTY_ACCEL, MFX_ACCEL_MODE_VIA_D3D11) {
unsafe { (d.unload)(loader) };
return Err(e);
}
let mut session: *mut c_void = ptr::null_mut();
let status = unsafe { (d.create_session)(loader, 0, &mut session) };
if status != MFX_ERR_NONE || session.is_null() {
unsafe { (d.unload)(loader) };
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-create-session",
status: status as i64,
});
}
assert!(!session.is_null(), "modern session non-null");
Ok((loader, session))
}
fn set_filter_u32(
d: &DispatcherTable,
loader: *mut c_void,
property: &'static [u8],
value_u32: u32,
) -> Result<(), EncoderError> {
assert!(!loader.is_null(), "loader non-null");
assert!(!property.is_empty(), "property non-empty");
let cfg = unsafe { (d.create_config)(loader) };
if cfg.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-create-config",
status: -1,
});
}
let variant = MfxVariant {
version: MfxStructVersion {
minor: MFX_VARIANT_VERSION_MINOR,
major: MFX_VARIANT_VERSION_MAJOR,
},
type_: MFX_VARIANT_TYPE_U32,
data: MfxVariantData { u32_: value_u32 },
};
let status = unsafe { (d.set_config_property)(cfg, property.as_ptr(), variant) };
if status != MFX_ERR_NONE {
let vendor = if property == FILTER_PROPERTY_ACCEL {
"qsv-accel-mode-rejected"
} else {
"qsv-set-config-property"
};
return Err(EncoderError::SessionInitFailed {
vendor,
status: status as i64,
});
}
Ok(())
}
fn init_session_legacy(api: &ApiTable) -> Result<*mut c_void, EncoderError> {
let session_via_d3d11 = try_mfxinit(api, MFX_IMPL_HARDWARE | MFX_IMPL_VIA_D3D11);
if let Ok(s) = session_via_d3d11 {
return Ok(s);
}
try_mfxinit(api, MFX_IMPL_HARDWARE)
}
fn try_mfxinit(api: &ApiTable, impl_flags: i32) -> Result<*mut c_void, EncoderError> {
let mut version = MfxVersion { major: 1, minor: 0 };
let mut session: *mut c_void = ptr::null_mut();
let status = unsafe { (api.init)(impl_flags, &mut version, &mut session) };
if status != MFX_ERR_NONE {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-mfxinit",
status: status as i64,
});
}
if session.is_null() {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-mfxinit",
status: -1,
});
}
assert!(!session.is_null(), "session non-null after MFXInit");
Ok(session)
}
fn set_d3d11_handle(
api: &ApiTable,
session: *mut c_void,
device: &ID3D11Device,
) -> Result<(), EncoderError> {
assert!(!session.is_null(), "session non-null");
let raw = device.as_raw();
assert!(!raw.is_null(), "device raw non-null");
if let Ok(mt) = device.cast::<ID3D11Multithread>() {
let _ = unsafe { mt.SetMultithreadProtected(true) };
}
let status = unsafe { (api.set_handle)(session, MFX_HANDLE_D3D11_DEVICE, raw) };
if status != MFX_ERR_NONE && status != MFX_WRN_IN_EXECUTION {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-set-handle",
status: status as i64,
});
}
Ok(())
}
fn build_video_params(
dims: EncoderDims,
target_kbps: u16,
frame_rate: EncoderFrameRate,
) -> MfxVideoParam {
let info = MfxFrameInfo {
four_cc: MFX_FOURCC_NV12,
width: align16(dims.width as u16),
height: align16(dims.height as u16),
crop_w: dims.width as u16,
crop_h: dims.height as u16,
frame_rate_extn: frame_rate.numerator,
frame_rate_extd: frame_rate.denominator,
aspect_ratio_w: 1,
aspect_ratio_h: 1,
pic_struct: MFX_PICSTRUCT_PROGRESSIVE,
chroma_format: MFX_CHROMAFORMAT_YUV420,
..Default::default()
};
let mfx = MfxInfoMfx {
frame_info: info,
codec_id: MFX_CODEC_AVC,
target_usage: 4,
gop_pic_size: frame_rate.gop_pic_size(),
gop_ref_dist: 1,
rate_control_method: MFX_RATECONTROL_CBR,
target_kbps,
max_kbps: target_kbps,
num_slice: 1,
num_ref_frame: 1,
..Default::default()
};
MfxVideoParam {
alloc_id: 0,
reserved: [0; 2],
reserved3: 0,
async_depth: 1,
mfx,
protected: 0,
io_pattern: MFX_IOPATTERN_IN_VIDEO_MEMORY,
ext_param: ptr::null_mut(),
num_ext_param: 0,
reserved2: 0,
}
}
fn align16(v: u16) -> u16 {
(v + 15) & !15
}
fn encode_init(
api: &ApiTable,
session: *mut c_void,
dims: EncoderDims,
target_kbps: u16,
frame_rate: EncoderFrameRate,
) -> Result<(), EncoderError> {
let mut params = build_video_params(dims, target_kbps, frame_rate);
let mut query_out = build_video_params(dims, target_kbps, frame_rate);
let q_status = unsafe { (api.encode_query)(session, &mut params, &mut query_out) };
let q_ok = q_status == MFX_ERR_NONE
|| q_status == MFX_WRN_IN_EXECUTION
|| q_status == -3
|| q_status > 0;
if !q_ok {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-encode-query",
status: q_status as i64,
});
}
let status = unsafe { (api.encode_init)(session, &mut query_out) };
if status != MFX_ERR_NONE && status != MFX_WRN_IN_EXECUTION {
return Err(EncoderError::SessionInitFailed {
vendor: "qsv-encode-init",
status: status as i64,
});
}
Ok(())
}
fn build_slot_state(
shared_handle: u64,
dims: EncoderDims,
target_kbps: u16,
frame_rate: EncoderFrameRate,
) -> SlotState {
let max_bs = (dims.width as usize * dims.height as usize * 3 / 2).max(512 * 1024);
let mut bitstream_buf = vec![0u8; max_bs];
let bs_ptr = bitstream_buf.as_mut_ptr();
let bitstream = MfxBitstream {
encrypted_data: ptr::null_mut(),
num_extparam: 0,
ext_param: ptr::null_mut(),
reserved: [0; 6],
decode_time_stamp: 0,
time_stamp: 0,
data: bs_ptr,
data_offset: 0,
data_length: 0,
max_length: max_bs as u32,
pic_struct: 0,
frame_type: 0,
data_flag: 0,
reserved2: 0,
};
let info = MfxFrameInfo {
four_cc: MFX_FOURCC_NV12,
width: align16(dims.width as u16),
height: align16(dims.height as u16),
crop_w: dims.width as u16,
crop_h: dims.height as u16,
frame_rate_extn: frame_rate.numerator,
frame_rate_extd: frame_rate.denominator,
aspect_ratio_w: 1,
aspect_ratio_h: 1,
pic_struct: MFX_PICSTRUCT_PROGRESSIVE,
chroma_format: MFX_CHROMAFORMAT_YUV420,
..Default::default()
};
let _ = target_kbps;
let surface = MfxFrameSurface1 {
reserved: [0; 4],
interface_ptr: ptr::null_mut(),
info,
data: MfxFrameData {
ext_param: ptr::null_mut(),
num_extparam: 0,
reserved: [0; 8],
mem_type: 0,
pitch_high: 0,
time_stamp: 0,
frame_order: 0,
locked: 0,
pitch_low: 0,
plane_ptrs: [ptr::null_mut(); 7],
mem_id: shared_handle as *mut c_void,
corrupted: 0,
data_flag: 0,
},
};
SlotState {
pending_pts_us: 0,
pending_force_keyframe: false,
sync_point: ptr::null_mut(),
bitstream,
bitstream_buf,
surface,
in_flight: false,
}
}
impl Drop for QsvD3D11Handoff {
fn drop(&mut self) {
self.slots.clear();
if !self.session.is_null() {
let _ = unsafe { (self.api.encode_close)(self.session) };
let _ = unsafe { (self.api.close)(self.session) };
self.session = ptr::null_mut();
}
if let Some(d) = self.dispatcher.take()
&& !self.loader.is_null()
{
unsafe { (d.unload)(self.loader) };
self.loader = ptr::null_mut();
}
}
}
impl QsvHandoff for QsvD3D11Handoff {
fn register_slot(
&mut self,
shared_handle: u64,
_key: u64,
dims: EncoderDims,
) -> Result<HandoffSlot, EncoderError> {
assert!(shared_handle != 0, "shared_handle non-zero");
assert!(dims.width > 0, "width positive");
let slot_index = self.next_slot_index;
self.next_slot_index = self.next_slot_index.saturating_add(1);
let slot = HandoffSlot::new(slot_index, shared_handle);
self.slots.insert(
slot_index,
build_slot_state(shared_handle, dims, self.target_kbps, self.frame_rate),
);
assert!(self.slots.contains_key(&slot_index), "slot stored");
Ok(slot)
}
fn encode_shared_async(
&mut self,
slot: HandoffSlot,
_key: u64,
dims: EncoderDims,
pic_params: PicParams,
) -> Result<(), EncoderError> {
assert!(slot.shared_handle != 0, "slot handle non-zero");
assert!(dims.width > 0, "width positive");
let state = self
.slots
.get_mut(&slot.slot_index)
.ok_or(EncoderError::SlotUnknown {
slot_index: slot.slot_index,
})?;
state.surface.data.time_stamp = pic_params.pts_us;
state.bitstream.data_length = 0;
state.bitstream.data_offset = 0;
let mut sync: *mut c_void = ptr::null_mut();
let status = unsafe {
(self.api.encode_frame_async)(
self.session,
ptr::null_mut(),
&mut state.surface,
&mut state.bitstream,
&mut sync,
)
};
state.pending_pts_us = pic_params.pts_us;
state.pending_force_keyframe = pic_params.force_keyframe;
if status == MFX_ERR_MORE_DATA {
state.in_flight = false;
return Ok(());
}
if status != MFX_ERR_NONE {
return Err(EncoderError::EncodeFailed {
vendor: "qsv",
status: status as i64,
});
}
state.sync_point = sync;
state.in_flight = !sync.is_null();
Ok(())
}
fn poll_completed(&mut self, slot: HandoffSlot) -> Option<EncodedBitstream> {
let session_ptr = self.session;
let dts_offset_us = self.dts_offset_us;
let completed_count = self.completed_count;
let state = self.slots.get_mut(&slot.slot_index)?;
if !state.in_flight || state.sync_point.is_null() {
return None;
}
let status = unsafe { (self.api.sync_operation)(session_ptr, state.sync_point, 0) };
if status == MFX_WRN_IN_EXECUTION || status != MFX_ERR_NONE {
return None;
}
let len = state.bitstream.data_length as usize;
if len == 0 {
state.in_flight = false;
return None;
}
let mut data: Vec<u8> = Vec::with_capacity(len);
let offset = state.bitstream.data_offset as usize;
data.extend_from_slice(&state.bitstream_buf[offset..offset + len]);
let pts = state.bitstream.time_stamp;
let is_keyframe = (state.bitstream.frame_type & 0x1) != 0 || completed_count == 0;
let dts = apply_dts_offset(pts, dts_offset_us);
state.in_flight = false;
state.sync_point = ptr::null_mut();
self.completed_count = self.completed_count.saturating_add(1);
Some(EncodedBitstream::new(data, pts, dts, is_keyframe))
}
fn unregister_slot(&mut self, slot: HandoffSlot) {
self.slots.remove(&slot.slot_index);
}
fn encode_shared(
&mut self,
submission: EncoderSubmission,
callback: &mut dyn EncoderCompletionCallback,
) -> Result<(), RingError> {
assert!(submission.shared_handle != 0, "submission handle non-zero");
assert!(submission.dims.width > 0, "submission width positive");
let slot = self
.register_slot(
submission.shared_handle,
submission.keyed_mutex_key,
submission.dims,
)
.map_err(|_| RingError::NotImplemented {
what: "qsv::register_slot in encode_shared",
})?;
let pts_us = submission.capture_pts_us.unwrap_or_else(|| {
submission
.sequence
.saturating_mul(self.frame_rate.frame_interval_us())
});
let pic = PicParams::new(pts_us, false);
QsvHandoff::encode_shared_async(
self,
slot,
submission.keyed_mutex_key,
submission.dims,
pic,
)
.map_err(|_| RingError::NotImplemented {
what: "qsv::encode_shared_async",
})?;
if let Some(bs) = QsvHandoff::poll_completed(self, slot) {
callback.on_complete(submission.sequence, bs.data.len() as u32);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sdk_not_found_when_dll_missing() {
let dummy_path = "/this/path/does/not/exist/fake-libmfxhw64.dll";
let result = unsafe { Library::new(dummy_path) };
assert!(result.is_err());
}
#[test]
fn fourcc_nv12_packs_correctly() {
assert_eq!(MFX_FOURCC_NV12, 0x3231564E);
}
#[test]
fn align16_rounds_up() {
assert_eq!(align16(1080), 1088);
assert_eq!(align16(1920), 1920);
assert_eq!(align16(0), 0);
}
#[test]
fn target_kbps_clamps_against_overflow() {
let bitrate: u32 = u32::MAX;
let kbps = (bitrate / 1000).clamp(500, 60_000) as u16;
assert_eq!(kbps, 60_000);
}
#[test]
fn accel_mode_constant_matches_onevpl_spec() {
assert_eq!(MFX_ACCEL_MODE_VIA_D3D11, 0x0300);
assert_ne!(MFX_ACCEL_MODE_VIA_D3D11, 0x0200);
}
#[test]
fn impl_type_hardware_constant_matches_onevpl_spec() {
assert_eq!(MFX_IMPL_TYPE_HARDWARE, 2);
}
#[test]
fn variant_type_u32_constant_matches_onevpl_spec() {
assert_eq!(MFX_VARIANT_TYPE_U32, 5);
assert_ne!(MFX_VARIANT_TYPE_U32, 8);
}
#[test]
fn d3d11_device_handle_type_matches_onevpl_spec() {
assert_eq!(MFX_HANDLE_D3D11_DEVICE, 3);
assert_ne!(MFX_HANDLE_D3D11_DEVICE, 2);
}
#[test]
fn variant_version_matches_onevpl_spec() {
assert_eq!(MFX_VARIANT_VERSION_MAJOR, 1);
assert_eq!(MFX_VARIANT_VERSION_MINOR, 1);
}
#[test]
fn filter_property_name_is_null_terminated_accel() {
assert!(FILTER_PROPERTY_ACCEL.ends_with(b"\0"));
let view = &FILTER_PROPERTY_ACCEL[..FILTER_PROPERTY_ACCEL.len() - 1];
assert_eq!(view, b"mfxImplDescription.AccelerationMode");
}
#[test]
fn filter_property_name_is_null_terminated_impl() {
assert!(FILTER_PROPERTY_IMPL.ends_with(b"\0"));
let view = &FILTER_PROPERTY_IMPL[..FILTER_PROPERTY_IMPL.len() - 1];
assert_eq!(view, b"mfxImplDescription.Impl");
}
#[test]
fn variant_payload_carries_u32_value() {
let v = MfxVariant {
version: MfxStructVersion {
minor: MFX_VARIANT_VERSION_MINOR,
major: MFX_VARIANT_VERSION_MAJOR,
},
type_: MFX_VARIANT_TYPE_U32,
data: MfxVariantData {
u32_: MFX_ACCEL_MODE_VIA_D3D11,
},
};
let read = unsafe { v.data.u32_ };
assert_eq!(read, MFX_ACCEL_MODE_VIA_D3D11);
assert_eq!(v.type_, 5);
}
#[derive(Default)]
struct FilterRecord {
property: Vec<u8>,
value_u32: u32,
variant_type: u32,
}
#[test]
fn mock_set_filter_records_property_and_value() {
let mut record = FilterRecord::default();
let property = FILTER_PROPERTY_ACCEL;
let variant = MfxVariant {
version: MfxStructVersion {
minor: MFX_VARIANT_VERSION_MINOR,
major: MFX_VARIANT_VERSION_MAJOR,
},
type_: MFX_VARIANT_TYPE_U32,
data: MfxVariantData {
u32_: MFX_ACCEL_MODE_VIA_D3D11,
},
};
let mut len: usize = 0;
while *property.get(len).unwrap_or(&1) != 0 {
len += 1;
}
record.property.extend_from_slice(&property[..len]);
record.variant_type = variant.type_;
record.value_u32 = unsafe { variant.data.u32_ };
assert_eq!(record.property, b"mfxImplDescription.AccelerationMode");
assert_eq!(record.variant_type, MFX_VARIANT_TYPE_U32);
assert_eq!(record.value_u32, MFX_ACCEL_MODE_VIA_D3D11);
assert_eq!(record.value_u32, 0x0300);
}
}
@@ -0,0 +1,966 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::backend::{BackendError, KeyedMutexBackend, TextureFormat};
pub const RING_SIZE: usize = 8;
pub const DUPLICATE_COUNT_MAX: u32 = 30;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SlotState {
Free,
Filling,
Submitted,
Dispatched,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RingError {
FullDropped {
dropped_so_far: u64,
},
BackendFailed {
source: BackendError,
},
NotInitialised,
AlreadyInitialised,
UnknownSlot,
UnexpectedSlotState {
slot_index: u32,
observed: SlotState,
},
PlatformUnsupported {
reason: &'static str,
},
NotImplemented {
what: &'static str,
},
SlotsExhausted,
}
impl std::fmt::Display for RingError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::FullDropped { dropped_so_far } => {
write!(f, "ring full; total dropped={dropped_so_far}")
}
Self::BackendFailed { source } => write!(f, "backend failed: {source}"),
Self::NotInitialised => write!(f, "ring not initialised"),
Self::AlreadyInitialised => write!(f, "ring already initialised"),
Self::UnknownSlot => write!(f, "slot handle does not belong to this ring"),
Self::UnexpectedSlotState {
slot_index,
observed,
} => {
write!(f, "slot {slot_index} in unexpected state {observed:?}")
}
Self::PlatformUnsupported { reason } => write!(f, "platform unsupported: {reason}"),
Self::NotImplemented { what } => write!(f, "not implemented: {what}"),
Self::SlotsExhausted => write!(f, "slot id space exhausted (all candidates in use)"),
}
}
}
impl std::error::Error for RingError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct RingMetrics {
pub submitted_count: u64,
pub completed_count: u64,
pub dropped_count: u64,
pub dispatched_count: u64,
pub lagged_count: u64,
}
pub struct EncoderReady<H: Clone> {
pub handle: H,
pub sequence: u64,
pub slot_index: u32,
pub duplicate_count: u32,
}
pub struct FillReservation<H: Clone> {
pub handle: H,
slot_index: u32,
key: u64,
}
impl<H: Clone> FillReservation<H> {
pub fn slot_index(&self) -> u32 {
let index = self.slot_index;
assert!((index as usize) < RING_SIZE_MAX, "slot_index within max");
assert!(self.key < u64::MAX, "reservation key plausible");
index
}
}
struct SlotMeta {
state: SlotState,
sequence: u64,
key: u64,
duplicate_count: u32,
}
impl SlotMeta {
const fn fresh() -> Self {
Self {
state: SlotState::Free,
sequence: 0,
key: 0,
duplicate_count: 0,
}
}
}
pub struct EncoderInputRing<B: KeyedMutexBackend> {
backend: B,
slots: Vec<B::SlotHandle>,
meta: Vec<SlotMeta>,
metrics: RingMetrics,
pending_lagged: u32,
initialised: bool,
width: u32,
height: u32,
format: TextureFormat,
}
impl<B: KeyedMutexBackend> EncoderInputRing<B> {
pub fn new(backend: B) -> Self {
let ring = Self {
backend,
slots: Vec::with_capacity(B::NUM_SLOTS),
meta: Vec::with_capacity(B::NUM_SLOTS),
metrics: RingMetrics::default(),
pending_lagged: 0,
initialised: false,
width: 0,
height: 0,
format: TextureFormat::Nv12,
};
assert!(!ring.initialised, "fresh ring is uninitialised");
assert_eq!(ring.slots.len(), 0, "fresh ring has no slots");
ring
}
pub fn initialise(
&mut self,
width: u32,
height: u32,
format: TextureFormat,
) -> Result<(), RingError> {
if self.initialised {
return Err(RingError::AlreadyInitialised);
}
let handles = self
.backend
.create_slots(width, height, format)
.map_err(|source| RingError::BackendFailed { source })?;
assert_eq!(
handles.len(),
B::NUM_SLOTS,
"backend returns NUM_SLOTS handles"
);
assert!(handles.len() <= RING_SIZE_MAX, "NUM_SLOTS within max");
self.slots = handles;
self.meta = (0..B::NUM_SLOTS).map(|_| SlotMeta::fresh()).collect();
self.width = width;
self.height = height;
self.format = format;
self.initialised = true;
assert!(self.initialised, "initialised flipped");
assert_eq!(self.slots.len(), self.meta.len(), "slots and meta align");
Ok(())
}
pub fn submit<F>(&mut self, fill: F) -> Result<(), RingError>
where
F: FnOnce(&mut B::SlotHandle),
{
let mut reservation = self.reserve()?;
fill(&mut reservation.handle);
let sequence = self.commit(reservation)?;
assert!(sequence > 0, "committed sequence positive");
assert!(
self.metrics.submitted_count >= sequence,
"monotonic submitted"
);
Ok(())
}
pub fn submit_skip_oldest<F>(&mut self, fill: F) -> Result<(), RingError>
where
F: FnOnce(&mut B::SlotHandle),
{
let mut reservation = self.reserve_skip_oldest()?;
fill(&mut reservation.handle);
let sequence = self.commit(reservation)?;
assert!(sequence > 0, "skip-oldest: committed sequence positive");
assert!(
self.metrics.submitted_count >= sequence,
"skip-oldest: monotonic submitted"
);
Ok(())
}
pub fn reserve(&mut self) -> Result<FillReservation<B::SlotHandle>, RingError> {
if !self.initialised {
return Err(RingError::NotInitialised);
}
self.acquire_free_slot()
}
pub fn reserve_skip_oldest(&mut self) -> Result<FillReservation<B::SlotHandle>, RingError> {
if !self.initialised {
return Err(RingError::NotInitialised);
}
if self.find_free_slot().is_none() {
let _ = self.evict_oldest_submitted();
}
self.acquire_free_slot()
}
fn acquire_free_slot(&mut self) -> Result<FillReservation<B::SlotHandle>, RingError> {
assert!(self.initialised, "acquire requires initialised ring");
assert_eq!(self.slots.len(), self.meta.len(), "slots and meta align");
for index in 0..self.meta.len() {
if self.meta[index].state != SlotState::Free {
continue;
}
let key = self.meta[index].key;
match self.backend.acquire_write(&self.slots[index], key) {
Ok(()) => {
self.meta[index].state = SlotState::Filling;
return Ok(FillReservation {
handle: self.slots[index].clone(),
slot_index: index as u32,
key,
});
}
Err(BackendError::WouldBlock { .. }) => continue,
Err(source) => return Err(RingError::BackendFailed { source }),
}
}
self.fold_lagged(1);
self.metrics.lagged_count = self.metrics.lagged_count.saturating_add(1);
self.metrics.dropped_count = self.metrics.dropped_count.saturating_add(1);
Err(RingError::FullDropped {
dropped_so_far: self.metrics.dropped_count,
})
}
pub fn commit(
&mut self,
reservation: FillReservation<B::SlotHandle>,
) -> Result<u64, RingError> {
if !self.initialised {
return Err(RingError::NotInitialised);
}
let index = reservation.slot_index as usize;
if index >= self.meta.len() {
return Err(RingError::UnknownSlot);
}
let observed = self.meta[index].state;
if observed != SlotState::Filling {
return Err(RingError::UnexpectedSlotState {
slot_index: reservation.slot_index,
observed,
});
}
assert_eq!(
self.meta[index].key, reservation.key,
"reservation key matches slot"
);
let next_key = reservation.key.wrapping_add(1);
self.backend
.release_write(&self.slots[index], next_key)
.map_err(|source| RingError::BackendFailed { source })?;
let sequence = self.metrics.submitted_count.saturating_add(1);
self.meta[index].state = SlotState::Submitted;
self.meta[index].sequence = sequence;
self.meta[index].key = next_key;
self.meta[index].duplicate_count = self.pending_lagged;
self.pending_lagged = 0;
self.metrics.submitted_count = sequence;
self.metrics.completed_count = self.metrics.completed_count.saturating_add(1);
assert_eq!(
self.meta[index].state,
SlotState::Submitted,
"post-commit submitted"
);
assert!(
self.metrics.submitted_count >= sequence,
"commit: monotonic submitted"
);
Ok(sequence)
}
pub fn cancel(&mut self, reservation: FillReservation<B::SlotHandle>) -> Result<(), RingError> {
if !self.initialised {
return Err(RingError::NotInitialised);
}
let index = reservation.slot_index as usize;
if index >= self.meta.len() {
return Err(RingError::UnknownSlot);
}
let observed = self.meta[index].state;
if observed != SlotState::Filling {
return Err(RingError::UnexpectedSlotState {
slot_index: reservation.slot_index,
observed,
});
}
assert_eq!(
self.meta[index].key, reservation.key,
"cancel: reservation key matches slot"
);
let next_key = reservation.key.wrapping_add(1);
self.backend
.release_write(&self.slots[index], next_key)
.map_err(|source| RingError::BackendFailed { source })?;
self.backend.mark_consumed(&self.slots[index]);
self.meta[index].state = SlotState::Free;
self.meta[index].key = next_key;
assert_eq!(self.meta[index].state, SlotState::Free, "post-cancel free");
assert!(
!self.backend.poll_complete(&self.slots[index]),
"cancelled slot not complete"
);
Ok(())
}
fn fold_lagged(&mut self, amount: u32) {
assert!(amount > 0, "fold amount positive");
assert!(
amount <= DUPLICATE_COUNT_MAX.saturating_add(1),
"fold amount bounded"
);
let mut newest: Option<usize> = None;
let mut newest_sequence: u64 = 0;
for (i, meta) in self.meta.iter().enumerate() {
if meta.state != SlotState::Submitted {
continue;
}
if meta.sequence >= newest_sequence {
newest_sequence = meta.sequence;
newest = Some(i);
}
}
match newest {
Some(index) => {
let total = self.meta[index].duplicate_count.saturating_add(amount);
self.meta[index].duplicate_count = total.min(DUPLICATE_COUNT_MAX);
}
None => {
let total = self.pending_lagged.saturating_add(amount);
self.pending_lagged = total.min(DUPLICATE_COUNT_MAX);
}
}
assert!(
self.pending_lagged <= DUPLICATE_COUNT_MAX,
"pending lag bounded"
);
}
fn evict_oldest_submitted(&mut self) -> bool {
let mut chosen: Option<usize> = None;
let mut chosen_sequence: u64 = u64::MAX;
for (i, meta) in self.meta.iter().enumerate() {
if meta.state != SlotState::Submitted {
continue;
}
if meta.sequence < chosen_sequence {
chosen_sequence = meta.sequence;
chosen = Some(i);
}
}
let Some(index) = chosen else {
return false;
};
assert_eq!(
self.meta[index].state,
SlotState::Submitted,
"evict candidate is submitted"
);
assert!(
chosen_sequence != u64::MAX,
"evict candidate had real sequence"
);
let folded = self.meta[index].duplicate_count.saturating_add(1);
self.backend.mark_consumed(&self.slots[index]);
self.meta[index].state = SlotState::Free;
self.meta[index].duplicate_count = 0;
self.fold_lagged(folded);
self.metrics.lagged_count = self.metrics.lagged_count.saturating_add(1);
self.metrics.dropped_count = self.metrics.dropped_count.saturating_add(1);
true
}
pub fn poll_next_ready(&mut self) -> Option<EncoderReady<B::SlotHandle>> {
if !self.initialised {
return None;
}
let mut chosen: Option<usize> = None;
let mut chosen_sequence: u64 = u64::MAX;
for i in 0..self.meta.len() {
if self.meta[i].state != SlotState::Submitted {
continue;
}
if !self.backend.poll_complete(&self.slots[i]) {
continue;
}
if self.meta[i].sequence < chosen_sequence {
chosen_sequence = self.meta[i].sequence;
chosen = Some(i);
}
}
let index = chosen?;
assert_eq!(
self.meta[index].state,
SlotState::Submitted,
"ready slot was submitted"
);
assert!(
self.backend.poll_complete(&self.slots[index]),
"ready slot is complete"
);
self.meta[index].state = SlotState::Dispatched;
self.metrics.dispatched_count = self.metrics.dispatched_count.saturating_add(1);
Some(EncoderReady {
handle: self.slots[index].clone(),
sequence: self.meta[index].sequence,
slot_index: index as u32,
duplicate_count: self.meta[index].duplicate_count,
})
}
pub fn release_completed(
&mut self,
ready: EncoderReady<B::SlotHandle>,
) -> Result<(), RingError> {
if !self.initialised {
return Err(RingError::NotInitialised);
}
let index = ready.slot_index as usize;
if index >= self.meta.len() {
return Err(RingError::UnknownSlot);
}
let observed = self.meta[index].state;
if observed != SlotState::Dispatched {
return Err(RingError::UnexpectedSlotState {
slot_index: ready.slot_index,
observed,
});
}
self.backend.mark_consumed(&self.slots[index]);
self.meta[index].state = SlotState::Free;
assert_eq!(self.meta[index].state, SlotState::Free, "post-release free");
assert!(
!self.backend.poll_complete(&self.slots[index]),
"no longer reports complete"
);
Ok(())
}
pub fn metrics(&self) -> RingMetrics {
assert!(
self.metrics.completed_count <= self.metrics.submitted_count,
"complete<=submit"
);
assert!(
self.metrics.dispatched_count <= self.metrics.completed_count,
"dispatch<=complete"
);
self.metrics
}
pub fn submitted_count(&self) -> u64 {
self.metrics.submitted_count
}
pub fn completed_count(&self) -> u64 {
self.metrics.completed_count
}
pub fn dropped_count(&self) -> u64 {
self.metrics.dropped_count
}
pub fn dispatched_count(&self) -> u64 {
self.metrics.dispatched_count
}
pub fn capacity(&self) -> usize {
let cap = B::NUM_SLOTS;
assert!(cap > 0, "NUM_SLOTS must be positive");
assert!(cap <= RING_SIZE_MAX, "NUM_SLOTS within max");
cap
}
pub fn free_count(&self) -> usize {
let mut count: usize = 0;
for meta in self.meta.iter() {
if meta.state == SlotState::Free {
count = count.saturating_add(1);
}
}
assert!(count <= self.meta.len(), "free count within capacity");
count
}
pub fn backend_mut(&mut self) -> &mut B {
assert!(self.initialised, "backend access requires init");
&mut self.backend
}
fn find_free_slot(&self) -> Option<usize> {
for (i, meta) in self.meta.iter().enumerate() {
if meta.state == SlotState::Free {
return Some(i);
}
}
None
}
}
pub const RING_SIZE_MAX: usize = 16;
const _: () = assert!(RING_SIZE <= RING_SIZE_MAX, "RING_SIZE within max");
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::{CpuMemcpyBackend, CpuSlotHandle};
fn make_ring() -> EncoderInputRing<CpuMemcpyBackend> {
let mut ring = EncoderInputRing::new(CpuMemcpyBackend::new());
ring.initialise(64, 64, TextureFormat::Nv12).expect("init");
ring
}
fn fill_noop(_: &mut CpuSlotHandle) {}
#[test]
fn submit_then_poll_returns_some() {
let mut ring = make_ring();
ring.submit(fill_noop).expect("submit");
let ready = ring.poll_next_ready().expect("poll yields ready");
assert_eq!(ready.sequence, 1);
ring.release_completed(ready).expect("release");
}
#[test]
fn eight_submits_dispatch_in_fifo_order() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
assert_eq!(ring.submitted_count(), 8);
let mut observed_seq: Vec<u64> = Vec::new();
for _ in 0..8 {
let ready = ring.poll_next_ready().expect("ready");
observed_seq.push(ready.sequence);
ring.release_completed(ready).expect("release");
}
assert_eq!(observed_seq, vec![1, 2, 3, 4, 5, 6, 7, 8]);
}
#[test]
fn ninth_submit_when_full_returns_full_dropped() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let err = ring.submit(fill_noop).err();
assert!(matches!(
err,
Some(RingError::FullDropped { dropped_so_far: 1 })
));
assert_eq!(ring.dropped_count(), 1);
let err2 = ring.submit(fill_noop).err();
assert!(matches!(
err2,
Some(RingError::FullDropped { dropped_so_far: 2 })
));
assert_eq!(ring.dropped_count(), 2);
}
#[test]
fn release_completed_returns_slot_to_pool() {
let mut ring = make_ring();
ring.submit(fill_noop).expect("submit");
let ready = ring.poll_next_ready().expect("ready");
assert_eq!(ring.free_count(), 7);
ring.release_completed(ready).expect("release");
assert_eq!(ring.free_count(), 8);
ring.submit(fill_noop).expect("re-submit after release");
}
#[test]
fn pair_asserts_pass_under_random_submit_poll_release() {
let mut ring = make_ring();
let mut state: u64 = 0xcafef00d;
let mut in_flight: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..1000 {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
let action = state % 3;
match action {
0 => {
let _ = ring.submit(fill_noop);
}
1 => {
if let Some(r) = ring.poll_next_ready() {
in_flight.push(r);
}
}
_ => {
if let Some(r) = in_flight.pop() {
ring.release_completed(r).expect("release");
}
}
}
let metrics = ring.metrics();
assert!(metrics.completed_count <= metrics.submitted_count);
assert!(metrics.dispatched_count <= metrics.completed_count);
}
while let Some(r) = in_flight.pop() {
ring.release_completed(r).expect("drain release");
}
while let Some(r) = ring.poll_next_ready() {
ring.release_completed(r).expect("drain release post poll");
}
assert_eq!(ring.free_count(), ring.capacity());
}
#[test]
#[allow(clippy::panic)]
fn submit_with_panicking_closure_leaves_state_consistent() {
use std::panic::{AssertUnwindSafe, catch_unwind};
let mut ring = make_ring();
ring.submit(fill_noop).expect("first submit ok");
let result = catch_unwind(AssertUnwindSafe(|| {
let _ = ring.submit(|_h: &mut CpuSlotHandle| panic!("user fill panicked"));
}));
assert!(result.is_err());
let metrics = ring.metrics();
assert!(metrics.submitted_count >= 1);
assert!(metrics.completed_count <= metrics.submitted_count);
assert!(metrics.dispatched_count <= metrics.completed_count);
let ready = ring.poll_next_ready().expect("first frame still pollable");
assert_eq!(ready.sequence, 1);
ring.release_completed(ready).expect("release first");
}
#[test]
fn determinism_same_sequence_yields_same_release_order() {
fn run() -> Vec<u64> {
let mut ring = make_ring();
let mut released: Vec<u64> = Vec::new();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
for _ in 0..8 {
let r = ring.poll_next_ready().expect("ready");
released.push(r.sequence);
ring.release_completed(r).expect("release");
}
released
}
let a = run();
let b = run();
assert_eq!(a, b);
assert_eq!(a, vec![1, 2, 3, 4, 5, 6, 7, 8]);
}
#[test]
fn capacity_is_eight() {
let ring = make_ring();
assert_eq!(ring.capacity(), 8);
assert_eq!(ring.free_count(), 8);
}
#[test]
fn double_release_of_same_ready_rejected() {
let mut ring = make_ring();
ring.submit(fill_noop).expect("submit");
let ready = ring.poll_next_ready().expect("ready");
let cloned = EncoderReady {
handle: ready.handle.clone(),
sequence: ready.sequence,
slot_index: ready.slot_index,
duplicate_count: ready.duplicate_count,
};
ring.release_completed(ready).expect("first release");
let err = ring.release_completed(cloned).err();
assert!(matches!(err, Some(RingError::UnexpectedSlotState { .. })));
}
#[test]
fn submit_before_init_returns_not_initialised() {
let mut ring: EncoderInputRing<CpuMemcpyBackend> =
EncoderInputRing::new(CpuMemcpyBackend::new());
let err = ring.submit(fill_noop).err();
assert!(matches!(err, Some(RingError::NotInitialised)));
}
#[test]
fn re_initialise_rejected() {
let mut ring = make_ring();
let err = ring.initialise(64, 64, TextureFormat::Nv12).err();
assert!(matches!(err, Some(RingError::AlreadyInitialised)));
}
#[test]
fn poll_when_empty_returns_none() {
let mut ring = make_ring();
assert!(ring.poll_next_ready().is_none());
}
#[test]
fn skip_oldest_rejects_when_every_slot_is_dispatched() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let mut in_flight: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..8 {
in_flight.push(ring.poll_next_ready().expect("ready"));
}
assert_eq!(ring.free_count(), 0);
let err = ring.submit_skip_oldest(fill_noop).err();
assert!(matches!(
err,
Some(RingError::FullDropped { dropped_so_far: 1 })
));
assert_eq!(ring.dropped_count(), 1);
assert_eq!(ring.dispatched_count(), 8);
for ready in in_flight.drain(..) {
ring.release_completed(ready).expect("release");
}
assert_eq!(ring.free_count(), 8);
ring.submit_skip_oldest(fill_noop)
.expect("submit succeeds once dispatched slots are released");
}
#[test]
fn skip_oldest_evicts_oldest_submitted_never_dispatched() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let mut dispatched: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..3 {
dispatched.push(ring.poll_next_ready().expect("ready"));
}
assert_eq!(dispatched[0].sequence, 1);
assert_eq!(dispatched[2].sequence, 3);
ring.submit_skip_oldest(fill_noop)
.expect("skip-oldest evicts a submitted slot");
assert_eq!(ring.dropped_count(), 1);
assert_eq!(ring.submitted_count(), 9);
let mut remaining: Vec<u64> = Vec::new();
while let Some(ready) = ring.poll_next_ready() {
remaining.push(ready.sequence);
ring.release_completed(ready).expect("release");
}
assert_eq!(remaining, vec![5, 6, 7, 8, 9], "sequence 4 was evicted");
for ready in dispatched.drain(..) {
ring.release_completed(ready).expect("release dispatched");
}
assert_eq!(ring.free_count(), 8);
}
#[test]
fn reserve_then_commit_matches_submit_semantics() {
let mut ring = make_ring();
let reservation = ring.reserve().expect("reserve");
assert_eq!(reservation.slot_index(), 0);
assert_eq!(ring.free_count(), 7);
assert_eq!(ring.submitted_count(), 0, "sequence assigned at commit");
let sequence = ring.commit(reservation).expect("commit");
assert_eq!(sequence, 1);
let ready = ring.poll_next_ready().expect("ready");
assert_eq!(ready.sequence, 1);
assert_eq!(ready.duplicate_count, 0);
ring.release_completed(ready).expect("release");
}
#[test]
fn cancel_returns_slot_to_free_without_sequence() {
let mut ring = make_ring();
let reservation = ring.reserve().expect("reserve");
ring.cancel(reservation).expect("cancel");
assert_eq!(ring.free_count(), 8);
assert_eq!(ring.submitted_count(), 0);
assert!(ring.poll_next_ready().is_none());
ring.submit(fill_noop).expect("submit after cancel");
let ready = ring.poll_next_ready().expect("ready");
assert_eq!(ready.sequence, 1);
ring.release_completed(ready).expect("release");
}
#[test]
fn commit_of_freed_reservation_rejected() {
let mut ring = make_ring();
let first = ring.reserve().expect("reserve");
let index = first.slot_index();
ring.cancel(first).expect("cancel");
let second = ring.reserve().expect("re-reserve");
assert_eq!(second.slot_index(), index, "same slot reused");
ring.commit(second).expect("commit reused slot");
let ready = ring.poll_next_ready().expect("ready");
let stale = EncoderReady {
handle: ready.handle.clone(),
sequence: ready.sequence,
slot_index: ready.slot_index,
duplicate_count: ready.duplicate_count,
};
ring.release_completed(ready).expect("release");
let err = ring.release_completed(stale).err();
assert!(matches!(err, Some(RingError::UnexpectedSlotState { .. })));
}
#[test]
fn skip_oldest_never_evicts_filling_slot() {
let mut ring = make_ring();
for _ in 0..7 {
ring.submit(fill_noop).expect("submit");
}
let reservation = ring.reserve().expect("reserve eighth slot");
let reserved_index = reservation.slot_index();
ring.submit_skip_oldest(fill_noop)
.expect("skip-oldest evicts a submitted slot");
assert_eq!(ring.dropped_count(), 1);
let sequence = ring.commit(reservation).expect("commit survives eviction");
assert_eq!(sequence, 9);
let mut seen_indices: Vec<u32> = Vec::new();
while let Some(ready) = ring.poll_next_ready() {
seen_indices.push(ready.slot_index);
ring.release_completed(ready).expect("release");
}
assert!(seen_indices.contains(&reserved_index));
}
#[test]
fn full_ring_submit_folds_lag_into_newest_submitted() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let err = ring.submit(fill_noop).err();
assert!(matches!(
err,
Some(RingError::FullDropped { dropped_so_far: 1 })
));
assert_eq!(ring.metrics().lagged_count, 1);
let mut by_sequence: Vec<(u64, u32)> = Vec::new();
while let Some(ready) = ring.poll_next_ready() {
by_sequence.push((ready.sequence, ready.duplicate_count));
ring.release_completed(ready).expect("release");
}
assert_eq!(by_sequence.len(), 8);
for (sequence, duplicate_count) in by_sequence.iter().take(7) {
assert_eq!(*duplicate_count, 0, "sequence {sequence} not duplicated");
}
assert_eq!(by_sequence[7], (8, 1), "newest carries the lagged frame");
}
#[test]
fn eviction_conserves_duplicate_timing_slots() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let mut dispatched: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..7 {
dispatched.push(ring.poll_next_ready().expect("ready"));
}
let _ = ring.submit(fill_noop).err();
ring.submit_skip_oldest(fill_noop)
.expect("evicts the only submitted slot");
let ready = ring.poll_next_ready().expect("new frame ready");
assert_eq!(ready.sequence, 9);
assert_eq!(
ready.duplicate_count, 2,
"evicted frame plus its duplicate folded into successor"
);
assert_eq!(ring.metrics().lagged_count, 2);
ring.release_completed(ready).expect("release");
for ready in dispatched.drain(..) {
ring.release_completed(ready).expect("release dispatched");
}
}
#[test]
fn all_dispatched_lag_attaches_to_next_submission() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let mut in_flight: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..8 {
in_flight.push(ring.poll_next_ready().expect("ready"));
}
for _ in 0..2 {
let err = ring.submit_skip_oldest(fill_noop).err();
assert!(matches!(err, Some(RingError::FullDropped { .. })));
}
assert_eq!(ring.metrics().lagged_count, 2);
let first = in_flight.remove(0);
ring.release_completed(first).expect("release one");
ring.submit_skip_oldest(fill_noop)
.expect("submit after free");
let ready = ring.poll_next_ready().expect("ready");
assert_eq!(ready.sequence, 9);
assert_eq!(ready.duplicate_count, 2, "pending lag attached");
ring.release_completed(ready).expect("release");
for ready in in_flight.drain(..) {
ring.release_completed(ready).expect("release in flight");
}
}
#[test]
fn duplicate_count_saturates_at_named_cap() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
for _ in 0..(DUPLICATE_COUNT_MAX + 5) {
let _ = ring.submit(fill_noop).err();
}
let mut last: Option<(u64, u32)> = None;
while let Some(ready) = ring.poll_next_ready() {
last = Some((ready.sequence, ready.duplicate_count));
ring.release_completed(ready).expect("release");
}
assert_eq!(
last,
Some((8, DUPLICATE_COUNT_MAX)),
"duplicates saturate at cap"
);
}
#[test]
fn skip_oldest_repeated_rejection_counts_every_drop() {
let mut ring = make_ring();
for _ in 0..8 {
ring.submit(fill_noop).expect("submit");
}
let mut in_flight: Vec<EncoderReady<CpuSlotHandle>> = Vec::new();
for _ in 0..8 {
in_flight.push(ring.poll_next_ready().expect("ready"));
}
for expected_drops in 1..=3_u64 {
let err = ring.submit_skip_oldest(fill_noop).err();
assert!(matches!(
err,
Some(RingError::FullDropped { dropped_so_far }) if dropped_so_far == expected_drops
));
}
assert_eq!(ring.dropped_count(), 3);
while let Some(ready) = in_flight.pop() {
ring.release_completed(ready).expect("release");
}
}
}
File diff suppressed because it is too large Load Diff