Add native self-hosted instance connection to fluxer_desktop

Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 +
tools/ci) with a "Connect to a Different Server" menu item and popout
that lets the desktop app switch to any self-hosted Fluxer instance,
plus fixes for well-known discovery on single-domain self-hosted
deployments and a false-positive ERR_ABORTED on same-origin client
redirects during the switch. Defaults to chat.fluxr.chat and uses an
isolated userData directory from the official build.
This commit is contained in:
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,135 @@
[package]
name = "fluxer_webrtc_sender"
version = "0.0.0"
edition = "2024"
license = "AGPL-3.0-or-later"
publish = false
[workspace]
resolver = "2"
exclude = [
"vendor/tract-linalg-0.19.16",
"vendor/tract-linalg-0.23.1",
]
[lib]
crate-type = ["cdylib", "rlib"]
[features]
# `publisher` pulls the LiveKit native SDK (libwebrtc). It is on by default for
# the real addon build, but `cargo test --no-default-features` builds only the
# pure YUV-conversion core so the unit tests run anywhere (no libwebrtc).
default = ["publisher"]
publisher = ["dep:livekit", "dep:tokio", "dep:futures-util"]
camera-native = ["dep:nokhwa"]
# Exposes pub re-exports of private camera-background hot-path internals so the
# criterion bench target can call them; never enabled for the shipped addon.
bench-internals = []
[dependencies]
napi = {version = "3.9.1", default-features = false, features = ["dyn-symbols", "napi8", "tokio_rt"]}
napi-derive = "3.5.6"
crossbeam-queue = "0.3"
# LiveKit native SDK (wraps libwebrtc). Optional / feature-gated. The exact
# version is resolved on the Windows build host (where the libwebrtc prebuilt
# links); pinned loosely here.
# Keep LiveKit's default features (default = ["tokio"] -> the runtime +
# signal-client-tokio) AND add a TLS backend for the signal WebSocket -- without
# one, connecting to a wss:// SFU fails with "TLS support not compiled in".
# rustls-tls-native-roots = pure-Rust TLS using the OS root store (no OpenSSL).
livekit = {version = "0.7", optional = true, features = ["rustls-tls-native-roots"]}
tokio = {version = "1", optional = true, features = ["rt-multi-thread", "sync", "macros", "time"]}
parking_lot = "0.12"
fluxer_screen_frame_bus = { path = "../screen-frame-bus" }
futures-core = "0.3"
# StreamExt::next() for draining inbound NativeAudioStream/NativeVideoStream.
futures-util = {version = "0.3", optional = true}
libloading = "0.9.0"
fluxer_desktop_native = { path = "../rust" }
image = {version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp", "gif"]}
# System-camera capture for the native camera-publish path. Optional /
# feature-gated for the shipped desktop builds. `input-native` maps to V4L2 on
# Linux, AVFoundation on macOS, and Media Foundation on Windows.
# BUILD-HOST NOTE: `decoding` pulls `mozjpeg-sys`, which statically builds
# libjpeg and -- with its default `nasm_simd` -- needs NASM + a C compiler on the
# build host.
nokhwa = {version = "0.10", optional = true, default-features = false, features = ["input-native", "decoding"]}
# Person segmentation for camera background blur / replacement on platforms
# without a native segmentation API (Windows, Linux) and as the macOS fallback
# when Vision is unavailable. tract is a pure-Rust ONNX inference engine: no
# C++ runtime to link or ship per target, deterministic results, and the unit
# tests exercise the real model on any build host. It runs the bundled Apache-2.0
# MediaPipe Selfie Segmenter landscape model (models/, ~450 KB, embedded into the
# addon) in ~10 ms per 256x144 frame on a laptop core; an inference-cadence
# governor halves the rate on machines that miss the frame budget. Cost: ~40
# transitive crates, all pure Rust, compile-time only impact elsewhere.
tract-onnx = "0.23"
# DeepFilterNet3 noise suppression for the native device-microphone publish
# path, replacing the WebRTC-APM fallback when the renderer requests the
# deepFilter profile (parity with the web build's deepfilternet3-noise-filter
# WASM chain). The crates.io release (0.2.5, 2022) predates DFN3, so this pins
# the upstream git repo at the v0.5.6 release tag -- the combination the shipped
# deep-filter binaries were built from. `tract` + `default-model` run the
# bundled DFN3 ONNX models (~2 MB, embedded into the addon) on the same
# pure-Rust tract inference engine already used for person segmentation, albeit
# at tract 0.19 (deep_filter's tested series) alongside our 0.23 tree:
# compile-time cost only, no new native or C++ dependencies.
deep_filter = {git = "https://github.com/Rikorose/DeepFilterNet", rev = "978576aa8400552a4ce9730838c635aa30db5e61", default-features = false, features = ["tract", "default-model"]}
# deep_filter declares tract ^0.19.4, but newer releases in that semver range
# regress DFN3 model codegen (0.21.4 fails with "duplicate name .../Conv.bias"
# on the transposed convolutions). These renamed dependency keys exist solely
# to pin deep_filter's tract subtree to 0.19.16, the version in upstream's own
# v0.5.6 lockfile; tract's internal `=` pins drag tract-core/hir/data/linalg
# along. Never import these directly: the 0.23 `tract-onnx` entry above is the
# one the rest of this crate uses.
tract-onnx-deepfilter-pin = {package = "tract-onnx", version = "=0.19.16"}
tract-pulse-deepfilter-pin = {package = "tract-pulse", version = "=0.19.16"}
# Pinned to the 0.15 family to match deep_filter's public API surface
# (DfTract::process takes ndarray 0.15 ArrayView2 arguments).
ndarray = "0.15"
# macOS person segmentation for camera background blur / replacement. Vision's
# VNGeneratePersonSegmentationRequest produces the per-pixel person mask that the
# camera background transform composites with; without it the transform falls
# back to a fixed portrait ellipse. Pinned to the objc2 0.6 / framework 0.3
# family already used across fluxer_desktop/native, so no new dependency tree.
[target.'cfg(target_os = "macos")'.dependencies]
objc2 = "0.6"
objc2-foundation = {version = "0.3", features = ["NSArray", "NSDictionary", "NSError", "NSObject", "NSString"]}
objc2-vision = {version = "0.3", default-features = false, features = ["std", "VNRequest", "VNStatefulRequest", "VNGeneratePersonSegmentationRequest", "VNRequestHandler", "VNObservation", "VNTypes", "objc2-core-video", "objc2-core-foundation"]}
objc2-core-video = {version = "0.3", features = ["CVPixelBuffer", "CVImageBuffer", "CVBuffer", "CVReturn", "CVBase"]}
objc2-core-foundation = {version = "0.3", features = ["CFDictionary", "CFBase"]}
[build-dependencies]
napi-build = "2.3.2"
[dev-dependencies]
# Test-only (never linked into the shipped addon): parses the committed golden
# event fixtures so the renderer's (eventType, jsonPayload) parser contract is
# regression-locked offline, without a live SFU or a second identity.
serde_json = "1"
criterion = {version = "0.8", default-features = false, features = ["cargo_bench_support", "html_reports"]}
[[bench]]
name = "frame_bus"
harness = false
required-features = []
[[bench]]
name = "camera_background"
harness = false
required-features = ["bench-internals"]
[[bench]]
name = "deep_filter"
harness = false
required-features = ["bench-internals"]
[patch.crates-io]
libwebrtc = { path = "vendor/libwebrtc" }
webrtc-sys = { path = "vendor/webrtc-sys" }
tract-linalg-019 = { package = "tract-linalg", path = "vendor/tract-linalg-0.19.16" }
tract-linalg-023 = { package = "tract-linalg", path = "vendor/tract-linalg-0.23.1" }
@@ -0,0 +1,95 @@
{
"measured_at": "85e057a273fd",
"host": "darwin-arm64-apple-silicon",
"regression_budget_percent": 5.0,
"criterion_args": {
"warm_up_time_sec": 2,
"measurement_time_sec": 5
},
"benches": {
"frame_bus::get_sink": {
"median_ns": 13.292,
"low_ns": 13.189,
"high_ns": 13.408,
"budget_percent_override": 10.0,
"note": "~13ns op; noise floor reasoning."
},
"frame_bus::get_sink_miss": {
"median_ns": 3.9658,
"low_ns": 3.9444,
"high_ns": 3.9918,
"budget_percent_override": 15.0,
"note": "~4ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::native_handle_enqueue/nv12_320x180": {
"median_ns": 1.808,
"low_ns": 1.7961,
"high_ns": 1.8205,
"budget_percent_override": 15.0,
"note": "~2ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::native_handle_enqueue/nv12_1920x1080": {
"median_ns": 1.8086,
"low_ns": 1.8008,
"high_ns": 1.8171,
"budget_percent_override": 15.0,
"note": "~2ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::native_handle_enqueue/nv12_3840x2160": {
"median_ns": 1.8141,
"low_ns": 1.7964,
"high_ns": 1.8333,
"budget_percent_override": 15.0,
"note": "~2ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::native_handle_enqueue/bgra_320x180": {
"median_ns": 1.8261,
"low_ns": 1.8172,
"high_ns": 1.8361,
"budget_percent_override": 15.0,
"note": "~2ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::native_handle_enqueue/bgra_1920x1080": {
"median_ns": 1.7952,
"low_ns": 1.7877,
"high_ns": 1.8027,
"budget_percent_override": 15.0,
"note": "~2ns op; below the practical resolution of repeated criterion runs."
},
"frame_bus::enqueue_discard/nv12_320x180": {
"median_ns": 1391.4,
"low_ns": 1342.4,
"high_ns": 1455.8,
"budget_percent_override": 20.0,
"note": "Allocator-bound: bench does vec![0; total] per iter. Run-to-run sigma observed ~15%; budget set above that."
},
"frame_bus::enqueue_discard/nv12_1920x1080": {
"median_ns": 29502.0,
"low_ns": 28397.0,
"high_ns": 30819.0,
"budget_percent_override": 20.0,
"note": "Allocator-bound; see nv12_320x180 note."
},
"frame_bus::enqueue_discard/nv12_3840x2160": {
"median_ns": 121900.0,
"low_ns": 114880.0,
"high_ns": 129680.0,
"budget_percent_override": 25.0,
"note": "Largest 4K allocator-bound bench; widest budget."
},
"frame_bus::enqueue_discard/bgra_320x180": {
"median_ns": 2807.2,
"low_ns": 2695.1,
"high_ns": 2937.6,
"budget_percent_override": 20.0,
"note": "Allocator-bound; see nv12_320x180 note."
},
"frame_bus::enqueue_discard/bgra_1920x1080": {
"median_ns": 83692.0,
"low_ns": 78328.0,
"high_ns": 90200.0,
"budget_percent_override": 25.0,
"note": "Allocator-bound 1080p BGRA bench; widest budget."
}
}
}
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
use fluxer_webrtc_sender::bench_internals::{
BlurScratch, MaskRefiner, blur_plane_masked, composite_masked_plane,
};
use std::hint::black_box;
const BENCH_WIDTH: usize = 1280;
const BENCH_HEIGHT: usize = 720;
const BENCH_BLUR_RADIUS_PASS: usize = 6;
const BENCH_MASK_FEATHER_PX: usize = 64;
fn synth_plane(seed: usize) -> Vec<u8> {
let mut plane = vec![0u8; BENCH_WIDTH * BENCH_HEIGHT];
for (index, value) in plane.iter_mut().enumerate() {
*value = ((index * 31 + seed) % 251) as u8;
}
plane
}
fn synth_person_mask() -> Vec<u8> {
let mut mask = vec![0u8; BENCH_WIDTH * BENCH_HEIGHT];
let person_edge = BENCH_WIDTH / 2;
for row in mask.chunks_exact_mut(BENCH_WIDTH) {
for (x, value) in row.iter_mut().enumerate() {
*value = if x < person_edge {
255
} else if x < person_edge + BENCH_MASK_FEATHER_PX {
(255 - (x - person_edge) * 255 / BENCH_MASK_FEATHER_PX) as u8
} else {
0
};
}
}
mask
}
fn bench_blur_plane_masked(c: &mut Criterion) {
let source = synth_plane(7);
let mask = synth_person_mask();
let mut plane = source.clone();
let mut scratch = BlurScratch::new(BENCH_WIDTH, BENCH_HEIGHT);
let mut group = c.benchmark_group("camera_background::blur_plane_masked");
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
group.bench_function("1280x720_radius_pass6", |b| {
b.iter(|| {
plane.copy_from_slice(&source);
blur_plane_masked(
black_box(&mut plane),
BENCH_WIDTH,
BENCH_HEIGHT,
black_box(&mask),
BENCH_BLUR_RADIUS_PASS,
&mut scratch,
);
black_box(&plane);
})
});
group.finish();
}
fn bench_mask_refine(c: &mut Criterion) {
let luma = synth_plane(13);
let raw_mask = synth_person_mask();
let mut mask = raw_mask.clone();
let mut refiner = MaskRefiner::new(BENCH_WIDTH, BENCH_HEIGHT);
let mut group = c.benchmark_group("camera_background::mask_refine");
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
group.bench_function("1280x720_guided_refine", |b| {
b.iter(|| {
mask.copy_from_slice(&raw_mask);
refiner.refine(black_box(&luma), black_box(&mut mask));
black_box(&mask);
})
});
group.finish();
}
fn bench_composite_masked_plane(c: &mut Criterion) {
let source = synth_plane(11);
let background = synth_plane(151);
let mask = synth_person_mask();
let mut plane = source.clone();
let mut group = c.benchmark_group("camera_background::composite_masked_plane");
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
group.bench_function("1280x720_feathered_mask", |b| {
b.iter(|| {
plane.copy_from_slice(&source);
composite_masked_plane(
black_box(&mut plane),
black_box(&background),
BENCH_WIDTH,
BENCH_HEIGHT,
&mask,
);
black_box(&plane);
})
});
group.finish();
}
criterion_group!(
benches,
bench_blur_plane_masked,
bench_mask_refine,
bench_composite_masked_plane
);
criterion_main!(benches);
@@ -0,0 +1,44 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
use fluxer_webrtc_sender::bench_internals::{
DEEP_FILTER_FRAME_SAMPLES, DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX, DeepFilterProcessor,
};
use std::hint::black_box;
fn next_noise_sample(seed: &mut u32) -> i16 {
*seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
((*seed >> 16) as u16 as i16) / 4
}
fn noise_frame(seed: &mut u32) -> Vec<i16> {
let mut frame = vec![0i16; DEEP_FILTER_FRAME_SAMPLES];
for sample in frame.iter_mut() {
*sample = next_noise_sample(seed);
}
frame
}
fn bench_deep_filter_process_frame(c: &mut Criterion) {
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
.expect("embedded model must initialize");
let mut seed = 0x9e37_79b9u32;
let mut group = c.benchmark_group("deep_filter");
group.throughput(Throughput::Elements(DEEP_FILTER_FRAME_SAMPLES as u64));
group.bench_function("process_frame_10ms", |bencher| {
bencher.iter_batched(
|| noise_frame(&mut seed),
|mut frame| {
processor
.process_frame(black_box(&mut frame))
.expect("processing must succeed");
frame
},
criterion::BatchSize::SmallInput,
);
});
group.finish();
}
criterion_group!(benches, bench_deep_filter_process_frame);
criterion_main!(benches);
@@ -0,0 +1,220 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use fluxer_screen_frame_bus::{
BgraFrame, EnqueueOutcome, NATIVE_SCREEN_FRAME_SINK_ACCEPTED,
NATIVE_SCREEN_FRAME_SINK_HANDLE_MAGIC, NATIVE_SCREEN_FRAME_SINK_HANDLE_VERSION,
NativeScreenFrameSinkHandle, Nv12Frame, ScreenFrame, ScreenFrameSink, get_sink, register_sink,
unregister_sink,
};
use std::ffi::c_void;
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
struct DiscardSink(AtomicU64);
impl ScreenFrameSink for DiscardSink {
fn enqueue(&self, _frame: ScreenFrame) -> EnqueueOutcome {
self.0.fetch_add(1, Ordering::Relaxed);
EnqueueOutcome::Accepted
}
}
struct NativeDiscardSink(AtomicU64);
unsafe extern "C" fn retain_native_discard_sink(context: *const c_void) {
unsafe { Arc::increment_strong_count(context.cast::<NativeDiscardSink>()) };
}
unsafe extern "C" fn release_native_discard_sink(context: *const c_void) {
unsafe { drop(Arc::from_raw(context.cast::<NativeDiscardSink>())) };
}
unsafe extern "C" fn enqueue_native_discard_nv12(
context: *const c_void,
data: *const u8,
data_len: usize,
_width: u32,
_height: u32,
_stride_y: u32,
_stride_uv: u32,
_timestamp_us: i64,
) -> u32 {
if !context.is_null() && !data.is_null() && data_len > 0 {
unsafe {
(*context.cast::<NativeDiscardSink>())
.0
.fetch_add(1, Ordering::Relaxed)
};
}
NATIVE_SCREEN_FRAME_SINK_ACCEPTED
}
unsafe extern "C" fn enqueue_native_discard_bgra(
context: *const c_void,
data: *const u8,
data_len: usize,
_width: u32,
_height: u32,
_stride: u32,
_timestamp_us: i64,
) -> u32 {
if !context.is_null() && !data.is_null() && data_len > 0 {
unsafe {
(*context.cast::<NativeDiscardSink>())
.0
.fetch_add(1, Ordering::Relaxed)
};
}
NATIVE_SCREEN_FRAME_SINK_ACCEPTED
}
fn native_discard_handle() -> (
Arc<NativeDiscardSink>,
fluxer_screen_frame_bus::NativeScreenFrameSinkHandleRef,
) {
let sink = Arc::new(NativeDiscardSink(AtomicU64::new(0)));
let raw_context = Arc::into_raw(sink.clone()).cast::<c_void>();
let handle = NativeScreenFrameSinkHandle {
magic: NATIVE_SCREEN_FRAME_SINK_HANDLE_MAGIC,
version: NATIVE_SCREEN_FRAME_SINK_HANDLE_VERSION,
context: raw_context,
retain: retain_native_discard_sink,
release: release_native_discard_sink,
enqueue_nv12: Some(enqueue_native_discard_nv12),
enqueue_bgra: Some(enqueue_native_discard_bgra),
enqueue_mac_cv_pixel_buffer: None,
enqueue_dmabuf: None,
enqueue_shared_texture: None,
};
let retained = unsafe { handle.retain_ref().expect("valid native sink handle") };
unsafe { release_native_discard_sink(raw_context) };
(sink, retained)
}
fn bench_registry_lookup(c: &mut Criterion) {
let id = format!("bench:registry:{}", std::process::id());
register_sink(
id.clone(),
Arc::new(DiscardSink(AtomicU64::new(0))) as Arc<dyn ScreenFrameSink>,
);
c.bench_function("frame_bus::get_sink", |b| {
b.iter(|| {
let s = get_sink(black_box(id.as_str()));
black_box(s);
})
});
unregister_sink(&id);
}
fn synth_nv12(width: u32, height: u32) -> ScreenFrame {
let total = (width * height) as usize + (width * (height / 2)) as usize;
let data = vec![0x80u8; total];
ScreenFrame::Nv12(Nv12Frame {
data: data.into(),
width,
height,
stride_y: width,
stride_uv: width,
timestamp_us: 1,
})
}
fn synth_bgra(width: u32, height: u32) -> ScreenFrame {
let total = (width * height * 4) as usize;
let data = vec![0xff; total];
ScreenFrame::Bgra(BgraFrame {
data,
width,
height,
stride: width * 4,
timestamp_us: 1,
})
}
fn bench_discard_sink_enqueue(c: &mut Criterion) {
let id = format!("bench:discard:{}", std::process::id());
let sink = Arc::new(DiscardSink(AtomicU64::new(0)));
register_sink(id.clone(), sink.clone() as Arc<dyn ScreenFrameSink>);
let mut group = c.benchmark_group("frame_bus::enqueue_discard");
for (label, w, h) in [
("nv12_320x180", 320u32, 180u32),
("nv12_1920x1080", 1920, 1080),
("nv12_3840x2160", 3840, 2160),
("bgra_320x180", 320, 180),
("bgra_1920x1080", 1920, 1080),
] {
let bytes = if label.starts_with("nv12") {
((w * h) + (w * (h / 2))) as u64
} else {
(w * h * 4) as u64
};
group.throughput(Throughput::Bytes(bytes));
group.bench_with_input(BenchmarkId::from_parameter(label), &(), |b, _| {
b.iter(|| {
let frame = if label.starts_with("nv12") {
synth_nv12(w, h)
} else {
synth_bgra(w, h)
};
let s = get_sink(id.as_str()).unwrap();
let outcome = s.enqueue(black_box(frame));
black_box(outcome);
})
});
}
group.finish();
unregister_sink(&id);
}
fn bench_native_handle_enqueue(c: &mut Criterion) {
let (_sink, handle) = native_discard_handle();
let mut group = c.benchmark_group("frame_bus::native_handle_enqueue");
for (label, w, h) in [
("nv12_320x180", 320u32, 180u32),
("nv12_1920x1080", 1920, 1080),
("nv12_3840x2160", 3840, 2160),
("bgra_320x180", 320, 180),
("bgra_1920x1080", 1920, 1080),
] {
let bytes = if label.starts_with("nv12") {
((w * h) + (w * (h / 2))) as usize
} else {
(w * h * 4) as usize
};
let data = vec![0x80u8; bytes];
group.throughput(Throughput::Bytes(bytes as u64));
group.bench_with_input(BenchmarkId::from_parameter(label), &(), |b, _| {
b.iter(|| {
let outcome = if label.starts_with("nv12") {
handle.enqueue_nv12_copy(black_box(&data), w, h, w, w, 1)
} else {
handle.enqueue_bgra_copy(black_box(&data), w, h, w * 4, 1)
};
black_box(outcome);
})
});
}
group.finish();
}
fn bench_registry_lookup_miss(c: &mut Criterion) {
c.bench_function("frame_bus::get_sink_miss", |b| {
b.iter(|| {
let s = get_sink(black_box("nonexistent"));
black_box(s);
})
});
}
criterion_group!(
benches,
bench_registry_lookup,
bench_registry_lookup_miss,
bench_native_handle_enqueue,
bench_discard_sink_enqueue
);
criterion_main!(benches);
@@ -0,0 +1,99 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::env;
use std::path::{Path, PathBuf};
fn main() {
println!("cargo:rerun-if-changed=vendor/webrtc-sys/src/nvidia/NvCodec/include/cuda.h");
println!("cargo:rustc-check-cfg=cfg(fluxer_linux_nvenc)");
println!("cargo:rustc-check-cfg=cfg(fluxer_windows_nvenc)");
println!("cargo:rustc-check-cfg=cfg(fluxer_windows_nvenc_encoder)");
println!("cargo:rustc-check-cfg=cfg(fluxer_macos_videotoolbox)");
let cuda_include_dir = vendored_cuda_include_dir();
let cuda_version = cuda_include_dir
.as_deref()
.and_then(read_cuda_version)
.unwrap_or(0);
let is_macos = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos");
let has_linux_nvenc = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux")
&& supports_webrtc_sys_nvenc_arch()
&& cuda_version > 0;
let has_windows_nvenc = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows")
&& supports_webrtc_sys_nvenc_arch();
let has_windows_nvenc_encoder = has_windows_nvenc;
println!(
"cargo:rustc-env=FLUXER_LINUX_NVENC_COMPILED={}",
if has_linux_nvenc { "1" } else { "0" }
);
println!(
"cargo:rustc-env=FLUXER_WINDOWS_NVENC_COMPILED={}",
if has_windows_nvenc { "1" } else { "0" }
);
println!(
"cargo:rustc-env=FLUXER_WINDOWS_NVENC_ENCODER_COMPILED={}",
if has_windows_nvenc_encoder { "1" } else { "0" }
);
println!("cargo:rustc-env=FLUXER_CUDA_VERSION={cuda_version}");
if has_linux_nvenc {
println!("cargo:rustc-cfg=fluxer_linux_nvenc");
}
if has_windows_nvenc {
println!("cargo:rustc-cfg=fluxer_windows_nvenc");
}
if has_windows_nvenc_encoder {
println!("cargo:rustc-cfg=fluxer_windows_nvenc_encoder");
}
println!(
"cargo:rustc-env=FLUXER_MACOS_VIDEOTOOLBOX_COMPILED={}",
if is_macos { "1" } else { "0" }
);
if is_macos {
println!("cargo:rustc-cfg=fluxer_macos_videotoolbox");
println!("cargo:rustc-link-lib=framework=CoreFoundation");
println!("cargo:rustc-link-lib=framework=CoreMedia");
println!("cargo:rustc-link-lib=framework=VideoToolbox");
}
configure_darwin_objc_linking();
napi_build::setup();
}
fn configure_darwin_objc_linking() {
if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") {
return;
}
println!("cargo:rustc-link-arg=-ObjC");
}
fn vendored_cuda_include_dir() -> Option<PathBuf> {
let manifest_dir = env::var("CARGO_MANIFEST_DIR").ok()?;
Some(
PathBuf::from(manifest_dir)
.join("vendor")
.join("webrtc-sys")
.join("src")
.join("nvidia")
.join("NvCodec")
.join("include"),
)
}
fn read_cuda_version(include_dir: &Path) -> Option<u32> {
let content = std::fs::read_to_string(include_dir.join("cuda.h")).ok()?;
content.lines().find_map(|line| {
let mut parts = line.split_whitespace();
match (parts.next(), parts.next(), parts.next()) {
(Some("#define"), Some("CUDA_VERSION"), Some(value)) => value.parse().ok(),
_ => None,
}
})
}
fn supports_webrtc_sys_nvenc_arch() -> bool {
let Ok(arch) = env::var("CARGO_CFG_TARGET_ARCH") else {
return false;
};
matches!(arch.as_str(), "x86_64" | "i686" | "aarch64") || arch.contains("arm")
}
+363
View File
@@ -0,0 +1,363 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export interface AudioOutputDevice {
deviceId: string;
label: string;
isDefault: boolean;
}
export interface AudioInputDevice {
deviceId: string;
label: string;
isDefault: boolean;
}
export interface PublishMicrophoneOptions {
deviceId?: string;
echoCancellation?: boolean;
noiseSuppression?: boolean;
autoGainControl?: boolean;
deepFilter?: boolean;
deepFilterNoiseReductionLevel?: number;
maxBitrateBps?: number;
}
export interface VoiceEngineV2BridgeCapabilities {
microphoneCapture: boolean;
syntheticMicrophonePcm: boolean;
cameraCapture: boolean;
nativeCameraBackgrounds: boolean;
screenShare: boolean;
screenShareEncodingUpdate: boolean;
screenShareAudio: boolean;
deviceLists: boolean;
outputDeviceSelection: boolean;
participantVolume: boolean;
remoteTrackSubscription: boolean;
dataChannel: boolean;
connectionStats: boolean;
nativeVideoFrames: boolean;
hardwareEncoderCapabilities: boolean;
}
export type VoiceEngineTrackKind = 'audio' | 'video';
export type VoiceEngineTrackSource =
| 'unknown'
| 'camera'
| 'microphone'
| 'screen_share'
| 'screen_share_audio'
| 'screenshare'
| 'screenshareAudio';
export type VoiceEngineSubscriptionStatus = 'desired' | 'subscribed' | 'unsubscribed';
export type VoiceEngineConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost';
export interface VoiceEngineParticipantEventPayload {
sid: string;
identity: string;
name: string;
}
export interface VoiceEngineTrackEventPayload {
participantSid: string;
identity: string;
participantName: string;
trackSid: string;
trackName: string;
kind: VoiceEngineTrackKind;
source: VoiceEngineTrackSource;
muted: boolean;
}
export interface VoiceEngineSubscribedTrackEventPayload extends VoiceEngineTrackEventPayload {
subscribed: boolean;
subscriptionStatus: VoiceEngineSubscriptionStatus;
}
export interface VoiceEngineTrackSubscriptionFailedEventPayload {
participantSid: string;
identity: string;
participantName: string;
trackSid: string;
error: string;
trackName?: string;
kind?: VoiceEngineTrackKind;
source?: VoiceEngineTrackSource;
muted?: boolean;
subscribed?: boolean;
subscriptionStatus?: VoiceEngineSubscriptionStatus;
}
export interface VoiceEngineLocalTrackRepublishedEventPayload extends VoiceEngineTrackEventPayload {
previousTrackSid: string;
}
export interface VoiceEngineV2BridgeEventPayloads {
connected: Record<keyof any, never>;
connectionState: {state: string};
disconnected: {reason: string};
participantJoined: VoiceEngineParticipantEventPayload;
participantLeft: VoiceEngineParticipantEventPayload;
participantNameChanged: {sid: string; identity: string; oldName: string; name: string};
participantMetadataChanged: {
sid: string;
identity: string;
name: string;
oldMetadata: string;
metadata: string;
attributes: Record<string, string>;
};
participantAttributesChanged: {
sid: string;
identity: string;
name: string;
attributes: Record<string, string>;
changedAttributes: Record<string, string>;
};
trackPublished: VoiceEngineSubscribedTrackEventPayload;
trackUnpublished: VoiceEngineSubscribedTrackEventPayload;
trackSubscribed: VoiceEngineSubscribedTrackEventPayload;
trackUnsubscribed: VoiceEngineSubscribedTrackEventPayload;
trackSubscriptionFailed: VoiceEngineTrackSubscriptionFailedEventPayload;
trackMuted: VoiceEngineTrackEventPayload;
trackUnmuted: VoiceEngineTrackEventPayload;
localTrackPublished: VoiceEngineTrackEventPayload;
localTrackUnpublished: VoiceEngineTrackEventPayload;
localTrackRepublished: VoiceEngineLocalTrackRepublishedEventPayload;
activeSpeakers: {sids: Array<string>; participants: Array<VoiceEngineParticipantEventPayload>};
connectionQuality: {sid: string; identity: string; name: string; quality: VoiceEngineConnectionQuality};
dataReceived: {
payloadBytes: Array<number>;
payloadText?: string;
topic?: string;
reliable: boolean;
kind: 'reliable' | 'lossy';
participantSid?: string;
identity?: string;
participantName?: string;
};
e2eeState: {sid: string; identity: string; name: string; state: string};
stats: VoiceEngineV2BridgeStats;
audioPlaybackUnavailable: {message: string};
}
export type VoiceEngineKnownEventType = keyof VoiceEngineV2BridgeEventPayloads;
export type VoiceEngineV2BridgeEventType = VoiceEngineKnownEventType | (string & {});
export interface VoiceEngineOutboundStats {
trackSid: string;
source: string;
kind: VoiceEngineTrackKind;
codec?: string;
bitrateKbps: number;
packetsLost: number;
fps?: number;
}
export interface VoiceEngineInboundStats {
participantSid: string;
trackSid: string;
kind: VoiceEngineTrackKind;
codec?: string;
bitrateKbps: number;
packetsLost: number;
jitterMs?: number;
audioLevel?: number;
}
export interface VoiceEngineV2BridgeStats {
rttMs: number | null;
outbound: Array<VoiceEngineOutboundStats>;
inbound: Array<VoiceEngineInboundStats>;
droppedVideoFrameCallbacks?: number;
send?: VoiceEngineSendStats | null;
}
export interface VoiceEngineSendStats {
outgoingVideoQueueDepth: number;
outgoingVideoFramesProduced: number;
outgoingVideoFramesAccepted: number;
outgoingVideoFramesDropped: number;
outgoingVideoFramesCoalesced: number;
outgoingVideoFramesCaptured: number;
outgoingVideoCaptureFailures: number;
outgoingVideoEffectiveFps: number;
outgoingVideoTargetFps: number;
outgoingVideoMaxQueueAgeMs: number;
outgoingVideoMaxPushLatencyMs: number;
outgoingAudioBufferTargetMs: number;
outgoingAudioBufferMaxMs: number;
outgoingAudioUnderruns: number;
outgoingAudioRebuffers: number;
outgoingAudioMaxFrameGapMs: number;
adaptiveSendTier: string;
adaptiveSendReason: string;
}
export interface PublishScreenShareOptions {
adaptiveSend?: boolean;
minVideoFps?: number;
maxAudioBufferMs?: number;
pacing?: 'sender' | 'source';
captureId: string;
trackName?: string;
}
export type VoiceEngineRemoteTrackSubscriptionQuality = 'low' | 'medium' | 'high';
export interface VoiceEngineV2BridgeRemoteTrackSubscriptionOptions {
participantIdentity: string;
source: string;
subscribed: boolean;
enabled?: boolean;
quality?: VoiceEngineRemoteTrackSubscriptionQuality;
}
export interface PublishCameraOptions {
deviceId?: string;
width?: number;
height?: number;
frameRate?: number;
mirror?: boolean;
backgroundMode?: 'none' | 'non' | 'blur' | 'custom';
backgroundCustomMediaPath?: string;
backgroundCustomMediaKind?: 'static' | 'animated' | 'video';
backgroundBlurStrength?: number;
codec?: '' | 'vp8' | 'vp9' | 'h264' | 'av1' | 'h265' | 'hevc';
maxBitrateBps?: number;
maxFramerate?: number;
}
export interface PublishProcessedCameraOptions {
width: number;
height: number;
frameRate: number;
}
export interface PublishProcessedCameraResult {
trackSid: string;
}
export type NativeCameraFrameSinkHandle = object;
export interface ProcessedCameraFrame {
format: 'i420';
width: number;
height: number;
timestampUs: number;
data: Buffer;
}
export interface CameraDeviceInfo {
deviceId: string;
label: string;
description: string;
index?: number | null;
deviceIdAliases: Array<string>;
}
export interface HardwareEncoderCapability {
available: boolean;
backend: 'nvenc' | 'videotoolbox' | 'none';
compiled: boolean;
runtime: boolean;
codecs: Array<string>;
zeroCopy: boolean;
nativeInputs: Array<'dmabuf' | 'd3d11-texture' | string>;
reason?: string;
detail?: string;
}
export interface VoiceEngineV2BridgeConnectOptions {
autoSubscribe?: boolean;
adaptiveStream?: boolean;
dynacast?: boolean;
}
export declare class VoiceEngine {
constructor();
setEventCallback(callback: (eventType: VoiceEngineV2BridgeEventType, jsonPayload: string) => void): void;
setVideoFrameCallback(callback: (metaJson: string, data: Buffer) => void): void;
clearVideoFrameCallback(): void;
setCountInboundAudio(enabled: boolean): void;
connect(url: string, token: string, e2eeKey?: Buffer, options?: VoiceEngineV2BridgeConnectOptions): Promise<void>;
disconnect(): Promise<void>;
isConnected(): boolean;
publishScreenShare(
width: number,
height: number,
codec: '' | 'vp8' | 'vp9' | 'h264' | 'av1' | 'h265' | 'hevc' | undefined,
maxBitrateBps: number | undefined,
maxFramerate: number | undefined,
simulcast: boolean | undefined,
options: PublishScreenShareOptions,
): Promise<void>;
updateScreenShareEncoding(
width: number,
height: number,
maxBitrateBps: number | undefined,
maxFramerate: number | undefined,
options: PublishScreenShareOptions,
): Promise<void>;
createScreenFrameSinkHandle(captureId: string): unknown | null;
unpublishScreenShare(): Promise<void>;
isPublishingScreen(): boolean;
publishScreenShareAudio(sampleRate: number, numChannels: number): Promise<void>;
pushScreenSharePcm(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
pushScreenShareFloat(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
unpublishScreenShareAudio(): Promise<void>;
isPublishingScreenAudio(): boolean;
publishMicrophone(sampleRate: number, numChannels: number): Promise<void>;
publishDeviceMicrophone(opts?: PublishMicrophoneOptions): Promise<void>;
pushPcm(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
setMicEnabled(enabled: boolean): Promise<void>;
setSpeakingDetection(localThresholdRms: number, remoteThresholdRms: number): void;
publishCamera(opts?: PublishCameraOptions): Promise<void>;
updateCameraCapture(opts?: PublishCameraOptions): Promise<void>;
publishProcessedCamera(opts: PublishProcessedCameraOptions): Promise<PublishProcessedCameraResult>;
publishNativeCameraSink(opts?: PublishCameraOptions): Promise<PublishProcessedCameraResult>;
createCameraFrameSinkHandle(): NativeCameraFrameSinkHandle | null;
pushProcessedCameraFrame(frame: ProcessedCameraFrame): Promise<boolean>;
publishDeviceScreenShare(opts?: PublishCameraOptions): Promise<void>;
listCameraDevices(): Array<CameraDeviceInfo>;
unpublishCamera(): Promise<void>;
isPublishingCamera(): boolean;
listAudioInputDevices(): Promise<Array<AudioInputDevice>>;
listAudioOutputDevices(): Promise<Array<AudioOutputDevice>>;
setAudioOutputDevice(deviceId: string): Promise<void>;
ensurePlatformAudio(): Promise<void>;
setParticipantVolume(participantSid: string, volume: number): Promise<void>;
setRemoteTrackSubscription(options: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise<void>;
publishData(
payload: Buffer | ArrayBuffer | Uint8Array,
options?: {reliable?: boolean; topic?: string; destinationIdentities?: Array<string>},
): Promise<void>;
getConnectionStats(): Promise<VoiceEngineV2BridgeStats>;
inboundAudioFrames(): number;
inboundVideoFrames(): number;
droppedVideoFrameCallbacks(): number;
droppedEngineEvents(): number;
}
export declare function isSupported(): boolean;
export declare function getEngineBridgeVersion(): number | null;
export declare function assertEngineBridgeVersion(version: number): void;
export declare function getHardwareEncoderCapability(): HardwareEncoderCapability;
export declare function getHardwareEncoderCapabilities(): HardwareEncoderCapability;
export declare function getCapabilities(): VoiceEngineV2BridgeCapabilities;
export declare function hasNativeCameraBackgrounds(): boolean;
export declare function prewarmVoiceEngine(): void;
export declare function probeAudioDeviceModule(): Promise<boolean>;
export declare const loadError: Error | null;
export declare function __nativeFileNameForTests(platform: string, arch: string): string;
export declare function __setBindingForTests(binding: unknown): void;
@@ -0,0 +1,509 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const {existsSync} = require('node:fs');
const {join, sep} = require('node:path');
const MODULE_NAME = '@fluxer/webrtc-sender';
function resolveNativeRoot() {
const asarSegment = `${sep}app.asar${sep}`;
if (!__dirname.includes(asarSegment)) return __dirname;
const unpackedDir = __dirname.replace(asarSegment, `${sep}app.asar.unpacked${sep}`);
return existsSync(unpackedDir) ? unpackedDir : __dirname;
}
function nativeFileName(platform = process.platform, arch = process.arch) {
if ((arch !== 'x64' && arch !== 'arm64') || !['darwin', 'linux', 'win32'].includes(platform)) {
throw new Error(`${MODULE_NAME} not supported on ${platform}-${arch}`);
}
if (platform === 'darwin') return `webrtc-sender.darwin-${arch}.node`;
if (platform === 'linux') return `webrtc-sender.linux-${arch}-gnu.node`;
if (platform === 'win32') return `webrtc-sender.win32-${arch}-msvc.node`;
throw new Error(`${MODULE_NAME} not supported on ${platform}-${arch}`);
}
let binding = null;
let loadError = null;
try {
const nativePath = join(resolveNativeRoot(), nativeFileName());
if (existsSync(nativePath)) {
try {
binding = require(nativePath);
} catch (error) {
loadError = error instanceof Error ? error : new Error(String(error));
}
} else {
loadError = new Error(`${MODULE_NAME} native binary missing: ${nativePath}`);
}
} catch (error) {
loadError = error instanceof Error ? error : new Error(String(error));
}
function isSupported() {
return Boolean(binding);
}
function unavailableHardwareEncoderCapability(reason, detail) {
return {
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason,
detail,
};
}
function getHardwareEncoderCapability() {
if (!binding) {
return unavailableHardwareEncoderCapability(
'native_binding_unavailable',
loadError ? loadError.message : `${MODULE_NAME} binding unavailable`,
);
}
if (typeof binding.getHardwareEncoderCapability !== 'function') {
return unavailableHardwareEncoderCapability(
'native_capability_unavailable',
`${MODULE_NAME} native binding does not export getHardwareEncoderCapability`,
);
}
return binding.getHardwareEncoderCapability();
}
function getHardwareEncoderCapabilities() {
return getHardwareEncoderCapability();
}
function hasVoiceEngineMethod(name) {
const prototype = binding?.VoiceEngine?.prototype;
return Boolean(prototype && typeof prototype[name] === 'function');
}
function hasNativeCameraBackgrounds() {
if (!binding) return false;
if (typeof binding.hasNativeCameraBackgrounds !== 'function') return false;
return binding.hasNativeCameraBackgrounds() === true;
}
function getCapabilities() {
const hasVoiceEngine = Boolean(binding && typeof binding.VoiceEngine === 'function');
return {
microphoneCapture: hasVoiceEngineMethod('publishDeviceMicrophone'),
syntheticMicrophonePcm: hasVoiceEngineMethod('publishMicrophone'),
cameraCapture: hasVoiceEngineMethod('publishCamera') && hasVoiceEngineMethod('listCameraDevices'),
nativeCameraBackgrounds: hasNativeCameraBackgrounds(),
screenShare: hasVoiceEngineMethod('publishScreenShare') && hasVoiceEngineMethod('unpublishScreenShare'),
screenShareEncodingUpdate: hasVoiceEngineMethod('updateScreenShareEncoding'),
screenShareAudio:
hasVoiceEngineMethod('publishScreenShareAudio') &&
hasVoiceEngineMethod('pushScreenSharePcm') &&
hasVoiceEngineMethod('pushScreenShareFloat') &&
hasVoiceEngineMethod('unpublishScreenShareAudio'),
deviceLists: hasVoiceEngineMethod('listAudioInputDevices') && hasVoiceEngineMethod('listAudioOutputDevices'),
outputDeviceSelection: hasVoiceEngineMethod('setAudioOutputDevice'),
participantVolume: hasVoiceEngineMethod('setParticipantVolume'),
remoteTrackSubscription: hasVoiceEngineMethod('setRemoteTrackSubscription'),
dataChannel: hasVoiceEngineMethod('publishData'),
connectionStats: hasVoiceEngineMethod('getConnectionStats'),
nativeVideoFrames: hasVoiceEngineMethod('setVideoFrameCallback'),
hardwareEncoderCapabilities: hasVoiceEngine && typeof getHardwareEncoderCapability === 'function',
};
}
function requireScreenShareCaptureId(options, operation) {
if (!options || typeof options.captureId !== 'string' || options.captureId.trim().length === 0) {
throw new Error(`${operation} requires a non-empty captureId`);
}
}
function getEngineBridgeVersion() {
if (!binding) return null;
if (typeof binding.getEngineBridgeVersion !== 'function') return null;
return binding.getEngineBridgeVersion();
}
function assertEngineBridgeVersion(version) {
if (!binding) {
throw loadError || new Error(`${MODULE_NAME} binding unavailable`);
}
if (typeof binding.assertEngineBridgeVersion !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export assertEngineBridgeVersion`);
}
binding.assertEngineBridgeVersion(version);
}
function prewarmVoiceEngine() {
if (!binding) {
if (loadError) throw loadError;
return;
}
if (typeof binding.prewarmVoiceEngine === 'function') {
return binding.prewarmVoiceEngine();
}
}
function probeAudioDeviceModule() {
if (!binding) {
if (loadError) throw loadError;
return Promise.resolve(false);
}
if (typeof binding.probeAudioDeviceModule === 'function') {
return Promise.resolve(binding.probeAudioDeviceModule());
}
return Promise.resolve(true);
}
function normalizeCameraOptions(opts = {}) {
return {
deviceId: opts.deviceId,
width: opts.width,
height: opts.height,
frameRate: opts.frameRate,
mirror: opts.mirror,
backgroundMode: opts.backgroundMode,
backgroundCustomMediaPath: opts.backgroundCustomMediaPath,
backgroundCustomMediaKind: opts.backgroundCustomMediaKind,
backgroundBlurStrength: opts.backgroundBlurStrength,
codec: opts.codec,
maxBitrateBps: opts.maxBitrateBps,
maxFramerate: opts.maxFramerate,
};
}
class VoiceEngine {
constructor() {
if (!binding) {
throw loadError || new Error(`${MODULE_NAME} binding unavailable`);
}
this.native = new binding.VoiceEngine();
}
setEventCallback(callback) {
return this.native.setEventCallback(callback);
}
setVideoFrameCallback(callback) {
return this.native.setVideoFrameCallback(callback);
}
clearVideoFrameCallback() {
if (typeof this.native.clearVideoFrameCallback === 'function') {
return this.native.clearVideoFrameCallback();
}
if (typeof this.native.setVideoFrameCallback === 'function') {
return this.native.setVideoFrameCallback(() => {});
}
return undefined;
}
setCountInboundAudio(enabled) {
return this.native.setCountInboundAudio(enabled);
}
connect(url, token, e2eeKey, options) {
return this.native.connect(url, token, e2eeKey, options);
}
disconnect() {
return this.native.disconnect();
}
isConnected() {
return this.native.isConnected();
}
publishScreenShare(width, height, codec = '', maxBitrateBps, maxFramerate, simulcast, options) {
requireScreenShareCaptureId(options, 'Native screen-share publish');
return this.native.publishScreenShare(width, height, codec, maxBitrateBps, maxFramerate, simulcast, options);
}
updateScreenShareEncoding(width, height, maxBitrateBps, maxFramerate, options) {
requireScreenShareCaptureId(options, 'Native screen-share encoding update');
if (typeof this.native.updateScreenShareEncoding === 'function') {
return this.native.updateScreenShareEncoding(width, height, maxBitrateBps, maxFramerate, options);
}
return Promise.reject(new Error('Native screen-share encoding update is unavailable'));
}
createScreenFrameSinkHandle(captureId) {
if (typeof this.native.createScreenFrameSinkHandle !== 'function') return null;
return this.native.createScreenFrameSinkHandle(captureId);
}
createScreenAudioSinkHandle() {
if (typeof this.native.createScreenAudioSinkHandle !== 'function') return null;
return this.native.createScreenAudioSinkHandle();
}
unpublishScreenShare() {
return this.native.unpublishScreenShare();
}
isPublishingScreen() {
return this.native.isPublishingScreen();
}
publishScreenShareAudio(sampleRate, numChannels) {
return this.native.publishScreenShareAudio(sampleRate, numChannels);
}
pushScreenSharePcm(buffer, sampleRate, numChannels) {
return this.native.pushScreenSharePcm(buffer, sampleRate, numChannels);
}
pushScreenShareFloat(buffer, sampleRate, numChannels) {
return this.native.pushScreenShareFloat(buffer, sampleRate, numChannels);
}
unpublishScreenShareAudio() {
return this.native.unpublishScreenShareAudio();
}
isPublishingScreenAudio() {
return this.native.isPublishingScreenAudio();
}
publishMicrophone(sampleRate, numChannels) {
return this.native.publishMicrophone(sampleRate, numChannels);
}
publishDeviceMicrophone(opts = {}) {
if (typeof this.native.publishDeviceMicrophone !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export publishDeviceMicrophone`);
}
return this.native.publishDeviceMicrophone({
deviceId: opts.deviceId,
echoCancellation: opts.echoCancellation,
noiseSuppression: opts.noiseSuppression,
autoGainControl: opts.autoGainControl,
...(opts.deepFilter !== undefined ? {deepFilter: opts.deepFilter} : {}),
...(opts.deepFilterNoiseReductionLevel !== undefined
? {deepFilterNoiseReductionLevel: opts.deepFilterNoiseReductionLevel}
: {}),
maxBitrateBps: opts.maxBitrateBps,
});
}
pushPcm(buffer, sampleRate, numChannels) {
return this.native.pushPcm(buffer, sampleRate, numChannels);
}
setMicEnabled(enabled) {
return this.native.setMicEnabled(enabled);
}
setSpeakingDetection(localThresholdRms, remoteThresholdRms) {
if (typeof this.native.setSpeakingDetection !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export setSpeakingDetection`);
}
return this.native.setSpeakingDetection(localThresholdRms, remoteThresholdRms);
}
publishCamera(opts = {}) {
return this.native.publishCamera(normalizeCameraOptions(opts));
}
updateCameraCapture(opts = {}) {
if (typeof this.native.updateCameraCapture !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export updateCameraCapture`);
}
return this.native.updateCameraCapture(normalizeCameraOptions(opts));
}
publishProcessedCamera(opts) {
if (typeof this.native.publishProcessedCamera !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export publishProcessedCamera`);
}
return this.native.publishProcessedCamera({
width: opts.width,
height: opts.height,
frameRate: opts.frameRate,
});
}
publishNativeCameraSink(opts = {}) {
if (typeof this.native.publishNativeCameraSink !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export publishNativeCameraSink`);
}
return this.native.publishNativeCameraSink(normalizeCameraOptions(opts));
}
createCameraFrameSinkHandle() {
if (typeof this.native.createCameraFrameSinkHandle !== 'function') return null;
return this.native.createCameraFrameSinkHandle();
}
pushProcessedCameraFrame(frame) {
if (typeof this.native.pushProcessedCameraFrame !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export pushProcessedCameraFrame`);
}
return this.native.pushProcessedCameraFrame({
format: frame.format,
width: frame.width,
height: frame.height,
timestampUs: frame.timestampUs,
data: frame.data,
});
}
startCameraPreview(opts = {}) {
if (typeof this.native.startCameraPreview !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export startCameraPreview`);
}
return this.native.startCameraPreview(normalizeCameraOptions(opts));
}
stopCameraPreview() {
if (typeof this.native.stopCameraPreview !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export stopCameraPreview`);
}
return this.native.stopCameraPreview();
}
pushCameraBackgroundFrame(frame) {
if (typeof this.native.pushCameraBackgroundFrame !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export pushCameraBackgroundFrame`);
}
return this.native.pushCameraBackgroundFrame({
format: frame.format,
width: frame.width,
height: frame.height,
timestampUs: frame.timestampUs,
data: frame.data,
});
}
clearCameraBackgroundFrame() {
if (typeof this.native.clearCameraBackgroundFrame !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export clearCameraBackgroundFrame`);
}
return this.native.clearCameraBackgroundFrame();
}
publishDeviceScreenShare(opts = {}) {
if (typeof this.native.publishDeviceScreenShare !== 'function') {
throw new Error(`${MODULE_NAME} native binding does not export publishDeviceScreenShare`);
}
return this.native.publishDeviceScreenShare(normalizeCameraOptions(opts));
}
listCameraDevices() {
return this.native.listCameraDevices();
}
unpublishCamera() {
return this.native.unpublishCamera();
}
isPublishingCamera() {
return this.native.isPublishingCamera();
}
async listAudioInputDevices() {
const json = await this.native.listAudioInputDevices();
return JSON.parse(json);
}
droppedEngineEvents() {
if (typeof this.native.droppedEngineEvents !== 'function') return 0;
return this.native.droppedEngineEvents();
}
async listAudioOutputDevices() {
const json = await this.native.listAudioOutputDevices();
return JSON.parse(json);
}
setAudioOutputDevice(deviceId) {
return this.native.setAudioOutputDevice(deviceId || '');
}
ensurePlatformAudio() {
if (typeof this.native.ensurePlatformAudio !== 'function') return Promise.resolve();
return this.native.ensurePlatformAudio();
}
setParticipantVolume(participantSid, volume) {
return this.native.setParticipantVolume(participantSid, volume);
}
setRemoteTrackSubscription(opts = {}) {
if (typeof this.native.setRemoteTrackSubscription !== 'function') return Promise.resolve();
return this.native.setRemoteTrackSubscription(
opts.participantIdentity || '',
opts.source || '',
opts.subscribed === true,
opts.enabled !== false,
opts.quality || undefined,
);
}
publishData(payload, opts = {}) {
if (typeof this.native.publishData !== 'function') {
return Promise.reject(new Error(`${MODULE_NAME} native binding does not export publishData`));
}
const buffer = Buffer.isBuffer(payload)
? payload
: payload instanceof ArrayBuffer
? Buffer.from(payload)
: ArrayBuffer.isView(payload)
? Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)
: null;
if (!buffer) {
return Promise.reject(new TypeError('publishData payload must be a Buffer, ArrayBuffer, or typed array'));
}
return this.native.publishData(
buffer,
opts.reliable !== false,
typeof opts.topic === 'string' ? opts.topic : undefined,
Array.isArray(opts.destinationIdentities) ? opts.destinationIdentities : undefined,
);
}
getConnectionStats() {
const json = this.native.getConnectionStats();
try {
return JSON.parse(json);
} catch {
return {rttMs: null, outbound: [], inbound: []};
}
}
inboundAudioFrames() {
return this.native.inboundAudioFrames();
}
inboundVideoFrames() {
return this.native.inboundVideoFrames();
}
droppedVideoFrameCallbacks() {
if (typeof this.native.droppedVideoFrameCallbacks !== 'function') return 0;
return this.native.droppedVideoFrameCallbacks();
}
}
module.exports = {
isSupported,
getEngineBridgeVersion,
assertEngineBridgeVersion,
getHardwareEncoderCapability,
getHardwareEncoderCapabilities,
getCapabilities,
hasNativeCameraBackgrounds,
prewarmVoiceEngine,
probeAudioDeviceModule,
VoiceEngine,
get loadError() {
return loadError;
},
__nativeFileNameForTests: nativeFileName,
__setBindingForTests(next) {
binding = next;
loadError = null;
},
};
@@ -0,0 +1,848 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {createRequire} from 'node:module';
import {describe, test} from 'node:test';
const require = createRequire(import.meta.url);
const webrtcSender = require('./index.js');
describe('webrtc-sender loader wrapper', () => {
test('resolves the native addon filename for every desktop OS/arch target', () => {
const cases = [
['win32', 'x64', 'webrtc-sender.win32-x64-msvc.node'],
['win32', 'arm64', 'webrtc-sender.win32-arm64-msvc.node'],
['darwin', 'x64', 'webrtc-sender.darwin-x64.node'],
['darwin', 'arm64', 'webrtc-sender.darwin-arm64.node'],
['linux', 'x64', 'webrtc-sender.linux-x64-gnu.node'],
['linux', 'arm64', 'webrtc-sender.linux-arm64-gnu.node'],
];
for (const [platform, arch, expected] of cases) {
assert.equal(webrtcSender.__nativeFileNameForTests(platform, arch), expected);
}
});
test('rejects unsupported platform/architecture pairs explicitly', () => {
assert.throws(() => webrtcSender.__nativeFileNameForTests('linux', 'ia32'), /not supported/);
assert.throws(() => webrtcSender.__nativeFileNameForTests('freebsd', 'x64'), /not supported/);
});
test('returns a native hardware encoder capability when the binding exports one', () => {
const expected = {
available: true,
backend: 'nvenc',
compiled: true,
runtime: true,
codecs: ['h264', 'h265'],
zeroCopy: true,
nativeInputs: ['dmabuf'],
};
webrtcSender.__setBindingForTests({
getHardwareEncoderCapability() {
return expected;
},
});
assert.deepEqual(webrtcSender.getHardwareEncoderCapability(), expected);
assert.deepEqual(webrtcSender.getHardwareEncoderCapabilities(), expected);
});
test('delegates dropped video callback metrics through the VoiceEngine wrapper', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {
inboundAudioFrames() {
return 2;
}
inboundVideoFrames() {
return 3;
}
droppedVideoFrameCallbacks() {
return 5;
}
},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.inboundAudioFrames(), 2);
assert.equal(engine.inboundVideoFrames(), 3);
assert.equal(engine.droppedVideoFrameCallbacks(), 5);
});
test('reports VoiceEngine feature capabilities from the wrapped native prototype', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishDeviceMicrophone() {}
publishMicrophone() {}
publishCamera() {}
listCameraDevices() {}
publishScreenShare() {}
updateScreenShareEncoding() {}
unpublishScreenShare() {}
publishScreenShareAudio() {}
pushScreenSharePcm() {}
pushScreenShareFloat() {}
unpublishScreenShareAudio() {}
listAudioInputDevices() {}
listAudioOutputDevices() {}
setAudioOutputDevice() {}
setParticipantVolume() {}
setRemoteTrackSubscription() {}
publishData() {}
getConnectionStats() {}
setVideoFrameCallback() {}
},
});
assert.deepEqual(webrtcSender.getCapabilities(), {
microphoneCapture: true,
syntheticMicrophonePcm: true,
cameraCapture: true,
nativeCameraBackgrounds: false,
screenShare: true,
screenShareEncodingUpdate: true,
screenShareAudio: true,
deviceLists: true,
outputDeviceSelection: true,
participantVolume: true,
remoteTrackSubscription: true,
dataChannel: true,
connectionStats: true,
nativeVideoFrames: true,
hardwareEncoderCapabilities: true,
});
});
test('does not report screen-share audio capability without the float push method', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishScreenShareAudio() {}
pushScreenSharePcm() {}
unpublishScreenShareAudio() {}
},
});
assert.equal(webrtcSender.getCapabilities().screenShareAudio, false);
});
test('reports native camera background support only from the explicit native probe', () => {
webrtcSender.__setBindingForTests({
hasNativeCameraBackgrounds() {
return true;
},
VoiceEngine: class {
publishCamera() {}
listCameraDevices() {}
},
});
assert.equal(webrtcSender.hasNativeCameraBackgrounds(), true);
assert.equal(webrtcSender.getCapabilities().nativeCameraBackgrounds, true);
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishCamera() {}
listCameraDevices() {}
},
});
assert.equal(webrtcSender.hasNativeCameraBackgrounds(), false);
assert.equal(webrtcSender.getCapabilities().nativeCameraBackgrounds, false);
});
test('delegates video-frame callback clearing to the native binding when exported', () => {
let clearCalls = 0;
const setCalls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
setVideoFrameCallback(callback) {
setCalls.push(callback);
}
clearVideoFrameCallback() {
clearCalls += 1;
}
},
});
const engine = new webrtcSender.VoiceEngine();
engine.clearVideoFrameCallback();
assert.equal(clearCalls, 1);
assert.deepEqual(setCalls, []);
});
test('delegates screen-share float audio through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
pushScreenShareFloat(buffer, sampleRate, numChannels) {
calls.push([buffer, sampleRate, numChannels]);
return Promise.resolve(true);
}
},
});
const engine = new webrtcSender.VoiceEngine();
const buffer = Buffer.from(new Float32Array([0.25, -0.25]).buffer);
const accepted = await engine.pushScreenShareFloat(buffer, 48000, 2);
assert.equal(accepted, true);
assert.deepEqual(calls, [[buffer, 48000, 2]]);
});
test('falls back to a no-op video-frame callback for older native bindings', () => {
const setCalls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
setVideoFrameCallback(callback) {
setCalls.push(callback);
}
},
});
const engine = new webrtcSender.VoiceEngine();
engine.clearVideoFrameCallback();
assert.equal(setCalls.length, 1);
assert.equal(typeof setCalls[0], 'function');
assert.equal(setCalls[0]('{}', Buffer.alloc(0)), undefined);
});
test('treats video-frame callback clearing as a no-op when the binding exports neither method', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.clearVideoFrameCallback(), undefined);
});
test('delegates engine bridge version reads and assertions to the native binding', () => {
const assertedVersions = [];
webrtcSender.__setBindingForTests({
getEngineBridgeVersion() {
return 9;
},
assertEngineBridgeVersion(version) {
assertedVersions.push(version);
if (version !== 9) {
throw new Error(`voice engine bridge version mismatch: host sent ${version}, native addon expects 9`);
}
},
});
assert.equal(webrtcSender.getEngineBridgeVersion(), 9);
webrtcSender.assertEngineBridgeVersion(9);
assert.throws(() => webrtcSender.assertEngineBridgeVersion(8), /bridge version mismatch/);
assert.deepEqual(assertedVersions, [9, 8]);
});
test('reports a null engine bridge version and throws on assertion when the binding lacks the exports', () => {
webrtcSender.__setBindingForTests({});
assert.equal(webrtcSender.getEngineBridgeVersion(), null);
assert.throws(() => webrtcSender.assertEngineBridgeVersion(9), /does not export assertEngineBridgeVersion/);
});
test('delegates native voice engine prewarm when the binding exports it', () => {
let prewarmCalls = 0;
webrtcSender.__setBindingForTests({
prewarmVoiceEngine() {
prewarmCalls += 1;
},
});
webrtcSender.prewarmVoiceEngine();
assert.equal(prewarmCalls, 1);
});
test('delegates device microphone publish through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishDeviceMicrophone(opts) {
calls.push(opts);
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.publishDeviceMicrophone({
deviceId: 'mic-guid',
echoCancellation: false,
noiseSuppression: true,
autoGainControl: false,
});
await engine.publishDeviceMicrophone({
deviceId: 'mic-guid',
echoCancellation: false,
noiseSuppression: true,
autoGainControl: false,
maxBitrateBps: 96_000,
});
assert.deepEqual(calls, [
{
deviceId: 'mic-guid',
echoCancellation: false,
noiseSuppression: true,
autoGainControl: false,
maxBitrateBps: undefined,
},
{
deviceId: 'mic-guid',
echoCancellation: false,
noiseSuppression: true,
autoGainControl: false,
maxBitrateBps: 96_000,
},
]);
});
test('delegates native camera devices and publish options through the VoiceEngine wrapper', async () => {
const publishCalls = [];
const processedPublishCalls = [];
const processedFrameCalls = [];
const nativeSinkPublishCalls = [];
const nativeSinkHandle = {};
const devices = [
{
deviceId: 'native-camera-id',
label: 'Studio Display Camera',
description: 'Apple Studio Display Camera',
index: 0,
deviceIdAliases: ['native-camera-id', '0'],
},
];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
listCameraDevices() {
return devices;
}
publishCamera(opts) {
publishCalls.push(opts);
return Promise.resolve();
}
publishProcessedCamera(opts) {
processedPublishCalls.push(opts);
return Promise.resolve({trackSid: 'TR_processed_camera'});
}
publishNativeCameraSink(opts) {
nativeSinkPublishCalls.push(opts);
return Promise.resolve({trackSid: 'TR_native_camera'});
}
createCameraFrameSinkHandle() {
return nativeSinkHandle;
}
pushProcessedCameraFrame(frame) {
processedFrameCalls.push(frame);
return Promise.resolve(true);
}
},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.listCameraDevices(), devices);
await engine.publishCamera({
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
mirror: true,
codec: 'h265',
maxBitrateBps: 6_000_000,
maxFramerate: 30,
});
const processedPublishResult = await engine.publishProcessedCamera({
width: 1280,
height: 720,
frameRate: 30,
ignored: true,
});
const nativeSinkPublishResult = await engine.publishNativeCameraSink({
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
backgroundMode: 'custom',
backgroundCustomMediaPath: '/tmp/bg.webp',
backgroundCustomMediaKind: 'animated',
codec: 'h264',
});
const cameraFrameSinkHandle = engine.createCameraFrameSinkHandle();
const frame = {
format: 'i420',
width: 4,
height: 2,
timestampUs: 12_345,
data: Buffer.alloc(12),
ignored: true,
};
const processedFrameResult = await engine.pushProcessedCameraFrame(frame);
assert.deepEqual(publishCalls, [
{
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
mirror: true,
backgroundMode: undefined,
backgroundCustomMediaPath: undefined,
backgroundCustomMediaKind: undefined,
backgroundBlurStrength: undefined,
codec: 'h265',
maxBitrateBps: 6_000_000,
maxFramerate: 30,
},
]);
assert.deepEqual(processedPublishResult, {trackSid: 'TR_processed_camera'});
assert.deepEqual(processedPublishCalls, [
{
width: 1280,
height: 720,
frameRate: 30,
},
]);
assert.deepEqual(nativeSinkPublishResult, {trackSid: 'TR_native_camera'});
assert.deepEqual(nativeSinkPublishCalls, [
{
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
mirror: undefined,
backgroundMode: 'custom',
backgroundCustomMediaPath: '/tmp/bg.webp',
backgroundCustomMediaKind: 'animated',
backgroundBlurStrength: undefined,
codec: 'h264',
maxBitrateBps: undefined,
maxFramerate: undefined,
},
]);
assert.equal(cameraFrameSinkHandle, nativeSinkHandle);
assert.equal(processedFrameResult, true);
assert.deepEqual(processedFrameCalls, [
{
format: 'i420',
width: 4,
height: 2,
timestampUs: 12_345,
data: frame.data,
},
]);
});
test('reports processed camera publishing as unavailable for older native bindings', async () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
assert.throws(
() => engine.publishProcessedCamera({width: 1280, height: 720, frameRate: 30}),
/native binding does not export publishProcessedCamera/,
);
assert.throws(
() => engine.publishNativeCameraSink({width: 1280, height: 720, frameRate: 30}),
/native binding does not export publishNativeCameraSink/,
);
assert.equal(engine.createCameraFrameSinkHandle(), null);
assert.throws(
() =>
engine.pushProcessedCameraFrame({
format: 'i420',
width: 4,
height: 2,
timestampUs: 12_345,
data: Buffer.alloc(12),
}),
/native binding does not export pushProcessedCameraFrame/,
);
});
test('delegates device screen-share publish through the VoiceEngine wrapper', async () => {
const publishCalls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishDeviceScreenShare(opts) {
publishCalls.push(opts);
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.publishDeviceScreenShare({
deviceId: 'studio-display-camera',
width: 1920,
height: 1080,
frameRate: 60,
codec: 'h264',
maxBitrateBps: 8_000_000,
maxFramerate: 60,
});
assert.deepEqual(publishCalls, [
{
deviceId: 'studio-display-camera',
width: 1920,
height: 1080,
frameRate: 60,
mirror: undefined,
backgroundMode: undefined,
backgroundCustomMediaPath: undefined,
backgroundCustomMediaKind: undefined,
backgroundBlurStrength: undefined,
codec: 'h264',
maxBitrateBps: 8_000_000,
maxFramerate: 60,
},
]);
});
test('delegates camera capture updates with effect strengths through the VoiceEngine wrapper', async () => {
const updateCalls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
updateCameraCapture(opts) {
updateCalls.push(opts);
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.updateCameraCapture({
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
mirror: false,
backgroundMode: 'blur',
backgroundBlurStrength: 90,
ignored: true,
});
assert.deepEqual(updateCalls, [
{
deviceId: 'native-camera-id',
width: 1280,
height: 720,
frameRate: 30,
mirror: false,
backgroundMode: 'blur',
backgroundCustomMediaPath: undefined,
backgroundCustomMediaKind: undefined,
backgroundBlurStrength: 90,
codec: undefined,
maxBitrateBps: undefined,
maxFramerate: undefined,
},
]);
});
test('reports camera capture updates as unavailable for older native bindings', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
assert.throws(
() => engine.updateCameraCapture({deviceId: 'native-camera-id'}),
/native binding does not export updateCameraCapture/,
);
});
test('reports device screen-share publish as unavailable for older native bindings', async () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
assert.throws(
() => engine.publishDeviceScreenShare({deviceId: 'studio-display-camera'}),
/native binding does not export publishDeviceScreenShare/,
);
});
test('delegates screen-share simulcast selection through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishScreenShare(width, height, codec, maxBitrateBps, maxFramerate, simulcast, options) {
calls.push({width, height, codec, maxBitrateBps, maxFramerate, simulcast, options});
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.publishScreenShare(3840, 2160, 'h265', 50_000_000, 60, false, {
adaptiveSend: true,
minVideoFps: 15,
minResolutionScale: 0.5,
maxAudioBufferMs: 750,
captureId: 'screen-harness-primary',
});
assert.deepEqual(calls, [
{
width: 3840,
height: 2160,
codec: 'h265',
maxBitrateBps: 50_000_000,
maxFramerate: 60,
simulcast: false,
options: {
adaptiveSend: true,
minVideoFps: 15,
minResolutionScale: 0.5,
maxAudioBufferMs: 750,
captureId: 'screen-harness-primary',
},
},
]);
});
test('rejects screen-share publish and encoding update without capture IDs', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishScreenShare(...args) {
calls.push(['publish', args]);
return Promise.resolve();
}
updateScreenShareEncoding(...args) {
calls.push(['update', args]);
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
assert.throws(
() => engine.publishScreenShare(1280, 720, 'h264', undefined, 30, false, undefined),
/screen-share publish requires a non-empty captureId/,
);
assert.throws(
() => engine.publishScreenShare(1280, 720, 'h264', undefined, 30, false, {captureId: ''}),
/screen-share publish requires a non-empty captureId/,
);
assert.throws(
() => engine.updateScreenShareEncoding(1280, 720, undefined, 30, undefined),
/screen-share encoding update requires a non-empty captureId/,
);
assert.throws(
() => engine.updateScreenShareEncoding(1280, 720, undefined, 30, {captureId: ''}),
/screen-share encoding update requires a non-empty captureId/,
);
assert.deepEqual(calls, []);
});
test('delegates connect options through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
connect(url, token, e2eeKey, options) {
calls.push({url, token, e2eeKey, options});
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
const key = Buffer.from('secret');
await engine.connect('ws://localhost:7880', 'token', key, {
autoSubscribe: false,
adaptiveStream: true,
dynacast: true,
});
assert.deepEqual(calls, [
{
url: 'ws://localhost:7880',
token: 'token',
e2eeKey: key,
options: {
autoSubscribe: false,
adaptiveStream: true,
dynacast: true,
},
},
]);
});
test('delegates native screen frame sink handle creation through the VoiceEngine wrapper', () => {
const calls = [];
const handle = {native: true};
webrtcSender.__setBindingForTests({
VoiceEngine: class {
createScreenFrameSinkHandle(captureId) {
calls.push(captureId);
return handle;
}
},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.createScreenFrameSinkHandle('capture-1'), handle);
assert.deepEqual(calls, ['capture-1']);
});
test('treats native screen frame sink handles as unavailable for older voice bindings', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.createScreenFrameSinkHandle('capture-1'), null);
});
test('defaults dropped video callback metrics to zero for older native bindings', () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {
inboundAudioFrames() {
return 2;
}
inboundVideoFrames() {
return 3;
}
},
});
const engine = new webrtcSender.VoiceEngine();
assert.equal(engine.droppedVideoFrameCallbacks(), 0);
});
test('delegates remote track subscription updates through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
setRemoteTrackSubscription(participantIdentity, source, subscribed, enabled, quality) {
calls.push({participantIdentity, source, subscribed, enabled, quality});
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.setRemoteTrackSubscription({
participantIdentity: 'user_1_conn',
source: 'screen_share',
subscribed: true,
enabled: false,
quality: 'high',
});
assert.deepEqual(calls, [
{
participantIdentity: 'user_1_conn',
source: 'screen_share',
subscribed: true,
enabled: false,
quality: 'high',
},
]);
});
test('ignores remote track subscription updates for older native bindings', async () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {},
});
const engine = new webrtcSender.VoiceEngine();
await engine.setRemoteTrackSubscription({
participantIdentity: 'user_1_conn',
source: 'camera',
subscribed: true,
});
});
test('delegates data packets through the VoiceEngine wrapper', async () => {
const calls = [];
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishData(payload, reliable, topic, destinationIdentities) {
calls.push({payload, reliable, topic, destinationIdentities});
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await engine.publishData(new Uint8Array([1, 2, 3]), {
reliable: true,
topic: 'screen-share-codec',
destinationIdentities: ['user_1_conn'],
});
assert.equal(Buffer.isBuffer(calls[0].payload), true);
assert.deepEqual([...calls[0].payload], [1, 2, 3]);
assert.deepEqual(calls, [
{
payload: calls[0].payload,
reliable: true,
topic: 'screen-share-codec',
destinationIdentities: ['user_1_conn'],
},
]);
});
test('rejects invalid data packet payloads in the wrapper', async () => {
webrtcSender.__setBindingForTests({
VoiceEngine: class {
publishData() {
return Promise.resolve();
}
},
});
const engine = new webrtcSender.VoiceEngine();
await assert.rejects(() => engine.publishData('not-bytes'), /payload must be/);
});
test('returns an unavailable hardware encoder capability without a binding', () => {
webrtcSender.__setBindingForTests(null);
assert.deepEqual(webrtcSender.getHardwareEncoderCapability(), {
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason: 'native_binding_unavailable',
detail: '@fluxer/webrtc-sender binding unavailable',
});
});
test('exposes every VoiceEngine method declared in index.d.ts on the wrapper', async () => {
const {readFileSync} = await import('node:fs');
const dts = readFileSync(new URL('./index.d.ts', import.meta.url), 'utf8');
const classStart = dts.indexOf('export declare class VoiceEngine {');
assert.ok(classStart >= 0);
const classEnd = dts.indexOf('\n}', classStart);
assert.ok(classEnd > classStart);
const classBody = dts.slice(classStart, classEnd);
const declaredMethods = [...classBody.matchAll(/^\t([A-Za-z0-9_]+)\(/gm)]
.map((match) => match[1])
.filter((name) => name !== 'constructor');
assert.ok(declaredMethods.length >= 30);
const wrapperMethods = new Set(Object.getOwnPropertyNames(webrtcSender.VoiceEngine.prototype));
const missing = declaredMethods.filter((name) => !wrapperMethods.has(name));
assert.deepEqual(missing, []);
});
});
@@ -0,0 +1,10 @@
# Bundled segmentation models
## selfie_segmenter_landscape.onnx
- **Source**: Google MediaPipe Selfie Segmenter (landscape), `selfie_segmenter_landscape.tflite`, downloaded from `https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite` (sha256 `490e9ea734313e0de10fa0cd9e3c6133e36ea4db2b7a49bde9ef019f72796b8e`).
- **License**: Apache License 2.0, per the official model card ("Model Card MediaPipe Selfie Segmentation", Google, 2021; `https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf`). This is the Apache-licensed Selfie model, not the ToS-restricted Google Meet model (`segm_full_v679.tflite`), which must never be shipped.
- **Conversion**: `tf2onnx` (`python -m tf2onnx.convert --tflite selfie_segmenter_landscape.tflite --output model.onnx --opset 13`), followed by graph surgery that replaces the single `TFL_Convolution2DTransposeBias` custom op with a standard `ConvTranspose` (weights transposed from `[out, kh, kw, in]` to `[in, out, kh, kw]`, strides 2x2, no padding) wrapped in NHWC/NCHW transposes, and bumps the default opset domain to 14 for `HardSwish`.
- **Verification**: output of the converted model matches the original tflite interpreter to a max abs diff of 8.7e-8 on random input. On a portrait test image the output is person confidence (1.0 on the subject, 0.0 in background corners), despite the output tensor name `segment_back`.
- **Signature**: input `input_1` `[1, 144, 256, 3]` f32 RGB scaled to 0..1, output `segment_back` `[1, 144, 256, 1]` f32 person confidence 0..1.
- **sha256**: `e8224061bba6031282bfd00cf23a2563fa11a1e28baae0a9052cef6b4e7f3321`
@@ -0,0 +1,33 @@
{
"name": "@fluxer/webrtc-sender",
"version": "0.0.0",
"description": "Native main-process WebRTC video sender (LiveKit/libwebrtc) for publishing game/screen capture without a renderer hop",
"private": true,
"license": "AGPL-3.0-or-later",
"os": [
"darwin",
"linux",
"win32"
],
"cpu": [
"x64",
"arm64"
],
"main": "index.js",
"types": "index.d.ts",
"files": [
"index.js",
"index.d.ts",
"webrtc-sender.darwin-x64.node",
"webrtc-sender.darwin-arm64.node",
"webrtc-sender.linux-x64-gnu.node",
"webrtc-sender.linux-arm64-gnu.node",
"webrtc-sender.win32-x64-msvc.node",
"webrtc-sender.win32-arm64-msvc.node"
],
"scripts": {
"build": "cargo run --locked --quiet --manifest-path ../../../tools/ci/Cargo.toml -- build-desktop-native-addon",
"test": "cargo run --locked --quiet --manifest-path ../../../tools/ci/Cargo.toml -- test-webrtc-sender-rust && node --test index.test.mjs scripts/livekit-harness.test.mjs",
"test:livekit": "node scripts/livekit-harness.mjs"
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,842 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {describe, test} from 'node:test';
import {
buildReport,
buildScenarioConfigs,
createLiveKitAdminToken,
expectedCodecMime,
jwtSubject,
liveKitApiUrl,
liveKitTcpTarget,
parseCodecList,
parseConfig,
sanitizeConfig,
scenarioRoomName,
serverPublishedTrackChecks,
statsHasOutbound,
statsHasOutboundCodec,
strictFailures,
} from './livekit-harness.mjs';
function tokenForSubject(subject) {
const payload = Buffer.from(JSON.stringify({sub: subject}), 'utf8').toString('base64url');
return `header.${payload}.signature`;
}
function jwtPayload(token) {
const parts = token.split('.');
return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
}
async function withHarnessEnv(env, callback) {
const previous = {...process.env};
for (const key of Object.keys(process.env)) {
if (
key.startsWith('LIVEKIT_') ||
key.startsWith('FLUXER_WEBRTC_SENDER_LIVEKIT') ||
key.startsWith('FLUXER_NATIVE_MEDIA')
) {
delete process.env[key];
}
}
Object.assign(process.env, env);
try {
return await callback();
} finally {
for (const key of Object.keys(process.env)) {
delete process.env[key];
}
Object.assign(process.env, previous);
}
}
function strictReportFor(
config,
{
videoFrameTimes = null,
videoFrameRecords = null,
audioFrameTimes = [],
droppedVideoFrameCallbacks = 0,
subscriberStatsSamples = [],
} = {},
) {
const strictStartedAtMs = 1_000;
const strictEndedAtMs = 2_000;
const effectiveVideoFrameRecords =
videoFrameRecords ??
(videoFrameTimes ?? []).map((atMs) => ({
atMs,
identity: config.publisherIdentity,
kind: 'video',
source: 'screen_share',
trackName: 'screen',
trackSid: 'TR_screen',
width: config.expectedWidth,
height: config.expectedHeight,
}));
const effectiveVideoFrameTimes = videoFrameTimes ?? effectiveVideoFrameRecords.map((record) => record.atMs);
return buildReport({
status: 'pass',
config,
context: {
startedAtMs: strictStartedAtMs - 500,
startedAtIso: new Date(strictStartedAtMs - 500).toISOString(),
strictStartedAtMs,
strictEndedAtMs,
publisherState: {statsSamples: [], stats: null},
subscriberState: {
statsSamples: subscriberStatsSamples,
stats: subscriberStatsSamples.at(-1)?.payload ?? null,
videoCallbacks: effectiveVideoFrameTimes.length,
videoBytes: effectiveVideoFrameTimes.length * 4,
videoFrameTimes: effectiveVideoFrameTimes,
videoFrameRecords: effectiveVideoFrameRecords,
audioFrameTimes,
lastVideoMeta: effectiveVideoFrameRecords.at(-1) ?? null,
lastVideoFrameAtMs: effectiveVideoFrameTimes.at(-1) ?? null,
lastAudioFrameAtMs: audioFrameTimes.at(-1) ?? null,
},
subscriber: {
inboundAudioFrames: () => audioFrameTimes.length,
inboundVideoFrames: () => effectiveVideoFrameTimes.length,
droppedVideoFrameCallbacks: () => droppedVideoFrameCallbacks,
},
},
});
}
function findCheck(checks, name) {
const check = checks.find((entry) => entry.name === name);
assert.ok(check, `missing check: ${name}`);
return check;
}
function serverChecksFor(config, serverState) {
return serverPublishedTrackChecks({
config,
serverState,
publisherIdentity: config.publisherIdentity,
secondaryPublisherIdentity: config.secondaryPublisherIdentity,
});
}
describe('livekit harness setup matrix', () => {
test('normalizes LiveKit screenshare source spelling in outbound stats', () => {
const stats = {
outbound: [
{
trackSid: 'TR_screen',
source: 'screenshare',
kind: 'video',
codec: 'video/VP8',
bitrateKbps: 400,
packetsLost: 0,
},
],
};
assert.equal(statsHasOutbound(stats, 'video', 'screen_share'), true);
assert.equal(statsHasOutboundCodec(stats, 'video', 'screen_share', 'video/VP8'), true);
assert.equal(statsHasOutboundCodec(stats, 'video', 'screen_share', 'video/H264'), false);
});
test('maps all supported codec spellings to expected mime types', () => {
assert.equal(expectedCodecMime('vp8'), 'video/VP8');
assert.equal(expectedCodecMime('VP9'), 'video/VP9');
assert.equal(expectedCodecMime('h264'), 'video/H264');
assert.equal(expectedCodecMime('av1'), 'video/AV1');
assert.equal(expectedCodecMime('h265'), 'video/H265');
assert.equal(expectedCodecMime('hevc'), 'video/H265');
assert.equal(expectedCodecMime(''), null);
assert.throws(() => expectedCodecMime('h266'), /unsupported/);
});
test('parses codec lists with fallback and rejects empty effective lists', () => {
assert.deepEqual(parseCodecList(' vp8, h264,, hevc ', 'av1'), ['vp8', 'h264', 'hevc']);
assert.deepEqual(parseCodecList('', 'vp9'), ['vp9']);
assert.throws(() => parseCodecList('', ''), /at least one/);
assert.throws(() => parseCodecList('vp8,h266', 'vp8'), /unsupported/);
});
test('default non-strict config keeps the fast local live setup small', async () => {
await withHarnessEnv({}, () => {
const config = parseConfig();
assert.equal(config.url, 'ws://localhost:7880');
assert.equal(config.serverApiUrl, 'http://localhost:7880');
assert.deepEqual(config.screenCodecs, ['vp8']);
assert.equal(config.expectedScreenCodec, 'video/VP8');
assert.equal(config.secondaryPublisher, false);
assert.equal(config.microphone, true);
assert.equal(config.screenAudio, false);
assert.equal(config.dataPacket, false);
assert.equal(config.subscriptionCycle, false);
assert.equal(config.validateServerPublishing, false);
assert.equal(config.durationMs, 0);
assert.equal(config.externalTokens, false);
assert.equal(config.screenSimulcast, true);
assert.equal(config.secondaryPublisherScreenSimulcast, true);
assert.equal(config.adaptiveSend, true);
assert.equal(config.minVideoFps, 15);
assert.equal(config.minResolutionScale, 0.5);
assert.equal(config.maxAudioBufferMs, 750);
});
});
test('strict config expands to multi-codec secondary-publisher scenarios', async () => {
await withHarnessEnv(
{
LIVEKIT_HARNESS_STRICT: '1',
LIVEKIT_ROOM: 'matrix-room',
LIVEKIT_SCREEN_CODECS: 'vp8,h264,hevc',
LIVEKIT_EXPECT_SCREEN_CODECS: 'vp8,h264,h265',
},
() => {
const config = parseConfig();
assert.equal(config.secondaryPublisher, true);
assert.equal(config.screenAudio, true);
assert.equal(config.dataPacket, true);
assert.equal(config.subscriptionCycle, true);
assert.equal(config.validateServerPublishing, true);
assert.equal(config.durationMs, 10 * 60 * 1000);
assert.equal(config.maxAudioFrameGapMs, 250);
assert.equal(config.maxPacketLoss, null);
assert.equal(config.requireStableResolution, true);
assert.equal(config.videoPattern, 'gradient');
const scenarios = buildScenarioConfigs(config);
assert.deepEqual(
scenarios.map((scenario) => scenario.scenarioName),
['codec-vp8-with-h264', 'codec-h264-with-hevc', 'codec-hevc-with-vp8'],
);
assert.deepEqual(
scenarios.map((scenario) => scenario.expectedScreenCodec),
['video/VP8', 'video/H264', 'video/H265'],
);
assert.deepEqual(
scenarios.map((scenario) => scenario.expectedSecondaryPublisherCodec),
['video/H264', 'video/H265', 'video/VP8'],
);
assert.equal(scenarios[0].room, 'matrix-room-codec-vp8-with-h264');
assert.equal(scenarios[1].reportPath, null);
},
);
});
test('stress knobs parse packet loss, resolution stability, audio gap, and video pattern', async () => {
await withHarnessEnv(
{
LIVEKIT_MAX_PACKET_LOSS: '0',
LIVEKIT_REQUIRE_STABLE_RESOLUTION: '0',
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '180',
LIVEKIT_VIDEO_PATTERN: 'fast',
LIVEKIT_VIDEO_INPUT: 'nv12',
LIVEKIT_SUBSCRIBER_VIDEO_QUALITY: 'high',
LIVEKIT_SCREEN_SIMULCAST: '0',
LIVEKIT_SECOND_PUBLISHER_SCREEN_SIMULCAST: '1',
LIVEKIT_SCREEN_FPS: '60',
LIVEKIT_ADAPTIVE_SEND: '0',
LIVEKIT_MIN_VIDEO_FPS: '24',
LIVEKIT_MIN_RESOLUTION_SCALE: '0.75',
LIVEKIT_MAX_AUDIO_BUFFER_MS: '640',
},
() => {
const config = parseConfig();
assert.equal(config.maxPacketLoss, 0);
assert.equal(config.requireStableResolution, false);
assert.equal(config.maxAudioFrameGapMs, 180);
assert.equal(config.videoPattern, 'fast');
assert.equal(config.videoInput, 'nv12');
assert.equal(config.subscriberVideoQuality, 'high');
assert.equal(config.screenSimulcast, false);
assert.equal(config.secondaryPublisherScreenSimulcast, true);
assert.equal(config.adaptiveSend, false);
assert.equal(config.minVideoFps, 24);
assert.equal(config.minResolutionScale, 0.75);
assert.equal(config.maxAudioBufferMs, 640);
},
);
});
test('stress knobs reject impossible send pacing configurations', async () => {
await withHarnessEnv({LIVEKIT_SCREEN_FPS: '0'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_SCREEN_FPS must be a positive number/),
);
await withHarnessEnv({LIVEKIT_MIN_VIDEO_FPS: '0'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_MIN_VIDEO_FPS must be a positive number/),
);
await withHarnessEnv({LIVEKIT_SCREEN_FPS: '30', LIVEKIT_MIN_VIDEO_FPS: '60'}, () =>
assert.throws(() => parseConfig(), /MIN_VIDEO_FPS must be less than or equal/),
);
await withHarnessEnv({LIVEKIT_MAX_AUDIO_BUFFER_MS: '0'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_MAX_AUDIO_BUFFER_MS must be a positive integer/),
);
await withHarnessEnv({LIVEKIT_MIN_RESOLUTION_SCALE: '0'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_MIN_RESOLUTION_SCALE must be a positive number/),
);
await withHarnessEnv({LIVEKIT_MIN_RESOLUTION_SCALE: '1.5'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_MIN_RESOLUTION_SCALE must be greater than 0/),
);
await withHarnessEnv({LIVEKIT_ENABLE_SCREEN_AUDIO: 'maybe'}, () =>
assert.throws(() => parseConfig(), /LIVEKIT_ENABLE_SCREEN_AUDIO must be a boolean flag/),
);
await withHarnessEnv({LIVEKIT_SCREEN_CODECS: 'vp8,h264', LIVEKIT_EXPECT_SCREEN_CODECS: 'vp8'}, () =>
assert.throws(() => parseConfig(), /EXPECT_SCREEN_CODECS length must match/),
);
});
test('explicit secondary codec expectations override rotating defaults', async () => {
await withHarnessEnv(
{
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
LIVEKIT_SECOND_PUBLISHER_CODEC: 'av1',
LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC: 'hevc',
},
() => {
const scenarios = buildScenarioConfigs(parseConfig());
assert.deepEqual(
scenarios.map((scenario) => scenario.secondaryPublisherCodec),
['av1', 'av1'],
);
assert.deepEqual(
scenarios.map((scenario) => scenario.expectedSecondaryPublisherCodec),
['video/H265', 'video/H265'],
);
},
);
});
test('secondary screen codec aliases are accepted and conflicting aliases are rejected', async () => {
await withHarnessEnv(
{
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_SCREEN_CODECS: 'h264',
LIVEKIT_SECOND_PUBLISHER_SCREEN_CODECS: 'vp8',
LIVEKIT_EXPECT_SECONDARY_PUBLISHER_SCREEN_CODEC: 'vp8',
},
() => {
const config = parseConfig();
assert.equal(config.secondaryPublisherCodec, 'vp8');
assert.equal(config.expectedSecondaryPublisherCodec, 'video/VP8');
},
);
await withHarnessEnv(
{
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_SECOND_PUBLISHER_CODEC: 'h264',
LIVEKIT_SECOND_PUBLISHER_SCREEN_CODEC: 'vp8',
},
() => assert.throws(() => parseConfig(), /conflicts with LIVEKIT_SECOND_PUBLISHER_CODEC/),
);
await withHarnessEnv(
{
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC: 'h264',
LIVEKIT_EXPECT_SECONDARY_PUBLISHER_SCREEN_CODEC: 'vp8',
},
() => assert.throws(() => parseConfig(), /conflicts with LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC/),
);
});
test('external token setup derives identities, requires matching tokens, and redacts reports', async () => {
await withHarnessEnv(
{
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
},
() => assert.throws(() => parseConfig(), /both required/),
);
await withHarnessEnv(
{
LIVEKIT_ROOM: 'shared-token-room',
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
LIVEKIT_SUBSCRIBER_TOKEN: tokenForSubject('subscriber-subject'),
},
() => assert.throws(() => parseConfig(), /SECONDARY_PUBLISHER_TOKEN/),
);
await withHarnessEnv(
{
LIVEKIT_ROOM: 'shared-token-room',
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
LIVEKIT_SUBSCRIBER_TOKEN: tokenForSubject('subscriber-subject'),
LIVEKIT_SECONDARY_PUBLISHER_TOKEN: tokenForSubject('secondary-subject'),
LIVEKIT_E2EE_KEY: 'secret-key',
},
() => {
const config = parseConfig();
assert.equal(config.externalTokens, true);
assert.equal(config.publisherIdentity, 'publisher-subject');
assert.equal(config.subscriberIdentity, 'subscriber-subject');
assert.equal(config.secondaryPublisherIdentity, 'secondary-subject');
assert.deepEqual(
buildScenarioConfigs(config).map((scenario) => scenario.room),
['shared-token-room', 'shared-token-room'],
);
const sanitized = sanitizeConfig(config);
assert.equal(sanitized.url, 'ws://localhost:7880');
assert.equal(sanitized.apiSecret, '<redacted>');
assert.equal(sanitized.e2eeKey, '<present>');
assert.equal(sanitized.publisherToken, '<present>');
assert.equal(sanitized.subscriberToken, '<present>');
assert.equal(sanitized.secondaryPublisherToken, '<present>');
},
);
});
test('report sanitization strips URL credentials, query strings, and fragments', async () => {
await withHarnessEnv(
{
LIVEKIT_URL: 'wss://user:password@example.test:443/rtc?access_token=secret#fragment',
},
() => {
const sanitized = sanitizeConfig(parseConfig());
assert.equal(sanitized.url, 'wss://example.test/rtc');
assert.doesNotMatch(JSON.stringify(sanitized), /user|password|access_token|secret|fragment/);
},
);
});
test('jwt subject and TCP target parsing catch malformed live setups early', () => {
assert.equal(jwtSubject(tokenForSubject('user_1_connection')), 'user_1_connection');
assert.throws(() => jwtSubject('not-a-jwt'), /must be a JWT/);
assert.throws(() => jwtSubject('header.bad-json.signature'), /failed to decode/);
assert.throws(() => jwtSubject(tokenForSubject('')), /does not contain/);
assert.deepEqual(liveKitTcpTarget('ws://localhost:7880'), {host: 'localhost', port: 7880});
assert.deepEqual(liveKitTcpTarget('wss://[::1]/rtc'), {host: '::1', port: 443});
assert.equal(liveKitApiUrl('ws://localhost:7880/rtc?token=secret'), 'http://localhost:7880');
assert.equal(liveKitApiUrl('wss://livekit.example.test/rtc'), 'https://livekit.example.test');
assert.throws(() => liveKitTcpTarget('https://localhost'), /must use ws/);
});
test('admin token has the room-scoped grant needed for server publishing validation', () => {
const token = createLiveKitAdminToken({apiKey: 'devkey', apiSecret: 'secret', room: 'room-a'});
const payload = jwtPayload(token);
assert.equal(payload.iss, 'devkey');
assert.equal(payload.video.room, 'room-a');
assert.equal(payload.video.roomAdmin, true);
assert.equal(payload.video.roomJoin, undefined);
});
test('scenario room names are stable and bounded for generated-token suites', () => {
assert.equal(scenarioRoomName('room', 'codec-vp8', 1, false), 'room');
assert.equal(scenarioRoomName('room', 'codec:vP8 with H264', 2, false), 'room-codec-vP8-with-H264');
assert.equal(scenarioRoomName('room', 'codec-vp8', 2, true), 'room');
const longRoom = scenarioRoomName('r'.repeat(120), 'codec-h264-with-hevc', 2, false);
assert.equal(longRoom.length, 128);
assert.ok(longRoom.startsWith('r'.repeat(120)));
});
test('strict report checks fail when the strict window has too few samples to measure gaps', async () => {
await withHarnessEnv(
{
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
LIVEKIT_MAX_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AV_DRIFT_MS: '0',
},
() => {
const config = parseConfig();
const report = strictReportFor(config, {
videoFrameTimes: [1_250],
audioFrameTimes: [1_300],
});
assert.equal(report.metrics.strictVideoFrames, 1);
assert.equal(report.metrics.strictAudioFrames, 1);
assert.match(strictFailures(report, config).join('\n'), /video frame gap unavailable/);
assert.match(strictFailures(report, config).join('\n'), /audio frame gap unavailable/);
},
);
});
test('strict report checks use only samples inside the measured window', async () => {
await withHarnessEnv(
{
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
LIVEKIT_MAX_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AV_DRIFT_MS: '0',
},
() => {
const config = parseConfig();
const report = strictReportFor(config, {
videoFrameTimes: [500, 1_050, 1_200, 2_500],
audioFrameTimes: [600, 1_100, 1_240, 2_600],
});
assert.equal(report.metrics.strictVideoFrames, 2);
assert.equal(report.metrics.strictAudioFrames, 2);
assert.equal(report.metrics.maxVideoFrameGapMs, 150);
assert.equal(report.metrics.maxAudioFrameGapMs, 140);
assert.deepEqual(strictFailures(report, config), []);
},
);
});
test('strict screen report ignores camera video callbacks for resolution and drift', async () => {
await withHarnessEnv(
{
LIVEKIT_ENABLE_CAMERA: '1',
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
LIVEKIT_MAX_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
LIVEKIT_MAX_AV_DRIFT_MS: '80',
},
() => {
const config = parseConfig();
const report = strictReportFor(config, {
videoFrameRecords: [
{
atMs: 1_010,
identity: config.publisherIdentity,
kind: 'video',
source: 'screen_share',
trackName: 'screen',
trackSid: 'TR_screen',
width: 320,
height: 180,
},
{
atMs: 1_025,
identity: config.publisherIdentity,
kind: 'video',
source: 'camera',
trackName: 'camera',
trackSid: 'TR_camera',
width: 480,
height: 360,
},
{
atMs: 1_080,
identity: config.publisherIdentity,
kind: 'video',
source: 'screen_share',
trackName: 'screen',
trackSid: 'TR_screen',
width: 320,
height: 180,
},
{
atMs: 1_220,
identity: config.publisherIdentity,
kind: 'video',
source: 'camera',
trackName: 'camera',
trackSid: 'TR_camera',
width: 480,
height: 360,
},
],
audioFrameTimes: [1_040, 1_100],
});
assert.equal(report.metrics.videoCallbacks, 4);
assert.equal(report.metrics.strictVideoFrames, 2);
assert.deepEqual(report.metrics.videoResolutionCounts, {'320x180': 2});
assert.equal(report.metrics.videoResolutionMismatchCount, 0);
assert.equal(report.metrics.avDriftMs, 20);
assert.deepEqual(strictFailures(report, config), []);
},
);
});
test('strict packet loss gate uses loss deltas inside the strict window', async () => {
await withHarnessEnv(
{
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
LIVEKIT_MAX_FRAME_GAP_MS: '0',
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '0',
LIVEKIT_MAX_AV_DRIFT_MS: '0',
LIVEKIT_MAX_PACKET_LOSS: '0',
},
() => {
const config = parseConfig();
const report = strictReportFor(config, {
videoFrameTimes: [1_100, 1_200],
subscriberStatsSamples: [
{atMs: 900, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
{atMs: 1_500, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
],
});
assert.equal(report.metrics.maxObservedPacketLoss, 4);
assert.equal(report.metrics.maxObservedPacketLossDelta, 0);
assert.deepEqual(strictFailures(report, config), []);
const failingReport = strictReportFor(config, {
videoFrameTimes: [1_100, 1_200],
subscriberStatsSamples: [
{atMs: 900, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
{atMs: 1_500, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 5}]}},
],
});
assert.equal(failingReport.metrics.maxObservedPacketLossDelta, 1);
assert.match(strictFailures(failingReport, config).join('\n'), /packet loss delta 1/);
},
);
});
test('server publishing checks cover primary and secondary screen, audio, and camera tracks', async () => {
await withHarnessEnv(
{
LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1',
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_ENABLE_SCREEN_AUDIO: '1',
LIVEKIT_ENABLE_CAMERA: '1',
},
() => {
const config = parseConfig();
const serverState = {
participants: [
{
identity: config.publisherIdentity,
tracks: [
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false, sid: 'TR_V1'},
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false, sid: 'TR_A1'},
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false, sid: 'TR_A2'},
{type: 'VIDEO', source: 'CAMERA', muted: false, sid: 'TR_V2'},
],
},
{
identity: config.secondaryPublisherIdentity,
tracks: [
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false, sid: 'TR_V3'},
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false, sid: 'TR_A3'},
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false, sid: 'TR_A4'},
],
},
],
error: null,
};
const checks = serverChecksFor(config, serverState);
assert.equal(checks.length, 8);
assert.deepEqual(
checks.map((check) => [check.name, check.pass]),
[
['server publishing API participants listed', true],
['server sees publisher screenshare publication', true],
['server sees publisher microphone publication', true],
['server sees publisher screen-share audio publication', true],
['server sees publisher camera publication', true],
['server sees secondary publisher screenshare publication', true],
['server sees secondary publisher microphone publication', true],
['server sees secondary publisher screen-share audio publication', true],
],
);
},
);
});
test('server publishing checks fail closed on missing server tracks', async () => {
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1', LIVEKIT_ENABLE_SCREEN_AUDIO: '1'}, () => {
const config = parseConfig();
const checks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, true);
assert.equal(findCheck(checks, 'server sees publisher screen-share audio publication').pass, false);
});
});
test('server publishing checks reject wrong participant identity', async () => {
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
const config = parseConfig();
const checks = serverChecksFor(config, {
participants: [
{
identity: config.subscriberIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(checks, 'server publishing API participants listed').pass, true);
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, false);
});
});
test('server publishing checks reject wrong kind, source, and screen mime', async () => {
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1', LIVEKIT_EXPECT_SCREEN_CODEC: 'h264'}, () => {
const config = parseConfig();
const wrongKindChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'AUDIO', source: 'SCREEN_SHARE', mimeType: 'video/H264', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(wrongKindChecks, 'server sees publisher screenshare publication').pass, false);
const wrongSourceChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'CAMERA', mimeType: 'video/H264', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(wrongSourceChecks, 'server sees publisher screenshare publication').pass, false);
const wrongMimeChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(wrongMimeChecks, 'server sees publisher screenshare publication').pass, false);
const matchingChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mime_type: 'video/h264', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(matchingChecks, 'server sees publisher screenshare publication').pass, true);
});
});
test('server publishing checks reject muted or missing mute state on expected tracks', async () => {
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
const config = parseConfig();
const mutedChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: true}],
},
],
error: null,
});
assert.equal(findCheck(mutedChecks, 'server sees publisher screenshare publication').pass, false);
const missingMutedChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8'}],
},
],
error: null,
});
assert.equal(findCheck(missingMutedChecks, 'server sees publisher screenshare publication').pass, false);
});
});
test('server publishing checks handle protobuf numeric enums without accepting data tracks as video', async () => {
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
const config = parseConfig();
const dataTrackChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 2, source: 3, mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(dataTrackChecks, 'server sees publisher screenshare publication').pass, false);
const videoTrackChecks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 1, source: 3, mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(videoTrackChecks, 'server sees publisher screenshare publication').pass, true);
});
});
test('server publishing checks fail closed on missing secondary, audio, camera, and screen-audio tracks', async () => {
await withHarnessEnv(
{
LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1',
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
LIVEKIT_ENABLE_SCREEN_AUDIO: '1',
LIVEKIT_ENABLE_CAMERA: '1',
},
() => {
const config = parseConfig();
const checks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
},
{
identity: config.secondaryPublisherIdentity,
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
},
],
error: null,
});
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, true);
assert.equal(findCheck(checks, 'server sees publisher microphone publication').pass, false);
assert.equal(findCheck(checks, 'server sees publisher screen-share audio publication').pass, false);
assert.equal(findCheck(checks, 'server sees publisher camera publication').pass, false);
assert.equal(findCheck(checks, 'server sees secondary publisher screenshare publication').pass, true);
assert.equal(findCheck(checks, 'server sees secondary publisher microphone publication').pass, false);
assert.equal(findCheck(checks, 'server sees secondary publisher screen-share audio publication').pass, false);
},
);
});
test('strict server publishing validation does not pass open on Twirp errors', async () => {
await withHarnessEnv({LIVEKIT_HARNESS_STRICT: '1'}, () => {
const config = parseConfig();
const checks = serverChecksFor(config, {
participants: [
{
identity: config.publisherIdentity,
tracks: [
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false},
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false},
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false},
],
},
{
identity: config.secondaryPublisherIdentity,
tracks: [
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false},
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false},
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false},
],
},
],
error: 'RoomService.ListParticipants failed with HTTP 401',
});
assert.equal(config.validateServerPublishing, true);
assert.equal(findCheck(checks, 'server publishing API participants listed').pass, false);
assert.equal(
checks.every((check) => check.pass),
false,
);
});
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use napi_derive::napi;
pub const ENGINE_BRIDGE_VERSION: u32 = 18;
const _: () = assert!(ENGINE_BRIDGE_VERSION > 0);
fn check_engine_bridge_version(version: u32) -> Result<(), String> {
if version == ENGINE_BRIDGE_VERSION {
return Ok(());
}
Err(format!(
"voice engine bridge version mismatch: host sent {version}, native addon expects {ENGINE_BRIDGE_VERSION}"
))
}
#[napi]
pub fn get_engine_bridge_version() -> u32 {
ENGINE_BRIDGE_VERSION
}
#[napi]
pub fn assert_engine_bridge_version(version: u32) -> napi::Result<()> {
check_engine_bridge_version(version).map_err(napi::Error::from_reason)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn matching_version_passes() {
assert!(check_engine_bridge_version(ENGINE_BRIDGE_VERSION).is_ok());
assert!(assert_engine_bridge_version(ENGINE_BRIDGE_VERSION).is_ok());
}
#[test]
fn mismatched_version_fails_with_both_versions_in_message() {
let error = check_engine_bridge_version(ENGINE_BRIDGE_VERSION + 1).unwrap_err();
assert!(error.contains("voice engine bridge version mismatch"));
assert!(error.contains(&(ENGINE_BRIDGE_VERSION + 1).to_string()));
assert!(error.contains(&ENGINE_BRIDGE_VERSION.to_string()));
}
#[test]
fn zero_version_fails() {
assert!(check_engine_bridge_version(0).is_err());
}
#[test]
fn mismatch_surfaces_as_napi_error() {
let error = assert_engine_bridge_version(ENGINE_BRIDGE_VERSION - 1).unwrap_err();
assert!(
error
.reason
.contains("voice engine bridge version mismatch")
);
}
#[test]
fn exported_getter_reports_the_constant() {
assert_eq!(get_engine_bridge_version(), ENGINE_BRIDGE_VERSION);
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,200 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#[cfg(test)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PublishConfig {
pub url: String,
pub token: String,
pub width: u32,
pub height: u32,
pub fps: u32,
pub codec: String,
}
#[cfg(test)]
impl PublishConfig {
pub fn validate(&self) -> Result<(), String> {
if self.url.trim().is_empty() {
return Err("livekit url is empty".into());
}
if !(self.url.starts_with("ws://") || self.url.starts_with("wss://")) {
return Err("livekit url must be ws:// or wss://".into());
}
if self.token.trim().is_empty() {
return Err("livekit token is empty".into());
}
if self.width < 2 || self.height < 2 {
return Err("capture dimensions too small".into());
}
if !self.width.is_multiple_of(2) || !self.height.is_multiple_of(2) {
return Err("capture dimensions must be even".into());
}
if self.width > 8192 || self.height > 8192 {
return Err("capture dimensions too large".into());
}
if self.fps == 0 {
return Err("capture fps must be positive".into());
}
if !self.codec.trim().is_empty() && canonical_codec_name(&self.codec).is_none() {
return Err("unsupported video codec".into());
}
Ok(())
}
}
pub const SUPPORTED_CODECS: &[&str] = &["vp8", "h264", "vp9", "av1", "h265"];
pub fn canonical_codec_name(name: &str) -> Option<&'static str> {
let lower = name.trim().to_ascii_lowercase();
if lower == "hevc" {
return Some("h265");
}
SUPPORTED_CODECS.iter().copied().find(|&c| c == lower)
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PublisherState {
Idle,
Connecting,
Publishing,
Closed,
Failed,
}
#[cfg(test)]
impl PublisherState {
pub fn accepts_frames(self) -> bool {
matches!(self, PublisherState::Publishing)
}
pub fn can_connect(self) -> bool {
matches!(
self,
PublisherState::Idle | PublisherState::Closed | PublisherState::Failed
)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn cfg() -> PublishConfig {
PublishConfig {
url: "wss://sfu.example/rtc".into(),
token: "jwt".into(),
width: 1920,
height: 1080,
fps: 30,
codec: String::new(),
}
}
#[test]
fn valid_config_passes() {
assert!(cfg().validate().is_ok());
}
#[test]
fn rejects_bad_url() {
let mut c = cfg();
c.url = "https://sfu".into();
assert!(c.validate().is_err());
c.url = String::new();
assert!(c.validate().is_err());
}
#[test]
fn rejects_empty_token() {
let mut c = cfg();
c.token = " ".into();
assert!(c.validate().is_err());
}
#[test]
fn rejects_odd_or_oob_dimensions() {
let mut c = cfg();
c.width = 1921;
assert!(c.validate().is_err());
c.width = 1920;
c.height = 0;
assert!(c.validate().is_err());
c.height = 16384;
assert!(c.validate().is_err());
}
#[test]
fn accepts_dimension_boundaries_and_rejects_zero_fps() {
let mut c = cfg();
c.width = 2;
c.height = 2;
c.fps = 1;
assert!(c.validate().is_ok());
c.width = 8192;
c.height = 8192;
assert!(c.validate().is_ok());
c.fps = 0;
assert_eq!(
c.validate(),
Err("capture fps must be positive".to_string())
);
}
#[test]
fn rejects_unknown_non_empty_codec_but_allows_empty_default() {
let mut c = cfg();
c.codec = String::new();
assert!(c.validate().is_ok());
c.codec = " ".into();
assert!(c.validate().is_ok());
c.codec = "h266".into();
assert_eq!(c.validate(), Err("unsupported video codec".to_string()));
}
#[test]
fn canonical_codec_name_accepts_all_five_case_insensitively() {
for (input, expected) in [
("vp8", "vp8"),
(" vp8 ", "vp8"),
("VP8", "vp8"),
("h264", "h264"),
("H264", "h264"),
("vp9", "vp9"),
("Vp9", "vp9"),
("av1", "av1"),
("AV1", "av1"),
("h265", "h265"),
("H265", "h265"),
("hevc", "h265"),
("HEVC", "h265"),
] {
assert_eq!(canonical_codec_name(input), Some(expected), "codec {input}");
}
assert_eq!(SUPPORTED_CODECS.len(), 5);
}
#[test]
fn canonical_codec_name_rejects_empty_and_unknown() {
assert_eq!(canonical_codec_name(""), None);
assert_eq!(canonical_codec_name("h266"), None);
assert_eq!(canonical_codec_name("rubbish"), None);
}
#[test]
fn state_gates_frames_and_connect() {
assert!(PublisherState::Publishing.accepts_frames());
assert!(!PublisherState::Connecting.accepts_frames());
assert!(!PublisherState::Idle.accepts_frames());
assert!(PublisherState::Idle.can_connect());
assert!(PublisherState::Closed.can_connect());
assert!(PublisherState::Failed.can_connect());
assert!(!PublisherState::Failed.accepts_frames());
assert!(!PublisherState::Publishing.can_connect());
assert!(!PublisherState::Connecting.can_connect());
}
}
@@ -0,0 +1,181 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::audio::{
DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX, DEEP_FILTER_NOISE_REDUCTION_LEVEL_MIN,
clamp_deep_filter_noise_reduction_level,
};
use df::tract::{DfParams, DfTract, RuntimeParams};
use ndarray::Array2;
pub const DEEP_FILTER_SAMPLE_RATE_HZ: u32 = 48_000;
pub const DEEP_FILTER_NUM_CHANNELS: u32 = 1;
pub const DEEP_FILTER_FRAME_SAMPLES: usize = 480;
const SAMPLE_SCALE_I16_TO_F32: f32 = 1.0 / 32_768.0;
const SAMPLE_SCALE_F32_TO_I16: f32 = 32_767.0;
const _: () = assert!(DEEP_FILTER_FRAME_SAMPLES == DEEP_FILTER_SAMPLE_RATE_HZ as usize / 100);
const _: () = assert!(DEEP_FILTER_NUM_CHANNELS == 1);
pub struct DeepFilterProcessor {
model: DfTract,
input: Array2<f32>,
output: Array2<f32>,
}
impl DeepFilterProcessor {
pub fn new(noise_reduction_level: f64) -> Result<DeepFilterProcessor, String> {
let level_db = clamp_deep_filter_noise_reduction_level(noise_reduction_level);
assert!(level_db >= DEEP_FILTER_NOISE_REDUCTION_LEVEL_MIN);
assert!(level_db <= DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX);
let params = RuntimeParams::default_with_ch(DEEP_FILTER_NUM_CHANNELS as usize)
.with_atten_lim(level_db as f32);
let model = DfTract::new(DfParams::default(), &params)
.map_err(|error| format!("deep filter model init: {error:#}"))?;
if model.sr != DEEP_FILTER_SAMPLE_RATE_HZ as usize {
return Err(format!(
"deep filter model sample rate {} != {DEEP_FILTER_SAMPLE_RATE_HZ}",
model.sr
));
}
if model.ch != DEEP_FILTER_NUM_CHANNELS as usize {
return Err(format!(
"deep filter model channels {} != {DEEP_FILTER_NUM_CHANNELS}",
model.ch
));
}
if model.hop_size != DEEP_FILTER_FRAME_SAMPLES {
return Err(format!(
"deep filter model hop {} != {DEEP_FILTER_FRAME_SAMPLES}",
model.hop_size
));
}
Ok(DeepFilterProcessor {
model,
input: Array2::zeros((1, DEEP_FILTER_FRAME_SAMPLES)),
output: Array2::zeros((1, DEEP_FILTER_FRAME_SAMPLES)),
})
}
pub fn process_frame(&mut self, samples: &mut [i16]) -> Result<(), String> {
assert_eq!(samples.len(), DEEP_FILTER_FRAME_SAMPLES);
assert_eq!(self.input.len(), DEEP_FILTER_FRAME_SAMPLES);
assert_eq!(self.output.len(), DEEP_FILTER_FRAME_SAMPLES);
for (target, sample) in self.input.iter_mut().zip(samples.iter()) {
*target = f32::from(*sample) * SAMPLE_SCALE_I16_TO_F32;
}
self.model
.process(self.input.view(), self.output.view_mut())
.map_err(|error| format!("deep filter process: {error:#}"))?;
for (sample, enhanced) in samples.iter_mut().zip(self.output.iter()) {
*sample = sample_f32_to_i16(*enhanced);
}
Ok(())
}
}
fn sample_f32_to_i16(sample: f32) -> i16 {
if !sample.is_finite() {
return 0;
}
let clamped = sample.clamp(-1.0, 1.0);
assert!(clamped >= -1.0);
assert!(clamped <= 1.0);
(clamped * SAMPLE_SCALE_F32_TO_I16) as i16
}
#[cfg(test)]
mod tests {
use super::*;
fn next_noise_sample(seed: &mut u32) -> i16 {
*seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
((*seed >> 16) as u16 as i16) / 4
}
fn noise_frame(seed: &mut u32) -> [i16; DEEP_FILTER_FRAME_SAMPLES] {
let mut frame = [0i16; DEEP_FILTER_FRAME_SAMPLES];
for sample in frame.iter_mut() {
*sample = next_noise_sample(seed);
}
frame
}
fn frame_rms(samples: &[i16]) -> f64 {
assert!(!samples.is_empty());
let sum_squares: f64 = samples
.iter()
.map(|sample| {
let normalized = f64::from(*sample) / 32_768.0;
normalized * normalized
})
.sum();
(sum_squares / samples.len() as f64).sqrt()
}
#[test]
fn sample_conversion_holds_the_contract_range() {
assert_eq!(sample_f32_to_i16(0.0), 0);
assert_eq!(sample_f32_to_i16(1.0), 32_767);
assert_eq!(sample_f32_to_i16(-1.0), -32_767);
assert_eq!(sample_f32_to_i16(2.0), 32_767);
assert_eq!(sample_f32_to_i16(-2.0), -32_767);
assert_eq!(sample_f32_to_i16(0.5), 16_383);
}
#[test]
fn sample_conversion_maps_non_finite_to_silence() {
assert_eq!(sample_f32_to_i16(f32::NAN), 0);
assert_eq!(sample_f32_to_i16(f32::INFINITY), 0);
assert_eq!(sample_f32_to_i16(f32::NEG_INFINITY), 0);
}
#[test]
fn zero_level_passes_audio_through() {
let mut processor = DeepFilterProcessor::new(0.0).expect("embedded model must initialize");
let mut seed = 0x2545_f491u32;
for _ in 0..5 {
let original = noise_frame(&mut seed);
let mut processed = original;
processor
.process_frame(&mut processed)
.expect("processing must succeed");
for (output, input) in processed.iter().zip(original.iter()) {
assert!((i32::from(*output) - i32::from(*input)).abs() <= 1);
}
}
}
#[test]
fn full_level_attenuates_steady_noise() {
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
.expect("embedded model must initialize");
let mut seed = 0x9e37_79b9u32;
let mut input_rms = 0.0;
let mut output_rms = 0.0;
for frame_index in 0..30 {
let mut frame = noise_frame(&mut seed);
let frame_input_rms = frame_rms(&frame);
processor
.process_frame(&mut frame)
.expect("processing must succeed");
if frame_index >= 20 {
input_rms += frame_input_rms;
output_rms += frame_rms(&frame);
}
}
assert!(input_rms > 0.0);
assert!(output_rms < input_rms * 0.5);
}
#[test]
fn frame_length_contract_is_enforced() {
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
.expect("embedded model must initialize");
let mut short_frame = [0i16; DEEP_FILTER_FRAME_SAMPLES - 1];
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = processor.process_frame(&mut short_frame);
}));
assert!(result.is_err());
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,643 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub fn push_json_string(out: &mut String, value: &str) {
out.push('"');
for ch in value.chars() {
match ch {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
'\u{08}' => out.push_str("\\b"),
'\u{0C}' => out.push_str("\\f"),
c if (c as u32) < 0x20 => {
out.push_str(&format!("\\u{:04x}", c as u32));
}
c => out.push(c),
}
}
out.push('"');
}
pub enum JsonValue {
Str(String),
Raw(String),
}
pub fn json_object(fields: &[(&str, JsonValue)]) -> String {
let mut out = String::from("{");
for (i, (key, value)) in fields.iter().enumerate() {
if i > 0 {
out.push(',');
}
push_json_string(&mut out, key);
out.push(':');
match value {
JsonValue::Str(s) => push_json_string(&mut out, s),
JsonValue::Raw(r) => out.push_str(r),
}
}
out.push('}');
out
}
pub fn json_string_array(items: &[String]) -> String {
let mut out = String::from("[");
for (i, item) in items.iter().enumerate() {
if i > 0 {
out.push(',');
}
push_json_string(&mut out, item);
}
out.push(']');
out
}
pub fn json_u8_array(items: &[u8]) -> String {
let mut out = String::from("[");
for (i, item) in items.iter().enumerate() {
if i > 0 {
out.push(',');
}
out.push_str(&item.to_string());
}
out.push(']');
out
}
pub fn json_raw_array(items: &[String]) -> String {
let mut out = String::from("[");
for (i, item) in items.iter().enumerate() {
if i > 0 {
out.push(',');
}
out.push_str(item);
}
out.push(']');
out
}
pub fn json_string_map(items: &std::collections::HashMap<String, String>) -> String {
let mut entries: Vec<(&String, &String)> = items.iter().collect();
entries.sort_by(|left, right| left.0.cmp(right.0));
let mut out = String::from("{");
for (i, (key, value)) in entries.iter().enumerate() {
if i > 0 {
out.push(',');
}
push_json_string(&mut out, key);
out.push(':');
push_json_string(&mut out, value);
}
out.push('}');
out
}
#[cfg(feature = "publisher")]
mod live {
use super::{
JsonValue, json_object, json_raw_array, json_string_array, json_string_map, json_u8_array,
};
use livekit::participant::{
ConnectionQuality, LocalParticipant, Participant, RemoteParticipant,
};
use livekit::publication::{
LocalTrackPublication, RemoteTrackPublication, SubscriptionStatus, TrackPublication,
};
use livekit::track::{TrackKind, TrackSource};
use livekit::{DataPacketKind, RoomEvent};
pub fn track_kind_str(kind: TrackKind) -> &'static str {
match kind {
TrackKind::Audio => "audio",
TrackKind::Video => "video",
}
}
pub fn track_source_str(source: TrackSource) -> &'static str {
match source {
TrackSource::Unknown => "unknown",
TrackSource::Camera => "camera",
TrackSource::Microphone => "microphone",
TrackSource::Screenshare => "screen_share",
TrackSource::ScreenshareAudio => "screen_share_audio",
}
}
pub fn connection_quality_str(quality: ConnectionQuality) -> &'static str {
match quality {
ConnectionQuality::Excellent => "excellent",
ConnectionQuality::Good => "good",
ConnectionQuality::Poor => "poor",
ConnectionQuality::Lost => "lost",
}
}
fn s(value: impl Into<String>) -> JsonValue {
JsonValue::Str(value.into())
}
fn b(value: bool) -> JsonValue {
JsonValue::Raw(value.to_string())
}
fn subscription_status_str(status: SubscriptionStatus) -> &'static str {
match status {
SubscriptionStatus::Desired => "desired",
SubscriptionStatus::Subscribed => "subscribed",
SubscriptionStatus::Unsubscribed => "unsubscribed",
}
}
fn participant_snapshot(participant: &Participant) -> String {
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
])
}
fn push_remote_participant_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
participant: &RemoteParticipant,
) {
fields.push(("participantSid", s(participant.sid().to_string())));
fields.push(("identity", s(participant.identity().to_string())));
fields.push(("participantName", s(participant.name())));
}
fn push_local_participant_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
participant: &LocalParticipant,
) {
fields.push(("participantSid", s(participant.sid().to_string())));
fields.push(("identity", s(participant.identity().to_string())));
fields.push(("participantName", s(participant.name())));
}
fn push_participant_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
participant: &Participant,
) {
fields.push(("participantSid", s(participant.sid().to_string())));
fields.push(("identity", s(participant.identity().to_string())));
fields.push(("participantName", s(participant.name())));
}
fn push_remote_publication_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
publication: &RemoteTrackPublication,
) {
fields.push(("trackSid", s(publication.sid().to_string())));
fields.push(("trackName", s(publication.name())));
fields.push(("kind", s(track_kind_str(publication.kind()))));
fields.push(("source", s(track_source_str(publication.source()))));
fields.push(("muted", b(publication.is_muted())));
fields.push(("subscribed", b(publication.is_subscribed())));
fields.push((
"subscriptionStatus",
s(subscription_status_str(publication.subscription_status())),
));
}
fn push_local_publication_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
publication: &LocalTrackPublication,
) {
fields.push(("trackSid", s(publication.sid().to_string())));
fields.push(("trackName", s(publication.name())));
fields.push(("kind", s(track_kind_str(publication.kind()))));
fields.push(("source", s(track_source_str(publication.source()))));
fields.push(("muted", b(publication.is_muted())));
}
fn push_publication_fields(
fields: &mut Vec<(&'static str, JsonValue)>,
publication: &TrackPublication,
) {
fields.push(("trackSid", s(publication.sid().to_string())));
fields.push(("trackName", s(publication.name())));
fields.push(("kind", s(track_kind_str(publication.kind()))));
fields.push(("source", s(track_source_str(publication.source()))));
fields.push(("muted", b(publication.is_muted())));
}
fn remote_track_payload(
participant: &RemoteParticipant,
publication: &RemoteTrackPublication,
) -> String {
let mut fields = Vec::new();
push_remote_participant_fields(&mut fields, participant);
push_remote_publication_fields(&mut fields, publication);
json_object(&fields)
}
const CONNECTED_ROSTER_PARTICIPANTS_MAX: usize = 1024;
const CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX: usize = 16;
fn connected_payload(
participants_with_tracks: &[(RemoteParticipant, Vec<RemoteTrackPublication>)],
) -> String {
let participant_count = participants_with_tracks
.len()
.min(CONNECTED_ROSTER_PARTICIPANTS_MAX);
let mut entries = Vec::with_capacity(participant_count);
for (participant, publications) in participants_with_tracks
.iter()
.take(CONNECTED_ROSTER_PARTICIPANTS_MAX)
{
let track_count = publications
.len()
.min(CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX);
let mut tracks = Vec::with_capacity(track_count);
for publication in publications
.iter()
.take(CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX)
{
tracks.push(remote_track_payload(participant, publication));
}
entries.push(json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
("tracks", JsonValue::Raw(json_raw_array(&tracks))),
]));
}
assert!(entries.len() <= CONNECTED_ROSTER_PARTICIPANTS_MAX);
json_object(&[("participants", JsonValue::Raw(json_raw_array(&entries)))])
}
fn local_track_payload(
participant: &LocalParticipant,
publication: &LocalTrackPublication,
) -> String {
let mut fields = Vec::new();
push_local_participant_fields(&mut fields, participant);
push_local_publication_fields(&mut fields, publication);
json_object(&fields)
}
pub fn map_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
map_participant_lifecycle_room_event(event)
.or_else(|| map_participant_profile_room_event(event))
.or_else(|| map_track_room_event(event))
.or_else(|| map_local_track_room_event(event))
.or_else(|| map_connection_room_event(event))
}
fn map_participant_lifecycle_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
match event {
RoomEvent::ParticipantConnected(p) | RoomEvent::ParticipantActive(p) => Some((
"participantJoined",
json_object(&[
("sid", s(p.sid().to_string())),
("identity", s(p.identity().to_string())),
("name", s(p.name())),
]),
)),
RoomEvent::ParticipantDisconnected(p) => Some((
"participantLeft",
json_object(&[
("sid", s(p.sid().to_string())),
("identity", s(p.identity().to_string())),
("name", s(p.name())),
]),
)),
RoomEvent::ParticipantNameChanged {
participant,
old_name,
name,
} => Some((
"participantNameChanged",
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("oldName", s(old_name.to_string())),
("name", s(name.to_string())),
]),
)),
_ => None,
}
}
fn map_participant_profile_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
match event {
RoomEvent::ParticipantMetadataChanged {
participant,
old_metadata,
metadata,
} => Some((
"participantMetadataChanged",
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
("oldMetadata", s(old_metadata.to_string())),
("metadata", s(metadata.to_string())),
(
"attributes",
JsonValue::Raw(json_string_map(&participant.attributes())),
),
]),
)),
RoomEvent::ParticipantAttributesChanged {
participant,
changed_attributes,
} => Some((
"participantAttributesChanged",
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
(
"attributes",
JsonValue::Raw(json_string_map(&participant.attributes())),
),
(
"changedAttributes",
JsonValue::Raw(json_string_map(changed_attributes)),
),
]),
)),
_ => None,
}
}
fn map_track_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
match event {
RoomEvent::TrackSubscribed {
publication,
participant,
..
} => Some((
"trackSubscribed",
remote_track_payload(participant, publication),
)),
RoomEvent::TrackUnsubscribed {
publication,
participant,
..
} => Some((
"trackUnsubscribed",
remote_track_payload(participant, publication),
)),
RoomEvent::TrackSubscriptionFailed {
participant,
error,
track_sid,
} => {
let mut fields = Vec::new();
push_remote_participant_fields(&mut fields, participant);
if let Some(publication) = participant.get_track_publication(track_sid) {
push_remote_publication_fields(&mut fields, &publication);
} else {
fields.push(("trackSid", s(track_sid.to_string())));
}
fields.push(("error", s(format!("{error}"))));
Some(("trackSubscriptionFailed", json_object(&fields)))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some((
"trackPublished",
remote_track_payload(participant, publication),
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some((
"trackUnpublished",
remote_track_payload(participant, publication),
)),
RoomEvent::TrackMuted {
participant,
publication,
} => {
let mut fields = Vec::new();
push_participant_fields(&mut fields, participant);
push_publication_fields(&mut fields, publication);
Some(("trackMuted", json_object(&fields)))
}
RoomEvent::TrackUnmuted {
participant,
publication,
} => {
let mut fields = Vec::new();
push_participant_fields(&mut fields, participant);
push_publication_fields(&mut fields, publication);
Some(("trackUnmuted", json_object(&fields)))
}
_ => None,
}
}
fn map_connection_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
match event {
RoomEvent::ActiveSpeakersChanged { speakers } => {
let sids: Vec<String> = speakers.iter().map(|p| p.sid().to_string()).collect();
let participants: Vec<String> = speakers.iter().map(participant_snapshot).collect();
Some((
"activeSpeakers",
json_object(&[
("sids", JsonValue::Raw(json_string_array(&sids))),
(
"participants",
JsonValue::Raw(json_raw_array(&participants)),
),
]),
))
}
RoomEvent::ConnectionQualityChanged {
quality,
participant,
} => Some((
"connectionQuality",
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
("quality", s(connection_quality_str(*quality))),
]),
)),
RoomEvent::DataReceived {
payload,
topic,
kind,
participant,
} => Some((
"dataReceived",
data_received_payload(
payload.as_ref().as_slice(),
topic.as_deref(),
kind,
participant.as_ref(),
),
)),
RoomEvent::E2eeStateChanged { participant, state } => Some((
"e2eeState",
json_object(&[
("sid", s(participant.sid().to_string())),
("identity", s(participant.identity().to_string())),
("name", s(participant.name())),
("state", s(format!("{state:?}").to_lowercase())),
]),
)),
RoomEvent::ConnectionStateChanged(state) => Some((
"connectionState",
json_object(&[("state", s(format!("{state:?}").to_lowercase()))]),
)),
RoomEvent::Disconnected { reason } => Some((
"disconnected",
json_object(&[("reason", s(format!("{reason:?}").to_lowercase()))]),
)),
RoomEvent::Connected {
participants_with_tracks,
} => Some(("connected", connected_payload(participants_with_tracks))),
_ => None,
}
}
fn data_received_payload(
payload: &[u8],
topic: Option<&str>,
kind: &DataPacketKind,
participant: Option<&RemoteParticipant>,
) -> String {
let mut fields = vec![
("payloadBytes", JsonValue::Raw(json_u8_array(payload))),
(
"reliable",
JsonValue::Raw(matches!(kind, DataPacketKind::Reliable).to_string()),
),
(
"kind",
s(match kind {
DataPacketKind::Reliable => "reliable",
DataPacketKind::Lossy => "lossy",
}),
),
];
if let Some(topic) = topic {
fields.push(("topic", s(topic.to_string())));
}
if let Ok(payload_text) = std::str::from_utf8(payload) {
fields.push(("payloadText", s(payload_text.to_string())));
}
if let Some(participant) = participant {
push_remote_participant_fields(&mut fields, participant);
}
json_object(&fields)
}
fn map_local_track_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
match event {
RoomEvent::LocalTrackPublished {
publication,
participant,
..
} => Some((
"localTrackPublished",
local_track_payload(participant, publication),
)),
RoomEvent::LocalTrackUnpublished {
publication,
participant,
} => Some((
"localTrackUnpublished",
local_track_payload(participant, publication),
)),
RoomEvent::LocalTrackRepublished {
previous_sid,
publication,
participant,
..
} => {
let mut fields = Vec::new();
push_local_participant_fields(&mut fields, participant);
fields.push(("previousTrackSid", s(previous_sid.to_string())));
push_local_publication_fields(&mut fields, publication);
Some(("localTrackRepublished", json_object(&fields)))
}
_ => None,
}
}
}
#[cfg(feature = "publisher")]
pub use live::{map_room_event, track_kind_str, track_source_str};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn escapes_quote_backslash_and_control_chars() {
let mut out = String::new();
push_json_string(&mut out, "a\"b\\c\nd\te");
assert_eq!(out, "\"a\\\"b\\\\c\\nd\\te\"");
let mut ctrl = String::new();
push_json_string(&mut ctrl, "\u{01}");
assert_eq!(ctrl, "\"\\u0001\"");
}
#[test]
fn leaves_plain_ascii_and_unicode_untouched() {
let mut out = String::new();
push_json_string(&mut out, "PA_abc123");
assert_eq!(out, "\"PA_abc123\"");
}
#[test]
fn json_object_preserves_order_and_mixes_str_and_raw() {
let json = json_object(&[
("sid", JsonValue::Str("PA_1".into())),
(
"sids",
JsonValue::Raw(json_string_array(&["PA_1".into(), "PA_2".into()])),
),
]);
assert_eq!(json, "{\"sid\":\"PA_1\",\"sids\":[\"PA_1\",\"PA_2\"]}");
}
#[cfg(feature = "publisher")]
#[test]
fn track_source_strings_match_livekit_js_sources() {
use livekit::track::TrackSource;
assert_eq!(track_source_str(TrackSource::Camera), "camera");
assert_eq!(track_source_str(TrackSource::Microphone), "microphone");
assert_eq!(track_source_str(TrackSource::Screenshare), "screen_share");
assert_eq!(
track_source_str(TrackSource::ScreenshareAudio),
"screen_share_audio"
);
}
#[test]
fn empty_object_and_array() {
assert_eq!(json_object(&[]), "{}");
assert_eq!(json_string_array(&[]), "[]");
assert_eq!(json_u8_array(&[]), "[]");
assert_eq!(json_raw_array(&[]), "[]");
}
#[test]
fn json_u8_array_serializes_bytes_as_numbers() {
assert_eq!(json_u8_array(&[0, 1, 127, 255]), "[0,1,127,255]");
}
#[test]
fn raw_array_preserves_prebuilt_json_objects() {
let items = vec![
json_object(&[("sid", JsonValue::Str("PA_1".into()))]),
json_object(&[("sid", JsonValue::Str("PA_2".into()))]),
];
assert_eq!(
json_raw_array(&items),
"[{\"sid\":\"PA_1\"},{\"sid\":\"PA_2\"}]"
);
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,415 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use futures_core::Stream;
use napi::tokio;
use napi::tokio::task::AbortHandle;
use parking_lot::Mutex;
use std::collections::HashMap;
use std::future::poll_fn;
use std::pin::pin;
pub const INBOUND_FORWARDERS_MAX: usize = 512;
struct ForwarderEntry {
participant_sid: String,
handle: AbortHandle,
}
pub struct InboundForwarderRegistry {
entries: Mutex<HashMap<String, ForwarderEntry>>,
}
impl InboundForwarderRegistry {
pub fn new() -> Self {
Self {
entries: Mutex::new(HashMap::new()),
}
}
#[cfg(test)]
pub fn len(&self) -> usize {
self.entries.lock().len()
}
#[cfg(test)]
pub fn contains(&self, track_sid: &str) -> bool {
self.entries.lock().contains_key(track_sid)
}
pub fn register(&self, track_sid: &str, participant_sid: &str, handle: AbortHandle) -> bool {
if track_sid.is_empty() {
handle.abort();
return false;
}
let mut entries = self.entries.lock();
if let Some(previous) = entries.remove(track_sid) {
previous.handle.abort();
}
if entries.len() >= INBOUND_FORWARDERS_MAX {
drop(entries);
handle.abort();
eprintln!(
"webrtc-sender: inbound forwarder registry at cap {INBOUND_FORWARDERS_MAX}; \
refusing forwarder for track {track_sid}"
);
return false;
}
entries.insert(
track_sid.to_string(),
ForwarderEntry {
participant_sid: participant_sid.to_string(),
handle,
},
);
assert!(entries.len() <= INBOUND_FORWARDERS_MAX);
true
}
pub fn cancel(&self, track_sid: &str) {
let removed = self.entries.lock().remove(track_sid);
if let Some(entry) = removed {
entry.handle.abort();
}
}
pub fn cancel_for_participant(&self, participant_sid: &str) {
let aborted: Vec<ForwarderEntry> = {
let mut entries = self.entries.lock();
let matching: Vec<String> = entries
.iter()
.filter(|(_, entry)| entry.participant_sid == participant_sid)
.map(|(track_sid, _)| track_sid.clone())
.collect();
matching
.into_iter()
.filter_map(|track_sid| entries.remove(&track_sid))
.collect()
};
for entry in aborted {
entry.handle.abort();
}
}
pub fn clear(&self) {
let drained: Vec<ForwarderEntry> = {
let mut entries = self.entries.lock();
entries.drain().map(|(_, entry)| entry).collect()
};
for entry in drained {
entry.handle.abort();
}
}
}
impl Default for InboundForwarderRegistry {
fn default() -> Self {
Self::new()
}
}
pub fn spawn_drain_forwarder<S, F>(stream: S, mut on_item: F) -> AbortHandle
where
S: Stream + Send + 'static,
S::Item: Send,
F: FnMut(S::Item) + Send + 'static,
{
let task = tokio::spawn(async move {
let mut stream = pin!(stream);
loop {
let item = poll_fn(|cx| stream.as_mut().poll_next(cx)).await;
match item {
Some(item) => on_item(item),
None => return,
}
}
});
task.abort_handle()
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::VecDeque;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::task::{Context, Poll};
use tokio::runtime::Builder;
use tokio::sync::Notify;
struct CloseSignal {
closed: AtomicBool,
notify: Notify,
}
impl CloseSignal {
fn new() -> Arc<Self> {
Arc::new(Self {
closed: AtomicBool::new(false),
notify: Notify::new(),
})
}
fn is_closed(&self) -> bool {
self.closed.load(Ordering::SeqCst)
}
fn fire(&self) {
self.closed.store(true, Ordering::SeqCst);
self.notify.notify_one();
}
async fn wait(&self) {
if self.is_closed() {
return;
}
self.notify.notified().await;
assert!(self.is_closed());
}
}
struct FakeVideoStream {
signal: Arc<CloseSignal>,
}
impl Stream for FakeVideoStream {
type Item = u64;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Pending
}
}
impl Drop for FakeVideoStream {
fn drop(&mut self) {
self.signal.fire();
}
}
struct CountedStream {
items: VecDeque<u64>,
count: Arc<AtomicUsize>,
done: Arc<Notify>,
}
impl Stream for CountedStream {
type Item = u64;
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match self.items.pop_front() {
Some(item) => {
self.count.fetch_add(1, Ordering::SeqCst);
Poll::Ready(Some(item))
}
None => {
self.done.notify_one();
Poll::Ready(None)
}
}
}
}
fn runtime() -> tokio::runtime::Runtime {
Builder::new_multi_thread()
.worker_threads(1)
.build()
.expect("tokio runtime")
}
fn dispatch_track_subscribed(
registry: &InboundForwarderRegistry,
track_sid: &str,
participant_sid: &str,
) -> Arc<CloseSignal> {
let (signal, registered) = dispatch_with_outcome(registry, track_sid, participant_sid);
assert!(registered);
signal
}
fn dispatch_with_outcome(
registry: &InboundForwarderRegistry,
track_sid: &str,
participant_sid: &str,
) -> (Arc<CloseSignal>, bool) {
let signal = CloseSignal::new();
let stream = FakeVideoStream {
signal: signal.clone(),
};
let handle = spawn_drain_forwarder(stream, |_frame| {});
let registered = registry.register(track_sid, participant_sid, handle);
(signal, registered)
}
#[test]
fn teardown_symmetry_closes_the_stream() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let signal = dispatch_track_subscribed(&registry, "TR_x", "PA_one");
assert_eq!(registry.len(), 1);
assert!(!signal.is_closed());
registry.cancel("TR_x");
assert_eq!(registry.len(), 0);
signal.wait().await;
assert!(signal.is_closed());
});
}
#[test]
fn repeated_subscribe_unsubscribe_never_accumulates() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let cycles = 32usize;
let mut signals = Vec::with_capacity(cycles);
for _ in 0..cycles {
let signal = dispatch_track_subscribed(&registry, "TR_x", "PA_one");
assert_eq!(registry.len(), 1);
registry.cancel("TR_x");
assert_eq!(registry.len(), 0);
signal.wait().await;
signals.push(signal);
}
assert_eq!(registry.len(), 0);
let closed_count = signals.iter().filter(|signal| signal.is_closed()).count();
assert_eq!(closed_count, cycles);
});
}
#[test]
fn double_subscribe_keeps_one_live_forwarder() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let first_signal = dispatch_track_subscribed(&registry, "TR_x", "PA_one");
let second_signal = dispatch_track_subscribed(&registry, "TR_x", "PA_one");
assert_eq!(registry.len(), 1);
first_signal.wait().await;
assert!(first_signal.is_closed());
assert!(!second_signal.is_closed());
registry.cancel("TR_x");
assert_eq!(registry.len(), 0);
second_signal.wait().await;
assert!(second_signal.is_closed());
});
}
#[test]
fn participant_disconnect_tears_down_all_forwarders() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let video_signal = dispatch_track_subscribed(&registry, "TR_video", "PA_one");
let audio_signal = dispatch_track_subscribed(&registry, "TR_audio", "PA_one");
let other_signal = dispatch_track_subscribed(&registry, "TR_other", "PA_two");
assert_eq!(registry.len(), 3);
registry.cancel_for_participant("PA_one");
assert_eq!(registry.len(), 1);
assert!(registry.contains("TR_other"));
video_signal.wait().await;
audio_signal.wait().await;
assert!(video_signal.is_closed());
assert!(audio_signal.is_closed());
assert!(!other_signal.is_closed());
registry.clear();
assert_eq!(registry.len(), 0);
other_signal.wait().await;
assert!(other_signal.is_closed());
});
}
#[test]
fn clear_tears_down_every_forwarder() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let mut signals = Vec::new();
for index in 0..8 {
let track_sid = format!("TR_{index}");
signals.push(dispatch_track_subscribed(&registry, &track_sid, "PA_one"));
}
assert_eq!(registry.len(), 8);
registry.clear();
assert_eq!(registry.len(), 0);
for signal in &signals {
signal.wait().await;
}
assert!(signals.iter().all(|signal| signal.is_closed()));
});
}
#[test]
fn drain_forwarder_invokes_callback_per_item() {
let runtime = runtime();
runtime.block_on(async {
let count = Arc::new(AtomicUsize::new(0));
let polled = Arc::new(AtomicUsize::new(0));
let polled_in_task = polled.clone();
let done = Arc::new(Notify::new());
let stream = CountedStream {
items: VecDeque::from(vec![1u64, 2, 3, 4]),
count: count.clone(),
done: done.clone(),
};
let _handle = spawn_drain_forwarder(stream, move |_item| {
polled_in_task.fetch_add(1, Ordering::SeqCst);
});
done.notified().await;
assert_eq!(count.load(Ordering::SeqCst), 4);
assert_eq!(polled.load(Ordering::SeqCst), 4);
});
}
#[test]
fn register_refuses_and_closes_forwarder_at_cap() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
for index in 0..INBOUND_FORWARDERS_MAX {
let track_sid = format!("TR_{index}");
let (_signal, registered) = dispatch_with_outcome(&registry, &track_sid, "PA_one");
assert!(registered);
}
assert_eq!(registry.len(), INBOUND_FORWARDERS_MAX);
let (overflow_signal, registered) =
dispatch_with_outcome(&registry, "TR_overflow", "PA_one");
assert!(!registered);
assert_eq!(registry.len(), INBOUND_FORWARDERS_MAX);
assert!(!registry.contains("TR_overflow"));
overflow_signal.wait().await;
assert!(overflow_signal.is_closed());
registry.clear();
assert_eq!(registry.len(), 0);
});
}
#[test]
fn register_refuses_and_closes_forwarder_for_empty_sid() {
let runtime = runtime();
runtime.block_on(async {
let registry = InboundForwarderRegistry::new();
let (signal, registered) = dispatch_with_outcome(&registry, "", "PA_one");
assert!(!registered);
assert_eq!(registry.len(), 0);
signal.wait().await;
assert!(signal.is_closed());
});
}
}
@@ -0,0 +1,73 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#![allow(unsafe_op_in_unsafe_fn)]
#![cfg_attr(
not(all(feature = "publisher", feature = "camera-native")),
allow(dead_code)
)]
mod audio;
mod bridge_version;
mod camera;
mod camera_background;
mod config;
mod deep_filter;
mod events;
mod hardware_encoder;
mod inbound_forwarder;
mod mask_refine;
mod native_camera;
mod person_segmentation;
mod send_control;
mod speaking;
mod stats;
mod texture_source;
mod yuv;
#[cfg(feature = "publisher")]
mod engine;
#[cfg(feature = "bench-internals")]
pub mod bench_internals {
pub use crate::audio::DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX;
pub use crate::deep_filter::{DEEP_FILTER_FRAME_SAMPLES, DeepFilterProcessor};
pub use crate::mask_refine::MaskRefiner;
pub struct BlurScratch(crate::camera_background::BlurScratch);
impl BlurScratch {
pub fn new(width: usize, height: usize) -> Self {
Self(crate::camera_background::BlurScratch::new(width, height))
}
}
pub fn blur_plane_masked(
plane: &mut [u8],
width: usize,
height: usize,
mask: &[u8],
radius_pass: usize,
scratch: &mut BlurScratch,
) {
let mask = crate::camera_background::plane_mask(mask, width, 1);
crate::camera_background::blur_plane_masked(
plane,
width,
height,
mask,
radius_pass,
&mut scratch.0,
);
}
pub fn composite_masked_plane(
plane: &mut [u8],
background: &[u8],
width: usize,
height: usize,
mask: &[u8],
) {
let mask = crate::camera_background::plane_mask(mask, width, 1);
crate::camera_background::composite_masked_plane(plane, background, width, height, mask);
}
}
@@ -0,0 +1,626 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const FRAME_EDGE_MAX: usize = 8192;
const MASK_REFINE_DOWNSAMPLE: usize = 4;
const GUIDED_FILTER_RADIUS_LOW: usize = 4;
const GUIDED_FILTER_EPSILON: f32 = 1e-4;
const TEMPORAL_COMBINE_RATIO: f32 = 0.7;
const TEMPORAL_UNCERTAINTY_C1: f32 = 5.68842;
const TEMPORAL_UNCERTAINTY_C2: f32 = -0.748699;
const TEMPORAL_UNCERTAINTY_C3: f32 = -57.8051;
const TEMPORAL_UNCERTAINTY_C4: f32 = 291.309;
const TEMPORAL_UNCERTAINTY_C5: f32 = -624.717;
const SHAPE_SMOOTHSTEP_EDGE_LOW: f32 = 0.55;
const SHAPE_SMOOTHSTEP_EDGE_HIGH: f32 = 0.85;
const LUT_LEN: usize = 256;
pub struct MaskRefiner {
width: usize,
height: usize,
low_width: usize,
low_height: usize,
previous_mask: Vec<u8>,
previous_mask_valid: bool,
temporal_weight_lut: [u16; LUT_LEN],
shape_lut: [u8; LUT_LEN],
column_fixed: Vec<u32>,
guide_low: Vec<f32>,
mask_low: Vec<f32>,
mean_guide: Vec<f32>,
mean_mask: Vec<f32>,
corr_guide_guide: Vec<f32>,
corr_guide_mask: Vec<f32>,
coeff_a: Vec<f32>,
coeff_b: Vec<f32>,
scratch: Vec<f32>,
}
impl MaskRefiner {
pub fn new(width: usize, height: usize) -> Self {
assert!(width >= 2);
assert!(height >= 2);
assert!(width <= FRAME_EDGE_MAX);
assert!(height <= FRAME_EDGE_MAX);
let low_width = (width / MASK_REFINE_DOWNSAMPLE).max(1);
let low_height = (height / MASK_REFINE_DOWNSAMPLE).max(1);
let low_len = low_width * low_height;
let mut column_fixed = vec![0u32; width];
for (x, slot) in column_fixed.iter_mut().enumerate() {
*slot = bilinear_fixed_coord(x, width, low_width);
}
Self {
width,
height,
low_width,
low_height,
previous_mask: vec![0; width * height],
previous_mask_valid: false,
temporal_weight_lut: temporal_weight_lut(),
shape_lut: shape_lut(),
column_fixed,
guide_low: vec![0.0; low_len],
mask_low: vec![0.0; low_len],
mean_guide: vec![0.0; low_len],
mean_mask: vec![0.0; low_len],
corr_guide_guide: vec![0.0; low_len],
corr_guide_mask: vec![0.0; low_len],
coeff_a: vec![0.0; low_len],
coeff_b: vec![0.0; low_len],
scratch: vec![0.0; low_len],
}
}
pub fn refine(&mut self, luma: &[u8], mask: &mut [u8]) {
assert!(luma.len() >= self.width * self.height);
assert!(mask.len() >= self.width * self.height);
self.blend_temporal(mask);
self.downsample(luma, mask);
self.close_mask_low();
self.solve_guided_coefficients();
self.apply_guided_coefficients(luma, mask);
}
fn blend_temporal(&mut self, mask: &mut [u8]) {
let len = self.width * self.height;
assert!(mask.len() >= len);
assert_eq!(self.previous_mask.len(), len);
if self.previous_mask_valid {
for (current, previous) in mask[..len].iter_mut().zip(self.previous_mask.iter()) {
let new_value = i32::from(*current);
let weight = i32::from(self.temporal_weight_lut[usize::from(*current)]);
let delta = (i32::from(*previous) - new_value) * weight;
*current = (new_value + ((delta + 128) >> 8)).clamp(0, 255) as u8;
}
}
self.previous_mask.copy_from_slice(&mask[..len]);
self.previous_mask_valid = true;
}
fn downsample(&mut self, luma: &[u8], mask: &[u8]) {
assert_eq!(self.guide_low.len(), self.low_width * self.low_height);
assert_eq!(self.mask_low.len(), self.guide_low.len());
for low_y in 0..self.low_height {
let y_start = low_y * MASK_REFINE_DOWNSAMPLE;
let y_end = (y_start + MASK_REFINE_DOWNSAMPLE).min(self.height);
for low_x in 0..self.low_width {
let x_start = low_x * MASK_REFINE_DOWNSAMPLE;
let x_end = (x_start + MASK_REFINE_DOWNSAMPLE).min(self.width);
let mut guide_sum: u32 = 0;
let mut mask_sum: u32 = 0;
for y in y_start..y_end {
let row = y * self.width;
for x in x_start..x_end {
guide_sum += u32::from(luma[row + x]);
mask_sum += u32::from(mask[row + x]);
}
}
let count = ((y_end - y_start) * (x_end - x_start)) as f32;
assert!(count >= 1.0);
let low_offset = low_y * self.low_width + low_x;
self.guide_low[low_offset] = guide_sum as f32 / (count * 255.0);
self.mask_low[low_offset] = mask_sum as f32 / (count * 255.0);
}
}
}
fn close_mask_low(&mut self) {
morph_pass_low(
&self.mask_low,
&mut self.scratch,
&mut self.coeff_a,
self.low_width,
self.low_height,
f32::max,
);
morph_pass_low(
&self.coeff_a,
&mut self.scratch,
&mut self.mask_low,
self.low_width,
self.low_height,
f32::min,
);
}
fn solve_guided_coefficients(&mut self) {
let len = self.low_width * self.low_height;
assert_eq!(self.coeff_a.len(), len);
assert_eq!(self.coeff_b.len(), len);
let radius = GUIDED_FILTER_RADIUS_LOW;
let w = self.low_width;
let h = self.low_height;
box_filter_low(
&self.guide_low,
&mut self.scratch,
&mut self.mean_guide,
w,
h,
radius,
);
box_filter_low(
&self.mask_low,
&mut self.scratch,
&mut self.mean_mask,
w,
h,
radius,
);
for i in 0..len {
self.coeff_a[i] = self.guide_low[i] * self.guide_low[i];
self.coeff_b[i] = self.guide_low[i] * self.mask_low[i];
}
box_filter_low(
&self.coeff_a,
&mut self.scratch,
&mut self.corr_guide_guide,
w,
h,
radius,
);
box_filter_low(
&self.coeff_b,
&mut self.scratch,
&mut self.corr_guide_mask,
w,
h,
radius,
);
for i in 0..len {
let variance = self.corr_guide_guide[i] - self.mean_guide[i] * self.mean_guide[i];
let covariance = self.corr_guide_mask[i] - self.mean_guide[i] * self.mean_mask[i];
let a = covariance / (variance.max(0.0) + GUIDED_FILTER_EPSILON);
self.coeff_a[i] = a;
self.coeff_b[i] = self.mean_mask[i] - a * self.mean_guide[i];
}
box_filter_low(
&self.coeff_a,
&mut self.scratch,
&mut self.mean_guide,
w,
h,
radius,
);
box_filter_low(
&self.coeff_b,
&mut self.scratch,
&mut self.mean_mask,
w,
h,
radius,
);
}
fn apply_guided_coefficients(&self, luma: &[u8], mask: &mut [u8]) {
assert!(luma.len() >= self.width * self.height);
assert!(mask.len() >= self.width * self.height);
let low_w = self.low_width;
for y in 0..self.height {
let row_fixed = bilinear_fixed_coord(y, self.height, self.low_height);
let sy = (row_fixed / 256) as usize;
let fy = (row_fixed % 256) as f32 / 256.0;
let sy_next = (sy + 1).min(self.low_height - 1);
let row = y * self.width;
for x in 0..self.width {
let col_fixed = self.column_fixed[x];
let sx = (col_fixed / 256) as usize;
let fx = (col_fixed % 256) as f32 / 256.0;
let sx_next = (sx + 1).min(low_w - 1);
let a = bilinear_sample(&self.mean_guide, low_w, sx, sx_next, sy, sy_next, fx, fy);
let b = bilinear_sample(&self.mean_mask, low_w, sx, sx_next, sy, sy_next, fx, fy);
let q = a * (f32::from(luma[row + x]) / 255.0) + b;
let shaped = (q * 255.0 + 0.5).clamp(0.0, 255.0) as usize;
mask[row + x] = self.shape_lut[shaped.min(LUT_LEN - 1)];
}
}
}
}
pub(crate) fn bilinear_fixed_coord(index: usize, full_len: usize, low_len: usize) -> u32 {
assert!(full_len >= 1);
assert!(low_len >= 1);
if full_len == 1 {
return 0;
}
(index * (low_len - 1) * 256 / (full_len - 1)) as u32
}
#[expect(clippy::too_many_arguments)]
pub(crate) fn bilinear_sample(
plane: &[f32],
width: usize,
sx: usize,
sx_next: usize,
sy: usize,
sy_next: usize,
fx: f32,
fy: f32,
) -> f32 {
assert!(sy * width + sx_next < plane.len());
assert!(sy_next * width + sx_next < plane.len());
let top = plane[sy * width + sx] * (1.0 - fx) + plane[sy * width + sx_next] * fx;
let bottom = plane[sy_next * width + sx] * (1.0 - fx) + plane[sy_next * width + sx_next] * fx;
top * (1.0 - fy) + bottom * fy
}
fn temporal_uncertainty(probability: f32) -> f32 {
assert!(probability >= 0.0);
assert!(probability <= 1.0);
let x = (probability - 0.5) * (probability - 0.5);
let polynomial = x
* (TEMPORAL_UNCERTAINTY_C1
+ x * (TEMPORAL_UNCERTAINTY_C2
+ x * (TEMPORAL_UNCERTAINTY_C3
+ x * (TEMPORAL_UNCERTAINTY_C4 + x * TEMPORAL_UNCERTAINTY_C5))));
1.0 - polynomial.min(1.0)
}
fn temporal_weight_lut() -> [u16; LUT_LEN] {
let mut lut = [0u16; LUT_LEN];
for (value, slot) in lut.iter_mut().enumerate() {
let probability = value as f32 / 255.0;
let weight = temporal_uncertainty(probability) * TEMPORAL_COMBINE_RATIO;
assert!(weight >= 0.0);
assert!(weight <= 1.0);
*slot = (weight * 256.0 + 0.5) as u16;
}
lut
}
fn shape_lut() -> [u8; LUT_LEN] {
let span = SHAPE_SMOOTHSTEP_EDGE_HIGH - SHAPE_SMOOTHSTEP_EDGE_LOW;
assert!(span > 0.0);
let mut lut = [0u8; LUT_LEN];
for (value, slot) in lut.iter_mut().enumerate() {
let probability = value as f32 / 255.0;
let t = ((probability - SHAPE_SMOOTHSTEP_EDGE_LOW) / span).clamp(0.0, 1.0);
let smooth = t * t * (3.0 - 2.0 * t);
*slot = (smooth * 255.0 + 0.5) as u8;
}
assert_eq!(lut[0], 0);
assert_eq!(lut[LUT_LEN - 1], 255);
lut
}
pub(crate) fn box_filter_low(
src: &[f32],
scratch: &mut [f32],
dst: &mut [f32],
width: usize,
height: usize,
radius: usize,
) {
assert!(width >= 1);
assert!(height >= 1);
assert!(src.len() >= width * height);
assert!(scratch.len() >= width * height);
assert!(dst.len() >= width * height);
box_filter_rows_low(src, scratch, width, height, radius);
box_filter_columns_low(scratch, dst, width, height, radius);
}
fn box_filter_rows_low(src: &[f32], dst: &mut [f32], width: usize, height: usize, radius: usize) {
assert!(width >= 1);
assert!(src.len() >= width * height);
for y in 0..height {
let row = y * width;
let mut start = 0usize;
let mut end = radius.min(width - 1);
let mut sum: f32 = src[row..=row + end].iter().sum();
for x in 0..width {
dst[row + x] = sum / ((end - start + 1) as f32);
let next_end = (x + 1 + radius).min(width - 1);
if next_end > end {
sum += src[row + next_end];
end = next_end;
}
let next_start = (x + 1).saturating_sub(radius);
if next_start > start {
sum -= src[row + start];
start = next_start;
}
}
}
}
fn box_filter_columns_low(
src: &[f32],
dst: &mut [f32],
width: usize,
height: usize,
radius: usize,
) {
assert!(width >= 1);
assert!(width <= FRAME_EDGE_MAX);
assert!(height >= 1);
let mut sums = [0.0f32; FRAME_EDGE_MAX];
let mut start = 0usize;
let mut end = radius.min(height - 1);
for y in 0..=end {
let row = y * width;
for x in 0..width {
sums[x] += src[row + x];
}
}
for y in 0..height {
let scale = 1.0 / ((end - start + 1) as f32);
let row = y * width;
for x in 0..width {
dst[row + x] = sums[x] * scale;
}
let next_end = (y + 1 + radius).min(height - 1);
if next_end > end {
let next_row = next_end * width;
for x in 0..width {
sums[x] += src[next_row + x];
}
end = next_end;
}
let next_start = (y + 1).saturating_sub(radius);
if next_start > start {
let previous_row = start * width;
for x in 0..width {
sums[x] -= src[previous_row + x];
}
start = next_start;
}
}
}
fn morph_pass_low(
src: &[f32],
scratch: &mut [f32],
dst: &mut [f32],
width: usize,
height: usize,
select: fn(f32, f32) -> f32,
) {
assert!(width >= 1);
assert!(height >= 1);
assert!(src.len() >= width * height);
assert!(scratch.len() >= width * height);
assert!(dst.len() >= width * height);
for y in 0..height {
let row = y * width;
for x in 0..width {
let left = src[row + x.saturating_sub(1)];
let right = src[row + (x + 1).min(width - 1)];
scratch[row + x] = select(select(left, src[row + x]), right);
}
}
for y in 0..height {
let above = y.saturating_sub(1) * width;
let below = (y + 1).min(height - 1) * width;
let row = y * width;
for x in 0..width {
dst[row + x] = select(
select(scratch[above + x], scratch[row + x]),
scratch[below + x],
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn gradient_luma(width: usize, height: usize) -> Vec<u8> {
let mut luma = vec![0u8; width * height];
for (index, value) in luma.iter_mut().enumerate() {
*value = ((index % width) * 255 / (width - 1).max(1)) as u8;
}
luma
}
fn left_half_mask(width: usize, height: usize) -> Vec<u8> {
let mut mask = vec![0u8; width * height];
for y in 0..height {
for x in 0..width / 2 {
mask[y * width + x] = 255;
}
}
mask
}
#[test]
fn temporal_weight_lut_smooths_uncertain_values_and_trusts_confident_ones() {
let lut = temporal_weight_lut();
assert!(lut[128] >= 170);
assert!(lut[128] <= 182);
assert!(lut[0] <= 8);
assert!(lut[255] <= 8);
assert!(lut[64] > lut[16]);
}
#[test]
fn temporal_blend_pulls_uncertain_pixels_toward_previous_mask() {
let mut refiner = MaskRefiner::new(8, 8);
let mut first = vec![128u8; 64];
refiner.blend_temporal(&mut first);
let mut second = vec![128u8; 64];
second[0] = 255;
second[1] = 130;
refiner.blend_temporal(&mut second);
assert_eq!(second[0], 255);
assert!(second[1] < 130);
assert_eq!(second[63], 128);
}
#[test]
fn temporal_blend_first_frame_passes_mask_through_unchanged() {
let mut refiner = MaskRefiner::new(8, 8);
let mut mask = vec![37u8; 64];
refiner.blend_temporal(&mut mask);
assert!(mask.iter().all(|value| *value == 37));
assert!(refiner.previous_mask_valid);
}
#[test]
fn shape_lut_is_monotonic_and_saturates_at_both_ends() {
let lut = shape_lut();
for value in 1..LUT_LEN {
assert!(lut[value] >= lut[value - 1]);
}
assert_eq!(lut[(255.0 * SHAPE_SMOOTHSTEP_EDGE_LOW) as usize - 4], 0);
assert_eq!(lut[(255.0 * SHAPE_SMOOTHSTEP_EDGE_HIGH) as usize + 4], 255);
}
#[test]
fn refine_keeps_solid_person_and_background_regions_saturated() {
let width = 64usize;
let height = 48usize;
let mut refiner = MaskRefiner::new(width, height);
let luma = {
let mut luma = vec![32u8; width * height];
for y in 0..height {
for x in width / 2..width {
luma[y * width + x] = 224;
}
}
luma
};
let mut mask = left_half_mask(width, height);
refiner.refine(&luma, &mut mask);
assert_eq!(mask[24 * width], 255);
assert_eq!(mask[24 * width + 4], 255);
assert_eq!(mask[24 * width + width - 1], 0);
assert_eq!(mask[24 * width + width - 5], 0);
}
#[test]
fn refine_snaps_mask_transition_to_the_luma_edge() {
let width = 64usize;
let height = 48usize;
let mut refiner = MaskRefiner::new(width, height);
let mut luma = vec![16u8; width * height];
for y in 0..height {
for x in 0..width / 2 {
luma[y * width + x] = 240;
}
}
let mut blurry_mask = vec![0u8; width * height];
for y in 0..height {
for x in 0..width {
let distance = (width as i32 / 2 - x as i32).clamp(-12, 12);
blurry_mask[y * width + x] = (127 + distance * 10).clamp(0, 255) as u8;
}
}
refiner.refine(&luma, &mut blurry_mask);
let row = 24 * width;
assert!(blurry_mask[row + width / 2 - 8] > 220);
assert!(blurry_mask[row + width / 2 + 8] < 35);
}
#[test]
fn refine_fills_small_holes_inside_the_person() {
let width = 64usize;
let height = 48usize;
let mut refiner = MaskRefiner::new(width, height);
let luma = vec![128u8; width * height];
let mut mask = vec![255u8; width * height];
mask[24 * width + 32] = 0;
refiner.refine(&luma, &mut mask);
assert!(mask[24 * width + 32] > 200);
assert_eq!(mask[0], 255);
}
#[test]
fn refine_handles_minimum_dimensions_without_panicking() {
let mut refiner = MaskRefiner::new(2, 2);
let luma = vec![128u8; 4];
let mut mask = vec![255u8; 4];
refiner.refine(&luma, &mut mask);
assert_eq!(mask.len(), 4);
assert!(mask.iter().all(|value| *value == 255));
}
#[test]
fn box_filter_low_preserves_constant_planes_exactly() {
let width = 9usize;
let height = 7usize;
let src = vec![0.625f32; width * height];
let mut scratch = vec![0.0f32; width * height];
let mut dst = vec![0.0f32; width * height];
box_filter_low(&src, &mut scratch, &mut dst, width, height, 4);
for value in dst {
assert!((value - 0.625).abs() < 1e-6);
}
}
#[test]
fn morph_close_removes_single_pixel_pits_and_keeps_plateaus() {
let width = 8usize;
let height = 8usize;
let mut src = vec![1.0f32; width * height];
src[3 * width + 3] = 0.0;
let mut scratch = vec![0.0f32; width * height];
let mut maxed = vec![0.0f32; width * height];
let mut closed = vec![0.0f32; width * height];
morph_pass_low(&src, &mut scratch, &mut maxed, width, height, f32::max);
morph_pass_low(&maxed, &mut scratch, &mut closed, width, height, f32::min);
assert!(closed[3 * width + 3] > 0.99);
assert!(closed[0] > 0.99);
}
#[test]
fn refine_converges_to_stable_mask_over_repeated_identical_frames() {
let width = 32usize;
let height = 24usize;
let mut refiner = MaskRefiner::new(width, height);
let luma = gradient_luma(width, height);
let raw = left_half_mask(width, height);
let mut previous_output = vec![0u8; width * height];
for iteration in 0..8 {
let mut mask = raw.clone();
refiner.refine(&luma, &mut mask);
if iteration == 7 {
let drift: i32 = mask
.iter()
.zip(previous_output.iter())
.map(|(a, b)| (i32::from(*a) - i32::from(*b)).abs())
.sum();
assert!(drift <= (width * height) as i32);
}
previous_output.copy_from_slice(&mask);
}
assert_eq!(previous_output[12 * width], 255);
}
}
@@ -0,0 +1,163 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#![allow(dead_code)]
#[cfg(feature = "publisher")]
use livekit::webrtc::video_frame::{VideoBuffer, VideoFrame, VideoRotation, native::NativeBuffer};
#[cfg(feature = "publisher")]
use livekit::webrtc::video_source::native::NativeVideoSource;
use napi_derive::napi;
pub const NATIVE_CAMERA_FRAME_QUEUE_CAPACITY: usize = 3;
const MIN_NATIVE_CAMERA_EDGE: u32 = 2;
const MAX_NATIVE_CAMERA_EDGE: u32 = 8192;
const TRANSPORT_CV_PIXEL_BUFFER: &str = "cvPixelBuffer";
const TRANSPORT_D3D11_TEXTURE: &str = "d3d11Texture";
const TRANSPORT_DMABUF: &str = "dmabuf";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum NativeCameraTransport {
CvPixelBuffer,
D3d11Texture,
Dmabuf,
}
impl NativeCameraTransport {
pub fn as_str(self) -> &'static str {
match self {
Self::CvPixelBuffer => TRANSPORT_CV_PIXEL_BUFFER,
Self::D3d11Texture => TRANSPORT_D3D11_TEXTURE,
Self::Dmabuf => TRANSPORT_DMABUF,
}
}
}
pub fn required_transports() -> &'static [NativeCameraTransport] {
const TRANSPORTS: &[NativeCameraTransport] = &[
NativeCameraTransport::CvPixelBuffer,
NativeCameraTransport::D3d11Texture,
NativeCameraTransport::Dmabuf,
];
assert_eq!(TRANSPORTS.len(), 3);
TRANSPORTS
}
pub fn required_transport_names() -> [&'static str; 3] {
let transports = required_transports();
[
transports[0].as_str(),
transports[1].as_str(),
transports[2].as_str(),
]
}
pub fn platform_native_backgrounds_available() -> bool {
platform_unavailable_reason().is_none()
}
pub fn camera_backgrounds_available() -> bool {
cfg!(feature = "camera-native")
}
#[napi]
pub fn has_native_camera_backgrounds() -> bool {
camera_backgrounds_available()
}
pub fn platform_unavailable_reason() -> Option<&'static str> {
Some(match std::env::consts::OS {
"macos" => "macOS AVFoundation CVPixelBuffer camera backend is not compiled",
"windows" => "Windows Media Foundation D3D11 camera backend is not compiled",
"linux" => "Linux PipeWire/V4L2 dmabuf camera backend is not compiled",
_ => "native platform-buffer camera backend is unsupported on this platform",
})
}
pub fn validate_native_frame_dimensions(width: u32, height: u32) -> bool {
if width < MIN_NATIVE_CAMERA_EDGE {
return false;
}
if height < MIN_NATIVE_CAMERA_EDGE {
return false;
}
if !width.is_multiple_of(2) {
return false;
}
if !height.is_multiple_of(2) {
return false;
}
width <= MAX_NATIVE_CAMERA_EDGE && height <= MAX_NATIVE_CAMERA_EDGE
}
pub fn unavailable_error() -> String {
let reason = platform_unavailable_reason().unwrap_or("native camera backend unavailable");
format!(
"native camera backgrounds require platform camera buffers ({}, {}, {}): {reason}",
TRANSPORT_CV_PIXEL_BUFFER, TRANSPORT_D3D11_TEXTURE, TRANSPORT_DMABUF
)
}
#[cfg(feature = "publisher")]
pub fn publish_native_buffer(
source: &NativeVideoSource,
buffer: NativeBuffer,
timestamp_us: i64,
) -> bool {
assert!(timestamp_us >= 0);
let width = buffer.width();
let height = buffer.height();
if !validate_native_frame_dimensions(width, height) {
return false;
}
source.capture_frame(&VideoFrame {
rotation: VideoRotation::VideoRotation0,
timestamp_us,
frame_metadata: None,
buffer,
});
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn required_transports_are_stable() {
assert_eq!(
required_transport_names(),
["cvPixelBuffer", "d3d11Texture", "dmabuf"]
);
}
#[test]
fn queue_capacity_stays_bounded_for_realtime_capture() {
assert_eq!(NATIVE_CAMERA_FRAME_QUEUE_CAPACITY, 3);
assert!(NATIVE_CAMERA_FRAME_QUEUE_CAPACITY < 8);
}
#[test]
fn camera_background_capability_tracks_native_camera_feature() {
assert_eq!(
camera_backgrounds_available(),
cfg!(feature = "camera-native")
);
}
#[test]
fn native_frame_dimensions_require_even_reasonable_sizes() {
assert!(validate_native_frame_dimensions(1280, 720));
assert!(!validate_native_frame_dimensions(0, 720));
assert!(!validate_native_frame_dimensions(1280, 1));
assert!(!validate_native_frame_dimensions(1279, 720));
assert!(!validate_native_frame_dimensions(1280, 721));
assert!(!validate_native_frame_dimensions(16_384, 720));
}
#[test]
fn unavailable_error_names_all_required_native_transports() {
let error = unavailable_error();
assert!(error.contains("cvPixelBuffer"));
assert!(error.contains("d3d11Texture"));
assert!(error.contains("dmabuf"));
}
}
@@ -0,0 +1,681 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub const PERSON_MASK_BACKGROUND: u8 = 0;
pub const PERSON_MASK_PERSON: u8 = 255;
pub const SEGMENTATION_FRAME_BUDGET_MS: u64 = 12;
pub const SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX: u32 = 30;
pub trait PersonMaskSource {
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool;
}
#[derive(Debug, Default)]
pub struct SegmentationQualityGovernor {
consecutive_slow_frames: u32,
downgraded: bool,
}
impl SegmentationQualityGovernor {
pub fn new() -> Self {
Self::default()
}
pub fn record_frame_duration_ms(&mut self, duration_ms: u64) -> bool {
assert!(self.consecutive_slow_frames < SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX);
if self.downgraded {
return false;
}
if duration_ms <= SEGMENTATION_FRAME_BUDGET_MS {
self.consecutive_slow_frames = 0;
return false;
}
self.consecutive_slow_frames += 1;
assert!(self.consecutive_slow_frames <= SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX);
if self.consecutive_slow_frames < SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
return false;
}
self.consecutive_slow_frames = 0;
self.downgraded = true;
true
}
}
pub fn create_person_mask_source(width: u32, height: u32) -> Option<Box<dyn PersonMaskSource>> {
assert!(width >= 2);
assert!(height >= 2);
#[cfg(target_os = "macos")]
{
if let Some(source) = vision::VisionPersonMaskSource::new(width, height) {
return Some(Box::new(source) as Box<dyn PersonMaskSource>);
}
}
selfie::SelfieMaskSource::new(width, height)
.map(|source| Box::new(source) as Box<dyn PersonMaskSource>)
}
pub fn resize_mask_bilinear(
src: &[u8],
src_width: usize,
src_height: usize,
src_stride: usize,
dst: &mut [u8],
dst_width: usize,
dst_height: usize,
) {
assert!(src_width >= 1);
assert!(src_height >= 1);
assert!(src_stride >= src_width);
assert!(dst_width >= 1);
assert!(dst_height >= 1);
assert!(src.len() >= src_stride * (src_height - 1) + src_width);
assert!(dst.len() >= dst_width * dst_height);
for y in 0..dst_height {
let sy_fixed = if dst_height == 1 {
0
} else {
y * (src_height - 1) * 256 / (dst_height - 1)
};
let sy = sy_fixed / 256;
let fy = (sy_fixed % 256) as u32;
let sy_next = (sy + 1).min(src_height - 1);
for x in 0..dst_width {
let sx_fixed = if dst_width == 1 {
0
} else {
x * (src_width - 1) * 256 / (dst_width - 1)
};
let sx = sx_fixed / 256;
let fx = (sx_fixed % 256) as u32;
let sx_next = (sx + 1).min(src_width - 1);
let top = u32::from(src[sy * src_stride + sx]) * (256 - fx)
+ u32::from(src[sy * src_stride + sx_next]) * fx;
let bottom = u32::from(src[sy_next * src_stride + sx]) * (256 - fx)
+ u32::from(src[sy_next * src_stride + sx_next]) * fx;
dst[y * dst_width + x] = ((top * (256 - fy) + bottom * fy) >> 16) as u8;
}
}
}
mod selfie {
use super::PersonMaskSource;
use std::sync::{Arc, OnceLock};
use tract_onnx::prelude::*;
const MODEL_BYTES: &[u8] = include_bytes!("../models/selfie_segmenter_landscape.onnx");
const MODEL_INPUT_WIDTH: usize = 256;
const MODEL_INPUT_HEIGHT: usize = 144;
const MODEL_INPUT_CHANNELS: usize = 3;
const MODEL_INPUT_LEN: usize = MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT * MODEL_INPUT_CHANNELS;
const MODEL_CHROMA_WIDTH: usize = MODEL_INPUT_WIDTH / 2;
const MODEL_CHROMA_HEIGHT: usize = MODEL_INPUT_HEIGHT / 2;
const INFERENCE_FRAME_INTERVAL_FULL: u32 = 1;
const INFERENCE_FRAME_INTERVAL_DOWNGRADED: u32 = 2;
type SelfiePlan = TypedRunnableModel;
fn shared_plan() -> Option<Arc<SelfiePlan>> {
static PLAN: OnceLock<Option<Arc<SelfiePlan>>> = OnceLock::new();
PLAN.get_or_init(|| match load_plan() {
Ok(plan) => Some(plan),
Err(error) => {
eprintln!(
"webrtc-sender: selfie segmentation model failed to load; camera \
background effects fall back to the portrait ellipse: {error}"
);
None
}
})
.clone()
}
fn load_plan() -> TractResult<Arc<SelfiePlan>> {
let mut reader = std::io::Cursor::new(MODEL_BYTES);
tract_onnx::onnx()
.model_for_read(&mut reader)?
.with_input_fact(
0,
f32::fact([
1,
MODEL_INPUT_HEIGHT,
MODEL_INPUT_WIDTH,
MODEL_INPUT_CHANNELS,
])
.into(),
)?
.into_optimized()?
.into_runnable()
}
pub struct SelfieMaskSource {
width: u32,
height: u32,
plan: Arc<SelfiePlan>,
luma_low: Vec<u8>,
chroma_u_low: Vec<u8>,
chroma_v_low: Vec<u8>,
input_rgb: Vec<f32>,
raw_mask_low: Vec<u8>,
raw_mask_valid: bool,
frame_counter: u32,
inference_interval: u32,
inference_error_logged: bool,
governor: super::SegmentationQualityGovernor,
}
impl SelfieMaskSource {
pub fn new(width: u32, height: u32) -> Option<Self> {
assert!(width >= 2);
assert!(height >= 2);
let plan = shared_plan()?;
Some(Self {
width,
height,
plan,
luma_low: vec![0; MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT],
chroma_u_low: vec![128; MODEL_CHROMA_WIDTH * MODEL_CHROMA_HEIGHT],
chroma_v_low: vec![128; MODEL_CHROMA_WIDTH * MODEL_CHROMA_HEIGHT],
input_rgb: vec![0.0; MODEL_INPUT_LEN],
raw_mask_low: vec![0; MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT],
raw_mask_valid: false,
frame_counter: 0,
inference_interval: INFERENCE_FRAME_INTERVAL_FULL,
inference_error_logged: false,
governor: super::SegmentationQualityGovernor::new(),
})
}
fn fill_model_input(&mut self, frame: &crate::yuv::I420) {
assert_eq!(frame.width, self.width);
assert_eq!(frame.height, self.height);
let width = self.width as usize;
let height = self.height as usize;
super::resize_mask_bilinear(
&frame.y,
width,
height,
width,
&mut self.luma_low,
MODEL_INPUT_WIDTH,
MODEL_INPUT_HEIGHT,
);
super::resize_mask_bilinear(
&frame.u,
width / 2,
height / 2,
width / 2,
&mut self.chroma_u_low,
MODEL_CHROMA_WIDTH,
MODEL_CHROMA_HEIGHT,
);
super::resize_mask_bilinear(
&frame.v,
width / 2,
height / 2,
width / 2,
&mut self.chroma_v_low,
MODEL_CHROMA_WIDTH,
MODEL_CHROMA_HEIGHT,
);
for y in 0..MODEL_INPUT_HEIGHT {
let row = y * MODEL_INPUT_WIDTH;
let chroma_row = (y / 2) * MODEL_CHROMA_WIDTH;
for x in 0..MODEL_INPUT_WIDTH {
let luma = i32::from(self.luma_low[row + x]) - 16;
let cb = i32::from(self.chroma_u_low[chroma_row + x / 2]) - 128;
let cr = i32::from(self.chroma_v_low[chroma_row + x / 2]) - 128;
let r = ((298 * luma + 409 * cr + 128) >> 8).clamp(0, 255);
let g = ((298 * luma - 100 * cb - 208 * cr + 128) >> 8).clamp(0, 255);
let b = ((298 * luma + 516 * cb + 128) >> 8).clamp(0, 255);
let offset = (row + x) * MODEL_INPUT_CHANNELS;
self.input_rgb[offset] = r as f32 / 255.0;
self.input_rgb[offset + 1] = g as f32 / 255.0;
self.input_rgb[offset + 2] = b as f32 / 255.0;
}
}
}
fn run_inference(&mut self, frame: &crate::yuv::I420) -> bool {
self.fill_model_input(frame);
let produced = self.run_model();
if !produced && !self.inference_error_logged {
self.inference_error_logged = true;
eprintln!(
"webrtc-sender: selfie segmentation inference failed; reusing the \
previous person mask"
);
}
produced
}
fn run_model(&mut self) -> bool {
assert_eq!(self.input_rgb.len(), MODEL_INPUT_LEN);
let Ok(tensor) = Tensor::from_shape(
&[
1,
MODEL_INPUT_HEIGHT,
MODEL_INPUT_WIDTH,
MODEL_INPUT_CHANNELS,
],
&self.input_rgb,
) else {
return false;
};
let Ok(result) = self.plan.run(tvec!(tensor.into())) else {
return false;
};
let Some(output) = result.first() else {
return false;
};
let Ok(alphas) = output.to_plain_array_view::<f32>() else {
return false;
};
if alphas.len() != self.raw_mask_low.len() {
return false;
}
for (slot, alpha) in self.raw_mask_low.iter_mut().zip(alphas.iter()) {
*slot = (alpha * 255.0 + 0.5).clamp(0.0, 255.0) as u8;
}
true
}
fn record_inference_duration(&mut self, duration_ms: u64) {
if self.governor.record_frame_duration_ms(duration_ms) {
self.inference_interval = INFERENCE_FRAME_INTERVAL_DOWNGRADED;
eprintln!(
"webrtc-sender: selfie segmentation exceeded the {}ms frame budget for {} \
consecutive frames; downgrading to inference every {} frames",
super::SEGMENTATION_FRAME_BUDGET_MS,
super::SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX,
INFERENCE_FRAME_INTERVAL_DOWNGRADED
);
}
}
}
impl PersonMaskSource for SelfieMaskSource {
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
let width = self.width as usize;
let height = self.height as usize;
assert!(mask.len() >= width * height);
if frame.width != self.width || frame.height != self.height {
return false;
}
assert!(self.inference_interval >= 1);
let due = self.frame_counter.is_multiple_of(self.inference_interval);
self.frame_counter = self.frame_counter.wrapping_add(1);
if due || !self.raw_mask_valid {
let started = std::time::Instant::now();
if self.run_inference(frame) {
self.raw_mask_valid = true;
let duration_ms = started.elapsed().as_millis() as u64;
self.record_inference_duration(duration_ms);
}
}
if !self.raw_mask_valid {
return false;
}
super::resize_mask_bilinear(
&self.raw_mask_low,
MODEL_INPUT_WIDTH,
MODEL_INPUT_HEIGHT,
MODEL_INPUT_WIDTH,
mask,
width,
height,
);
true
}
}
#[cfg(test)]
mod tests {
use super::*;
fn synthetic_frame(width: u32, height: u32) -> crate::yuv::I420 {
let mut frame = crate::yuv::I420::new(width, height).unwrap();
for (index, value) in frame.y.iter_mut().enumerate() {
*value = ((index * 31 + 17) % 220) as u8 + 16;
}
frame.u.fill(128);
frame.v.fill(128);
frame
}
#[test]
fn selfie_source_produces_full_range_mask_for_synthetic_frames() {
let mut source = SelfieMaskSource::new(128, 96).expect("bundled model loads");
let frame = synthetic_frame(128, 96);
let mut mask = vec![0u8; 128 * 96];
assert!(source.mask_into(&frame, &mut mask));
assert_eq!(mask.len(), 128 * 96);
assert!(source.raw_mask_valid);
}
#[test]
fn selfie_source_rejects_mismatched_frame_dimensions() {
let mut source = SelfieMaskSource::new(128, 96).expect("bundled model loads");
let frame = synthetic_frame(64, 48);
let mut mask = vec![0u8; 128 * 96];
assert!(!source.mask_into(&frame, &mut mask));
}
#[test]
fn selfie_source_reuses_cached_mask_between_inference_frames() {
let mut source = SelfieMaskSource::new(64, 48).expect("bundled model loads");
source.inference_interval = INFERENCE_FRAME_INTERVAL_DOWNGRADED;
let frame = synthetic_frame(64, 48);
let mut first = vec![0u8; 64 * 48];
let mut second = vec![0u8; 64 * 48];
assert!(source.mask_into(&frame, &mut first));
assert!(source.mask_into(&frame, &mut second));
assert_eq!(first, second);
}
}
}
#[cfg(target_os = "macos")]
mod vision {
use super::PersonMaskSource;
use core::ptr::NonNull;
use objc2::rc::Retained;
use objc2_core_foundation::CFRetained;
use objc2_core_video::{
CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBaseAddressOfPlane,
CVPixelBufferGetBytesPerRow, CVPixelBufferGetBytesPerRowOfPlane, CVPixelBufferGetHeight,
CVPixelBufferGetPixelFormatType, CVPixelBufferGetWidth, CVPixelBufferLockBaseAddress,
CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress,
};
use objc2_vision::{
VNGeneratePersonSegmentationRequest, VNGeneratePersonSegmentationRequestQualityLevel,
VNRequest, VNSequenceRequestHandler,
};
const PIXEL_FORMAT_NV12_FULL_RANGE: u32 = u32::from_be_bytes(*b"420f");
const PIXEL_FORMAT_ONE_COMPONENT_8: u32 = u32::from_be_bytes(*b"L008");
const MASK_PIXELS_MAX: usize = 8192 * 8192;
pub struct VisionPersonMaskSource {
width: u32,
height: u32,
pixel_buffer: CFRetained<CVPixelBuffer>,
request: Retained<VNGeneratePersonSegmentationRequest>,
requests: Retained<objc2_foundation::NSArray<VNRequest>>,
handler: Retained<VNSequenceRequestHandler>,
quality_governor: super::SegmentationQualityGovernor,
}
impl VisionPersonMaskSource {
pub fn new(width: u32, height: u32) -> Option<Self> {
assert!(width >= 2);
assert!(height >= 2);
assert!(width.is_multiple_of(2));
assert!(height.is_multiple_of(2));
let mut pixel_buffer_out: *mut CVPixelBuffer = core::ptr::null_mut();
let status = unsafe {
objc2_core_video::CVPixelBufferCreate(
None,
width as usize,
height as usize,
PIXEL_FORMAT_NV12_FULL_RANGE,
None,
NonNull::new(&mut pixel_buffer_out)?,
)
};
if status != 0 {
return None;
}
let pixel_buffer = unsafe { CFRetained::from_raw(NonNull::new(pixel_buffer_out)?) };
let request = unsafe { VNGeneratePersonSegmentationRequest::new() };
unsafe {
request.setQualityLevel(VNGeneratePersonSegmentationRequestQualityLevel::Balanced);
request.setOutputPixelFormat(PIXEL_FORMAT_ONE_COMPONENT_8);
}
let request_as_base: Retained<VNRequest> =
Retained::into_super(Retained::into_super(Retained::into_super(request.clone())));
let requests = objc2_foundation::NSArray::from_retained_slice(&[request_as_base]);
assert_eq!(requests.len(), 1);
let handler = unsafe { VNSequenceRequestHandler::new() };
Some(Self {
width,
height,
pixel_buffer,
request,
requests,
handler,
quality_governor: super::SegmentationQualityGovernor::new(),
})
}
fn downgrade_to_fast_quality(&self) {
unsafe {
self.request
.setQualityLevel(VNGeneratePersonSegmentationRequestQualityLevel::Fast);
}
eprintln!(
"webrtc-sender: person segmentation exceeded the {}ms frame budget for {} \
consecutive frames; downgrading Vision quality from balanced to fast",
super::SEGMENTATION_FRAME_BUDGET_MS,
super::SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX
);
}
fn fill_pixel_buffer(&self, frame: &crate::yuv::I420) -> bool {
assert_eq!(frame.width, self.width);
assert_eq!(frame.height, self.height);
let width = self.width as usize;
let height = self.height as usize;
let lock_flags = CVPixelBufferLockFlags(0);
let lock_status =
unsafe { CVPixelBufferLockBaseAddress(&self.pixel_buffer, lock_flags) };
if lock_status != 0 {
return false;
}
let y_base = CVPixelBufferGetBaseAddressOfPlane(&self.pixel_buffer, 0);
let y_stride = CVPixelBufferGetBytesPerRowOfPlane(&self.pixel_buffer, 0);
let uv_base = CVPixelBufferGetBaseAddressOfPlane(&self.pixel_buffer, 1);
let uv_stride = CVPixelBufferGetBytesPerRowOfPlane(&self.pixel_buffer, 1);
if y_base.is_null() || uv_base.is_null() || y_stride < width || uv_stride < width {
let _ = unsafe { CVPixelBufferUnlockBaseAddress(&self.pixel_buffer, lock_flags) };
return false;
}
let chroma_width = width / 2;
let chroma_height = height / 2;
unsafe {
let y_base = y_base as *mut u8;
for row in 0..height {
let src = &frame.y[row * width..row * width + width];
core::ptr::copy_nonoverlapping(src.as_ptr(), y_base.add(row * y_stride), width);
}
let uv_base = uv_base as *mut u8;
for row in 0..chroma_height {
let dst_row = uv_base.add(row * uv_stride);
for col in 0..chroma_width {
let chroma_index = row * chroma_width + col;
dst_row.add(col * 2).write(frame.u[chroma_index]);
dst_row.add(col * 2 + 1).write(frame.v[chroma_index]);
}
}
}
let unlock_status =
unsafe { CVPixelBufferUnlockBaseAddress(&self.pixel_buffer, lock_flags) };
unlock_status == 0
}
fn copy_observation_mask(
mask_buffer: &CVPixelBuffer,
mask: &mut [u8],
width: usize,
height: usize,
) -> bool {
if CVPixelBufferGetPixelFormatType(mask_buffer) != PIXEL_FORMAT_ONE_COMPONENT_8 {
return false;
}
let lock_flags = CVPixelBufferLockFlags::ReadOnly;
if unsafe { CVPixelBufferLockBaseAddress(mask_buffer, lock_flags) } != 0 {
return false;
}
let src_width = CVPixelBufferGetWidth(mask_buffer);
let src_height = CVPixelBufferGetHeight(mask_buffer);
let src_stride = CVPixelBufferGetBytesPerRow(mask_buffer);
let base = CVPixelBufferGetBaseAddress(mask_buffer);
let valid = !base.is_null()
&& src_width >= 1
&& src_height >= 1
&& src_stride >= src_width
&& src_width * src_height <= MASK_PIXELS_MAX;
if valid {
let src = unsafe {
core::slice::from_raw_parts(
base as *const u8,
src_stride * (src_height - 1) + src_width,
)
};
super::resize_mask_bilinear(
src, src_width, src_height, src_stride, mask, width, height,
);
}
let _ = unsafe { CVPixelBufferUnlockBaseAddress(mask_buffer, lock_flags) };
valid
}
}
impl PersonMaskSource for VisionPersonMaskSource {
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
let started = std::time::Instant::now();
let produced = self.mask_into_timed(frame, mask);
let duration_ms = started.elapsed().as_millis() as u64;
if self.quality_governor.record_frame_duration_ms(duration_ms) {
self.downgrade_to_fast_quality();
}
produced
}
}
impl VisionPersonMaskSource {
fn mask_into_timed(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
let width = self.width as usize;
let height = self.height as usize;
assert!(mask.len() >= width * height);
if frame.width != self.width || frame.height != self.height {
return false;
}
if !self.fill_pixel_buffer(frame) {
return false;
}
let performed = unsafe {
self.handler
.performRequests_onCVPixelBuffer_error(&self.requests, &self.pixel_buffer)
};
if performed.is_err() {
return false;
}
let Some(results) = (unsafe { self.request.results() }) else {
return false;
};
let Some(observation) = results.firstObject() else {
return false;
};
let mask_buffer = unsafe { observation.pixelBuffer() };
Self::copy_observation_mask(&mask_buffer, mask, width, height)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resize_identity_returns_same_values() {
let src = vec![0u8, 64, 128, 255];
let mut dst = vec![0u8; 4];
resize_mask_bilinear(&src, 2, 2, 2, &mut dst, 2, 2);
assert_eq!(dst, src);
}
#[test]
fn resize_upscales_with_interpolated_midpoints() {
let src = vec![0u8, 255, 0, 255];
let mut dst = vec![0u8; 9];
resize_mask_bilinear(&src, 2, 2, 2, &mut dst, 3, 3);
assert_eq!(dst[0], 0);
assert_eq!(dst[2], 255);
assert!(dst[1] > 100);
assert!(dst[1] < 156);
}
#[test]
fn resize_honours_source_stride_padding() {
let src = vec![10u8, 20, 99, 99, 30, 40, 99, 99];
let mut dst = vec![0u8; 4];
resize_mask_bilinear(&src, 2, 2, 4, &mut dst, 2, 2);
assert_eq!(dst, vec![10, 20, 30, 40]);
}
#[test]
fn resize_collapses_to_single_pixel_average_free() {
let src = vec![200u8; 16];
let mut dst = vec![0u8; 1];
resize_mask_bilinear(&src, 4, 4, 4, &mut dst, 1, 1);
assert_eq!(dst, vec![200]);
}
#[test]
fn mask_constants_span_full_alpha_range() {
assert_eq!(PERSON_MASK_BACKGROUND, 0);
assert_eq!(PERSON_MASK_PERSON, 255);
}
#[test]
fn segmentation_governor_downgrades_after_consecutive_slow_frames() {
let mut governor = SegmentationQualityGovernor::new();
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
#[test]
fn segmentation_governor_resets_count_after_a_frame_within_budget() {
let mut governor = SegmentationQualityGovernor::new();
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS));
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
#[test]
fn segmentation_governor_downgrades_only_once() {
let mut governor = SegmentationQualityGovernor::new();
for _ in 0..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX - 1 {
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
}
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
for _ in 0..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX * 2 {
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 100));
}
}
}
@@ -0,0 +1,974 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use parking_lot::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
pub const DEFAULT_AUDIO_BUFFER_TARGET_MS: u32 = 200;
pub const DEFAULT_AUDIO_BUFFER_MAX_MS: u32 = 750;
pub const DEFAULT_MIN_VIDEO_FPS: f64 = 15.0;
const PRESSURE_WINDOW_MS: u64 = 5_000;
const RECOVERY_WINDOW_COUNT: u32 = 12;
const AUDIO_REBUFFER_GAP_MS: u64 = 120;
const AUDIO_STABLE_GAP_MS: u64 = 60;
const AUDIO_BUFFER_STEP_MS: u32 = 100;
#[derive(Clone, Debug, PartialEq)]
pub struct SendHealthSnapshot {
pub outgoing_video_queue_depth: u64,
pub outgoing_video_queue_capacity: u64,
pub outgoing_video_max_queue_depth: u64,
pub outgoing_video_frames_produced: u64,
pub outgoing_video_frames_accepted: u64,
pub outgoing_video_frames_dropped: u64,
pub outgoing_video_frames_coalesced: u64,
pub outgoing_video_frames_captured: u64,
pub outgoing_video_capture_failures: u64,
pub outgoing_video_effective_fps: f64,
pub outgoing_video_target_fps: f64,
pub outgoing_video_pacing_target_fps: f64,
pub outgoing_video_max_queue_age_ms: u64,
pub outgoing_video_max_push_latency_ms: u64,
pub outgoing_video_pacing_mode: String,
pub outgoing_video_bus_active: bool,
pub outgoing_audio_buffer_target_ms: u32,
pub outgoing_audio_buffer_max_ms: u32,
pub outgoing_audio_underruns: u64,
pub outgoing_audio_rebuffers: u64,
pub outgoing_audio_max_frame_gap_ms: u64,
pub adaptive_send_tier: String,
pub adaptive_send_reason: String,
}
impl SendHealthSnapshot {
pub fn idle(audio: &AdaptiveAudioStats) -> Self {
Self {
outgoing_video_queue_depth: 0,
outgoing_video_queue_capacity: 0,
outgoing_video_max_queue_depth: 0,
outgoing_video_frames_produced: 0,
outgoing_video_frames_accepted: 0,
outgoing_video_frames_dropped: 0,
outgoing_video_frames_coalesced: 0,
outgoing_video_frames_captured: 0,
outgoing_video_capture_failures: 0,
outgoing_video_effective_fps: 0.0,
outgoing_video_target_fps: 0.0,
outgoing_video_pacing_target_fps: 0.0,
outgoing_video_max_queue_age_ms: 0,
outgoing_video_max_push_latency_ms: 0,
outgoing_video_pacing_mode: "idle".to_string(),
outgoing_video_bus_active: false,
outgoing_audio_buffer_target_ms: audio.target_buffer_ms(),
outgoing_audio_buffer_max_ms: audio.max_buffer_ms(),
outgoing_audio_underruns: audio.underruns.load(Ordering::Relaxed),
outgoing_audio_rebuffers: audio.rebuffers.load(Ordering::Relaxed),
outgoing_audio_max_frame_gap_ms: audio.max_frame_gap_ms.load(Ordering::Relaxed),
adaptive_send_tier: "idle".to_string(),
adaptive_send_reason: "notPublishing".to_string(),
}
}
}
pub struct AdaptiveVideoController {
requested_fps: f64,
min_fps: f64,
adaptive: bool,
state: Mutex<AdaptiveVideoState>,
}
struct AdaptiveVideoState {
current_fps: f64,
tier: String,
reason: String,
window_started_ms: u64,
window_produced: u64,
window_coalesced: u64,
window_dropped: u64,
window_max_queue_age_ms: u64,
window_max_push_latency_ms: u64,
window_egress_fps_sum: f64,
window_egress_fps_samples: u32,
stable_windows: u32,
}
impl AdaptiveVideoController {
pub fn new(requested_fps: f64, min_fps: f64, adaptive: bool, now_ms: u64) -> Self {
let requested_fps = sanitize_fps(requested_fps, 30.0);
let min_fps = sanitize_fps(min_fps, DEFAULT_MIN_VIDEO_FPS).min(requested_fps);
Self {
requested_fps,
min_fps,
adaptive,
state: Mutex::new(AdaptiveVideoState {
current_fps: requested_fps,
tier: "full".to_string(),
reason: "stable".to_string(),
window_started_ms: now_ms,
window_produced: 0,
window_coalesced: 0,
window_dropped: 0,
window_max_queue_age_ms: 0,
window_max_push_latency_ms: 0,
window_egress_fps_sum: 0.0,
window_egress_fps_samples: 0,
stable_windows: 0,
}),
}
}
pub fn current_fps(&self) -> f64 {
self.state.lock().current_fps
}
pub fn tier_and_reason(&self) -> (String, String) {
let state = self.state.lock();
(state.tier.clone(), state.reason.clone())
}
pub fn record_enqueue(&self, now_ms: u64, coalesced: bool) {
let mut state = self.state.lock();
self.rotate_window(&mut state, now_ms);
state.window_produced += 1;
if coalesced {
state.window_coalesced += 1;
}
}
#[cfg(test)]
pub fn record_drop(&self, now_ms: u64) {
let mut state = self.state.lock();
self.rotate_window(&mut state, now_ms);
state.window_dropped += 1;
}
pub fn record_capture(&self, now_ms: u64, queue_age_ms: u64, push_latency_ms: u64) {
let mut state = self.state.lock();
self.rotate_window(&mut state, now_ms);
state.window_max_queue_age_ms = state.window_max_queue_age_ms.max(queue_age_ms);
state.window_max_push_latency_ms = state.window_max_push_latency_ms.max(push_latency_ms);
}
pub fn record_egress_fps(&self, now_ms: u64, fps: f64) {
if !fps.is_finite() || fps < 0.0 {
return;
}
let mut state = self.state.lock();
self.rotate_window(&mut state, now_ms);
state.window_egress_fps_sum += fps;
state.window_egress_fps_samples += 1;
}
fn rotate_window(&self, state: &mut AdaptiveVideoState, now_ms: u64) {
if now_ms.saturating_sub(state.window_started_ms) < PRESSURE_WINDOW_MS {
return;
}
self.apply_window(state);
state.window_started_ms = now_ms;
state.window_produced = 0;
state.window_coalesced = 0;
state.window_dropped = 0;
state.window_max_queue_age_ms = 0;
state.window_max_push_latency_ms = 0;
state.window_egress_fps_sum = 0.0;
state.window_egress_fps_samples = 0;
}
fn apply_window(&self, state: &mut AdaptiveVideoState) {
if !self.adaptive {
state.current_fps = self.requested_fps;
state.tier = "full".to_string();
state.reason = "adaptiveDisabled".to_string();
return;
}
let frame_interval_ms = (1000.0 / state.current_fps.max(1.0)).ceil() as u64;
let latency_pressure = state.window_max_queue_age_ms > frame_interval_ms * 2
|| state.window_max_push_latency_ms > frame_interval_ms * 2;
let drop_ratio = state.window_dropped as f64 / state.window_produced.max(1) as f64;
let encoder_drop_pressure = state.window_produced >= 10 && drop_ratio > 0.05;
let average_egress_fps = if state.window_egress_fps_samples == 0 {
None
} else {
Some(state.window_egress_fps_sum / state.window_egress_fps_samples as f64)
};
let egress_pressure = average_egress_fps.is_some_and(|fps| {
state.window_egress_fps_samples >= 2
&& state.window_produced >= 10
&& fps < state.current_fps * 0.75
});
let pressure = latency_pressure || encoder_drop_pressure || egress_pressure;
if pressure {
let next = if state.current_fps > 30.0 {
30.0
} else if state.current_fps > self.min_fps {
self.min_fps
} else {
state.current_fps
};
if next < state.current_fps {
state.current_fps = next;
state.tier = tier_for_fps(self.requested_fps, state.current_fps);
}
state.reason = if latency_pressure {
"sendLatencyPressure".to_string()
} else if encoder_drop_pressure {
"encoderDropPressure".to_string()
} else {
"encoderEgressPressure".to_string()
};
state.stable_windows = 0;
return;
}
state.stable_windows += 1;
if state.current_fps >= self.requested_fps {
state.reason = "stable".to_string();
}
if state.stable_windows >= RECOVERY_WINDOW_COUNT && state.current_fps < self.requested_fps {
state.current_fps = (state.current_fps * 2.0).min(self.requested_fps);
state.tier = tier_for_fps(self.requested_fps, state.current_fps);
state.stable_windows = 0;
if state.current_fps >= self.requested_fps {
state.reason = "stable".to_string();
}
}
}
}
pub struct AdaptiveVideoStats {
produced: AtomicU64,
accepted: AtomicU64,
dropped: AtomicU64,
coalesced: AtomicU64,
captured: AtomicU64,
capture_failures: AtomicU64,
queue_depth: AtomicU64,
max_queue_depth: AtomicU64,
max_queue_age_ms: AtomicU64,
max_push_latency_ms: AtomicU64,
first_capture_ms: AtomicU64,
last_capture_ms: AtomicU64,
controller: AdaptiveVideoController,
}
#[derive(Clone, Debug)]
pub struct VideoTelemetryExtras {
pub pacing_mode: String,
pub pacing_target_fps: f64,
pub queue_capacity: u64,
pub bus_active: bool,
}
impl Default for VideoTelemetryExtras {
fn default() -> Self {
Self {
pacing_mode: "unknown".to_string(),
pacing_target_fps: 0.0,
queue_capacity: 0,
bus_active: false,
}
}
}
impl AdaptiveVideoStats {
pub fn new(requested_fps: f64, min_fps: f64, adaptive: bool, now_ms: u64) -> Self {
Self {
produced: AtomicU64::new(0),
accepted: AtomicU64::new(0),
dropped: AtomicU64::new(0),
coalesced: AtomicU64::new(0),
captured: AtomicU64::new(0),
capture_failures: AtomicU64::new(0),
queue_depth: AtomicU64::new(0),
max_queue_depth: AtomicU64::new(0),
max_queue_age_ms: AtomicU64::new(0),
max_push_latency_ms: AtomicU64::new(0),
first_capture_ms: AtomicU64::new(0),
last_capture_ms: AtomicU64::new(0),
controller: AdaptiveVideoController::new(requested_fps, min_fps, adaptive, now_ms),
}
}
#[cfg(test)]
pub fn record_enqueue(&self, now_ms: u64, replaced_pending: bool) {
self.record_enqueue_with_depth(now_ms, replaced_pending, 1);
}
pub fn record_enqueue_with_depth(&self, now_ms: u64, replaced_pending: bool, queue_depth: u64) {
self.produced.fetch_add(1, Ordering::Relaxed);
self.accepted.fetch_add(1, Ordering::Relaxed);
self.queue_depth.store(queue_depth, Ordering::Relaxed);
update_max(&self.max_queue_depth, queue_depth);
if replaced_pending {
self.coalesced.fetch_add(1, Ordering::Relaxed);
}
self.controller.record_enqueue(now_ms, replaced_pending);
}
#[cfg(test)]
pub fn record_drop(&self, now_ms: u64) {
self.dropped.fetch_add(1, Ordering::Relaxed);
self.controller.record_drop(now_ms);
}
pub fn record_reject(&self) {
self.dropped.fetch_add(1, Ordering::Relaxed);
}
pub fn record_capture(&self, now_ms: u64, queue_age_ms: u64, push_latency_ms: u64) {
update_min_nonzero(&self.first_capture_ms, now_ms);
update_max(&self.last_capture_ms, now_ms);
self.captured.fetch_add(1, Ordering::Relaxed);
self.queue_depth.store(0, Ordering::Relaxed);
update_max(&self.max_queue_age_ms, queue_age_ms);
update_max(&self.max_push_latency_ms, push_latency_ms);
self.controller
.record_capture(now_ms, queue_age_ms, push_latency_ms);
}
pub fn record_egress_fps(&self, now_ms: u64, fps: f64) {
self.controller.record_egress_fps(now_ms, fps);
}
pub fn record_capture_failure(&self) {
self.capture_failures.fetch_add(1, Ordering::Relaxed);
self.queue_depth.store(0, Ordering::Relaxed);
}
pub fn record_queue_cleared(&self) {
self.queue_depth.store(0, Ordering::Relaxed);
}
pub fn current_fps(&self) -> f64 {
self.controller.current_fps()
}
fn effective_fps(&self) -> f64 {
let first = self.first_capture_ms.load(Ordering::Relaxed);
let last = self.last_capture_ms.load(Ordering::Relaxed);
let captured = self.captured.load(Ordering::Relaxed);
if first == 0 || last <= first || captured <= 1 {
return 0.0;
}
let elapsed_s = (last - first) as f64 / 1000.0;
((captured - 1) as f64 / elapsed_s * 100.0).round() / 100.0
}
pub fn snapshot(
&self,
audio: &AdaptiveAudioStats,
extras: VideoTelemetryExtras,
) -> SendHealthSnapshot {
let (tier, reason) = self.controller.tier_and_reason();
SendHealthSnapshot {
outgoing_video_queue_depth: self.queue_depth.load(Ordering::Relaxed),
outgoing_video_queue_capacity: extras.queue_capacity,
outgoing_video_max_queue_depth: self.max_queue_depth.load(Ordering::Relaxed),
outgoing_video_frames_produced: self.produced.load(Ordering::Relaxed),
outgoing_video_frames_accepted: self.accepted.load(Ordering::Relaxed),
outgoing_video_frames_dropped: self.dropped.load(Ordering::Relaxed),
outgoing_video_frames_coalesced: self.coalesced.load(Ordering::Relaxed),
outgoing_video_frames_captured: self.captured.load(Ordering::Relaxed),
outgoing_video_capture_failures: self.capture_failures.load(Ordering::Relaxed),
outgoing_video_effective_fps: self.effective_fps(),
outgoing_video_target_fps: (self.current_fps() * 100.0).round() / 100.0,
outgoing_video_pacing_target_fps: (extras.pacing_target_fps * 100.0).round() / 100.0,
outgoing_video_max_queue_age_ms: self.max_queue_age_ms.load(Ordering::Relaxed),
outgoing_video_max_push_latency_ms: self.max_push_latency_ms.load(Ordering::Relaxed),
outgoing_video_pacing_mode: extras.pacing_mode,
outgoing_video_bus_active: extras.bus_active,
outgoing_audio_buffer_target_ms: audio.target_buffer_ms(),
outgoing_audio_buffer_max_ms: audio.max_buffer_ms(),
outgoing_audio_underruns: audio.underruns.load(Ordering::Relaxed),
outgoing_audio_rebuffers: audio.rebuffers.load(Ordering::Relaxed),
outgoing_audio_max_frame_gap_ms: audio.max_frame_gap_ms.load(Ordering::Relaxed),
adaptive_send_tier: tier,
adaptive_send_reason: reason,
}
}
}
pub struct AdaptiveAudioStats {
max_buffer_ms: AtomicU64,
target_buffer_ms: AtomicU64,
underruns: AtomicU64,
rebuffers: AtomicU64,
max_frame_gap_ms: AtomicU64,
last_push_ms: AtomicU64,
stable_started_ms: AtomicU64,
}
impl AdaptiveAudioStats {
pub fn new(max_buffer_ms: u32, now_ms: u64) -> Self {
let max_buffer_ms = clamp_audio_buffer_ms(max_buffer_ms);
Self {
max_buffer_ms: AtomicU64::new(max_buffer_ms as u64),
target_buffer_ms: AtomicU64::new(
DEFAULT_AUDIO_BUFFER_TARGET_MS.min(max_buffer_ms) as u64
),
underruns: AtomicU64::new(0),
rebuffers: AtomicU64::new(0),
max_frame_gap_ms: AtomicU64::new(0),
last_push_ms: AtomicU64::new(0),
stable_started_ms: AtomicU64::new(now_ms),
}
}
pub fn reset(&self, max_buffer_ms: u32, now_ms: u64) {
let max_buffer_ms = clamp_audio_buffer_ms(max_buffer_ms);
self.max_buffer_ms
.store(max_buffer_ms as u64, Ordering::Relaxed);
self.target_buffer_ms.store(
DEFAULT_AUDIO_BUFFER_TARGET_MS.min(max_buffer_ms) as u64,
Ordering::Relaxed,
);
self.underruns.store(0, Ordering::Relaxed);
self.rebuffers.store(0, Ordering::Relaxed);
self.max_frame_gap_ms.store(0, Ordering::Relaxed);
self.last_push_ms.store(0, Ordering::Relaxed);
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
}
pub fn record_push(&self, now_ms: u64) {
let Some(previous) = advance_monotonic(&self.last_push_ms, now_ms) else {
return;
};
if previous == 0 {
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
return;
}
let gap = now_ms - previous;
update_max(&self.max_frame_gap_ms, gap);
if gap > AUDIO_REBUFFER_GAP_MS {
self.rebuffers.fetch_add(1, Ordering::Relaxed);
if gap > self.target_buffer_ms() as u64 {
self.underruns.fetch_add(1, Ordering::Relaxed);
}
let next = (self.target_buffer_ms() + AUDIO_BUFFER_STEP_MS).min(self.max_buffer_ms());
self.target_buffer_ms.store(next as u64, Ordering::Relaxed);
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
return;
}
if gap <= AUDIO_STABLE_GAP_MS {
let stable_started = self.stable_started_ms.load(Ordering::Relaxed);
if now_ms.saturating_sub(stable_started) >= 30_000 {
let current = self.target_buffer_ms();
let next = current
.saturating_sub(AUDIO_BUFFER_STEP_MS)
.max(DEFAULT_AUDIO_BUFFER_TARGET_MS.min(self.max_buffer_ms()));
self.target_buffer_ms.store(next as u64, Ordering::Relaxed);
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
}
} else {
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
}
}
pub fn max_buffer_ms(&self) -> u32 {
self.max_buffer_ms.load(Ordering::Relaxed) as u32
}
pub fn target_buffer_ms(&self) -> u32 {
self.target_buffer_ms.load(Ordering::Relaxed) as u32
}
}
pub fn clamp_audio_buffer_ms(value: u32) -> u32 {
let clamped = value.clamp(DEFAULT_AUDIO_BUFFER_TARGET_MS, DEFAULT_AUDIO_BUFFER_MAX_MS);
clamped - (clamped % 10)
}
fn sanitize_fps(value: f64, fallback: f64) -> f64 {
if value.is_finite() && value > 0.0 {
value
} else {
fallback
}
}
fn tier_for_fps(requested_fps: f64, current_fps: f64) -> String {
if current_fps >= requested_fps {
"full".to_string()
} else if current_fps >= 30.0 {
"fps30".to_string()
} else {
"fps15".to_string()
}
}
fn update_max(slot: &AtomicU64, value: u64) {
let mut current = slot.load(Ordering::Relaxed);
while value > current {
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => break,
Err(next) => current = next,
}
}
}
fn update_min_nonzero(slot: &AtomicU64, value: u64) {
if value == 0 {
return;
}
let mut current = slot.load(Ordering::Relaxed);
loop {
if current != 0 && current <= value {
return;
}
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return,
Err(next) => current = next,
}
}
}
fn advance_monotonic(slot: &AtomicU64, value: u64) -> Option<u64> {
let mut current = slot.load(Ordering::Relaxed);
loop {
if current != 0 && value < current {
return None;
}
if value == current {
return Some(current);
}
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return Some(current),
Err(next) => current = next,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[test]
fn video_controller_ignores_pure_coalescing_jitter() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
for _ in 0..200 {
controller.record_enqueue(1_000, true);
}
controller.record_enqueue(5_001, false);
assert_eq!(controller.current_fps(), 60.0);
assert_eq!(controller.tier_and_reason().0, "full");
assert_eq!(controller.tier_and_reason().1, "stable");
}
#[test]
fn video_controller_ignores_sustained_coalescing_jitter_across_windows() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
for _ in 0..200 {
controller.record_enqueue(1_000, true);
}
controller.record_enqueue(5_001, false);
assert_eq!(controller.current_fps(), 60.0);
for _ in 0..200 {
controller.record_enqueue(6_000, true);
}
controller.record_enqueue(10_002, false);
assert_eq!(controller.current_fps(), 60.0);
assert_eq!(controller.tier_and_reason().1, "stable");
}
#[test]
fn video_controller_degrades_on_encoder_drop_pressure() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
for _ in 0..100 {
controller.record_enqueue(1_000, false);
}
for _ in 0..20 {
controller.record_drop(1_000);
}
controller.record_enqueue(5_001, false);
assert_eq!(controller.current_fps(), 30.0);
assert_eq!(controller.tier_and_reason().1, "encoderDropPressure");
}
#[test]
fn video_controller_continues_degrading_under_sustained_latency_pressure() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
controller.record_capture(1_000, 80, 1);
controller.record_capture(5_001, 1, 1);
assert_eq!(controller.current_fps(), 30.0);
controller.record_capture(6_000, 80, 1);
controller.record_capture(10_002, 1, 1);
assert_eq!(controller.current_fps(), 15.0);
assert_eq!(controller.tier_and_reason().0, "fps15");
assert_eq!(controller.tier_and_reason().1, "sendLatencyPressure");
}
#[test]
fn video_controller_degrades_on_latency_pressure_without_coalescing() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
controller.record_capture(1_000, 80, 5);
controller.record_capture(5_001, 1, 1);
assert_eq!(controller.current_fps(), 30.0);
assert_eq!(controller.tier_and_reason().1, "sendLatencyPressure");
}
#[test]
fn video_controller_degrades_on_encoder_egress_pressure() {
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
for _ in 0..60 {
controller.record_enqueue(1_000, false);
}
controller.record_egress_fps(1_000, 7.0);
controller.record_egress_fps(2_000, 8.0);
controller.record_egress_fps(4_000, 7.0);
controller.record_egress_fps(5_001, 7.0);
assert_eq!(controller.current_fps(), 15.0);
assert_eq!(controller.tier_and_reason().0, "fps15");
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
}
#[test]
fn video_controller_preserves_pressure_reason_while_degraded_but_stable() {
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
for _ in 0..60 {
controller.record_enqueue(1_000, false);
}
controller.record_egress_fps(1_000, 7.0);
controller.record_egress_fps(2_000, 8.0);
controller.record_egress_fps(4_000, 7.0);
controller.record_egress_fps(5_001, 7.0);
assert_eq!(controller.current_fps(), 15.0);
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
controller.record_capture(10_002, 1, 1);
assert_eq!(controller.current_fps(), 15.0);
assert_eq!(controller.tier_and_reason().0, "fps15");
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
for index in 2..=12 {
controller.record_capture(10_002 + index * 5_001, 1, 1);
}
assert_eq!(controller.current_fps(), 30.0);
assert_eq!(
controller.tier_and_reason(),
("full".to_string(), "stable".to_string())
);
}
#[test]
fn video_controller_ignores_single_encoder_egress_sample() {
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
for _ in 0..60 {
controller.record_enqueue(1_000, false);
}
controller.record_egress_fps(1_000, 7.0);
controller.record_egress_fps(5_001, 7.0);
assert_eq!(controller.current_fps(), 30.0);
assert_eq!(controller.tier_and_reason().1, "stable");
}
#[test]
fn video_controller_keeps_requested_fps_when_adaptive_send_is_disabled() {
let controller = AdaptiveVideoController::new(60.0, 15.0, false, 0);
for _ in 0..100 {
controller.record_enqueue(1_000, false);
}
controller.record_capture(5_001, 200, 200);
assert_eq!(controller.current_fps(), 60.0);
assert_eq!(
controller.tier_and_reason(),
("full".to_string(), "adaptiveDisabled".to_string())
);
}
#[test]
fn video_controller_recovers_after_stable_windows() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
controller.record_capture(1_000, 80, 1);
controller.record_capture(5_001, 1, 1);
assert_eq!(controller.current_fps(), 30.0);
for index in 1..=12 {
controller.record_capture(5_001 + index * 5_001, 1, 1);
}
assert_eq!(controller.current_fps(), 60.0);
}
#[test]
fn video_controller_recovers_from_minimum_in_two_stable_steps() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
controller.record_capture(1_000, 80, 1);
controller.record_capture(5_001, 1, 1);
assert_eq!(controller.current_fps(), 30.0);
controller.record_capture(6_000, 70, 1);
controller.record_capture(10_002, 1, 1);
assert_eq!(controller.current_fps(), 15.0);
for index in 1..=12 {
controller.record_capture(10_002 + index * 5_001, 1, 1);
}
assert_eq!(controller.current_fps(), 30.0);
for index in 13..=24 {
controller.record_capture(10_002 + index * 5_001, 1, 1);
}
assert_eq!(controller.current_fps(), 60.0);
}
#[test]
fn video_controller_requires_a_full_stable_recovery_window() {
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
controller.record_capture(1_000, 80, 1);
controller.record_capture(5_001, 1, 1);
assert_eq!(controller.current_fps(), 30.0);
for index in 1..12 {
controller.record_capture(5_001 + index * 5_001, 1, 1);
}
assert_eq!(controller.current_fps(), 30.0);
controller.record_capture(5_001 + 12 * 5_001, 1, 1);
assert_eq!(controller.current_fps(), 60.0);
}
#[test]
fn video_stats_snapshot_counts_coalescing_drops_failures_and_effective_fps() {
let audio = AdaptiveAudioStats::new(750, 0);
let stats = AdaptiveVideoStats::new(60.0, 15.0, true, 0);
stats.record_enqueue(1_000, false);
stats.record_enqueue(1_010, true);
stats.record_drop(1_011);
stats.record_capture(1_020, 20, 24);
stats.record_enqueue(2_000, false);
stats.record_capture(2_020, 20, 26);
stats.record_capture_failure();
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
assert_eq!(snapshot.outgoing_video_frames_produced, 3);
assert_eq!(snapshot.outgoing_video_frames_accepted, 3);
assert_eq!(snapshot.outgoing_video_frames_dropped, 1);
assert_eq!(snapshot.outgoing_video_frames_coalesced, 1);
assert_eq!(snapshot.outgoing_video_frames_captured, 2);
assert_eq!(snapshot.outgoing_video_capture_failures, 1);
assert_eq!(snapshot.outgoing_video_effective_fps, 1.0);
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 20);
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 26);
assert_eq!(snapshot.outgoing_video_queue_depth, 0);
}
#[test]
fn video_stats_effective_fps_uses_capture_time_bounds_for_out_of_order_records() {
let audio = AdaptiveAudioStats::new(750, 0);
let stats = AdaptiveVideoStats::new(60.0, 15.0, true, 0);
stats.record_capture(2_000, 5, 6);
stats.record_capture(1_000, 7, 8);
stats.record_capture(3_000, 9, 10);
stats.record_capture(2_500, 11, 12);
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
assert_eq!(snapshot.outgoing_video_frames_captured, 4);
assert_eq!(snapshot.outgoing_video_effective_fps, 1.5);
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 11);
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 12);
}
#[test]
fn video_stats_handles_concurrent_recording_without_lost_counts() {
let audio = AdaptiveAudioStats::new(750, 0);
let stats = Arc::new(AdaptiveVideoStats::new(60.0, 15.0, true, 0));
let mut workers = Vec::new();
for worker in 0..8 {
let stats = stats.clone();
workers.push(std::thread::spawn(move || {
for index in 0..250 {
let now_ms = 1_000 + worker * 1_000 + index;
stats.record_enqueue(now_ms, index % 3 == 0);
if index % 5 == 0 {
stats.record_drop(now_ms);
}
if index % 7 == 0 {
stats.record_capture(now_ms + 1, 1, 2);
}
}
}));
}
for worker in workers {
worker.join().expect("worker should not panic");
}
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
assert_eq!(snapshot.outgoing_video_frames_produced, 2_000);
assert_eq!(snapshot.outgoing_video_frames_accepted, 2_000);
assert_eq!(snapshot.outgoing_video_frames_coalesced, 672);
assert_eq!(snapshot.outgoing_video_frames_dropped, 400);
assert_eq!(snapshot.outgoing_video_frames_captured, 288);
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 1);
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 2);
}
#[test]
fn idle_snapshot_reflects_audio_pressure_without_video_state() {
let audio = AdaptiveAudioStats::new(750, 0);
audio.record_push(1_000);
audio.record_push(1_300);
let snapshot = SendHealthSnapshot::idle(&audio);
assert_eq!(snapshot.outgoing_video_frames_produced, 0);
assert_eq!(snapshot.outgoing_audio_buffer_target_ms, 300);
assert_eq!(snapshot.outgoing_audio_buffer_max_ms, 750);
assert_eq!(snapshot.outgoing_audio_rebuffers, 1);
assert_eq!(snapshot.outgoing_audio_underruns, 1);
assert_eq!(snapshot.outgoing_audio_max_frame_gap_ms, 300);
assert_eq!(snapshot.adaptive_send_tier, "idle");
assert_eq!(snapshot.adaptive_send_reason, "notPublishing");
}
#[test]
fn video_controller_stress_keeps_target_inside_configured_bounds() {
let controller = AdaptiveVideoController::new(144.0, 24.0, true, 0);
for window in 0..240 {
let base = window * 5_001;
match window % 4 {
0 => controller.record_capture(base + 1_000, 200, 1),
1 => {
for _ in 0..30 {
controller.record_enqueue(base + 1_000, true);
}
controller.record_capture(base + 1_500, 1, 1);
}
2 => {
for _ in 0..60 {
controller.record_enqueue(base + 1_000, false);
}
controller.record_egress_fps(base + 1_500, 30.0);
controller.record_egress_fps(base + 2_500, 30.0);
}
_ => controller.record_capture(base + 1_000, 1, 1),
}
controller.record_capture(base + 5_001, 1, 1);
let fps = controller.current_fps();
assert!(
(24.0..=144.0).contains(&fps),
"fps target escaped configured bounds: {fps}"
);
}
}
#[test]
fn audio_buffer_expands_on_gaps_and_shrinks_after_stability() {
let audio = AdaptiveAudioStats::new(750, 0);
audio.record_push(10);
audio.record_push(200);
assert_eq!(audio.target_buffer_ms(), 300);
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 1);
for now_ms in (220..=30_240).step_by(20) {
audio.record_push(now_ms);
}
assert_eq!(audio.target_buffer_ms(), 200);
}
#[test]
fn audio_buffer_ignores_clock_regression_without_false_rebuffer() {
let audio = AdaptiveAudioStats::new(750, 0);
audio.record_push(1_000);
audio.record_push(1_020);
audio.record_push(900);
audio.record_push(1_040);
assert_eq!(audio.target_buffer_ms(), 200);
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 20);
}
#[test]
fn audio_buffer_ignores_concurrent_stale_pushes_without_moving_last_push_backwards() {
let audio = Arc::new(AdaptiveAudioStats::new(750, 0));
audio.record_push(10_000);
let mut workers = Vec::new();
for worker in 0..8 {
let audio = audio.clone();
workers.push(std::thread::spawn(move || {
for index in 0..100 {
audio.record_push(1_000 + worker * 100 + index);
}
}));
}
for worker in workers {
worker.join().expect("worker should not panic");
}
audio.record_push(10_020);
assert_eq!(audio.target_buffer_ms(), 200);
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 20);
}
#[test]
fn audio_buffer_growth_caps_at_configured_max_and_reset_clears_pressure() {
let audio = AdaptiveAudioStats::new(350, 0);
audio.record_push(10);
for index in 1..=10 {
audio.record_push(10 + index * 500);
}
assert_eq!(audio.target_buffer_ms(), 350);
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 10);
assert_eq!(audio.underruns.load(Ordering::Relaxed), 10);
audio.reset(250, 10_000);
assert_eq!(audio.target_buffer_ms(), 200);
assert_eq!(audio.max_buffer_ms(), 250);
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 0);
}
#[test]
fn audio_buffer_stress_stays_within_realtime_bounds_under_jitter() {
let audio = AdaptiveAudioStats::new(620, 0);
let mut now_ms = 10;
audio.record_push(now_ms);
for index in 0..5_000 {
now_ms += match index % 11 {
0 => 180,
1 | 2 => 80,
_ => 20,
};
audio.record_push(now_ms);
assert!(
(DEFAULT_AUDIO_BUFFER_TARGET_MS..=620).contains(&audio.target_buffer_ms()),
"audio target escaped configured bounds"
);
}
assert_eq!(audio.max_buffer_ms(), 620);
assert!(audio.rebuffers.load(Ordering::Relaxed) > 0);
assert!(audio.max_frame_gap_ms.load(Ordering::Relaxed) >= 180);
}
#[test]
fn audio_buffer_max_is_clamped_to_real_time_bounds() {
assert_eq!(clamp_audio_buffer_ms(50), 200);
assert_eq!(clamp_audio_buffer_ms(777), 750);
assert_eq!(clamp_audio_buffer_ms(333), 330);
}
}
@@ -0,0 +1,275 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::sync::atomic::{AtomicU64, Ordering};
pub const SPEAKING_ATTACK_MS: u64 = 30;
pub const SPEAKING_RELEASE_MS_LOCAL: u64 = 180;
pub const SPEAKING_RELEASE_MS_REMOTE: u64 = 220;
pub const SPEAKING_HEARTBEAT_INTERVAL_MS: u64 = 1_000;
pub const SPEAKING_FRAME_TIMEOUT_MS: u64 = 250;
pub const SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT: f64 = 0.008;
pub const SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT: f64 = 0.006;
pub const SPEAKING_THRESHOLD_RMS_MIN: f64 = 0.000_1;
pub const SPEAKING_THRESHOLD_RMS_MAX: f64 = 0.5;
pub const SPEAKING_FRAME_SAMPLES_MAX: usize = 1 << 20;
const _: () = assert!(SPEAKING_ATTACK_MS < SPEAKING_RELEASE_MS_LOCAL);
const _: () = assert!(SPEAKING_ATTACK_MS < SPEAKING_RELEASE_MS_REMOTE);
const _: () = assert!(SPEAKING_RELEASE_MS_REMOTE < SPEAKING_HEARTBEAT_INTERVAL_MS);
const _: () = assert!(SPEAKING_FRAME_TIMEOUT_MS < SPEAKING_HEARTBEAT_INTERVAL_MS);
pub fn clamp_speaking_threshold_rms(threshold_rms: f64) -> f64 {
if !threshold_rms.is_finite() {
return SPEAKING_THRESHOLD_RMS_MIN;
}
threshold_rms.clamp(SPEAKING_THRESHOLD_RMS_MIN, SPEAKING_THRESHOLD_RMS_MAX)
}
pub struct SpeakingThresholds {
local_rms_bits: AtomicU64,
remote_rms_bits: AtomicU64,
}
impl SpeakingThresholds {
pub fn new() -> Self {
Self {
local_rms_bits: AtomicU64::new(SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT.to_bits()),
remote_rms_bits: AtomicU64::new(SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT.to_bits()),
}
}
pub fn set(&self, local_rms: f64, remote_rms: f64) {
let local = clamp_speaking_threshold_rms(local_rms);
let remote = clamp_speaking_threshold_rms(remote_rms);
assert!(local >= SPEAKING_THRESHOLD_RMS_MIN);
assert!(remote >= SPEAKING_THRESHOLD_RMS_MIN);
self.local_rms_bits
.store(local.to_bits(), Ordering::Release);
self.remote_rms_bits
.store(remote.to_bits(), Ordering::Release);
}
pub fn local_rms(&self) -> f64 {
let value = f64::from_bits(self.local_rms_bits.load(Ordering::Acquire));
assert!(value.is_finite());
value
}
pub fn remote_rms(&self) -> f64 {
let value = f64::from_bits(self.remote_rms_bits.load(Ordering::Acquire));
assert!(value.is_finite());
value
}
}
impl Default for SpeakingThresholds {
fn default() -> Self {
Self::new()
}
}
pub fn frame_rms_i16(samples: &[i16]) -> f64 {
assert!(!samples.is_empty());
assert!(samples.len() <= SPEAKING_FRAME_SAMPLES_MAX);
let mut sum_squares: f64 = 0.0;
for sample in samples {
let normalized = f64::from(*sample) / 32_768.0;
sum_squares += normalized * normalized;
}
let rms = (sum_squares / samples.len() as f64).sqrt();
assert!(rms.is_finite());
assert!(rms >= 0.0);
rms.min(1.0)
}
pub struct SpeakingGate {
attack_ms: u64,
release_ms: u64,
speaking: bool,
above_since_ms: Option<u64>,
below_since_ms: Option<u64>,
last_now_ms: u64,
}
impl SpeakingGate {
pub fn new(attack_ms: u64, release_ms: u64) -> Self {
assert!(attack_ms < release_ms);
assert!(release_ms <= SPEAKING_HEARTBEAT_INTERVAL_MS);
Self {
attack_ms,
release_ms,
speaking: false,
above_since_ms: None,
below_since_ms: None,
last_now_ms: 0,
}
}
pub fn speaking(&self) -> bool {
self.speaking
}
pub fn update(&mut self, rms: f64, threshold_rms: f64, now_ms: u64) -> Option<bool> {
assert!(rms.is_finite());
assert!(rms >= 0.0);
assert!(threshold_rms >= SPEAKING_THRESHOLD_RMS_MIN);
assert!(threshold_rms <= SPEAKING_THRESHOLD_RMS_MAX);
assert!(now_ms >= self.last_now_ms);
self.last_now_ms = now_ms;
if rms >= threshold_rms {
self.below_since_ms = None;
let above_since_ms = *self.above_since_ms.get_or_insert(now_ms);
if self.speaking {
return None;
}
if now_ms - above_since_ms < self.attack_ms {
return None;
}
self.speaking = true;
return Some(true);
}
self.above_since_ms = None;
let below_since_ms = *self.below_since_ms.get_or_insert(now_ms);
if !self.speaking {
return None;
}
if now_ms - below_since_ms < self.release_ms {
return None;
}
self.speaking = false;
Some(false)
}
}
#[cfg(test)]
mod tests {
use super::*;
const THRESHOLD: f64 = 0.01;
fn gate() -> SpeakingGate {
SpeakingGate::new(SPEAKING_ATTACK_MS, SPEAKING_RELEASE_MS_LOCAL)
}
#[test]
fn stays_quiet_below_threshold() {
let mut gate = gate();
for tick in 0..100u64 {
assert_eq!(gate.update(0.001, THRESHOLD, tick * 10), None);
}
assert!(!gate.speaking());
}
#[test]
fn attack_requires_sustained_signal() {
let mut gate = gate();
assert_eq!(gate.update(0.5, THRESHOLD, 0), None);
assert_eq!(gate.update(0.5, THRESHOLD, 10), None);
assert_eq!(gate.update(0.5, THRESHOLD, 20), None);
assert_eq!(gate.update(0.5, THRESHOLD, 30), Some(true));
assert!(gate.speaking());
}
#[test]
fn single_frame_blip_does_not_trigger() {
let mut gate = gate();
assert_eq!(gate.update(0.5, THRESHOLD, 0), None);
assert_eq!(gate.update(0.001, THRESHOLD, 10), None);
assert_eq!(gate.update(0.5, THRESHOLD, 20), None);
assert_eq!(gate.update(0.001, THRESHOLD, 30), None);
assert!(!gate.speaking());
}
#[test]
fn release_bridges_inter_word_gaps() {
let mut gate = gate();
for tick in 0..=3u64 {
gate.update(0.5, THRESHOLD, tick * 10);
}
assert!(gate.speaking());
for tick in 4..=20u64 {
assert_eq!(gate.update(0.001, THRESHOLD, tick * 10), None);
}
assert!(gate.speaking());
assert_eq!(gate.update(0.5, THRESHOLD, 210), None);
assert!(gate.speaking());
}
#[test]
fn release_fires_after_sustained_silence() {
let mut gate = gate();
for tick in 0..=3u64 {
gate.update(0.5, THRESHOLD, tick * 10);
}
assert!(gate.speaking());
assert_eq!(gate.update(0.001, THRESHOLD, 40), None);
assert_eq!(gate.update(0.001, THRESHOLD, 219), None);
assert_eq!(gate.update(0.001, THRESHOLD, 220), Some(false));
assert!(!gate.speaking());
}
#[test]
fn retrigger_after_release_needs_full_attack() {
let mut gate = gate();
for tick in 0..=3u64 {
gate.update(0.5, THRESHOLD, tick * 10);
}
gate.update(0.001, THRESHOLD, 40);
assert_eq!(gate.update(0.001, THRESHOLD, 220), Some(false));
assert_eq!(gate.update(0.5, THRESHOLD, 230), None);
assert_eq!(gate.update(0.5, THRESHOLD, 260), Some(true));
}
#[test]
fn frame_rms_of_silence_is_zero() {
let samples = [0i16; 480];
assert_eq!(frame_rms_i16(&samples), 0.0);
}
#[test]
fn frame_rms_of_full_scale_square_wave_is_one() {
let mut samples = [i16::MIN; 480];
for (index, sample) in samples.iter_mut().enumerate() {
if index % 2 == 0 {
*sample = i16::MAX;
}
}
let rms = frame_rms_i16(&samples);
assert!(rms > 0.999);
assert!(rms <= 1.0);
}
#[test]
fn frame_rms_scales_with_amplitude() {
let loud = [8_192i16; 480];
let quiet = [1_024i16; 480];
assert!(frame_rms_i16(&loud) > frame_rms_i16(&quiet));
assert!((frame_rms_i16(&loud) - 0.25).abs() < 0.001);
}
#[test]
fn thresholds_default_and_clamp() {
let thresholds = SpeakingThresholds::new();
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT);
assert_eq!(
thresholds.remote_rms(),
SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT
);
thresholds.set(-1.0, f64::NAN);
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_MIN);
assert_eq!(thresholds.remote_rms(), SPEAKING_THRESHOLD_RMS_MIN);
thresholds.set(9.0, 0.02);
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_MAX);
assert_eq!(thresholds.remote_rms(), 0.02);
}
#[test]
fn gate_update_is_monotonic_in_time() {
let mut gate = gate();
gate.update(0.5, THRESHOLD, 100);
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
gate.update(0.5, THRESHOLD, 50);
}));
assert!(result.is_err());
}
}
@@ -0,0 +1,365 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::send_control::SendHealthSnapshot;
use fluxer_desktop_native::voice::stats as core_stats;
pub type ByteRateSample = core_stats::ByteRateSample;
pub type OutboundEntry = core_stats::OutboundStatsEntry;
pub type InboundEntry = core_stats::InboundStatsEntry;
#[derive(Clone, Debug, PartialEq, Default)]
pub struct ConnectionStats {
pub rtt_ms: Option<f64>,
pub outbound: Vec<OutboundEntry>,
pub inbound: Vec<InboundEntry>,
pub send: Option<SendHealthSnapshot>,
}
pub fn bitrate_kbps(prev: Option<ByteRateSample>, cur: ByteRateSample) -> f64 {
core_stats::bitrate_kbps(prev, cur)
}
#[cfg(test)]
pub fn sanitize_kbps(kbps: f64) -> f64 {
core_stats::sanitize_kbps(kbps)
}
pub fn jitter_seconds_to_ms(jitter_s: f64) -> Option<f64> {
core_stats::jitter_seconds_to_ms(jitter_s)
}
pub fn rtt_seconds_to_ms(rtt_s: f64) -> Option<f64> {
core_stats::rtt_seconds_to_ms(rtt_s)
}
pub fn sanitize_audio_level(level: f64) -> Option<f64> {
core_stats::sanitize_audio_level(level)
}
pub fn stats_to_json(stats: &ConnectionStats) -> String {
core_stats::stats_to_json(&core_stats::ConnectionStats {
rtt_ms: stats.rtt_ms,
outbound: stats.outbound.clone(),
inbound: stats.inbound.clone(),
send: stats.send.as_ref().map(send_health_to_core),
})
}
fn send_health_to_core(send: &SendHealthSnapshot) -> core_stats::SendHealthStats {
core_stats::SendHealthStats {
outgoing_video_queue_depth: send.outgoing_video_queue_depth,
outgoing_video_queue_capacity: send.outgoing_video_queue_capacity,
outgoing_video_max_queue_depth: send.outgoing_video_max_queue_depth,
outgoing_video_frames_produced: send.outgoing_video_frames_produced,
outgoing_video_frames_accepted: send.outgoing_video_frames_accepted,
outgoing_video_frames_dropped: send.outgoing_video_frames_dropped,
outgoing_video_frames_coalesced: send.outgoing_video_frames_coalesced,
outgoing_video_frames_captured: send.outgoing_video_frames_captured,
outgoing_video_capture_failures: send.outgoing_video_capture_failures,
outgoing_video_effective_fps: send.outgoing_video_effective_fps,
outgoing_video_target_fps: send.outgoing_video_target_fps,
outgoing_video_pacing_target_fps: send.outgoing_video_pacing_target_fps,
outgoing_video_max_queue_age_ms: send.outgoing_video_max_queue_age_ms,
outgoing_video_max_push_latency_ms: send.outgoing_video_max_push_latency_ms,
outgoing_video_pacing_mode: send.outgoing_video_pacing_mode.clone(),
outgoing_video_bus_active: send.outgoing_video_bus_active,
outgoing_audio_buffer_target_ms: send.outgoing_audio_buffer_target_ms,
outgoing_audio_buffer_max_ms: send.outgoing_audio_buffer_max_ms,
outgoing_audio_underruns: send.outgoing_audio_underruns,
outgoing_audio_rebuffers: send.outgoing_audio_rebuffers,
outgoing_audio_max_frame_gap_ms: send.outgoing_audio_max_frame_gap_ms,
adaptive_send_tier: send.adaptive_send_tier.clone(),
adaptive_send_reason: send.adaptive_send_reason.clone(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bitrate_first_sample_is_zero() {
let cur = ByteRateSample {
bytes: 1000,
timestamp_us: 1_000_000,
};
assert_eq!(bitrate_kbps(None, cur), 0.0);
}
#[test]
fn bitrate_computes_kbps_from_byte_delta() {
let prev = ByteRateSample {
bytes: 0,
timestamp_us: 0,
};
let cur = ByteRateSample {
bytes: 12_500,
timestamp_us: 1_000_000,
};
assert_eq!(bitrate_kbps(Some(prev), cur), 100.0);
}
#[test]
fn bitrate_half_second_doubles_rate() {
let prev = ByteRateSample {
bytes: 1000,
timestamp_us: 1_000_000,
};
let cur = ByteRateSample {
bytes: 13_500,
timestamp_us: 1_500_000,
};
assert_eq!(bitrate_kbps(Some(prev), cur), 200.0);
}
#[test]
fn bitrate_rejects_backwards_time_and_bytes() {
let prev = ByteRateSample {
bytes: 5000,
timestamp_us: 2_000_000,
};
assert_eq!(
bitrate_kbps(
Some(prev),
ByteRateSample {
bytes: 6000,
timestamp_us: 1_000_000
}
),
0.0
);
assert_eq!(
bitrate_kbps(
Some(prev),
ByteRateSample {
bytes: 6000,
timestamp_us: 2_000_000
}
),
0.0
);
assert_eq!(
bitrate_kbps(
Some(prev),
ByteRateSample {
bytes: 100,
timestamp_us: 3_000_000
}
),
0.0
);
}
#[test]
fn sanitize_kbps_drops_nan_inf_negative_and_rounds() {
assert_eq!(sanitize_kbps(f64::NAN), 0.0);
assert_eq!(sanitize_kbps(f64::INFINITY), 0.0);
assert_eq!(sanitize_kbps(-5.0), 0.0);
assert_eq!(sanitize_kbps(123.456), 123.5);
assert_eq!(sanitize_kbps(100.0), 100.0);
}
#[test]
fn unit_conversions_seconds_to_ms() {
assert_eq!(jitter_seconds_to_ms(0.012), Some(12.0));
assert_eq!(jitter_seconds_to_ms(-1.0), None);
assert_eq!(jitter_seconds_to_ms(f64::NAN), None);
assert_eq!(rtt_seconds_to_ms(0.045), Some(45.0));
assert_eq!(rtt_seconds_to_ms(0.0), None);
assert_eq!(rtt_seconds_to_ms(f64::INFINITY), None);
}
#[test]
fn sanitize_audio_level_clamps() {
assert_eq!(sanitize_audio_level(0.5), Some(0.5));
assert_eq!(sanitize_audio_level(2.0), Some(1.0));
assert_eq!(sanitize_audio_level(-0.1), Some(0.0));
assert_eq!(sanitize_audio_level(f64::NAN), None);
}
#[test]
fn empty_stats_serialise_to_null_rtt_and_empty_arrays() {
let stats = ConnectionStats::default();
assert_eq!(
stats_to_json(&stats),
"{\"rttMs\":null,\"outbound\":[],\"inbound\":[],\"send\":null}"
);
}
#[test]
fn full_stats_serialise_to_exact_contract_shape() {
let stats = ConnectionStats {
rtt_ms: Some(42.0),
outbound: vec![
OutboundEntry {
track_sid: "TR_mic1".into(),
source: "microphone".into(),
kind: "audio".into(),
codec: Some("audio/opus".into()),
bitrate_kbps: 32.0,
packets_lost: 0,
fps: None,
audio_level: Some(0.62),
..Default::default()
},
OutboundEntry {
track_sid: "TR_screen1".into(),
source: "screen_share".into(),
kind: "video".into(),
codec: Some("video/H265".into()),
bitrate_kbps: 2500.5,
packets_lost: 3,
packets_sent: 9000,
fps: Some(30.0),
audio_level: None,
width: Some(2176),
height: Some(1200),
source_width: Some(2176),
source_height: Some(1200),
target_bitrate_kbps: Some(50_000.0),
configured_fps: Some(60.0),
target_fps: Some(30.0),
effective_fps: Some(29.8),
frames_produced: Some(120),
frames_accepted: Some(118),
frames_dropped: Some(1),
frames_coalesced: Some(2),
frames_captured: Some(117),
capture_failures: Some(0),
max_queue_age_ms: Some(18),
max_push_latency_ms: Some(12),
adaptive_send_tier: Some("fps30".into()),
adaptive_send_reason: Some("encoderEgressPressure".into()),
},
],
inbound: vec![InboundEntry {
participant_sid: "PA_remote1".into(),
participant_identity: Some("user_2_connection_2".into()),
track_sid: "TR_remoteAudio".into(),
source: Some("microphone".into()),
kind: "audio".into(),
codec: Some("audio/opus".into()),
bitrate_kbps: 28.0,
packets_lost: 1,
packets_received: 990,
jitter_ms: Some(5.0),
audio_level: Some(0.75),
fps: None,
width: None,
height: None,
source_width: None,
source_height: None,
}],
send: None,
};
let json = stats_to_json(&stats);
assert_eq!(
json,
"{\"rttMs\":42,\"outbound\":[\
{\"trackSid\":\"TR_mic1\",\"source\":\"microphone\",\"kind\":\"audio\",\"bitrateKbps\":32,\"packetsLost\":0,\"packetsSent\":0,\"audioLevel\":0.62,\"codec\":\"audio/opus\"},\
{\"trackSid\":\"TR_screen1\",\"source\":\"screen_share\",\"kind\":\"video\",\"bitrateKbps\":2500.5,\"packetsLost\":3,\"packetsSent\":9000,\"fps\":30,\"width\":2176,\"height\":1200,\"sourceWidth\":2176,\"sourceHeight\":1200,\"targetBitrateKbps\":50000,\"configuredFps\":60,\"targetFps\":30,\"effectiveFps\":29.8,\"framesProduced\":120,\"framesAccepted\":118,\"framesDropped\":1,\"framesCoalesced\":2,\"framesCaptured\":117,\"captureFailures\":0,\"maxQueueAgeMs\":18,\"maxPushLatencyMs\":12,\"adaptiveSendTier\":\"fps30\",\"adaptiveSendReason\":\"encoderEgressPressure\",\"codec\":\"video/H265\"}\
],\"inbound\":[\
{\"participantSid\":\"PA_remote1\",\"trackSid\":\"TR_remoteAudio\",\"kind\":\"audio\",\"bitrateKbps\":28,\"packetsLost\":1,\"packetsReceived\":990,\"participantIdentity\":\"user_2_connection_2\",\"source\":\"microphone\",\"jitterMs\":5,\"audioLevel\":0.75,\"codec\":\"audio/opus\"}\
],\"send\":null}"
);
let _ = json;
}
#[test]
fn send_health_serialises_to_exact_contract_shape() {
let stats = ConnectionStats {
rtt_ms: None,
outbound: vec![],
inbound: vec![],
send: Some(SendHealthSnapshot {
outgoing_video_queue_depth: 1,
outgoing_video_queue_capacity: 8,
outgoing_video_max_queue_depth: 4,
outgoing_video_frames_produced: 2,
outgoing_video_frames_accepted: 3,
outgoing_video_frames_dropped: 4,
outgoing_video_frames_coalesced: 5,
outgoing_video_frames_captured: 6,
outgoing_video_capture_failures: 7,
outgoing_video_effective_fps: 59.94,
outgoing_video_target_fps: 30.0,
outgoing_video_pacing_target_fps: 60.0,
outgoing_video_max_queue_age_ms: 8,
outgoing_video_max_push_latency_ms: 9,
outgoing_video_pacing_mode: "source".to_string(),
outgoing_video_bus_active: true,
outgoing_audio_buffer_target_ms: 300,
outgoing_audio_buffer_max_ms: 750,
outgoing_audio_underruns: 10,
outgoing_audio_rebuffers: 11,
outgoing_audio_max_frame_gap_ms: 120,
adaptive_send_tier: "fps30".to_string(),
adaptive_send_reason: "sendLatencyPressure".to_string(),
}),
};
assert_eq!(
stats_to_json(&stats),
"{\"rttMs\":null,\"outbound\":[],\"inbound\":[],\"send\":{\
\"outgoingVideoQueueDepth\":1,\
\"outgoingVideoQueueCapacity\":8,\
\"outgoingVideoMaxQueueDepth\":4,\
\"outgoingVideoFramesProduced\":2,\
\"outgoingVideoFramesAccepted\":3,\
\"outgoingVideoFramesDropped\":4,\
\"outgoingVideoFramesCoalesced\":5,\
\"outgoingVideoFramesCaptured\":6,\
\"outgoingVideoCaptureFailures\":7,\
\"outgoingVideoEffectiveFps\":59.94,\
\"outgoingVideoTargetFps\":30,\
\"outgoingVideoPacingTargetFps\":60,\
\"outgoingVideoMaxQueueAgeMs\":8,\
\"outgoingVideoMaxPushLatencyMs\":9,\
\"outgoingVideoPacingMode\":\"source\",\
\"outgoingVideoBusActive\":true,\
\"outgoingAudioBufferTargetMs\":300,\
\"outgoingAudioBufferMaxMs\":750,\
\"outgoingAudioUnderruns\":10,\
\"outgoingAudioRebuffers\":11,\
\"outgoingAudioMaxFrameGapMs\":120,\
\"adaptiveSendTier\":\"fps30\",\
\"adaptiveSendReason\":\"sendLatencyPressure\"\
}}"
);
}
#[test]
fn video_inbound_omits_audio_level_audio_omits_fps() {
let stats = ConnectionStats {
rtt_ms: None,
outbound: vec![],
inbound: vec![InboundEntry {
participant_sid: "PA_x".into(),
participant_identity: None,
track_sid: "TR_v".into(),
source: Some("screen_share".into()),
kind: "video".into(),
codec: None,
bitrate_kbps: 1000.0,
packets_lost: 0,
packets_received: 5000,
jitter_ms: None,
audio_level: None,
fps: Some(29.94),
width: Some(3840),
height: Some(2160),
source_width: Some(3840),
source_height: Some(2160),
}],
send: None,
};
let json = stats_to_json(&stats);
assert!(!json.contains("audioLevel"));
assert!(!json.contains("jitterMs"));
assert!(json.contains("\"source\":\"screen_share\""));
assert!(json.contains("\"fps\":29.9"));
assert!(json.contains("\"width\":3840"));
assert!(json.contains("\"height\":2160"));
assert!(json.contains("\"rttMs\":null"));
}
}
@@ -0,0 +1,747 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#[cfg(any(target_os = "windows", test))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TextureFrameDesc {
pub handle: u64,
pub width: u32,
pub height: u32,
pub dxgi_format: u32,
pub timestamp_us: i64,
}
#[cfg(any(target_os = "linux", test))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct DmabufFrameDesc {
pub plane_count: u8,
pub width: u32,
pub height: u32,
pub drm_format: u32,
pub modifier: u64,
pub strides: [u32; 4],
pub offsets: [u32; 4],
pub device_uuid: [u8; 16],
pub timestamp_us: i64,
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TextureEncodeError {
NoTexture,
InvalidDimensions,
UnsupportedFormat,
InvalidPlanes,
UnsupportedCodec,
NoHardwareEncoder,
SdkNativeTextureUnsupported,
}
impl TextureEncodeError {
#[allow(dead_code)]
pub fn as_str(self) -> &'static str {
match self {
TextureEncodeError::NoTexture => "noTexture",
TextureEncodeError::InvalidDimensions => "invalidDimensions",
TextureEncodeError::UnsupportedFormat => "unsupportedFormat",
TextureEncodeError::InvalidPlanes => "invalidPlanes",
TextureEncodeError::UnsupportedCodec => "unsupportedCodec",
TextureEncodeError::NoHardwareEncoder => "noHardwareEncoder",
TextureEncodeError::SdkNativeTextureUnsupported => "sdkNativeTextureUnsupported",
}
}
}
#[cfg(any(target_os = "windows", test))]
const DXGI_FORMAT_R8G8B8A8_UNORM: u32 = 28;
#[cfg(any(target_os = "windows", test))]
const DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: u32 = 29;
#[cfg(any(target_os = "windows", test))]
const DXGI_FORMAT_B8G8R8A8_UNORM: u32 = 87;
#[cfg(any(target_os = "windows", test))]
const DXGI_FORMAT_B8G8R8A8_UNORM_SRGB: u32 = 91;
#[cfg(any(target_os = "windows", test))]
const DXGI_FORMAT_NV12: u32 = 103;
#[cfg(any(target_os = "windows", test))]
pub fn dxgi_format_supported(dxgi_format: u32) -> bool {
matches!(
dxgi_format,
DXGI_FORMAT_R8G8B8A8_UNORM
| DXGI_FORMAT_R8G8B8A8_UNORM_SRGB
| DXGI_FORMAT_B8G8R8A8_UNORM
| DXGI_FORMAT_B8G8R8A8_UNORM_SRGB
| DXGI_FORMAT_NV12
)
}
#[cfg(any(target_os = "linux", test))]
const fn fourcc(bytes: [u8; 4]) -> u32 {
u32::from_le_bytes(bytes)
}
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_XRGB8888: u32 = fourcc(*b"XR24");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_ARGB8888: u32 = fourcc(*b"AR24");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_XBGR8888: u32 = fourcc(*b"XB24");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_ABGR8888: u32 = fourcc(*b"AB24");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_XRGB2101010: u32 = fourcc(*b"XR30");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_ARGB2101010: u32 = fourcc(*b"AR30");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_XBGR2101010: u32 = fourcc(*b"XB30");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_ABGR2101010: u32 = fourcc(*b"AB30");
#[cfg(any(target_os = "linux", test))]
const DRM_FORMAT_NV12: u32 = fourcc(*b"NV12");
#[cfg(any(target_os = "linux", test))]
pub fn drm_format_supported(drm_format: u32) -> bool {
matches!(
drm_format,
DRM_FORMAT_XRGB8888
| DRM_FORMAT_ARGB8888
| DRM_FORMAT_XBGR8888
| DRM_FORMAT_ABGR8888
| DRM_FORMAT_XRGB2101010
| DRM_FORMAT_ARGB2101010
| DRM_FORMAT_XBGR2101010
| DRM_FORMAT_ABGR2101010
| DRM_FORMAT_NV12
)
}
#[cfg(any(target_os = "linux", target_os = "windows", test))]
const MAX_TEXTURE_EDGE: u32 = 8192;
#[cfg(any(target_os = "linux", target_os = "windows", test))]
fn validate_dimensions(width: u32, height: u32) -> Result<(), TextureEncodeError> {
if width < 2
|| height < 2
|| !width.is_multiple_of(2)
|| !height.is_multiple_of(2)
|| width > MAX_TEXTURE_EDGE
|| height > MAX_TEXTURE_EDGE
{
return Err(TextureEncodeError::InvalidDimensions);
}
Ok(())
}
#[cfg(any(target_os = "windows", test))]
pub fn validate_texture_desc(desc: &TextureFrameDesc) -> Result<(), TextureEncodeError> {
if desc.handle == 0 {
return Err(TextureEncodeError::NoTexture);
}
validate_dimensions(desc.width, desc.height)?;
if !dxgi_format_supported(desc.dxgi_format) {
return Err(TextureEncodeError::UnsupportedFormat);
}
Ok(())
}
#[cfg(any(target_os = "linux", test))]
pub fn validate_dmabuf_desc(desc: &DmabufFrameDesc) -> Result<(), TextureEncodeError> {
validate_dimensions(desc.width, desc.height)?;
if !drm_format_supported(desc.drm_format) {
return Err(TextureEncodeError::UnsupportedFormat);
}
let plane_count = desc.plane_count as usize;
if !(1..=4).contains(&plane_count) {
return Err(TextureEncodeError::InvalidPlanes);
}
for plane in 0..plane_count {
if desc.strides[plane] == 0 {
return Err(TextureEncodeError::InvalidPlanes);
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
#[cfg(test)]
pub fn dmabuf_desc_from_parts(
fds: &[i32],
plane_count: u32,
width: u32,
height: u32,
drm_format: u32,
modifier: u64,
strides: &[u32],
offsets: &[u32],
device_uuid: &[u8],
timestamp_us: f64,
) -> Option<(DmabufFrameDesc, [i32; 4])> {
let plane_count_u8 = u8::try_from(plane_count).ok()?;
let planes = plane_count as usize;
if !(1..=4).contains(&planes)
|| fds.len() < planes
|| strides.len() < planes
|| offsets.len() < planes
|| device_uuid.len() != 16
{
return None;
}
if fds.iter().take(planes).any(|fd| *fd < 0) {
return None;
}
let mut fd_array = [-1; 4];
let mut stride_array = [0; 4];
let mut offset_array = [0; 4];
fd_array[..planes].copy_from_slice(&fds[..planes]);
stride_array[..planes].copy_from_slice(&strides[..planes]);
offset_array[..planes].copy_from_slice(&offsets[..planes]);
let mut uuid = [0u8; 16];
uuid.copy_from_slice(device_uuid);
Some((
DmabufFrameDesc {
plane_count: plane_count_u8,
width,
height,
drm_format,
modifier,
strides: stride_array,
offsets: offset_array,
device_uuid: uuid,
timestamp_us: timestamp_us as i64,
},
fd_array,
))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TextureCapability {
pub available: bool,
pub reason: TextureEncodeError,
}
impl TextureCapability {
pub fn unavailable(reason: TextureEncodeError) -> Self {
Self {
available: false,
reason,
}
}
pub fn for_screen_codec(codec: &str, has_hardware_encoder: bool) -> Self {
if !codec_allows_native_gpu(codec) {
return Self::unavailable(TextureEncodeError::UnsupportedCodec);
}
if !has_hardware_encoder {
return Self {
available: false,
reason: TextureEncodeError::NoHardwareEncoder,
};
}
Self {
available: true,
reason: TextureEncodeError::NoTexture,
}
}
}
pub(crate) fn codec_allows_native_gpu(codec: &str) -> bool {
matches!(
codec.trim().to_ascii_lowercase().as_str(),
"h264" | "h265" | "hevc"
)
}
#[cfg(any(target_os = "windows", test))]
fn sdk_accepts_d3d11_texture_buffers() -> bool {
true
}
#[cfg(target_os = "linux")]
fn sdk_accepts_dmabuf_texture_buffers() -> bool {
cfg!(target_os = "linux")
}
#[cfg(any(target_os = "windows", test))]
pub fn should_attempt_texture_encode(
capability: &TextureCapability,
desc: &TextureFrameDesc,
) -> Result<(), TextureEncodeError> {
if !capability.available {
return Err(capability.reason);
}
if !sdk_accepts_d3d11_texture_buffers() {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
}
validate_texture_desc(desc)
}
#[cfg(target_os = "linux")]
pub fn should_attempt_dmabuf_encode(
capability: &TextureCapability,
desc: &DmabufFrameDesc,
) -> Result<(), TextureEncodeError> {
if !capability.available {
return Err(capability.reason);
}
if !sdk_accepts_dmabuf_texture_buffers() {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
}
validate_dmabuf_desc(desc)
}
#[cfg(test)]
pub fn should_attempt_texture_encode_for_tests(
capability: &TextureCapability,
desc: &TextureFrameDesc,
sdk_accepts_d3d11: bool,
) -> Result<(), TextureEncodeError> {
if !capability.available {
return Err(capability.reason);
}
if !sdk_accepts_d3d11 {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
}
validate_texture_desc(desc)
}
#[cfg(test)]
pub fn should_attempt_dmabuf_encode_for_tests(
capability: &TextureCapability,
desc: &DmabufFrameDesc,
sdk_accepts_dmabuf: bool,
) -> Result<(), TextureEncodeError> {
if !capability.available {
return Err(capability.reason);
}
if !sdk_accepts_dmabuf {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
}
validate_dmabuf_desc(desc)
}
#[cfg(all(feature = "publisher", any(target_os = "linux", target_os = "windows")))]
pub mod bridge {
#[cfg(target_os = "linux")]
use super::{DmabufFrameDesc, should_attempt_dmabuf_encode};
use super::{TextureCapability, TextureEncodeError};
#[cfg(target_os = "windows")]
use super::{TextureFrameDesc, should_attempt_texture_encode};
use livekit::webrtc::video_frame::{VideoFrame, VideoRotation, native::NativeBuffer};
use livekit::webrtc::video_source::native::NativeVideoSource;
#[cfg(target_os = "windows")]
pub fn try_publish_texture(
source: &NativeVideoSource,
capability: &TextureCapability,
desc: &TextureFrameDesc,
) -> Result<(), TextureEncodeError> {
should_attempt_texture_encode(capability, desc)?;
let Some(buffer) = NativeBuffer::from_fluxer_d3d11_texture(
desc.handle,
desc.width,
desc.height,
desc.dxgi_format,
) else {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
};
publish_native_buffer(source, buffer, desc.timestamp_us);
Ok(())
}
#[cfg(target_os = "linux")]
pub fn try_publish_dmabuf(
source: &NativeVideoSource,
capability: &TextureCapability,
desc: &DmabufFrameDesc,
fds: [i32; 4],
) -> Result<(), TextureEncodeError> {
should_attempt_dmabuf_encode(capability, desc)?;
let uuid_hi = u64::from_be_bytes(desc.device_uuid[0..8].try_into().unwrap_or([0; 8]));
let uuid_lo = u64::from_be_bytes(desc.device_uuid[8..16].try_into().unwrap_or([0; 8]));
let Some(buffer) = NativeBuffer::from_fluxer_dmabuf_texture(
fds,
desc.plane_count as u32,
desc.width,
desc.height,
desc.drm_format,
desc.modifier,
desc.strides,
desc.offsets,
uuid_hi,
uuid_lo,
) else {
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
};
publish_native_buffer(source, buffer, desc.timestamp_us);
Ok(())
}
fn publish_native_buffer(source: &NativeVideoSource, buffer: NativeBuffer, timestamp_us: i64) {
source.capture_frame(&VideoFrame {
rotation: VideoRotation::VideoRotation0,
timestamp_us,
frame_metadata: None,
buffer,
});
}
}
#[cfg(test)]
mod tests {
use super::*;
fn good_desc() -> TextureFrameDesc {
TextureFrameDesc {
handle: 0xDEAD_BEEF,
width: 1920,
height: 1080,
dxgi_format: DXGI_FORMAT_B8G8R8A8_UNORM,
timestamp_us: 123_456,
}
}
#[test]
fn dxgi_format_gate_accepts_8bit_rgba_bgra_only() {
assert!(dxgi_format_supported(DXGI_FORMAT_B8G8R8A8_UNORM));
assert!(dxgi_format_supported(DXGI_FORMAT_B8G8R8A8_UNORM_SRGB));
assert!(dxgi_format_supported(DXGI_FORMAT_R8G8B8A8_UNORM));
assert!(dxgi_format_supported(DXGI_FORMAT_R8G8B8A8_UNORM_SRGB));
assert!(dxgi_format_supported(DXGI_FORMAT_NV12));
assert!(!dxgi_format_supported(24));
assert!(!dxgi_format_supported(10));
assert!(!dxgi_format_supported(0));
}
#[test]
fn drm_format_gate_accepts_obs_vkcapture_texture_formats() {
assert!(drm_format_supported(DRM_FORMAT_ARGB8888));
assert!(drm_format_supported(DRM_FORMAT_ABGR8888));
assert!(drm_format_supported(DRM_FORMAT_ARGB2101010));
assert!(drm_format_supported(DRM_FORMAT_ABGR2101010));
assert!(drm_format_supported(DRM_FORMAT_NV12));
assert!(
!drm_format_supported(fourcc(*b"AB4H")),
"16-bit float DMA-BUF import is not encodable by the native NVENC bridge yet"
);
assert!(!drm_format_supported(0));
}
#[test]
fn validate_rejects_zero_handle_as_no_texture() {
let mut d = good_desc();
d.handle = 0;
assert_eq!(
validate_texture_desc(&d),
Err(TextureEncodeError::NoTexture)
);
}
#[test]
fn validate_rejects_odd_zero_and_oversized_dims() {
for (w, h) in [
(1921, 1080),
(1920, 1081),
(0, 1080),
(1920, 0),
(8194, 1080),
(1920, 8194),
] {
let mut d = good_desc();
d.width = w;
d.height = h;
assert_eq!(
validate_texture_desc(&d),
Err(TextureEncodeError::InvalidDimensions),
"dims {w}x{h} should be rejected"
);
}
}
#[test]
fn validate_rejects_unsupported_format() {
let mut d = good_desc();
d.dxgi_format = 24;
assert_eq!(
validate_texture_desc(&d),
Err(TextureEncodeError::UnsupportedFormat)
);
}
#[test]
fn validate_accepts_a_clean_bgra_texture() {
assert_eq!(validate_texture_desc(&good_desc()), Ok(()));
}
fn good_dmabuf_desc() -> DmabufFrameDesc {
DmabufFrameDesc {
plane_count: 1,
width: 1920,
height: 1080,
drm_format: DRM_FORMAT_ARGB8888,
modifier: 0,
strides: [1920 * 4, 0, 0, 0],
offsets: [0, 0, 0, 0],
device_uuid: [1; 16],
timestamp_us: 123_456,
}
}
#[test]
fn validate_dmabuf_accepts_supported_formats_with_optional_uuid() {
assert_eq!(validate_dmabuf_desc(&good_dmabuf_desc()), Ok(()));
let mut desc = good_dmabuf_desc();
desc.drm_format = DRM_FORMAT_NV12;
desc.strides[0] = 1920;
assert_eq!(validate_dmabuf_desc(&desc), Ok(()));
desc = good_dmabuf_desc();
desc.device_uuid = [0; 16];
assert_eq!(validate_dmabuf_desc(&desc), Ok(()));
}
#[test]
fn validate_dmabuf_rejects_invalid_planes() {
let mut desc = good_dmabuf_desc();
desc.plane_count = 0;
assert_eq!(
validate_dmabuf_desc(&desc),
Err(TextureEncodeError::InvalidPlanes)
);
desc = good_dmabuf_desc();
desc.strides[0] = 0;
assert_eq!(
validate_dmabuf_desc(&desc),
Err(TextureEncodeError::InvalidPlanes)
);
}
#[test]
fn dmabuf_desc_from_parts_rejects_negative_fds() {
assert!(
dmabuf_desc_from_parts(
&[-1],
1,
1920,
1080,
DRM_FORMAT_ARGB8888,
0,
&[1920 * 4],
&[0],
&[1; 16],
123.0,
)
.is_none()
);
}
#[test]
fn dmabuf_desc_from_parts_populates_all_plane_arrays() {
let uuid = [7u8; 16];
let (desc, fds) = dmabuf_desc_from_parts(
&[10, 11, 12, 99],
3,
1920,
1080,
DRM_FORMAT_NV12,
0xABCD,
&[1920, 960, 960, 777],
&[0, 2_073_600, 3_110_400, 999],
&uuid,
123_456.75,
)
.expect("valid multi-plane descriptor");
assert_eq!(desc.plane_count, 3);
assert_eq!(desc.width, 1920);
assert_eq!(desc.height, 1080);
assert_eq!(desc.drm_format, DRM_FORMAT_NV12);
assert_eq!(desc.modifier, 0xABCD);
assert_eq!(desc.strides, [1920, 960, 960, 0]);
assert_eq!(desc.offsets, [0, 2_073_600, 3_110_400, 0]);
assert_eq!(desc.device_uuid, uuid);
assert_eq!(desc.timestamp_us, 123_456);
assert_eq!(fds, [10, 11, 12, -1]);
}
#[test]
fn dmabuf_desc_from_parts_rejects_incomplete_native_inputs() {
let uuid = [1u8; 16];
for (fds, strides, offsets, uuid_bytes, label) in [
(&[4][..], &[128][..], &[0][..], &uuid[..], "too few fds"),
(
&[4, 5][..],
&[128][..],
&[0, 64][..],
&uuid[..],
"too few strides",
),
(
&[4, 5][..],
&[128, 128][..],
&[0][..],
&uuid[..],
"too few offsets",
),
(
&[4, 5][..],
&[128, 128][..],
&[0, 64][..],
&[1u8; 15][..],
"bad uuid",
),
] {
assert!(
dmabuf_desc_from_parts(
fds,
2,
128,
128,
DRM_FORMAT_ARGB8888,
0,
strides,
offsets,
uuid_bytes,
0.0,
)
.is_none(),
"{label} should be rejected"
);
}
assert!(
dmabuf_desc_from_parts(
&[4, 5, 6, 7, 8],
5,
128,
128,
DRM_FORMAT_ARGB8888,
0,
&[128; 5],
&[0; 5],
&uuid,
0.0,
)
.is_none()
);
}
#[test]
fn validate_precedence_handle_before_dims_before_format() {
let d = TextureFrameDesc {
handle: 0,
width: 1921,
height: 0,
dxgi_format: 999,
timestamp_us: 0,
};
assert_eq!(
validate_texture_desc(&d),
Err(TextureEncodeError::NoTexture)
);
}
#[test]
fn probe_is_available_only_for_explicit_hardware_codecs() {
for codec in ["h264", "H264", "h265", "hevc", "HEVC"] {
let cap = TextureCapability::for_screen_codec(codec, true);
assert!(cap.available, "{codec} should allow native GPU buffers");
assert_eq!(cap.reason, TextureEncodeError::NoTexture);
}
for codec in ["", "vp8", "vp9", "av1", "rubbish"] {
let cap = TextureCapability::for_screen_codec(codec, true);
assert!(
!cap.available,
"{codec} should not allow native GPU buffers"
);
assert_eq!(cap.reason, TextureEncodeError::UnsupportedCodec);
}
let cap = TextureCapability::for_screen_codec("h264", false);
assert!(!cap.available);
assert_eq!(cap.reason, TextureEncodeError::NoHardwareEncoder);
}
#[test]
fn should_attempt_falls_back_when_capability_unavailable() {
let cap = TextureCapability {
available: false,
reason: TextureEncodeError::SdkNativeTextureUnsupported,
};
assert_eq!(
should_attempt_texture_encode(&cap, &good_desc()),
Err(TextureEncodeError::SdkNativeTextureUnsupported)
);
}
#[test]
fn should_attempt_validates_frame_when_capability_available() {
let open = TextureCapability {
available: true,
reason: TextureEncodeError::NoTexture,
};
assert_eq!(
should_attempt_texture_encode_for_tests(&open, &good_desc(), true),
Ok(())
);
assert_eq!(should_attempt_texture_encode(&open, &good_desc()), Ok(()));
let mut bad = good_desc();
bad.handle = 0;
assert_eq!(
should_attempt_texture_encode_for_tests(&open, &bad, true),
Err(TextureEncodeError::NoTexture)
);
bad = good_desc();
bad.dxgi_format = 24;
assert_eq!(
should_attempt_texture_encode_for_tests(&open, &bad, true),
Err(TextureEncodeError::UnsupportedFormat)
);
}
#[test]
fn should_attempt_dmabuf_is_sdk_gated_after_validation_capability() {
let open = TextureCapability {
available: true,
reason: TextureEncodeError::NoTexture,
};
assert_eq!(
should_attempt_dmabuf_encode_for_tests(&open, &good_dmabuf_desc(), true),
Ok(())
);
assert_eq!(
should_attempt_dmabuf_encode_for_tests(&open, &good_dmabuf_desc(), false),
Err(TextureEncodeError::SdkNativeTextureUnsupported)
);
let mut invalid = good_dmabuf_desc();
invalid.plane_count = 5;
assert_eq!(
should_attempt_dmabuf_encode_for_tests(&open, &invalid, true),
Err(TextureEncodeError::InvalidPlanes)
);
}
#[test]
fn error_strings_are_stable() {
assert_eq!(TextureEncodeError::NoTexture.as_str(), "noTexture");
assert_eq!(
TextureEncodeError::InvalidDimensions.as_str(),
"invalidDimensions"
);
assert_eq!(
TextureEncodeError::UnsupportedFormat.as_str(),
"unsupportedFormat"
);
assert_eq!(TextureEncodeError::InvalidPlanes.as_str(), "invalidPlanes");
assert_eq!(
TextureEncodeError::UnsupportedCodec.as_str(),
"unsupportedCodec"
);
assert_eq!(
TextureEncodeError::NoHardwareEncoder.as_str(),
"noHardwareEncoder"
);
assert_eq!(
TextureEncodeError::SdkNativeTextureUnsupported.as_str(),
"sdkNativeTextureUnsupported"
);
}
}
@@ -0,0 +1,732 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct I420 {
pub width: u32,
pub height: u32,
pub y: Vec<u8>,
pub u: Vec<u8>,
pub v: Vec<u8>,
}
impl I420 {
pub fn new(width: u32, height: u32) -> Option<Self> {
if !dims_ok(width, height) {
return None;
}
let w = width as usize;
let h = height as usize;
Some(Self {
width,
height,
y: vec![0u8; w * h],
u: vec![0u8; (w / 2) * (h / 2)],
v: vec![0u8; (w / 2) * (h / 2)],
})
}
fn has_layout(&self, width: u32, height: u32) -> bool {
if self.width != width || self.height != height {
return false;
}
let w = width as usize;
let h = height as usize;
self.y.len() == w * h && self.u.len() == (w / 2) * (h / 2) && self.v.len() == self.u.len()
}
}
pub fn tight_i420_byte_len(width: u32, height: u32) -> Option<usize> {
if !dims_ok(width, height) {
return None;
}
let w = width as usize;
let h = height as usize;
let y_len = w.checked_mul(h)?;
let chroma_len = (w / 2).checked_mul(h / 2)?;
y_len.checked_add(chroma_len.checked_mul(2)?)
}
pub fn copy_tight_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
if !dst.has_layout(width, height) {
return false;
}
let Some(total_len) = tight_i420_byte_len(width, height) else {
return false;
};
if src.len() != total_len {
return false;
}
let y_len = (width as usize) * (height as usize);
let chroma_len = y_len / 4;
dst.y.copy_from_slice(&src[..y_len]);
dst.u.copy_from_slice(&src[y_len..y_len + chroma_len]);
dst.v.copy_from_slice(&src[y_len + chroma_len..]);
true
}
fn dims_ok(width: u32, height: u32) -> bool {
width >= 2 && height >= 2 && width.is_multiple_of(2) && height.is_multiple_of(2)
}
#[cfg_attr(not(test), allow(dead_code))]
pub fn nv12_to_i420(
src: &[u8],
width: u32,
height: u32,
stride_y: u32,
stride_uv: u32,
) -> Option<I420> {
let mut dst = I420::new(width, height)?;
if !nv12_to_i420_into(src, width, height, stride_y, stride_uv, &mut dst) {
return None;
}
Some(dst)
}
pub fn nv12_to_i420_into(
src: &[u8],
width: u32,
height: u32,
stride_y: u32,
stride_uv: u32,
dst: &mut I420,
) -> bool {
if !dims_ok(width, height) {
return false;
}
if !dst.has_layout(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let sy = stride_y.max(width) as usize;
let suv = stride_uv.max(width) as usize;
let Some(uv_offset) = sy.checked_mul(h) else {
return false;
};
let Some(uv_len) = suv.checked_mul(ch) else {
return false;
};
let Some(needed) = uv_offset.checked_add(uv_len) else {
return false;
};
if src.len() < needed {
return false;
}
for row in 0..h {
let s = row * sy;
dst.y[row * w..row * w + w].copy_from_slice(&src[s..s + w]);
}
for row in 0..ch {
let base = uv_offset + row * suv;
for x in 0..cw {
dst.u[row * cw + x] = src[base + 2 * x];
dst.v[row * cw + x] = src[base + 2 * x + 1];
}
}
true
}
#[allow(clippy::too_many_arguments)]
pub fn copy_nv12_planes(
src: &[u8],
width: u32,
height: u32,
stride_y: u32,
stride_uv: u32,
dst_y: &mut [u8],
dst_uv: &mut [u8],
dst_stride_y: u32,
dst_stride_uv: u32,
) -> bool {
if !dims_ok(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let ch = h / 2;
let sy = stride_y.max(width) as usize;
let suv = stride_uv.max(width) as usize;
let dsy = dst_stride_y as usize;
let dsuv = dst_stride_uv as usize;
if dsy < w || dsuv < w {
return false;
}
let Some(uv_offset) = sy.checked_mul(h) else {
return false;
};
let Some(uv_len) = suv.checked_mul(ch) else {
return false;
};
let Some(needed) = uv_offset.checked_add(uv_len) else {
return false;
};
if src.len() < needed || dst_y.len() < dsy * h || dst_uv.len() < dsuv * ch {
return false;
}
for row in 0..h {
let s = row * sy;
let d = row * dsy;
dst_y[d..d + w].copy_from_slice(&src[s..s + w]);
}
for row in 0..ch {
let s = uv_offset + row * suv;
let d = row * dsuv;
dst_uv[d..d + w].copy_from_slice(&src[s..s + w]);
}
true
}
#[cfg(test)]
pub fn yuyv_to_i420(src: &[u8], width: u32, height: u32, stride: u32) -> Option<I420> {
let mut dst = I420::new(width, height)?;
if !yuyv_to_i420_into(src, width, height, stride, &mut dst) {
return None;
}
Some(dst)
}
pub fn yuyv_to_i420_into(src: &[u8], width: u32, height: u32, stride: u32, dst: &mut I420) -> bool {
if !dims_ok(width, height) {
return false;
}
if !dst.has_layout(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let stride = stride.max(width * 2) as usize;
if src.len() < stride * h {
return false;
}
for row in 0..h {
let row_base = row * stride;
for pair in 0..cw {
let src_offset = row_base + pair * 4;
let dst_offset = row * w + pair * 2;
dst.y[dst_offset] = src[src_offset];
dst.y[dst_offset + 1] = src[src_offset + 2];
}
}
for cy in 0..ch {
for cx in 0..cw {
let top = (cy * 2) * stride + cx * 4;
let bottom = (cy * 2 + 1) * stride + cx * 4;
dst.u[cy * cw + cx] =
((u16::from(src[top + 1]) + u16::from(src[bottom + 1])) / 2) as u8;
dst.v[cy * cw + cx] =
((u16::from(src[top + 3]) + u16::from(src[bottom + 3])) / 2) as u8;
}
}
true
}
fn clamp_u8(value: i32) -> u8 {
value.clamp(0, 255) as u8
}
fn rgb_to_y(r: i32, g: i32, b: i32) -> u8 {
clamp_u8(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16)
}
fn rgb_to_u(r: i32, g: i32, b: i32) -> u8 {
clamp_u8(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128)
}
fn rgb_to_v(r: i32, g: i32, b: i32) -> u8 {
clamp_u8(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128)
}
#[cfg(test)]
pub fn bgra_to_i420(src: &[u8], width: u32, height: u32, stride: u32) -> Option<I420> {
let mut dst = I420::new(width, height)?;
if !bgra_to_i420_planes(
src,
width,
height,
stride,
&mut dst.y,
&mut dst.u,
&mut dst.v,
width,
width / 2,
width / 2,
) {
return None;
}
Some(dst)
}
#[allow(clippy::too_many_arguments)]
pub fn bgra_to_i420_planes(
src: &[u8],
width: u32,
height: u32,
stride: u32,
dst_y: &mut [u8],
dst_u: &mut [u8],
dst_v: &mut [u8],
dst_stride_y: u32,
dst_stride_u: u32,
dst_stride_v: u32,
) -> bool {
if !dims_ok(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let stride = stride.max(width * 4) as usize;
if src.len() < stride * h {
return false;
}
let dsy = dst_stride_y as usize;
let dsu = dst_stride_u as usize;
let dsv = dst_stride_v as usize;
if dsy < w || dsu < cw || dsv < cw {
return false;
}
if dst_y.len() < dsy * h || dst_u.len() < dsu * ch || dst_v.len() < dsv * ch {
return false;
}
let px = |row: usize, col: usize| -> (i32, i32, i32) {
let o = row * stride + col * 4;
let b = src[o] as i32;
let g = src[o + 1] as i32;
let r = src[o + 2] as i32;
(r, g, b)
};
for row in 0..h {
for col in 0..w {
let (r, g, b) = px(row, col);
dst_y[row * dsy + col] = rgb_to_y(r, g, b);
}
}
for cy in 0..ch {
for cx in 0..cw {
let mut rs = 0;
let mut gs = 0;
let mut bs = 0;
for dy in 0..2 {
for dx in 0..2 {
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
rs += r;
gs += g;
bs += b;
}
}
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
dst_u[cy * dsu + cx] = rgb_to_u(r, g, b);
dst_v[cy * dsv + cx] = rgb_to_v(r, g, b);
}
}
true
}
#[cfg_attr(not(feature = "camera-native"), allow(dead_code))]
pub fn rgb_to_i420(src: &[u8], width: u32, height: u32) -> Option<I420> {
let mut dst = I420::new(width, height)?;
if !rgb_to_i420_into(src, width, height, &mut dst) {
return None;
}
Some(dst)
}
pub fn rgb_to_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
if !dims_ok(width, height) {
return false;
}
if !dst.has_layout(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let stride = w * 3;
if src.len() < stride * h {
return false;
}
let px = |row: usize, col: usize| -> (i32, i32, i32) {
let o = row * stride + col * 3;
let r = src[o] as i32;
let g = src[o + 1] as i32;
let b = src[o + 2] as i32;
(r, g, b)
};
for row in 0..h {
for col in 0..w {
let (r, g, b) = px(row, col);
dst.y[row * w + col] = rgb_to_y(r, g, b);
}
}
for cy in 0..ch {
for cx in 0..cw {
let mut rs = 0;
let mut gs = 0;
let mut bs = 0;
for dy in 0..2 {
for dx in 0..2 {
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
rs += r;
gs += g;
bs += b;
}
}
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
dst.u[cy * cw + cx] = rgb_to_u(r, g, b);
dst.v[cy * cw + cx] = rgb_to_v(r, g, b);
}
}
true
}
#[cfg_attr(not(feature = "camera-native"), allow(dead_code))]
pub fn bgr_to_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
if !dims_ok(width, height) {
return false;
}
if !dst.has_layout(width, height) {
return false;
}
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let stride = w * 3;
if src.len() < stride * h {
return false;
}
let px = |row: usize, col: usize| -> (i32, i32, i32) {
let o = row * stride + col * 3;
let b = src[o] as i32;
let g = src[o + 1] as i32;
let r = src[o + 2] as i32;
(r, g, b)
};
for row in 0..h {
for col in 0..w {
let (r, g, b) = px(row, col);
dst.y[row * w + col] = rgb_to_y(r, g, b);
}
}
for cy in 0..ch {
for cx in 0..cw {
let mut rs = 0;
let mut gs = 0;
let mut bs = 0;
for dy in 0..2 {
for dx in 0..2 {
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
rs += r;
gs += g;
bs += b;
}
}
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
dst.u[cy * cw + cx] = rgb_to_u(r, g, b);
dst.v[cy * cw + cx] = rgb_to_v(r, g, b);
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_odd_or_tiny_dimensions() {
assert!(nv12_to_i420(&[0u8; 64], 3, 2, 3, 3).is_none());
assert!(nv12_to_i420(&[0u8; 64], 2, 1, 2, 2).is_none());
assert!(bgra_to_i420(&[0u8; 256], 2, 3, 8).is_none());
assert!(bgra_to_i420(&[0u8; 256], 0, 2, 0).is_none());
}
#[test]
fn nv12_short_buffer_is_rejected() {
assert!(nv12_to_i420(&[0u8; 5], 2, 2, 2, 2).is_none());
assert!(nv12_to_i420(&[0u8; 6], 2, 2, 2, 2).is_some());
}
#[test]
fn nv12_packed_2x2_deinterleaves() {
let src = [1u8, 2, 3, 4, 10, 20];
let out = nv12_to_i420(&src, 2, 2, 2, 2).unwrap();
assert_eq!(out.y, vec![1, 2, 3, 4]);
assert_eq!(out.u, vec![10]);
assert_eq!(out.v, vec![20]);
assert_eq!((out.width / 2, out.height / 2), (1, 1));
}
#[test]
fn yuyv_2x2_deinterleaves_and_vertically_averages_chroma() {
let src = [1u8, 10, 2, 20, 3, 30, 4, 40];
let out = yuyv_to_i420(&src, 2, 2, 4).unwrap();
assert_eq!(out.y, vec![1, 2, 3, 4]);
assert_eq!(out.u, vec![20]);
assert_eq!(out.v, vec![30]);
}
#[test]
fn yuyv_respects_row_padding() {
let src = [1u8, 10, 2, 20, 99, 99, 3, 30, 4, 40, 88, 88];
let out = yuyv_to_i420(&src, 2, 2, 6).unwrap();
assert_eq!(out.y, vec![1, 2, 3, 4]);
assert_eq!(out.u, vec![20]);
assert_eq!(out.v, vec![30]);
}
#[test]
fn nv12_4x4_deinterleaves_two_chroma_columns() {
let mut src = Vec::new();
src.extend(0u8..16);
src.extend([100, 101, 102, 103, 104, 105, 106, 107]);
let out = nv12_to_i420(&src, 4, 4, 4, 4).unwrap();
assert_eq!(out.y, (0u8..16).collect::<Vec<_>>());
assert_eq!(out.u, vec![100, 102, 104, 106]);
assert_eq!(out.v, vec![101, 103, 105, 107]);
}
#[test]
fn nv12_respects_row_padding() {
let src = [1u8, 2, 0xFF, 0xFF, 3, 4, 0xFF, 0xFF, 10, 20, 0xFF, 0xFF];
let out = nv12_to_i420(&src, 2, 2, 4, 4).unwrap();
assert_eq!(out.y, vec![1, 2, 3, 4]);
assert_eq!(out.u, vec![10]);
assert_eq!(out.v, vec![20]);
}
#[test]
fn copy_nv12_planes_preserves_nv12_layout() {
let src = [1u8, 2, 3, 4, 10, 20];
let mut y = [0u8; 4];
let mut uv = [0u8; 2];
assert!(copy_nv12_planes(&src, 2, 2, 2, 2, &mut y, &mut uv, 2, 2));
assert_eq!(y, [1, 2, 3, 4]);
assert_eq!(uv, [10, 20]);
}
#[test]
fn copy_nv12_planes_respects_destination_stride() {
let src = [1u8, 2, 0xFF, 0xFF, 3, 4, 0xFF, 0xFF, 10, 20, 0xFF, 0xFF];
let mut y = [0u8; 8];
let mut uv = [0u8; 4];
assert!(copy_nv12_planes(&src, 2, 2, 4, 4, &mut y, &mut uv, 4, 4));
assert_eq!(y, [1, 2, 0, 0, 3, 4, 0, 0]);
assert_eq!(uv, [10, 20, 0, 0]);
}
#[test]
fn copy_nv12_planes_rejects_short_buffers() {
let src = [0u8; 6];
let mut y = [0u8; 3];
let mut uv = [0u8; 2];
assert!(!copy_nv12_planes(&src, 2, 2, 2, 2, &mut y, &mut uv, 2, 2));
let mut y = [0u8; 4];
assert!(!copy_nv12_planes(
&src[..5],
2,
2,
2,
2,
&mut y,
&mut uv,
2,
2
));
}
fn solid_bgra(width: u32, height: u32, b: u8, g: u8, r: u8) -> Vec<u8> {
let mut v = Vec::with_capacity((width * height * 4) as usize);
for _ in 0..(width * height) {
v.extend([b, g, r, 255]);
}
v
}
fn near(a: u8, b: u8, tol: i32) -> bool {
(a as i32 - b as i32).abs() <= tol
}
#[test]
fn bgra_black_white_grey_levels() {
let black = bgra_to_i420(&solid_bgra(2, 2, 0, 0, 0), 2, 2, 8).unwrap();
assert!(near(black.y[0], 16, 1), "black Y={}", black.y[0]);
assert!(near(black.u[0], 128, 1) && near(black.v[0], 128, 1));
let white = bgra_to_i420(&solid_bgra(2, 2, 255, 255, 255), 2, 2, 8).unwrap();
assert!(near(white.y[0], 235, 2), "white Y={}", white.y[0]);
assert!(near(white.u[0], 128, 2) && near(white.v[0], 128, 2));
}
#[test]
fn bgra_primaries_have_expected_chroma_signs() {
let red = bgra_to_i420(&solid_bgra(2, 2, 0, 0, 255), 2, 2, 8).unwrap();
assert!(red.v[0] > 200, "red V={}", red.v[0]);
let blue = bgra_to_i420(&solid_bgra(2, 2, 255, 0, 0), 2, 2, 8).unwrap();
assert!(blue.u[0] > 200, "blue U={}", blue.u[0]);
let green = bgra_to_i420(&solid_bgra(2, 2, 0, 255, 0), 2, 2, 8).unwrap();
assert!(
green.u[0] < 60 && green.v[0] < 60,
"green U={} V={}",
green.u[0],
green.v[0]
);
}
#[test]
fn bgra_plane_sizes() {
let out = bgra_to_i420(&solid_bgra(8, 6, 10, 20, 30), 8, 6, 32).unwrap();
assert_eq!(out.y.len(), 8 * 6);
assert_eq!(out.u.len(), 4 * 3);
assert_eq!(out.v.len(), 4 * 3);
}
#[test]
fn bgra_golden_2x2_solid_colour_exact_bytes() {
let src = solid_bgra(2, 2, 32, 64, 128);
let out = bgra_to_i420(&src, 2, 2, 8).unwrap();
assert_eq!(out.width, 2);
assert_eq!(out.height, 2);
assert_eq!(out.y, vec![84, 84, 84, 84]);
assert_eq!(out.u, vec![105]);
assert_eq!(out.v, vec![158]);
}
#[test]
fn bgra_golden_strided_2x2_skips_row_padding() {
let mut src = vec![0xFFu8; 16 * 2];
for px in 0..2 {
let o = px * 4;
src[o..o + 4].copy_from_slice(&[0, 0, 0, 255]);
}
for px in 0..2 {
let o = 16 + px * 4;
src[o..o + 4].copy_from_slice(&[255, 255, 255, 255]);
}
let out = bgra_to_i420(&src, 2, 2, 16).unwrap();
assert!(
near(out.y[0], 16, 1) && near(out.y[1], 16, 1),
"row0 Y={:?}",
&out.y[0..2]
);
assert!(
near(out.y[2], 235, 2) && near(out.y[3], 235, 2),
"row1 Y={:?}",
&out.y[2..4]
);
assert!(
near(out.u[0], 128, 2) && near(out.v[0], 128, 2),
"U={} V={}",
out.u[0],
out.v[0]
);
}
#[test]
fn bgra_to_i420_planes_matches_tight_conversion_with_destination_padding() {
let src = solid_bgra(4, 2, 32, 64, 128);
let tight = bgra_to_i420(&src, 4, 2, 16).unwrap();
let mut y = [0u8; 10];
let mut u = [0u8; 4];
let mut v = [0u8; 4];
assert!(bgra_to_i420_planes(
&src, 4, 2, 16, &mut y, &mut u, &mut v, 5, 2, 2
));
assert_eq!(&y[0..4], &tight.y[0..4]);
assert_eq!(&y[5..9], &tight.y[4..8]);
assert_eq!(&u[0..2], &tight.u[0..2]);
assert_eq!(&v[0..2], &tight.v[0..2]);
}
#[test]
fn copy_tight_i420_into_reuses_existing_plane_storage() {
let mut dst = I420::new(4, 2).unwrap();
let ptrs = (dst.y.as_ptr(), dst.u.as_ptr(), dst.v.as_ptr());
let src: Vec<u8> = (0u8..12).collect();
assert_eq!(tight_i420_byte_len(4, 2), Some(12));
assert!(copy_tight_i420_into(&src, 4, 2, &mut dst));
assert_eq!(dst.y.as_ptr(), ptrs.0);
assert_eq!(dst.u.as_ptr(), ptrs.1);
assert_eq!(dst.v.as_ptr(), ptrs.2);
assert_eq!(dst.y, vec![0, 1, 2, 3, 4, 5, 6, 7]);
assert_eq!(dst.u, vec![8, 9]);
assert_eq!(dst.v, vec![10, 11]);
}
fn solid_rgb(width: u32, height: u32, r: u8, g: u8, b: u8) -> Vec<u8> {
let mut v = Vec::with_capacity((width * height * 3) as usize);
for _ in 0..(width * height) {
v.extend([r, g, b]);
}
v
}
#[test]
fn rgb_rejects_odd_dims_and_short_buffer() {
assert!(rgb_to_i420(&[0u8; 64], 3, 2).is_none());
assert!(rgb_to_i420(&[0u8; 64], 2, 1).is_none());
assert!(rgb_to_i420(&[0u8; 11], 2, 2).is_none());
assert!(rgb_to_i420(&[0u8; 12], 2, 2).is_some());
}
#[test]
fn rgb_golden_2x2_solid_colour_matches_bgra_path() {
let out = rgb_to_i420(&solid_rgb(2, 2, 128, 64, 32), 2, 2).unwrap();
assert_eq!(out.width, 2);
assert_eq!(out.height, 2);
assert_eq!(out.y, vec![84, 84, 84, 84]);
assert_eq!(out.u, vec![105]);
assert_eq!(out.v, vec![158]);
}
#[test]
fn rgb_plane_sizes_and_levels() {
let black = rgb_to_i420(&solid_rgb(2, 2, 0, 0, 0), 2, 2).unwrap();
assert!(near(black.y[0], 16, 1));
assert!(near(black.u[0], 128, 1) && near(black.v[0], 128, 1));
let white = rgb_to_i420(&solid_rgb(4, 4, 255, 255, 255), 4, 4).unwrap();
assert_eq!(white.y.len(), 16);
assert_eq!(white.u.len(), 4);
assert_eq!(white.v.len(), 4);
assert!(near(white.y[0], 235, 2));
}
#[test]
fn nv12_golden_4x2_packed_to_i420() {
let mut src = Vec::new();
src.extend(0u8..8);
src.extend([40, 41, 42, 43]);
let out = nv12_to_i420(&src, 4, 2, 4, 4).unwrap();
assert_eq!(out.width, 4);
assert_eq!(out.height, 2);
assert_eq!(out.y, (0u8..8).collect::<Vec<_>>());
assert_eq!(out.u, vec![40, 42]);
assert_eq!(out.v, vec![41, 43]);
}
}
@@ -0,0 +1,520 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use serde_json::Value;
const OUTBOUND: &str = include_str!("fixtures/outbound_events.json");
const INBOUND: &str = include_str!("fixtures/inbound_events.json");
const VIDEO_FRAME_META: &str = include_str!("fixtures/video_frame_meta.json");
const TRACK_KINDS: &[&str] = &["audio", "video"];
const TRACK_SOURCES: &[&str] = &[
"unknown",
"camera",
"microphone",
"screen_share",
"screen_share_audio",
];
const QUALITIES: &[&str] = &["excellent", "good", "poor", "lost"];
const SUBSCRIPTION_STATUSES: &[&str] = &["desired", "subscribed", "unsubscribed"];
fn records(json: &str) -> Vec<(String, Value)> {
let parsed: Value = serde_json::from_str(json).expect("fixture is valid JSON");
parsed
.as_array()
.expect("fixture is a JSON array")
.iter()
.map(|rec| {
let ty = rec
.get("eventType")
.and_then(Value::as_str)
.expect("record has an eventType string")
.to_string();
let payload = rec.get("payload").expect("record has a payload").clone();
(ty, payload)
})
.collect()
}
fn assert_exact_keys(event_type: &str, payload: &Value, expected: &[&str]) {
let obj = payload
.as_object()
.unwrap_or_else(|| panic!("{event_type}: payload is not a JSON object"));
let mut got: Vec<&str> = obj.keys().map(String::as_str).collect();
got.sort_unstable();
let mut want: Vec<&str> = expected.to_vec();
want.sort_unstable();
assert_eq!(got, want, "{event_type}: payload keys mismatch");
}
fn str_field<'a>(event_type: &str, payload: &'a Value, key: &str) -> &'a str {
payload
.get(key)
.and_then(Value::as_str)
.unwrap_or_else(|| panic!("{event_type}.{key} is not a string"))
}
fn bool_field(event_type: &str, payload: &Value, key: &str) -> bool {
payload
.get(key)
.and_then(Value::as_bool)
.unwrap_or_else(|| panic!("{event_type}.{key} is not a boolean"))
}
fn validate_string_map(event_type: &str, payload: &Value, key: &str) {
let obj = payload
.get(key)
.and_then(Value::as_object)
.unwrap_or_else(|| panic!("{event_type}.{key} is not an object"));
for (map_key, value) in obj {
assert!(
!map_key.is_empty(),
"{event_type}.{key} contains an empty key"
);
assert!(
value.as_str().is_some(),
"{event_type}.{key}.{map_key} is not a string"
);
}
}
fn validate_participant_snapshot(event_type: &str, payload: &Value) {
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
}
fn validate_track_payload(event_type: &str, payload: &Value, includes_subscription: bool) {
let mut keys = vec![
"participantSid",
"identity",
"participantName",
"trackSid",
"trackName",
"kind",
"source",
"muted",
];
if includes_subscription {
keys.extend(["subscribed", "subscriptionStatus"]);
}
assert_exact_keys(event_type, payload, &keys);
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "participantName");
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
let _ = str_field(event_type, payload, "trackName");
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
let _ = bool_field(event_type, payload, "muted");
if includes_subscription {
let _ = bool_field(event_type, payload, "subscribed");
assert!(SUBSCRIPTION_STATUSES.contains(&str_field(
event_type,
payload,
"subscriptionStatus"
)));
}
}
fn validate_record(event_type: &str, payload: &Value) {
match event_type {
"connected" | "Reconnecting" | "Reconnected" => assert_exact_keys(event_type, payload, &[]),
"connectionState" => {
assert_exact_keys(event_type, payload, &["state"]);
assert!(!str_field(event_type, payload, "state").is_empty());
}
"disconnected" => {
assert_exact_keys(event_type, payload, &["reason"]);
assert!(!str_field(event_type, payload, "reason").is_empty());
}
"participantJoined" => {
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
}
"participantLeft" => {
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "name");
}
"trackPublished" | "trackUnpublished" | "trackSubscribed" | "trackUnsubscribed" => {
validate_track_payload(event_type, payload, true);
}
"trackMuted" | "trackUnmuted" => {
validate_track_payload(event_type, payload, false);
assert_eq!(
bool_field(event_type, payload, "muted"),
event_type == "trackMuted"
);
}
"trackSubscriptionFailed" => {
let has_publication = payload.get("kind").is_some();
if has_publication {
assert_exact_keys(
event_type,
payload,
&[
"participantSid",
"identity",
"participantName",
"trackSid",
"trackName",
"kind",
"source",
"muted",
"subscribed",
"subscriptionStatus",
"error",
],
);
} else {
assert_exact_keys(
event_type,
payload,
&[
"participantSid",
"identity",
"participantName",
"trackSid",
"error",
],
);
}
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
if has_publication {
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
assert!(SUBSCRIPTION_STATUSES.contains(&str_field(
event_type,
payload,
"subscriptionStatus"
)));
let _ = bool_field(event_type, payload, "muted");
let _ = bool_field(event_type, payload, "subscribed");
}
assert!(!str_field(event_type, payload, "error").is_empty());
}
"localTrackPublished" => {
validate_track_payload(event_type, payload, false);
}
"localTrackUnpublished" => {
validate_track_payload(event_type, payload, false);
}
"localTrackRepublished" => {
let keys = [
"participantSid",
"identity",
"participantName",
"previousTrackSid",
"trackSid",
"trackName",
"kind",
"source",
"muted",
];
assert_exact_keys(event_type, payload, &keys);
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
assert!(str_field(event_type, payload, "previousTrackSid").starts_with("TR_"));
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
let _ = bool_field(event_type, payload, "muted");
}
"activeSpeakers" => {
assert_exact_keys(event_type, payload, &["sids", "participants"]);
let sids = payload
.get("sids")
.and_then(Value::as_array)
.expect("activeSpeakers.sids is an array");
for sid in sids {
let sid = sid.as_str().expect("activeSpeakers.sids entry is a string");
assert!(
sid.starts_with("PA_"),
"activeSpeakers sid {sid} lacks PA_ prefix"
);
}
let participants = payload
.get("participants")
.and_then(Value::as_array)
.expect("activeSpeakers.participants is an array");
for participant in participants {
validate_participant_snapshot("activeSpeakers.participants[]", participant);
}
}
"connectionQuality" => {
assert_exact_keys(event_type, payload, &["sid", "identity", "name", "quality"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
assert!(QUALITIES.contains(&str_field(event_type, payload, "quality")));
}
"e2eeState" => {
assert_exact_keys(event_type, payload, &["sid", "identity", "name", "state"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
assert!(!str_field(event_type, payload, "state").is_empty());
}
"stats" => {
assert_exact_keys(event_type, payload, &["rttMs", "outbound", "inbound"]);
let rtt = payload.get("rttMs").expect("stats.rttMs exists");
assert!(
rtt.is_null() || rtt.as_f64().is_some(),
"stats.rttMs is null or number"
);
let outbound = payload
.get("outbound")
.and_then(Value::as_array)
.expect("stats.outbound is an array");
for entry in outbound {
let obj = entry
.as_object()
.expect("stats.outbound[] is a JSON object");
assert!(obj.contains_key("trackSid"));
assert!(TRACK_SOURCES.contains(&str_field("stats.outbound[]", entry, "source")));
assert!(TRACK_KINDS.contains(&str_field("stats.outbound[]", entry, "kind")));
assert!(entry.get("bitrateKbps").and_then(Value::as_f64).is_some());
assert!(entry.get("packetsLost").and_then(Value::as_u64).is_some());
}
let inbound = payload
.get("inbound")
.and_then(Value::as_array)
.expect("stats.inbound is an array");
for entry in inbound {
let obj = entry.as_object().expect("stats.inbound[] is a JSON object");
assert!(obj.contains_key("participantSid"));
assert!(obj.contains_key("trackSid"));
assert!(TRACK_KINDS.contains(&str_field("stats.inbound[]", entry, "kind")));
assert!(entry.get("bitrateKbps").and_then(Value::as_f64).is_some());
assert!(entry.get("packetsLost").and_then(Value::as_u64).is_some());
}
}
"audioPlaybackUnavailable" => {
assert_exact_keys(event_type, payload, &["message"]);
assert!(!str_field(event_type, payload, "message").is_empty());
}
"participantNameChanged" => {
assert_exact_keys(event_type, payload, &["sid", "identity", "oldName", "name"]);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "oldName");
let _ = str_field(event_type, payload, "name");
}
"participantMetadataChanged" => {
assert_exact_keys(
event_type,
payload,
&[
"sid",
"identity",
"name",
"oldMetadata",
"metadata",
"attributes",
],
);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
let _ = str_field(event_type, payload, "oldMetadata");
let _ = str_field(event_type, payload, "metadata");
validate_string_map(event_type, payload, "attributes");
}
"participantAttributesChanged" => {
assert_exact_keys(
event_type,
payload,
&["sid", "identity", "name", "attributes", "changedAttributes"],
);
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
let _ = str_field(event_type, payload, "identity");
let _ = str_field(event_type, payload, "name");
validate_string_map(event_type, payload, "attributes");
validate_string_map(event_type, payload, "changedAttributes");
}
other => panic!("unknown eventType in fixture: {other}"),
}
}
#[test]
fn outbound_fixtures_match_contract() {
let recs = records(OUTBOUND);
assert!(!recs.is_empty(), "outbound fixture is non-empty");
for (ty, payload) in &recs {
validate_record(ty, payload);
}
}
#[test]
fn inbound_fixtures_match_contract() {
let recs = records(INBOUND);
assert!(!recs.is_empty(), "inbound fixture is non-empty");
for (ty, payload) in &recs {
validate_record(ty, payload);
}
}
#[test]
fn outbound_covers_the_single_identity_publish_path() {
let types: Vec<String> = records(OUTBOUND).into_iter().map(|(ty, _)| ty).collect();
for expected in [
"connected",
"connectionState",
"Reconnecting",
"Reconnected",
"localTrackPublished",
"localTrackUnpublished",
"localTrackRepublished",
"e2eeState",
"activeSpeakers",
"stats",
"audioPlaybackUnavailable",
] {
assert!(
types.iter().any(|t| t == expected),
"outbound missing {expected}"
);
}
let published: Vec<(String, String)> = records(OUTBOUND)
.into_iter()
.filter(|(ty, _)| ty == "localTrackPublished")
.map(|(_, p)| {
(
p.get("kind").and_then(Value::as_str).unwrap().to_string(),
p.get("source").and_then(Value::as_str).unwrap().to_string(),
)
})
.collect();
assert!(published.contains(&("video".into(), "screen_share".into())));
assert!(published.contains(&("audio".into(), "microphone".into())));
}
#[test]
fn inbound_covers_subscribe_and_lifecycle() {
let types: Vec<String> = records(INBOUND).into_iter().map(|(ty, _)| ty).collect();
for expected in [
"participantJoined",
"participantNameChanged",
"participantMetadataChanged",
"participantAttributesChanged",
"trackPublished",
"trackSubscribed",
"trackMuted",
"trackUnmuted",
"trackSubscriptionFailed",
"trackUnsubscribed",
"trackUnpublished",
"e2eeState",
"activeSpeakers",
"connectionQuality",
"participantLeft",
] {
assert!(
types.iter().any(|t| t == expected),
"inbound missing {expected}"
);
}
}
#[test]
fn video_frame_meta_matches_contract() {
let parsed: Value = serde_json::from_str(VIDEO_FRAME_META).expect("fixture is valid JSON");
let recs = parsed.as_array().expect("video_frame_meta is a JSON array");
assert!(!recs.is_empty(), "video_frame_meta fixture is non-empty");
for meta in recs {
let obj = meta.as_object().expect("meta is a JSON object");
let mut got: Vec<&str> = obj.keys().map(String::as_str).collect();
got.sort_unstable();
assert_eq!(
got,
vec![
"bridgeVersion",
"height",
"participantSid",
"source",
"timestampUs",
"trackName",
"trackSid",
"width"
],
"video_frame_meta key set mismatch"
);
let bridge_version = obj
.get("bridgeVersion")
.and_then(Value::as_u64)
.expect("bridgeVersion is an unsigned integer");
assert!(
bridge_version >= 1,
"bridgeVersion {bridge_version} must be positive"
);
let participant_sid = obj
.get("participantSid")
.and_then(Value::as_str)
.expect("participantSid is a string");
let track_sid = obj
.get("trackSid")
.and_then(Value::as_str)
.expect("trackSid is a string");
let track_name = obj
.get("trackName")
.and_then(Value::as_str)
.expect("trackName is a string");
let source = obj
.get("source")
.and_then(Value::as_str)
.expect("source is a string");
assert!(
participant_sid.starts_with("PA_"),
"participantSid {participant_sid} lacks PA_"
);
assert!(
track_sid.starts_with("TR_"),
"trackSid {track_sid} lacks TR_"
);
assert!(!track_name.is_empty(), "trackName is not empty");
assert!(
TRACK_SOURCES.contains(&source),
"source {source} is a known LiveKit source"
);
let width = obj
.get("width")
.and_then(Value::as_u64)
.expect("width is an integer");
let height = obj
.get("height")
.and_then(Value::as_u64)
.expect("height is an integer");
assert!(
width >= 2 && width % 2 == 0,
"width {width} must be even and >= 2"
);
assert!(
height >= 2 && height % 2 == 0,
"height {height} must be even and >= 2"
);
assert!(
obj.get("timestampUs").and_then(Value::as_i64).is_some(),
"timestampUs is an i64-range integer"
);
}
assert!(
recs.iter()
.filter_map(|m| m.get("timestampUs").and_then(Value::as_i64))
.any(|ts| ts > u32::MAX as i64),
"expect a >2^32 timestampUs in the fixture to lock the i64 contract"
);
}
#[test]
fn e2ee_state_is_ok_on_the_wire() {
for json in [OUTBOUND, INBOUND] {
for (ty, payload) in records(json) {
if ty == "e2eeState" {
assert_eq!(payload.get("state").and_then(Value::as_str), Some("ok"));
}
}
}
}
@@ -0,0 +1,234 @@
[
{
"eventType": "participantJoined",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B"
}
},
{
"eventType": "participantNameChanged",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"oldName": "Test User",
"name": "Test User B"
}
},
{
"eventType": "participantMetadataChanged",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B",
"oldMetadata": "",
"metadata": "{\"status\":\"presenting\"}",
"attributes": {
"role": "speaker"
}
}
},
{
"eventType": "participantAttributesChanged",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B",
"attributes": {
"role": "speaker",
"hand": "raised"
},
"changedAttributes": {
"hand": "raised"
}
}
},
{
"eventType": "trackPublished",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteVideo01",
"trackName": "remote-screen",
"kind": "video",
"source": "screen_share",
"muted": false,
"subscribed": false,
"subscriptionStatus": "desired"
}
},
{
"eventType": "trackSubscribed",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteVideo01",
"trackName": "remote-screen",
"kind": "video",
"source": "screen_share",
"muted": false,
"subscribed": true,
"subscriptionStatus": "subscribed"
}
},
{
"eventType": "trackPublished",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteAudio01",
"trackName": "remote-microphone",
"kind": "audio",
"source": "microphone",
"muted": false,
"subscribed": false,
"subscriptionStatus": "desired"
}
},
{
"eventType": "trackSubscribed",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteAudio01",
"trackName": "remote-microphone",
"kind": "audio",
"source": "microphone",
"muted": false,
"subscribed": true,
"subscriptionStatus": "subscribed"
}
},
{
"eventType": "trackMuted",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteAudio01",
"trackName": "remote-microphone",
"kind": "audio",
"source": "microphone",
"muted": true
}
},
{
"eventType": "trackUnmuted",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteAudio01",
"trackName": "remote-microphone",
"kind": "audio",
"source": "microphone",
"muted": false
}
},
{
"eventType": "trackSubscriptionFailed",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteCamera01",
"trackName": "remote-camera",
"kind": "video",
"source": "camera",
"muted": false,
"subscribed": false,
"subscriptionStatus": "desired",
"error": "could not find published track with sid: \"TR_remoteCamera01\""
}
},
{
"eventType": "trackSubscriptionFailed",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_missingPublication01",
"error": "could not find published track with sid: \"TR_missingPublication01\""
}
},
{
"eventType": "e2eeState",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B",
"state": "ok"
}
},
{
"eventType": "activeSpeakers",
"payload": {
"sids": ["PA_remoteB12345"],
"participants": [
{
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B"
}
]
}
},
{
"eventType": "connectionQuality",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B",
"quality": "excellent"
}
},
{
"eventType": "trackUnsubscribed",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteVideo01",
"trackName": "remote-screen",
"kind": "video",
"source": "screen_share",
"muted": false,
"subscribed": false,
"subscriptionStatus": "unsubscribed"
}
},
{
"eventType": "trackUnpublished",
"payload": {
"participantSid": "PA_remoteB12345",
"identity": "1428486264293441600",
"participantName": "Test User B",
"trackSid": "TR_remoteVideo01",
"trackName": "remote-screen",
"kind": "video",
"source": "screen_share",
"muted": false,
"subscribed": false,
"subscriptionStatus": "unsubscribed"
}
},
{
"eventType": "participantLeft",
"payload": {
"sid": "PA_remoteB12345",
"identity": "1428486264293441600",
"name": "Test User B"
}
},
{
"eventType": "disconnected",
"payload": {
"reason": "clientinitiated"
}
}
]
@@ -0,0 +1,134 @@
[
{
"eventType": "connected",
"payload": {}
},
{
"eventType": "connectionState",
"payload": {
"state": "connected"
}
},
{
"eventType": "Reconnecting",
"payload": {}
},
{
"eventType": "Reconnected",
"payload": {}
},
{
"eventType": "localTrackPublished",
"payload": {
"participantSid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"participantName": "Test User A",
"trackSid": "TR_VShJ9y2A49ai6i",
"trackName": "local-screen",
"kind": "video",
"source": "screen_share",
"muted": false
}
},
{
"eventType": "localTrackPublished",
"payload": {
"participantSid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"participantName": "Test User A",
"trackSid": "TR_AMannbg6PLtcVz",
"trackName": "local-microphone",
"kind": "audio",
"source": "microphone",
"muted": false
}
},
{
"eventType": "localTrackUnpublished",
"payload": {
"participantSid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"participantName": "Test User A",
"trackSid": "TR_AMannbg6PLtcVz",
"trackName": "local-microphone",
"kind": "audio",
"source": "microphone",
"muted": false
}
},
{
"eventType": "localTrackRepublished",
"payload": {
"participantSid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"participantName": "Test User A",
"previousTrackSid": "TR_VShJ9y2A49ai6i",
"trackSid": "TR_VSsTf93n5GqJrK",
"trackName": "local-screen",
"kind": "video",
"source": "screen_share",
"muted": false
}
},
{
"eventType": "e2eeState",
"payload": {
"sid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"name": "Test User A",
"state": "ok"
}
},
{
"eventType": "activeSpeakers",
"payload": {
"sids": ["PA_77jesWWtSMge"],
"participants": [
{
"sid": "PA_77jesWWtSMge",
"identity": "1428486264293441599",
"name": "Test User A"
}
]
}
},
{
"eventType": "stats",
"payload": {
"rttMs": 24.5,
"outbound": [
{
"trackSid": "TR_VShJ9y2A49ai6i",
"source": "screen_share",
"kind": "video",
"bitrateKbps": 2500,
"packetsLost": 0,
"fps": 60
},
{
"trackSid": "TR_AMannbg6PLtcVz",
"source": "microphone",
"kind": "audio",
"bitrateKbps": 48,
"packetsLost": 1
}
],
"inbound": [
{
"participantSid": "PA_Subscriber",
"trackSid": "TR_RemoteVideo",
"kind": "video",
"bitrateKbps": 1200,
"packetsLost": 0,
"jitterMs": 3.5
}
]
}
},
{
"eventType": "audioPlaybackUnavailable",
"payload": {
"message": "platform audio unavailable: test fixture"
}
}
]
@@ -0,0 +1,32 @@
[
{
"bridgeVersion": 14,
"participantSid": "PA_remoteB12345",
"trackSid": "TR_remoteVideo01",
"trackName": "screen",
"source": "screen_share",
"width": 1920,
"height": 1080,
"timestampUs": 123456789
},
{
"bridgeVersion": 14,
"participantSid": "PA_77jesWWtSMge",
"trackSid": "TR_VShJ9y2A49ai6i",
"trackName": "camera",
"source": "camera",
"width": 1280,
"height": 720,
"timestampUs": 0
},
{
"bridgeVersion": 14,
"participantSid": "PA_remoteB12345",
"trackSid": "TR_remoteVideo02",
"trackName": "screen",
"source": "screen_share",
"width": 640,
"height": 360,
"timestampUs": 4294967296
}
]
@@ -0,0 +1 @@
{"v":1}
@@ -0,0 +1,6 @@
{
"git": {
"sha1": "ba0c31290ad6f7836ef96efd62167c3501ed07fa"
},
"path_in_vcs": "libwebrtc"
}
@@ -0,0 +1,293 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
## [0.3.26](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.25...rust-sdks/libwebrtc@0.3.26) - 2026-02-16
### Other
- add is_screencast to VideoSource ([#896](https://github.com/livekit/rust-sdks/pull/896))
## [0.3.25](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.24...rust-sdks/libwebrtc@0.3.25) - 2026-02-09
### Fixed
- fix the 440->441 samples issue and pass a noop callback for release ([#848](https://github.com/livekit/rust-sdks/pull/848))
### Other
- Use workspace dependencies & settings ([#856](https://github.com/livekit/rust-sdks/pull/856))
- allow apm >=10ms frames ([#843](https://github.com/livekit/rust-sdks/pull/843))
## [0.3.24](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.23...rust-sdks/libwebrtc@0.3.24) - 2026-01-15
### Other
- updated the following local packages: webrtc-sys
## [0.3.23](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.22...rust-sdks/libwebrtc@0.3.23) - 2025-12-19
### Fixed
- Exclude the desktop-capturer module link for mobile. ([#817](https://github.com/livekit/rust-sdks/pull/817))
## [0.3.22](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.21...rust-sdks/libwebrtc@0.3.22) - 2025-12-17
### Other
- Expose WebRTC's audio_mixer ([#806](https://github.com/livekit/rust-sdks/pull/806))
## [0.3.21](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.20...rust-sdks/libwebrtc@0.3.21) - 2025-12-04
### Other
- move starting/stopping GLib event loop into libwebrtc crate ([#798](https://github.com/livekit/rust-sdks/pull/798))
- Expose desktop capturer ([#725](https://github.com/livekit/rust-sdks/pull/725))
## [0.3.20](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.19...rust-sdks/libwebrtc@0.3.20) - 2025-11-20
### Other
- Fix the fast path in capture_frame function, without buffering ([#778](https://github.com/livekit/rust-sdks/pull/778))
## [0.3.19](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.18...rust-sdks/libwebrtc@0.3.19) - 2025-10-27
### Other
- updated the following local packages: webrtc-sys
## [0.3.18](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.17...rust-sdks/libwebrtc@0.3.18) - 2025-10-22
### Other
- License check ([#746](https://github.com/livekit/rust-sdks/pull/746))
## [0.3.17](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.16...rust-sdks/libwebrtc@0.3.17) - 2025-10-13
### Added
- *(e2ee)* add data channel encryption ([#708](https://github.com/livekit/rust-sdks/pull/708))
### Other
- Enable buffer scaling ([#473](https://github.com/livekit/rust-sdks/pull/473))
## [0.3.16](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.15...rust-sdks/libwebrtc@0.3.16) - 2025-10-03
### Other
- updated the following local packages: webrtc-sys
## [0.3.15](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.14...rust-sdks/libwebrtc@0.3.15) - 2025-09-29
### Fixed
- fix Builds/E2E Tests CI. ([#715](https://github.com/livekit/rust-sdks/pull/715))
## [0.3.14](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.13...rust-sdks/libwebrtc@0.3.14) - 2025-09-09
### Other
- updated the following local packages: webrtc-sys
## [0.3.13](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.12...rust-sdks/libwebrtc@0.3.13) - 2025-09-03
### Other
- updated the following local packages: webrtc-sys
# Changelog
## [0.3.12](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.11...rust-sdks/libwebrtc@0.3.12) - 2025-06-17
### Other
- updated the following local packages: livekit-protocol, webrtc-sys
## [0.3.11](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.10...rust-sdks/libwebrtc@0.3.11) - 2025-06-11
### Fixed
- fix uint32 overflow ([#615](https://github.com/livekit/rust-sdks/pull/615))
### Other
- remove ([#633](https://github.com/livekit/rust-sdks/pull/633))
- expose apm stream_delay ([#616](https://github.com/livekit/rust-sdks/pull/616))
- Add i420_to_nv12 ([#605](https://github.com/livekit/rust-sdks/pull/605))
- ffi-v0.13.0 ([#590](https://github.com/livekit/rust-sdks/pull/590))
- add AudioProcessingModule ([#580](https://github.com/livekit/rust-sdks/pull/580))
## [0.3.10] - 2025-02-05
### Fixed
- Fix build issue
## [0.3.9] - 2025-01-17
### Added
- Expose DataChannel.bufferedAmount property
## [0.3.8] - 2024-12-14
### Added
- bump libwebrtc to m125
## 0.3.35 (2026-05-29)
### Fixes
- Add native video pipeline timing instrumentation for local video measurements, exposing local publish and subscribe timing through async streams and subscriber overlay GPU upload and receive-to-GPU latency metrics through explicit timing observers.
## 0.3.34 (2026-05-21)
### Fixes
#### feat: add Android application context initialization for PlatformAudio support.
Android requires `ContextUtils.initialize(applicationContext)` before WebRTC audio components can be created. This change:
- Adds `livekit_ffi_initialize_android_context()` C FFI function for Unity and other FFI consumers
- Uses `CreateAndroidAudioDeviceModule()` instead of generic `CreateAudioDeviceModule()` on Android
- Handles empty device GUIDs on Android (falls back to index 0)
- Documents Android-specific limitations: single default device, no app-level device selection
Platform notes:
- Android device enumeration returns only one "default" device with empty name/GUID
- Audio routing (speaker/earpiece/Bluetooth) is controlled by Android's AudioManager, not WebRTC
## 0.3.33 (2026-05-14)
### Fixes
- feat: add scalability mode for AV1/VP9. - #1076 (@cloudwebrtc)
- Add `LIVEKIT_PREFERRED_HW_ENCODER` to prefer `nvenc` or `vaapi` hardware video encoding when both are available.
- Relocate unrelated types out of `livekit-protocol`
#### Get WebRTC ADM into Rust - #1037 (@xianshijing-lk)
This PR introduces platform audio device management via WebRTC's Audio Device Module (ADM).
#### Features
- **ADM Proxy**: New `AdmProxy` class that switches between Dummy ADM (synthetic mode) and Platform ADM (real audio I/O)
- **PlatformAudio API**: High-level Rust API for microphone capture and speaker playout with AEC/AGC/NS
- **Device enumeration**: List and select recording/playout devices by index or GUID
- **Mode switching**: Seamlessly switch between synthetic mode (FFI callbacks) and platform mode (native speakers) while audio is active
- **FFI platform audio support**: Expose platform audio device enumeration and selection through `livekit-ffi`
- **Audio processing**: Configure echo cancellation, noise suppression, and auto gain control with platform-specific defaults (hardware on iOS, software elsewhere)
#### Audio Modes
| Mode | Recording | Playout | Use Case |
|------|-----------|---------|----------|
| Synthetic | NativeAudioSource | Dummy ADM + FFI | Unity audio, agents |
| Platform | Platform ADM mic | Platform ADM speakers | VoIP with AEC |
#### API
```rust
// Create PlatformAudio for microphone/speaker access
let audio = PlatformAudio::new()?;
// Enumerate and select devices
for i in 0..audio.recording_devices() as u16 {
println!("Mic {}: {}", i, audio.recording_device_name(i));
}
audio.set_recording_device(0)?;
// Create audio track for publishing
let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source());
```
## 0.3.32 (2026-05-11)
### Fixes
- Upgrade protocol to v1.45.8
## 0.3.31 (2026-05-10)
### Fixes
- Fix missing `libwebrtc.jar` for Android builds, harden build scripts
- fix race in download_webrtc to reduce flaky build - #1047 (@hechen-eng)
- Improve WebRTC build scripts and add external_audio_source patch - #1053 (@xianshijing-lk)
## 0.3.30 (2026-04-23)
### Features
#### Add support for frame level packet trailer
##890 by @chenosaurus
- Add support to attach/parse frame level timestamps & frame ID to VideoTracks as a custom payload trailer.
- Breaking change in VideoFrame API, must include `frame_metadata` or use VideoFrame::new().
## 0.3.29 (2026-04-02)
### Features
#### chore: upgrade libwebrtc to m144.
##965 by @cloudwebrtc
### Fixes
#### use the bounded buffer for video stream
##956 by @xianshijing-lk
Before this PR, it uses an unbounded buffer for video stream, that will cause multiple problems:
1, video will be lagged behind if rendering is slow or just wake up from background
2, it will be out of sync with audio
This PRs provides options to set a bounded buffer for video stream, and use 1 buffer as the default option.
## 0.3.28 (2026-03-31)
### Fixes
- Upgrade to thiserror 2
#### fix: fix unavailable sem symbol for Linux aarch64.
##975 by @cloudwebrtc
## 0.3.27 (2026-03-22)
### Features
#### E2EE: allow setting key_ring_size and key_derivation_algorithm, update webrtc to m144
##921 by @onestacked
This PR uses [this webrtc-sdk PR](https://github.com/webrtc-sdk/webrtc/pull/224) to configure the KDF.
I've tested this with https://codeberg.org/esoteric_programmer/matrix-jukebox and it is compatible with Element Call.
Fixed: https://github.com/livekit/rust-sdks/issues/796
### Fixes
- Fix H.264 codec matching
#### add bounded buffer to audio_stream, and use 10 frames as the default
##945 by @xianshijing-lk
#### fix clang build issue from zed patches (#949)
##950 by @cloudwebrtc
* webrtc-sys: Use clang instead of gcc
* Debug CI output for aarch64-linux
* ci: Install lld for aarch64-linux FFI builders
* webrtc-sys: Disable CREL
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,110 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
name = "libwebrtc"
version = "0.3.35"
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Livekit safe bindings to libwebrtc"
homepage = "https://livekit.io"
readme = false
license = "Apache-2.0"
repository = "https://github.com/livekit/rust-sdks"
[features]
default = ["glib-main-loop"]
glib-main-loop = ["dep:glib"]
[lib]
name = "libwebrtc"
path = "src/lib.rs"
[dependencies.log]
version = "0.4"
[dependencies.serde]
version = "1"
features = ["derive"]
[dependencies.serde_json]
version = "1.0"
[dependencies.thiserror]
version = "2"
[dev-dependencies.env_logger]
version = "0.11"
[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies.glib]
version = "0.21.3"
optional = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.cxx]
version = "1.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.lazy_static]
version = "1.4"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.livekit-runtime]
version = "0.4.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.parking_lot]
version = "0.12"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.rtrb]
version = "0.3.3"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.tokio]
version = "1"
features = [
"sync",
"macros",
]
default-features = false
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.webrtc-sys]
version = "0.3.33"
[target.'cfg(target_arch = "wasm32")'.dependencies.js-sys]
version = "0.3"
[target.'cfg(target_arch = "wasm32")'.dependencies.wasm-bindgen]
version = "0.2"
[target.'cfg(target_arch = "wasm32")'.dependencies.wasm-bindgen-futures]
version = "0.4"
[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
version = "0.3"
features = [
"MessageEvent",
"RtcPeerConnection",
"RtcSignalingState",
"RtcSdpType",
"RtcSessionDescriptionInit",
"RtcPeerConnectionIceEvent",
"RtcIceCandidate",
"RtcDataChannel",
"RtcDataChannelEvent",
"RtcDataChannelState",
"EventTarget",
"WebGlRenderingContext",
"WebGlTexture",
]
[target.'cfg(target_os = "android")'.dependencies.jni]
version = "0.21"
+60
View File
@@ -0,0 +1,60 @@
[package]
name = "libwebrtc"
version = "0.3.35"
edition.workspace = true
homepage = "https://livekit.io"
license.workspace = true
description = "Livekit safe bindings to libwebrtc"
repository.workspace = true
[features]
default = [ "glib-main-loop" ]
# On Wayland, libwebrtc uses GDBus to communicate with the XDG Desktop Portal.
# GDBus requires a GLib event loop to be running. If you already have a GLib
# event loop running in your application, for example if you are using the
# GTK or GStreamer Rust bindings, disable this feature.
glib-main-loop = [ "dep:glib" ]
[dependencies]
log = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies]
glib = { version = "0.21.3", optional = true }
[target.'cfg(target_os = "android")'.dependencies]
jni = "0.21"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
webrtc-sys = { workspace = true }
livekit-runtime = { workspace = true }
lazy_static = { workspace = true }
parking_lot = { workspace = true }
tokio = { workspace = true, default-features = false, features = ["sync", "macros"] }
cxx = "1.0"
rtrb = "0.3.3"
[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen = "0.2"
js-sys = "0.3"
wasm-bindgen-futures = "0.4"
web-sys = { version = "0.3", features = [
"MessageEvent",
"RtcPeerConnection",
"RtcSignalingState",
"RtcSdpType",
"RtcSessionDescriptionInit",
"RtcPeerConnectionIceEvent",
"RtcIceCandidate",
"RtcDataChannel",
"RtcDataChannelEvent",
"RtcDataChannelState",
"EventTarget",
"WebGlRenderingContext",
"WebGlTexture",
] }
[dev-dependencies]
env_logger = { workspace = true }
@@ -0,0 +1,35 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::borrow::Cow;
#[derive(Debug, Clone)]
pub struct AudioFrame<'a> {
pub data: Cow<'a, [i16]>,
pub sample_rate: u32,
pub num_channels: u32,
pub samples_per_channel: u32,
}
impl AudioFrame<'_> {
// Owned
pub fn new(sample_rate: u32, num_channels: u32, samples_per_channel: u32) -> Self {
Self {
data: vec![0; (num_channels * samples_per_channel) as usize].into(),
sample_rate,
num_channels,
samples_per_channel,
}
}
}
@@ -0,0 +1,222 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::audio_source as imp_as;
/// Default sample rate used by WebRTC audio pipelines (48kHz).
pub const DEFAULT_SAMPLE_RATE: u32 = 48000;
/// Default number of audio channels (mono).
pub const DEFAULT_NUM_CHANNELS: u32 = 1;
#[derive(Default, Debug)]
pub struct AudioSourceOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
/// Audio source type for creating audio tracks.
///
/// Choose the appropriate source based on your use case:
///
/// | Use Case | Source | Description |
/// |----------|--------|-------------|
/// | Manual audio (TTS, files) | `RtcAudioSource::Native(source)` | Push frames manually |
/// | Microphone capture | `RtcAudioSource::Device` | Automatic via Platform ADM |
/// | Both (mic + screen) | Use both types | Multiple tracks supported |
///
/// # Combining Sources
///
/// You can have multiple audio tracks with different source types:
/// - Track A: `RtcAudioSource::Device` for microphone (via `PlatformAudio`)
/// - Track B: `RtcAudioSource::Native` for screen capture or TTS
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcAudioSource {
/// Native audio source for manual audio frame capture.
///
/// Use this with Synthetic ADM mode (the default). You push audio frames
/// manually via `NativeAudioSource::capture_frame()`.
///
/// # Example
///
/// ```rust,ignore
/// use livekit::webrtc::audio_source::native::NativeAudioSource;
/// use livekit::webrtc::audio_source::{AudioSourceOptions, RtcAudioSource};
///
/// let source = NativeAudioSource::new(
/// AudioSourceOptions::default(),
/// 48000, 2, 100,
/// );
/// source.capture_frame(&frame).await?;
///
/// let track = LocalAudioTrack::create_audio_track(
/// "audio",
/// RtcAudioSource::Native(source),
/// );
/// ```
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeAudioSource),
/// Device audio source - uses Platform ADM for automatic microphone capture.
///
/// WebRTC automatically captures audio from the selected recording device
/// (microphone). You do NOT push frames manually.
///
/// # Usage
///
/// Use `PlatformAudio` from the `livekit` crate, which manages the Platform ADM
/// lifecycle and provides `RtcAudioSource::Device` via `rtc_source()`:
///
/// ```rust,ignore
/// use livekit::prelude::*;
///
/// // Create PlatformAudio (enables Platform ADM)
/// let audio = PlatformAudio::new()?;
///
/// // Optionally select a specific device
/// if let Some(device) = audio.recording_devices().next() {
/// audio.set_recording_device(&device.id)?;
/// }
///
/// // Create track using the device source
/// let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source());
/// ```
///
/// # Combining with NativeAudioSource
///
/// You CAN use `NativeAudioSource` alongside Platform ADM to have multiple
/// audio tracks with different sources (e.g., microphone + screen capture).
///
/// # Platform Support
///
/// - **iOS**: CoreAudio with VPIO (Voice Processing IO)
/// - **macOS**: CoreAudio
/// - **Windows**: WASAPI
/// - **Linux**: PulseAudio / ALSA
/// - **Android**: AAudio / OpenSL ES
#[cfg(not(target_arch = "wasm32"))]
Device,
}
impl RtcAudioSource {
/// Set audio processing options.
/// Note: For `Device` source, options are controlled by the Platform ADM.
pub fn set_audio_options(&self, options: AudioSourceOptions) {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.set_audio_options(options),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => {
// Device source options are managed by the Platform ADM
// This is a no-op
}
}
}
/// Get audio processing options.
/// Note: For `Device` source, returns default options (actual options are managed by ADM).
pub fn audio_options(&self) -> AudioSourceOptions {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.audio_options(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => AudioSourceOptions::default(),
}
}
/// Get the sample rate.
/// Note: For `Device` source, returns [`DEFAULT_SAMPLE_RATE`] (48kHz).
pub fn sample_rate(&self) -> u32 {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.sample_rate(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => DEFAULT_SAMPLE_RATE,
}
}
/// Get the number of channels.
/// Note: For `Device` source, returns [`DEFAULT_NUM_CHANNELS`] (mono).
pub fn num_channels(&self) -> u32 {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.num_channels(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => DEFAULT_NUM_CHANNELS,
}
}
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::{Debug, Formatter};
use super::*;
use crate::{audio_frame::AudioFrame, RtcError};
#[derive(Clone)]
pub struct NativeAudioSource {
pub(crate) handle: imp_as::NativeAudioSource,
}
impl Debug for NativeAudioSource {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeAudioSource").finish()
}
}
impl NativeAudioSource {
pub fn new(
options: AudioSourceOptions,
sample_rate: u32,
num_channels: u32,
queue_size_ms: u32,
) -> NativeAudioSource {
Self {
handle: imp_as::NativeAudioSource::new(
options,
sample_rate,
num_channels,
queue_size_ms,
),
}
}
pub fn clear_buffer(&self) {
self.handle.clear_buffer()
}
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
self.handle.capture_frame(frame).await
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.handle.set_audio_options(options)
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.handle.audio_options()
}
pub fn sample_rate(&self) -> u32 {
self.handle.sample_rate()
}
pub fn num_channels(&self) -> u32 {
self.handle.num_channels()
}
}
}
@@ -0,0 +1,113 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::audio_stream as stream_imp;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::{
fmt::{Debug, Formatter},
pin::Pin,
task::{Context, Poll},
};
use livekit_runtime::Stream;
use super::stream_imp;
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 10;
#[derive(Clone, Debug, Default)]
pub struct NativeAudioStreamOptions {
/// Maximum number of queued WebRTC sink frames after the audio callback.
///
/// Each queued frame corresponds to roughly 10 ms of decoded PCM audio
/// on the WebRTC sink path.
///
/// `None` uses the default bounded queue size of 10 frames. `Some(0)`
/// opts into unbounded buffering. Positive values bound the queue, and
/// the stream drops the oldest queued frames on overflow so latency
/// stays bounded.
///
/// If your application consumes both audio and video, keep the queue
/// sizing strategy coordinated across both streams. Using a much larger
/// queue, or unbounded buffering, for only one of them can increase
/// end-to-end latency for that stream and cause audio/video drift.
pub queue_size_frames: Option<usize>,
}
pub struct NativeAudioStream {
pub(crate) handle: stream_imp::NativeAudioStream,
}
impl Debug for NativeAudioStream {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeAudioStream").field("track", &self.track()).finish()
}
}
impl NativeAudioStream {
pub fn new(audio_track: RtcAudioTrack, sample_rate: i32, num_channels: i32) -> Self {
Self {
handle: stream_imp::NativeAudioStream::new(
audio_track,
sample_rate,
num_channels,
Some(DEFAULT_QUEUE_SIZE_FRAMES),
),
}
}
pub fn with_options(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
options: NativeAudioStreamOptions,
) -> Self {
Self {
handle: stream_imp::NativeAudioStream::new(
audio_track,
sample_rate,
num_channels,
normalize_queue_size_frames(options.queue_size_frames),
),
}
}
pub fn track(&self) -> RtcAudioTrack {
self.handle.track()
}
pub fn close(&mut self) {
self.handle.close()
}
}
impl Stream for NativeAudioStream {
type Item = AudioFrame<'static>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.get_mut().handle).poll_next(cx)
}
}
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
match queue_size_frames {
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
Some(0) => None,
Some(value) => Some(value),
}
}
}
@@ -0,0 +1,39 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::audio_track as imp_at,
media_stream_track::{media_stream_track, RtcTrackState},
};
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) handle: imp_at::RtcAudioTrack,
}
impl RtcAudioTrack {
media_stream_track!();
}
impl Debug for RtcAudioTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcAudioTrack")
.field("id", &self.id())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,125 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{fmt::Debug, str::Utf8Error};
use serde::Deserialize;
use thiserror::Error;
use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority};
#[derive(Clone, Debug)]
pub struct DataChannelInit {
pub ordered: bool,
pub max_retransmit_time: Option<i32>,
pub max_retransmits: Option<i32>,
pub protocol: String,
pub negotiated: bool,
pub id: i32,
pub priority: Option<Priority>,
}
impl Default for DataChannelInit {
fn default() -> Self {
Self {
ordered: true,
max_retransmit_time: None,
max_retransmits: None,
protocol: String::new(),
negotiated: false,
id: -1,
priority: None,
}
}
}
#[derive(Debug, Error)]
pub enum DataChannelError {
#[error("failed to send data, dc not open? send buffer is full ?")]
Send,
#[error("only utf8 strings can be sent")]
Utf8(#[from] Utf8Error),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DataChannelState {
Connecting,
Open,
Closing,
Closed,
}
#[derive(Debug)]
pub struct DataBuffer<'a> {
pub data: &'a [u8],
pub binary: bool,
}
pub type OnStateChange = Box<dyn FnMut(DataChannelState) + Send + Sync>;
pub type OnMessage = Box<dyn FnMut(DataBuffer) + Send + Sync>;
pub type OnBufferedAmountChange = Box<dyn FnMut(u64) + Send + Sync>;
#[derive(Clone)]
pub struct DataChannel {
pub(crate) handle: dc_imp::DataChannel,
}
impl DataChannel {
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
self.handle.send(data, binary)
}
pub fn id(&self) -> i32 {
self.handle.id()
}
pub fn label(&self) -> String {
self.handle.label()
}
pub fn state(&self) -> DataChannelState {
self.handle.state()
}
pub fn close(&self) {
self.handle.close()
}
pub fn buffered_amount(&self) -> u64 {
self.handle.buffered_amount()
}
pub fn on_state_change(&self, callback: Option<OnStateChange>) {
self.handle.on_state_change(callback)
}
pub fn on_message(&self, callback: Option<OnMessage>) {
self.handle.on_message(callback)
}
pub fn on_buffered_amount_change(&self, callback: Option<OnBufferedAmountChange>) {
self.handle.on_buffered_amount_change(callback)
}
}
impl Debug for DataChannel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DataChannel")
.field("id", &self.id())
.field("label", &self.label())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,230 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::desktop_capturer as imp_dc;
/// Configuration options for creating a desktop capturer.
///
/// It contains a subset of libwebrtc's DesktopCaptureOptions.
///
/// By default, it captures the entire screen and does not include the cursor.
///
/// # Example
/// ```no_run
/// use libwebrtc::desktop_capturer::{DesktopCapturerOptions, DesktopCaptureSourceType};
///
/// let mut options = DesktopCapturerOptions::new(DesktopCaptureSourceType::Screen);
/// options.set_include_cursor(true);
/// ```
pub struct DesktopCapturerOptions {
sys_handle: imp_dc::DesktopCapturerOptions,
}
/// Specifies the type of source that a desktop capturer should capture.
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum DesktopCaptureSourceType {
Screen,
Window,
#[cfg(any(target_os = "macos", target_os = "linux"))]
Generic,
}
impl DesktopCapturerOptions {
/// Creates a new `DesktopCapturerOptions` with default values.
///
/// # Arguments
///
/// * `source_type` - The type of source to capture (screen or window).
///
/// # Defaults
///
/// - Cursor is not included in captured frames (use [`set_include_cursor`](Self::set_include_cursor) to change)
/// - On macOS, the ScreenCaptureKit system picker is enabled (use [`set_sck_system_picker`](Self::set_sck_system_picker) to change)
pub fn new(source_type: DesktopCaptureSourceType) -> Self {
let source_type = match source_type {
DesktopCaptureSourceType::Screen => imp_dc::SourceType::Screen,
DesktopCaptureSourceType::Window => imp_dc::SourceType::Window,
#[cfg(any(target_os = "macos", target_os = "linux"))]
DesktopCaptureSourceType::Generic => imp_dc::SourceType::Generic,
};
Self { sys_handle: imp_dc::DesktopCapturerOptions::new(source_type) }
}
/// Sets whether to include the cursor in captured frames.
pub fn set_include_cursor(&mut self, include: bool) {
self.sys_handle = self.sys_handle.with_cursor(include);
}
/// Sets whether to allow the ScreenCaptureKit system picker on macOS.
///
/// This is enabled by default.
///
/// When disabled, for capturing displays the client should get the source id
/// via a different way as [`DesktopCapturer::get_source_list`] returns an empty vector.
#[cfg(target_os = "macos")]
pub fn set_sck_system_picker(&mut self, allow_sck_system_picker: bool) {
self.sys_handle = self.sys_handle.with_sck_system_picker(allow_sck_system_picker);
}
}
/// A desktop capturer for capturing screens or windows.
pub struct DesktopCapturer {
handle: imp_dc::DesktopCapturer,
}
impl DesktopCapturer {
/// Creates a new `DesktopCapturer` with the specified callback and options.
///
/// # Arguments
///
/// * `options` - Configuration options for the capturer
///
/// # Returns
///
/// Returns `Some(DesktopCapturer)` if the capturer was created successfully,
/// or `None` if creation failed (e.g., due to platform limitations or permissions).
pub fn new(options: DesktopCapturerOptions) -> Option<Self> {
let desktop_capturer = imp_dc::DesktopCapturer::new(options.sys_handle);
if desktop_capturer.is_none() {
return None;
}
Some(Self { handle: desktop_capturer.unwrap() })
}
/// Starts capturing from the specified source.
///
/// # Arguments
///
/// * `source` - The capture source to use. It should be None when the capturer
/// is configured to use the system picker (on platforms that support it).
/// * `callback` - A function that will be called for each captured frame. The callback
/// receives a [`CaptureResult`] indicating success or error, and a [`DesktopFrame`]
/// containing the captured image data.
///
/// # Note
///
/// After calling this method, you must call [`capture_frame`](Self::capture_frame)
/// to actually capture frames. This method only initializes the capture session.
pub fn start_capture<T>(&mut self, source: Option<CaptureSource>, mut callback: T)
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
{
if let Some(source) = source {
self.handle.select_source(source.sys_handle.id());
}
let inner_callback = move |result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>| {
callback(capture_result_from_sys(result));
};
self.handle.start(inner_callback);
}
/// Captures a single frame.
///
/// You must call [`start_capture`](Self::start_capture) before calling this method.
pub fn capture_frame(&mut self) {
self.handle.capture_frame();
}
/// Retrieves a list of available capture sources.
///
/// Returns a list of screens or windows that can be captured, depending
/// on whether the capturer was configured for window or screen capture.
///
/// # Returns
///
/// A vector of [`CaptureSource`] objects representing available capture sources.
pub fn get_source_list(&self) -> Vec<CaptureSource> {
let source_list = self.handle.get_source_list();
source_list.into_iter().map(|source| CaptureSource { sys_handle: source }).collect()
}
}
pub struct DesktopFrame {
sys_handle: imp_dc::DesktopFrame,
}
impl DesktopFrame {
fn new(sys_handle: imp_dc::DesktopFrame) -> Self {
Self { sys_handle }
}
pub fn width(&self) -> i32 {
self.sys_handle.width() as i32
}
pub fn height(&self) -> i32 {
self.sys_handle.height() as i32
}
pub fn stride(&self) -> u32 {
self.sys_handle.stride() as u32
}
pub fn left(&self) -> i32 {
self.sys_handle.left()
}
pub fn top(&self) -> i32 {
self.sys_handle.top()
}
pub fn data(&self) -> &[u8] {
self.sys_handle.data()
}
}
#[derive(Clone)]
pub struct CaptureSource {
sys_handle: imp_dc::CaptureSource,
}
impl CaptureSource {
pub fn id(&self) -> u64 {
self.sys_handle.id()
}
pub fn title(&self) -> String {
self.sys_handle.title()
}
pub fn display_id(&self) -> i64 {
self.sys_handle.display_id()
}
}
impl std::fmt::Display for CaptureSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CaptureSource")
.field("id", &self.id())
.field("title", &self.title())
.field("display_id", &self.display_id())
.finish()
}
}
#[derive(Debug, PartialEq)]
pub enum CaptureError {
Temporary,
Permanent,
}
fn capture_result_from_sys(
result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>,
) -> Result<DesktopFrame, CaptureError> {
match result {
Ok(frame) => Ok(DesktopFrame::new(frame)),
Err(error) => Err(match error {
imp_dc::CaptureError::Temporary => CaptureError::Temporary,
imp_dc::CaptureError::Permanent => CaptureError::Permanent,
}),
}
}
@@ -0,0 +1,41 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO(theomonnom): Async methods
#[macro_export]
macro_rules! enum_dispatch {
// This arm is used to avoid nested loops with the arguments
// The arguments are transformed to $combined_args tt
(@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => {
match $self {
$(
Self::$variant(inner) => inner.$fnc$combined_args,
)+
}
};
// Create the function and extract self fron the $args tt (little hack)
(@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => {
#[inline]
$vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret {
$crate::enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
}
};
($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
$(
$crate::enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
)+
};
}
@@ -0,0 +1,55 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError};
pub struct IceCandidate {
pub(crate) handle: imp_ic::IceCandidate,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<IceCandidate, SdpParseError> {
imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp)
}
pub fn sdp_mid(&self) -> String {
self.handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for IceCandidate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IceCandidate").field("candidate", &self.to_string()).finish()
}
}
@@ -0,0 +1,82 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use thiserror::Error;
#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")]
#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")]
mod imp;
mod enum_dispatch;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum MediaType {
Audio,
Video,
Data,
Unsupported,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcErrorType {
Internal,
InvalidSdp,
InvalidState,
}
#[derive(Error, Debug)]
#[error("an RtcError occurred: {error_type:?} - {message}")]
pub struct RtcError {
pub error_type: RtcErrorType,
pub message: String,
}
pub mod audio_frame;
pub mod audio_source;
pub mod audio_stream;
pub mod audio_track;
pub mod data_channel;
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
pub mod desktop_capturer;
pub mod ice_candidate;
pub mod media_stream;
pub mod media_stream_track;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod prelude;
pub mod recorded_audio;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod stats;
pub mod video_frame;
pub mod video_source;
pub mod video_stream;
pub mod video_track;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
pub use webrtc_sys::webrtc::ffi::create_random_uuid;
pub use crate::imp::{
apm, audio_mixer, audio_resampler, frame_cryptor, packet_trailer, yuv_helper,
};
}
#[cfg(target_os = "android")]
pub mod android {
pub use crate::imp::android::*;
}
@@ -0,0 +1,46 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{audio_track::RtcAudioTrack, imp::media_stream as imp_ms, video_track::RtcVideoTrack};
#[derive(Clone)]
pub struct MediaStream {
pub(crate) handle: imp_ms::MediaStream,
}
impl MediaStream {
pub fn id(&self) -> String {
self.handle.id()
}
pub fn audio_tracks(&self) -> Vec<RtcAudioTrack> {
self.handle.audio_tracks()
}
pub fn video_tracks(&self) -> Vec<RtcVideoTrack> {
self.handle.video_tracks()
}
}
impl Debug for MediaStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MediaStream")
.field("id", &self.id())
.field("audio_tracks", &self.audio_tracks())
.field("video_tracks", &self.video_tracks())
.finish()
}
}
@@ -0,0 +1,86 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{audio_track::RtcAudioTrack, enum_dispatch, video_track::RtcVideoTrack};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcTrackState {
Live,
Ended,
}
#[derive(Debug, Clone)]
pub enum MediaStreamTrack {
Video(RtcVideoTrack),
Audio(RtcAudioTrack),
}
#[cfg(not(target_arch = "wasm32"))]
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack>;
);
}
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub fn id(self: &Self) -> String;
pub fn enabled(self: &Self) -> bool;
pub fn set_enabled(self: &Self, enabled: bool) -> bool;
pub fn state(self: &Self) -> RtcTrackState;
);
}
macro_rules! media_stream_track {
() => {
pub fn id(&self) -> String {
self.handle.id()
}
pub fn enabled(&self) -> bool {
self.handle.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
self.handle.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
self.handle.state().into()
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn sys_handle(
&self,
) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack> {
self.handle.sys_handle()
}
};
}
pub(crate) use media_stream_track;
impl From<RtcAudioTrack> for MediaStreamTrack {
fn from(track: RtcAudioTrack) -> Self {
Self::Audio(track)
}
}
impl From<RtcVideoTrack> for MediaStreamTrack {
fn from(track: RtcVideoTrack) -> Self {
Self::Video(track)
}
}
@@ -0,0 +1,65 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use jni::objects::JObject;
use webrtc_sys::android::ffi as sys_android;
/// Initialize Android WebRTC with the JVM.
///
/// This is automatically called by [`initialize_android_context`], so you only
/// need to call this directly if you don't have access to an Android Context
/// (e.g., in `JNI_OnLoad`).
///
/// This function is idempotent - safe to call multiple times.
pub fn initialize_android(vm: &jni::JavaVM) {
unsafe {
sys_android::init_android(vm.get_java_vm_pointer() as *mut _);
}
}
/// Initialize Android WebRTC with the application context.
///
/// This is the main initialization function for Android. It performs both:
/// 1. JVM initialization (same as [`initialize_android`])
/// 2. Context initialization (required for PlatformAudio)
///
/// This function is idempotent - safe to call multiple times.
///
/// # Arguments
/// * `vm` - The JavaVM instance
/// * `context` - The Android application context
///
/// # Returns
/// `true` if context initialization succeeded, `false` otherwise.
/// Note: JVM initialization always happens regardless of return value.
///
/// # Example
/// ```ignore
/// use jni::JavaVM;
/// use jni::objects::JObject;
/// use livekit::webrtc::android::initialize_android_context;
///
/// fn init(vm: JavaVM, context: JObject) {
/// // Just one call needed - handles both JVM and context init
/// initialize_android_context(&vm, &context);
/// }
/// ```
pub fn initialize_android_context(vm: &jni::JavaVM, context: &JObject) -> bool {
unsafe {
sys_android::init_android_context(
vm.get_java_vm_pointer() as *mut _,
context.as_raw() as usize,
)
}
}
@@ -0,0 +1,119 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::apm::ffi as sys_apm;
use crate::{RtcError, RtcErrorType};
pub struct AudioProcessingModule {
sys_handle: UniquePtr<sys_apm::AudioProcessingModule>,
}
impl AudioProcessingModule {
pub fn new(
echo_canceller_enabled: bool,
gain_controller_enabled: bool,
high_pass_filter_enabled: bool,
noise_suppression_enabled: bool,
) -> Self {
Self {
sys_handle: sys_apm::create_apm(
echo_canceller_enabled,
gain_controller_enabled,
high_pass_filter_enabled,
noise_suppression_enabled,
),
}
}
pub fn process_stream(
&mut self,
data: &mut [i16],
sample_rate: i32,
num_channels: i32,
) -> Result<(), RtcError> {
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
assert!(
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
"slice must have a multiple of 10ms worth of samples"
);
for chunk in data.chunks_mut(samples_per_10ms) {
if unsafe {
self.sys_handle.pin_mut().process_stream(
chunk.as_mut_ptr(),
chunk.len(),
chunk.as_mut_ptr(),
chunk.len(),
sample_rate,
num_channels,
)
} != 0
{
return Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to process stream".to_string(),
});
}
}
Ok(())
}
pub fn process_reverse_stream(
&mut self,
data: &mut [i16],
sample_rate: i32,
num_channels: i32,
) -> Result<(), RtcError> {
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
assert!(
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
"slice must have a multiple of 10ms worth of samples"
);
for chunk in data.chunks_mut(samples_per_10ms) {
if unsafe {
self.sys_handle.pin_mut().process_reverse_stream(
chunk.as_mut_ptr(),
chunk.len(),
chunk.as_mut_ptr(),
chunk.len(),
sample_rate,
num_channels,
)
} != 0
{
return Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to process reverse stream".to_string(),
});
}
}
Ok(())
}
pub fn set_stream_delay_ms(&mut self, delay_ms: i32) -> Result<(), RtcError> {
if self.sys_handle.pin_mut().set_stream_delay_ms(delay_ms) == 0 {
Ok(())
} else {
Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to set stream delay".to_string(),
})
}
}
}
@@ -0,0 +1,108 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::audio_frame::AudioFrame;
use cxx::UniquePtr;
use std::sync::Arc;
use webrtc_sys::audio_mixer as sys;
use webrtc_sys::audio_mixer::ffi;
pub struct AudioMixer {
sys_handle: UniquePtr<ffi::AudioMixer>,
}
pub use ffi::AudioFrameInfo;
pub trait AudioMixerSource {
fn ssrc(&self) -> i32;
fn preferred_sample_rate(&self) -> u32;
fn get_audio_frame_with_info(&self, target_sample_rate: u32) -> Option<AudioFrame<'_>>;
}
struct AudioMixerSourceImpl<T> {
inner: T,
}
impl<T: AudioMixerSource> sys::AudioMixerSource for AudioMixerSourceImpl<T> {
fn ssrc(&self) -> i32 {
self.inner.ssrc()
}
fn preferred_sample_rate(&self) -> i32 {
self.inner.preferred_sample_rate() as i32
}
fn get_audio_frame_with_info(
&self,
target_sample_rate: i32,
native_frame: sys::NativeAudioFrame,
) -> AudioFrameInfo {
if let Some(frame) = self.inner.get_audio_frame_with_info(target_sample_rate as u32) {
let samples_count = (frame.sample_rate as usize / 100) as usize;
assert_eq!(
frame.sample_rate, target_sample_rate as u32,
"sample rate must match target_sample_rate"
);
assert_eq!(
frame.samples_per_channel as usize, samples_count,
"frame must contain 10ms of samples"
);
assert_eq!(
frame.data.len(),
samples_count * frame.num_channels as usize,
"slice must contain 10ms of samples"
);
unsafe {
native_frame.update_frame(
0,
frame.data.as_ptr(),
frame.samples_per_channel as usize,
frame.sample_rate as i32,
frame.num_channels as usize,
);
}
return ffi::AudioFrameInfo::Normal;
} else {
return ffi::AudioFrameInfo::Muted;
}
}
}
impl AudioMixer {
pub fn new() -> Self {
let sys_handle = ffi::create_audio_mixer();
Self { sys_handle }
}
pub fn add_source(&mut self, source: impl AudioMixerSource + 'static) {
let source_impl = AudioMixerSourceImpl { inner: source };
let wrapper = Box::new(sys::AudioMixerSourceWrapper::new(Arc::new(source_impl)));
unsafe {
self.sys_handle.pin_mut().add_source(wrapper);
}
}
pub fn remove_source(&mut self, ssrc: i32) {
unsafe {
self.sys_handle.pin_mut().remove_source(ssrc);
}
}
pub fn mix(&mut self, num_channels: usize) -> &[i16] {
unsafe {
let len = self.sys_handle.pin_mut().mix(num_channels);
std::slice::from_raw_parts(self.sys_handle.data(), len)
}
}
}
@@ -0,0 +1,53 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::audio_resampler as sys_ar;
pub struct AudioResampler {
sys_handle: UniquePtr<sys_ar::ffi::AudioResampler>,
}
impl Default for AudioResampler {
fn default() -> Self {
Self { sys_handle: sys_ar::ffi::create_audio_resampler() }
}
}
impl AudioResampler {
pub fn remix_and_resample<'a>(
&'a mut self,
src: &[i16],
samples_per_channel: u32,
num_channels: u32,
sample_rate: u32,
dst_num_channels: u32,
dst_sample_rate: u32,
) -> &'a [i16] {
assert!(src.len() >= (samples_per_channel * num_channels) as usize, "src buffer too small");
unsafe {
let len = self.sys_handle.pin_mut().remix_and_resample(
src.as_ptr(),
samples_per_channel as usize,
num_channels as usize,
sample_rate as i32,
dst_num_channels as usize,
dst_sample_rate as i32,
);
std::slice::from_raw_parts(self.sys_handle.data(), len / 2)
}
}
}
@@ -0,0 +1,205 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::audio_track as sys_at;
use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions, RtcError, RtcErrorType};
#[derive(Clone)]
pub struct NativeAudioSource {
sys_handle: SharedPtr<sys_at::ffi::AudioTrackSource>,
sample_rate: u32,
num_channels: u32,
queue_size_samples: u32,
}
impl NativeAudioSource {
/// Creates a new [`NativeAudioSource`].
///
/// # Arguments
/// * `options` Configuration options for the source (e.g. echo cancellation, noise suppression).
/// * `sample_rate` Sampling rate in Hz (for example, `48000`).
/// * `num_channels` Number of audio channels (`1` for mono, `2` for stereo, etc.).
/// * `queue_size_ms` Size of the internal buffering queue, in milliseconds.
///
/// # Behavior
/// - If `queue_size_ms` is **zero**, buffering is **disabled** and audio frames are
/// delivered directly to webrtc sinks. In this mode, the caller **must provide 10 ms frames**
/// (i.e., `sample_rate / 100` samples per channel) when calling [`capture_frame`].
/// - If `queue_size_ms` is **non-zero**, buffering is enabled. The value must be a
/// **multiple of 10**, representing the total buffering duration in milliseconds.
/// Frames will be queued and flushed to sinks asynchronously once the buffer
/// reaches the configured threshold.
///
/// # Panics
/// assert if `queue_size_ms` is not a multiple of 10.
pub fn new(
options: AudioSourceOptions,
sample_rate: u32,
num_channels: u32,
queue_size_ms: u32,
) -> NativeAudioSource {
assert!(queue_size_ms % 10 == 0, "queue_size_ms must be a multiple of 10");
let sys_handle = sys_at::ffi::new_audio_track_source(
options.into(),
sample_rate.try_into().unwrap(),
num_channels.try_into().unwrap(),
queue_size_ms.try_into().unwrap(),
);
let queue_size_samples = (queue_size_ms * sample_rate * num_channels) / 1000;
Self { sys_handle, sample_rate, num_channels, queue_size_samples }
}
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::AudioTrackSource> {
self.sys_handle.clone()
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.sys_handle.set_audio_options(&sys_at::ffi::AudioSourceOptions::from(options))
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.sys_handle.audio_options().into()
}
pub fn sample_rate(&self) -> u32 {
self.sample_rate
}
pub fn num_channels(&self) -> u32 {
self.num_channels
}
pub fn clear_buffer(&self) {
self.sys_handle.clear_buffer();
}
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "sample_rate and num_channels don't match".to_owned(),
});
}
// Fast path: no buffering
if self.queue_size_samples == 0 {
// frame size must be 10ms for fast path
let expected_frames_per_ch = (self.sample_rate / 100) as usize;
if frame.data.len() % (self.num_channels as usize) != 0 {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "frame.data length not divisible by channel count".to_owned(),
});
}
let nb_frames = frame.data.len() / (self.num_channels as usize);
if nb_frames != expected_frames_per_ch {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: format!(
"direct capture requires 10ms frames: got {} frames, expected {}",
nb_frames, expected_frames_per_ch
),
});
}
// Define a no-op callback for fast path (queue_size_ms=0)
// This is safer than passing null, which can cause UB in release mode optimizations
extern "C" fn noop_complete_callback(_ctx: *const sys_at::SourceContext) {
// No-op: fast path completes synchronously, no callback needed
}
unsafe {
let data: &[i16] = frame.data.as_ref();
// Use a valid no-op callback instead of null for safety
// In release mode, transmuting null pointers can cause UB
let noop_callback = sys_at::CompleteCallback(noop_complete_callback);
let ok = self.sys_handle.capture_frame(
data,
self.sample_rate,
self.num_channels,
nb_frames,
std::ptr::null(), // Context is still null - callback won't use it
noop_callback,
);
if !ok {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "failed to capture frame without buffering".to_owned(),
});
}
}
return Ok(());
}
// Buffered path.
extern "C" fn lk_audio_source_complete(userdata: *const sys_at::SourceContext) {
let tx = unsafe { Box::from_raw(userdata as *mut oneshot::Sender<()>) };
let _ = tx.send(());
}
// iterate over chunks of self._queue_size_samples
for chunk in frame.data.chunks(self.queue_size_samples as usize) {
let nb_frames = chunk.len() / self.num_channels as usize;
let (tx, rx) = oneshot::channel::<()>();
let ctx = Box::new(tx);
let ctx_ptr = Box::into_raw(ctx) as *const sys_at::SourceContext;
unsafe {
// In the fast path, C++ never store / invoke on_complete / ctx.
if !self.sys_handle.capture_frame(
chunk,
self.sample_rate,
self.num_channels,
nb_frames,
ctx_ptr,
sys_at::CompleteCallback(lk_audio_source_complete),
) {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "failed to capture frame".to_owned(),
});
}
}
let _ = rx.await;
}
Ok(())
}
}
impl From<sys_at::ffi::AudioSourceOptions> for AudioSourceOptions {
fn from(options: sys_at::ffi::AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
impl From<AudioSourceOptions> for sys_at::ffi::AudioSourceOptions {
fn from(options: AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
@@ -0,0 +1,314 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
collections::VecDeque,
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
task::{Context, Poll, Waker},
};
use cxx::SharedPtr;
use livekit_runtime::Stream;
use parking_lot::Mutex;
use rtrb::{Consumer, Producer, PushError, RingBuffer};
use webrtc_sys::audio_track as sys_at;
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
pub struct NativeAudioStream {
native_sink: SharedPtr<sys_at::ffi::NativeAudioSink>,
audio_track: RtcAudioTrack,
frame_queue: Arc<AudioFrameQueue>,
}
impl NativeAudioStream {
pub fn new(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
queue_size_frames: Option<usize>,
) -> Self {
let frame_queue = Arc::new(AudioFrameQueue::new(queue_size_frames));
let observer = Arc::new(AudioTrackObserver { frame_queue: frame_queue.clone() });
let native_sink = sys_at::ffi::new_native_audio_sink(
Box::new(sys_at::AudioSinkWrapper::new(observer.clone())),
sample_rate,
num_channels,
);
let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) };
audio.add_sink(&native_sink);
Self { native_sink, audio_track, frame_queue }
}
pub fn track(&self) -> RtcAudioTrack {
self.audio_track.clone()
}
pub fn close(&mut self) {
let audio = unsafe { sys_at::ffi::media_to_audio(self.audio_track.sys_handle()) };
audio.remove_sink(&self.native_sink);
self.frame_queue.close();
}
}
impl Drop for NativeAudioStream {
fn drop(&mut self) {
self.close();
}
}
impl Stream for NativeAudioStream {
type Item = AudioFrame<'static>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.frame_queue.poll_recv(cx)
}
}
pub struct AudioTrackObserver {
frame_queue: Arc<AudioFrameQueue>,
}
impl sys_at::AudioSink for AudioTrackObserver {
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
self.frame_queue.push(AudioFrame {
data: data.to_owned().into(),
sample_rate: sample_rate as u32,
num_channels: nb_channels as u32,
samples_per_channel: nb_frames as u32,
});
}
}
struct AudioFrameQueue {
kind: AudioFrameQueueKind,
closed: AtomicBool,
dropped_frames: AtomicU64,
waker: Mutex<Option<Waker>>,
}
enum AudioFrameQueueKind {
Bounded(BoundedAudioFrameQueue),
Unbounded(UnboundedAudioFrameQueue),
}
struct BoundedAudioFrameQueue {
producer: Mutex<Producer<AudioFrame<'static>>>,
consumer: Mutex<Consumer<AudioFrame<'static>>>,
}
struct UnboundedAudioFrameQueue {
frames: Mutex<VecDeque<AudioFrame<'static>>>,
}
impl AudioFrameQueue {
fn new(capacity: Option<usize>) -> Self {
let kind = match capacity.filter(|capacity| *capacity > 0) {
Some(capacity) => {
let (producer, consumer) = RingBuffer::new(capacity);
AudioFrameQueueKind::Bounded(BoundedAudioFrameQueue {
producer: Mutex::new(producer),
consumer: Mutex::new(consumer),
})
}
None => AudioFrameQueueKind::Unbounded(UnboundedAudioFrameQueue {
frames: Mutex::new(VecDeque::new()),
}),
};
Self {
kind,
closed: AtomicBool::new(false),
dropped_frames: AtomicU64::new(0),
waker: Mutex::new(None),
}
}
fn push(&self, frame: AudioFrame<'static>) {
if self.closed.load(Ordering::Acquire) {
return;
}
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
AudioFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().push_back(frame);
}
}
self.wake_receiver();
}
fn push_bounded(&self, queue: &BoundedAudioFrameQueue, mut frame: AudioFrame<'static>) {
loop {
let push_result = queue.producer.lock().push(frame);
match push_result {
Ok(()) => return,
Err(PushError::Full(returned_frame)) => {
frame = returned_frame;
let dropped = queue.consumer.lock().pop().is_ok();
if dropped {
self.record_drop();
} else {
return;
}
}
}
}
}
fn close(&self) {
self.closed.store(true, Ordering::Release);
self.wake_receiver();
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => {
let mut consumer = queue.consumer.lock();
while consumer.pop().is_ok() {}
}
AudioFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().clear();
}
}
}
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<AudioFrame<'static>>> {
if let Some(frame) = self.try_pop() {
return Poll::Ready(Some(frame));
}
if self.closed.load(Ordering::Acquire) {
return Poll::Ready(None);
}
*self.waker.lock() = Some(cx.waker().clone());
if let Some(frame) = self.try_pop() {
self.waker.lock().take();
Poll::Ready(Some(frame))
} else if self.closed.load(Ordering::Acquire) {
Poll::Ready(None)
} else {
Poll::Pending
}
}
fn try_pop(&self) -> Option<AudioFrame<'static>> {
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
AudioFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
}
}
fn wake_receiver(&self) {
let waker = self.waker.lock().take();
if let Some(waker) = waker {
waker.wake();
}
}
fn record_drop(&self) {
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
if dropped_frames == 1 || dropped_frames % 100 == 0 {
log::warn!(
"native audio stream queue overflow; dropped {} queued frames",
dropped_frames
);
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::Ordering;
use super::AudioFrameQueue;
use crate::audio_frame::AudioFrame;
fn test_frame(marker: i16) -> AudioFrame<'static> {
AudioFrame {
data: vec![marker].into(),
sample_rate: 48_000,
num_channels: 1,
samples_per_channel: 1,
}
}
fn pop_marker(queue: &AudioFrameQueue) -> Option<i16> {
queue.try_pop().map(|frame| frame.data[0])
}
#[test]
fn bounded_queue_preserves_fifo_order_under_capacity() {
let queue = AudioFrameQueue::new(Some(3));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(pop_marker(&queue), Some(1));
assert_eq!(pop_marker(&queue), Some(2));
assert_eq!(pop_marker(&queue), Some(3));
assert_eq!(pop_marker(&queue), None);
}
#[test]
fn bounded_queue_drops_oldest_when_full() {
let queue = AudioFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
assert_eq!(pop_marker(&queue), Some(2));
assert_eq!(pop_marker(&queue), Some(3));
assert_eq!(pop_marker(&queue), None);
}
#[test]
fn unbounded_queue_retains_all_frames() {
let queue = AudioFrameQueue::new(None);
for marker in 1..=4 {
queue.push(test_frame(marker));
}
for marker in 1..=4 {
assert_eq!(pop_marker(&queue), Some(marker));
}
assert_eq!(pop_marker(&queue), None);
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
}
#[test]
fn close_clears_buffer_and_rejects_future_pushes() {
let queue = AudioFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.close();
queue.push(test_frame(2));
assert_eq!(pop_marker(&queue), None);
}
}
@@ -0,0 +1,33 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use sys_at::ffi::audio_to_media;
use webrtc_sys::audio_track as sys_at;
use super::media_stream_track::impl_media_stream_track;
use crate::media_stream_track::RtcTrackState;
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) sys_handle: SharedPtr<sys_at::ffi::AudioTrack>,
}
impl RtcAudioTrack {
impl_media_stream_track!(audio_to_media);
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::MediaStreamTrack> {
audio_to_media(self.sys_handle.clone())
}
}
@@ -0,0 +1,142 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{str, sync::Arc};
use cxx::SharedPtr;
use parking_lot::Mutex;
use webrtc_sys::data_channel as sys_dc;
use crate::data_channel::{
DataBuffer, DataChannelError, DataChannelInit, DataChannelState, OnBufferedAmountChange,
OnMessage, OnStateChange,
};
impl From<sys_dc::ffi::DataState> for DataChannelState {
fn from(value: sys_dc::ffi::DataState) -> Self {
match value {
sys_dc::ffi::DataState::Connecting => Self::Connecting,
sys_dc::ffi::DataState::Open => Self::Open,
sys_dc::ffi::DataState::Closing => Self::Closing,
sys_dc::ffi::DataState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
fn from(value: DataChannelInit) -> Self {
Self {
ordered: value.ordered,
has_max_retransmit_time: value.max_retransmit_time.is_some(),
max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
has_max_retransmits: value.max_retransmits.is_some(),
max_retransmits: value.max_retransmits.unwrap_or_default(),
protocol: value.protocol,
id: value.id,
has_priority: false,
priority: sys_dc::ffi::Priority::Medium,
negotiated: value.negotiated,
}
}
}
#[derive(Clone)]
pub struct DataChannel {
observer: Arc<DataChannelObserver>,
pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
}
impl DataChannel {
pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
let observer = Arc::new(DataChannelObserver::default());
let dc = Self { sys_handle: sys_handle.clone(), observer: observer.clone() };
dc.sys_handle
.register_observer(Box::new(sys_dc::DataChannelObserverWrapper::new(observer)));
dc
}
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if !binary {
str::from_utf8(data)?;
}
let buffer = sys_dc::ffi::DataBuffer { ptr: data.as_ptr(), len: data.len(), binary };
self.sys_handle.send(&buffer).then_some(()).ok_or(DataChannelError::Send)
}
pub fn id(&self) -> i32 {
self.sys_handle.id()
}
pub fn label(&self) -> String {
self.sys_handle.label()
}
pub fn state(&self) -> DataChannelState {
self.sys_handle.state().into()
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn buffered_amount(&self) -> u64 {
self.sys_handle.buffered_amount()
}
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
*self.observer.state_change_handler.lock() = handler;
}
pub fn on_message(&self, handler: Option<OnMessage>) {
*self.observer.message_handler.lock() = handler;
}
pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
*self.observer.buffered_amount_change_handler.lock() = handler;
}
}
#[derive(Default)]
struct DataChannelObserver {
state_change_handler: Mutex<Option<OnStateChange>>,
message_handler: Mutex<Option<OnMessage>>,
buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
}
impl sys_dc::DataChannelObserver for DataChannelObserver {
fn on_state_change(&self, state: sys_dc::ffi::DataState) {
let mut handler = self.state_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(state.into());
}
}
fn on_message(&self, data: &[u8], binary: bool) {
let mut handler = self.message_handler.lock();
if let Some(f) = handler.as_mut() {
f(DataBuffer { data, binary });
}
}
fn on_buffered_amount_change(&self, sent_data_size: u64) {
let mut handler = self.buffered_amount_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(sent_data_size);
}
}
}
@@ -0,0 +1,241 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::desktop_capturer::{self as sys_dc, ffi::new_desktop_capturer};
#[derive(Debug, Copy, Clone, PartialEq)]
pub(crate) enum SourceType {
Screen,
Window,
Generic,
}
#[derive(Copy, Clone, Debug)]
pub(crate) struct DesktopCapturerOptions {
source_type: SourceType,
include_cursor: bool,
#[cfg(target_os = "macos")]
allow_sck_system_picker: bool,
}
impl Default for DesktopCapturerOptions {
fn default() -> Self {
Self {
source_type: SourceType::Screen,
include_cursor: false,
#[cfg(target_os = "macos")]
allow_sck_system_picker: true,
}
}
}
impl DesktopCapturerOptions {
pub(crate) fn new(source_type: SourceType) -> Self {
Self { source_type, ..Default::default() }
}
pub(crate) fn with_cursor(mut self, include: bool) -> Self {
self.include_cursor = include;
self
}
#[cfg(target_os = "macos")]
pub(crate) fn with_sck_system_picker(mut self, allow_sck_system_picker: bool) -> Self {
self.allow_sck_system_picker = allow_sck_system_picker;
self
}
pub(crate) fn to_sys_handle(&self) -> sys_dc::ffi::DesktopCapturerOptions {
let source_type = match self.source_type {
SourceType::Screen => sys_dc::ffi::SourceType::Screen,
SourceType::Window => sys_dc::ffi::SourceType::Window,
SourceType::Generic => sys_dc::ffi::SourceType::Generic,
};
let mut sys_handle = sys_dc::ffi::DesktopCapturerOptions {
source_type,
include_cursor: self.include_cursor,
allow_sck_system_picker: false,
};
#[cfg(target_os = "macos")]
{
sys_handle.allow_sck_system_picker = self.allow_sck_system_picker;
}
sys_handle
}
}
pub(crate) struct DesktopCapturer {
sys_handle: UniquePtr<sys_dc::ffi::DesktopCapturer>,
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
glib_loop: Option<glib::MainLoop>,
}
impl DesktopCapturer {
pub(crate) fn new(options: DesktopCapturerOptions) -> Option<Self> {
let sys_handle = new_desktop_capturer(options.to_sys_handle());
if sys_handle.is_null() {
None
} else {
Some(Self {
sys_handle,
#[cfg(all(
any(target_os = "linux", target_os = "freebsd"),
feature = "glib-main-loop"
))]
glib_loop: None,
})
}
}
pub(crate) fn capture_frame(&self) {
self.sys_handle.capture_frame();
}
pub(crate) fn start<T>(&mut self, callback: T)
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
{
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
if std::env::var("WAYLAND_DISPLAY").is_ok() {
let main_loop = glib::MainLoop::new(None, false);
self.glib_loop = Some(main_loop.clone());
let _handle = std::thread::spawn(move || {
main_loop.run();
});
}
let pin_handle = self.sys_handle.pin_mut();
let callback = DesktopCallback::new(callback);
let callback_wrapper = sys_dc::DesktopCapturerCallbackWrapper::new(Box::new(callback));
pin_handle.start(Box::new(callback_wrapper));
}
pub(crate) fn select_source(&self, id: u64) -> bool {
self.sys_handle.select_source(id)
}
pub(crate) fn get_source_list(&self) -> Vec<CaptureSource> {
let mut sources = Vec::new();
let source_list = self.sys_handle.get_source_list();
for source in source_list.iter() {
sources.push(CaptureSource { sys_handle: source.clone() });
}
sources
}
}
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
impl Drop for DesktopCapturer {
fn drop(&mut self) {
if let Some(glib_loop) = &self.glib_loop {
glib_loop.quit();
}
}
}
pub(crate) struct DesktopFrame {
sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>,
}
impl DesktopFrame {
fn new(sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>) -> Self {
Self { sys_handle }
}
pub(crate) fn width(&self) -> i32 {
self.sys_handle.width()
}
pub(crate) fn height(&self) -> i32 {
self.sys_handle.height()
}
pub(crate) fn stride(&self) -> u32 {
self.sys_handle.stride() as u32
}
pub(crate) fn left(&self) -> i32 {
self.sys_handle.left()
}
pub(crate) fn top(&self) -> i32 {
self.sys_handle.top()
}
pub(crate) fn data(&self) -> &[u8] {
let data = self.sys_handle.data();
unsafe { std::slice::from_raw_parts(data, self.stride() as usize * self.height() as usize) }
}
}
struct DesktopCallback<T: FnMut(Result<DesktopFrame, CaptureError>) + Send> {
callback: T,
}
impl<T> DesktopCallback<T>
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
{
fn new(callback: T) -> Self {
Self { callback }
}
fn capture_result_from_sys(
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
) -> Result<DesktopFrame, CaptureError> {
match result {
Ok(frame) => Ok(DesktopFrame::new(frame)),
Err(error) => Err(match error {
sys_dc::CaptureError::Temporary => CaptureError::Temporary,
sys_dc::CaptureError::Permanent => CaptureError::Permanent,
}),
}
}
}
impl<T> sys_dc::DesktopCapturerCallback for DesktopCallback<T>
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
{
fn on_capture_result(
&mut self,
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
) {
(self.callback)(DesktopCallback::<T>::capture_result_from_sys(result));
}
}
#[derive(Clone)]
pub(crate) struct CaptureSource {
sys_handle: sys_dc::ffi::Source,
}
impl CaptureSource {
pub(crate) fn id(&self) -> u64 {
self.sys_handle.id
}
pub(crate) fn title(&self) -> String {
self.sys_handle.title.clone()
}
pub(crate) fn display_id(&self) -> i64 {
self.sys_handle.display_id
}
}
pub(crate) enum CaptureError {
Temporary,
Permanent,
}
@@ -0,0 +1,316 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use webrtc_sys::frame_cryptor::{self as sys_fc};
use crate::{
native::packet_trailer::PacketTrailerHandler, peer_connection_factory::PeerConnectionFactory,
rtp_receiver::RtpReceiver, rtp_sender::RtpSender,
};
pub type OnStateChange = Box<dyn FnMut(String, EncryptionState) + Send + Sync>;
#[derive(Copy, Clone, Debug)]
#[non_exhaustive]
pub enum KeyDerivationAlgorithm {
PBKDF2,
HKDF,
}
impl Into<sys_fc::ffi::KeyDerivationAlgorithm> for KeyDerivationAlgorithm {
fn into(self) -> sys_fc::ffi::KeyDerivationAlgorithm {
match self {
KeyDerivationAlgorithm::PBKDF2 => sys_fc::ffi::KeyDerivationAlgorithm::PBKDF2,
KeyDerivationAlgorithm::HKDF => sys_fc::ffi::KeyDerivationAlgorithm::HKDF,
}
}
}
#[derive(Debug, Clone)]
pub struct KeyProviderOptions {
pub shared_key: bool,
pub ratchet_window_size: i32,
pub ratchet_salt: Vec<u8>,
pub failure_tolerance: i32,
pub key_ring_size: i32,
pub key_derivation_algorithm: KeyDerivationAlgorithm,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionAlgorithm {
AesGcm,
AesCbc,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionState {
New,
Ok,
EncryptionFailed,
DecryptionFailed,
MissingKey,
KeyRatcheted,
InternalError,
}
#[derive(Debug, Clone)]
pub struct EncryptedPacket {
pub data: Vec<u8>,
pub iv: Vec<u8>,
pub key_index: u32,
}
#[derive(Clone)]
pub struct KeyProvider {
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::KeyProvider>,
}
impl KeyProvider {
pub fn new(options: KeyProviderOptions) -> Self {
Self { sys_handle: sys_fc::ffi::new_key_provider(options.into()) }
}
pub fn set_shared_key(&self, key_index: i32, key: Vec<u8>) -> bool {
self.sys_handle.set_shared_key(key_index, key)
}
pub fn ratchet_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.ratchet_shared_key(key_index).ok()
}
pub fn get_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.get_shared_key(key_index).ok()
}
pub fn set_key(&self, participant_id: String, key_index: i32, key: Vec<u8>) -> bool {
self.sys_handle.set_key(participant_id, key_index, key)
}
pub fn ratchet_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.ratchet_key(participant_id, key_index).ok()
}
pub fn get_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.get_key(participant_id, key_index).ok()
}
pub fn set_sif_trailer(&self, trailer: Vec<u8>) {
self.sys_handle.set_sif_trailer(trailer);
}
}
#[derive(Clone)]
pub struct FrameCryptor {
observer: Arc<RtcFrameCryptorObserver>,
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::FrameCryptor>,
}
impl FrameCryptor {
pub fn new_for_rtp_sender(
peer_factory: &PeerConnectionFactory,
participant_id: String,
algorithm: EncryptionAlgorithm,
key_provider: KeyProvider,
sender: RtpSender,
) -> Self {
let observer = Arc::new(RtcFrameCryptorObserver::default());
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_sender(
peer_factory.handle.sys_handle.clone(),
participant_id,
algorithm.into(),
key_provider.sys_handle,
sender.handle.sys_handle,
);
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
fc.sys_handle
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
fc
}
pub fn new_for_rtp_receiver(
peer_factory: &PeerConnectionFactory,
participant_id: String,
algorithm: EncryptionAlgorithm,
key_provider: KeyProvider,
receiver: RtpReceiver,
) -> Self {
let observer = Arc::new(RtcFrameCryptorObserver::default());
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_receiver(
peer_factory.handle.sys_handle.clone(),
participant_id,
algorithm.into(),
key_provider.sys_handle,
receiver.handle.sys_handle,
);
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
fc.sys_handle
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
fc
}
pub fn set_enabled(self: &FrameCryptor, enabled: bool) {
self.sys_handle.set_enabled(enabled);
}
pub fn enabled(self: &FrameCryptor) -> bool {
self.sys_handle.enabled()
}
pub fn set_key_index(self: &FrameCryptor, index: i32) {
self.sys_handle.set_key_index(index);
}
pub fn key_index(self: &FrameCryptor) -> i32 {
self.sys_handle.key_index()
}
pub fn participant_id(self: &FrameCryptor) -> String {
self.sys_handle.participant_id()
}
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
*self.observer.state_change_handler.lock() = handler;
}
pub fn set_packet_trailer_handler(&self, handler: &PacketTrailerHandler) {
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
}
}
#[derive(Clone)]
pub struct DataPacketCryptor {
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::DataPacketCryptor>,
}
impl DataPacketCryptor {
pub fn new(algorithm: EncryptionAlgorithm, key_provider: KeyProvider) -> Self {
Self {
sys_handle: sys_fc::ffi::new_data_packet_cryptor(
algorithm.into(),
key_provider.sys_handle,
),
}
}
pub fn encrypt(
&self,
participant_id: &str,
key_index: u32,
data: &[u8],
) -> Result<EncryptedPacket, Box<dyn std::error::Error>> {
let data_vec: Vec<u8> = data.to_vec();
match self.sys_handle.encrypt_data_packet(participant_id.to_string(), key_index, data_vec) {
Ok(packet) => Ok(packet.into()),
Err(e) => Err(format!("Encryption failed: {}", e).into()),
}
}
pub fn decrypt(
&self,
participant_id: &str,
encrypted_packet: &EncryptedPacket,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
match self
.sys_handle
.decrypt_data_packet(participant_id.to_string(), &encrypted_packet.clone().into())
{
Ok(data) => Ok(data.into_iter().collect()),
Err(e) => Err(format!("Decryption failed: {}", e).into()),
}
}
}
#[derive(Default)]
struct RtcFrameCryptorObserver {
state_change_handler: Mutex<Option<OnStateChange>>,
}
impl sys_fc::RtcFrameCryptorObserver for RtcFrameCryptorObserver {
fn on_frame_cryption_state_change(
&self,
participant_id: String,
state: sys_fc::ffi::FrameCryptionState,
) {
let mut handler = self.state_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(participant_id, state.into());
}
}
}
impl From<sys_fc::ffi::Algorithm> for EncryptionAlgorithm {
fn from(value: sys_fc::ffi::Algorithm) -> Self {
match value {
sys_fc::ffi::Algorithm::AesGcm => Self::AesGcm,
sys_fc::ffi::Algorithm::AesCbc => Self::AesCbc,
_ => panic!("unknown frame cyrptor Algorithm"),
}
}
}
impl From<EncryptionAlgorithm> for sys_fc::ffi::Algorithm {
fn from(value: EncryptionAlgorithm) -> Self {
match value {
EncryptionAlgorithm::AesGcm => Self::AesGcm,
EncryptionAlgorithm::AesCbc => Self::AesCbc,
}
}
}
impl From<sys_fc::ffi::FrameCryptionState> for EncryptionState {
fn from(value: sys_fc::ffi::FrameCryptionState) -> Self {
match value {
sys_fc::ffi::FrameCryptionState::New => Self::New,
sys_fc::ffi::FrameCryptionState::Ok => Self::Ok,
sys_fc::ffi::FrameCryptionState::EncryptionFailed => Self::EncryptionFailed,
sys_fc::ffi::FrameCryptionState::DecryptionFailed => Self::DecryptionFailed,
sys_fc::ffi::FrameCryptionState::MissingKey => Self::MissingKey,
sys_fc::ffi::FrameCryptionState::KeyRatcheted => Self::KeyRatcheted,
sys_fc::ffi::FrameCryptionState::InternalError => Self::InternalError,
_ => panic!("unknown frame cyrptor FrameCryptionState"),
}
}
}
impl From<KeyProviderOptions> for sys_fc::ffi::KeyProviderOptions {
fn from(value: KeyProviderOptions) -> Self {
Self {
shared_key: value.shared_key,
ratchet_window_size: value.ratchet_window_size,
ratchet_salt: value.ratchet_salt,
failure_tolerance: value.failure_tolerance,
key_ring_size: value.key_ring_size,
key_derivation_algorithm: value.key_derivation_algorithm.into(),
}
}
}
impl From<sys_fc::ffi::EncryptedPacket> for EncryptedPacket {
fn from(value: sys_fc::ffi::EncryptedPacket) -> Self {
Self {
data: value.data.into_iter().collect(),
iv: value.iv.into_iter().collect(),
key_index: value.key_index,
}
}
}
impl From<EncryptedPacket> for sys_fc::ffi::EncryptedPacket {
fn from(value: EncryptedPacket) -> Self {
Self { data: value.data, iv: value.iv, key_index: value.key_index }
}
}
@@ -0,0 +1,60 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::jsep as sys_jsep;
use crate::{ice_candidate as ic, session_description::SdpParseError};
#[derive(Clone)]
pub struct IceCandidate {
pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<ic::IceCandidate, SdpParseError> {
let res = sys_jsep::ffi::create_ice_candidate(
sdp_mid.to_string(),
sdp_mline_index,
sdp.to_string(),
);
match res {
Ok(sys_handle) => Ok(ic::IceCandidate { handle: IceCandidate { sys_handle } }),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_mid(&self) -> String {
self.sys_handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.sys_handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.sys_handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
@@ -0,0 +1,49 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::media_stream as sys_ms;
use crate::{
audio_track,
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
video_track,
};
#[derive(Clone)]
pub struct MediaStream {
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
}
impl MediaStream {
pub fn id(&self) -> String {
self.sys_handle.id()
}
pub fn audio_tracks(&self) -> Vec<audio_track::RtcAudioTrack> {
self.sys_handle
.get_audio_tracks()
.into_iter()
.map(|t| audio_track::RtcAudioTrack { handle: RtcAudioTrack { sys_handle: t.ptr } })
.collect()
}
pub fn video_tracks(&self) -> Vec<video_track::RtcVideoTrack> {
self.sys_handle
.get_video_tracks()
.into_iter()
.map(|t| video_track::RtcVideoTrack { handle: RtcVideoTrack::new(t.ptr) })
.collect()
}
}
@@ -0,0 +1,78 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::{
audio_track::ffi::media_to_audio, media_stream_track as sys_mst,
video_track::ffi::media_to_video, MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO,
};
use crate::{
audio_track,
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
media_stream_track::{MediaStreamTrack, RtcTrackState},
video_track,
};
impl From<sys_mst::ffi::TrackState> for RtcTrackState {
fn from(state: sys_mst::ffi::TrackState) -> Self {
match state {
sys_mst::ffi::TrackState::Live => RtcTrackState::Live,
sys_mst::ffi::TrackState::Ended => RtcTrackState::Ended,
_ => panic!("unknown TrackState"),
}
}
}
pub fn new_media_stream_track(
sys_handle: SharedPtr<sys_mst::ffi::MediaStreamTrack>,
) -> MediaStreamTrack {
if sys_handle.kind() == MEDIA_TYPE_AUDIO {
MediaStreamTrack::Audio(audio_track::RtcAudioTrack {
handle: RtcAudioTrack { sys_handle: unsafe { media_to_audio(sys_handle) } },
})
} else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
MediaStreamTrack::Video(video_track::RtcVideoTrack {
handle: RtcVideoTrack::new(unsafe { media_to_video(sys_handle) }),
})
} else {
panic!("unknown track kind")
}
}
macro_rules! impl_media_stream_track {
($cast:expr) => {
pub fn id(&self) -> String {
let ptr = $cast(self.sys_handle.clone());
ptr.id()
}
pub fn enabled(&self) -> bool {
let ptr = $cast(self.sys_handle.clone());
ptr.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
let ptr = $cast(self.sys_handle.clone());
ptr.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
let ptr = $cast(self.sys_handle.clone());
ptr.state().into()
}
};
}
pub(super) use impl_media_stream_track;
@@ -0,0 +1,72 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(target_os = "android")]
pub mod android;
pub mod apm;
pub mod audio_mixer;
pub mod audio_resampler;
pub mod audio_source;
pub mod audio_stream;
pub mod audio_track;
pub mod data_channel;
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
pub mod desktop_capturer;
pub mod frame_cryptor;
pub mod ice_candidate;
pub mod media_stream;
pub mod media_stream_track;
pub mod packet_trailer;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod video_frame;
pub mod video_source;
pub mod video_stream;
pub mod video_track;
pub mod yuv_helper;
use webrtc_sys::{rtc_error as sys_err, webrtc as sys_rtc};
use crate::{MediaType, RtcError, RtcErrorType};
impl From<sys_err::ffi::RtcErrorType> for RtcErrorType {
fn from(value: sys_err::ffi::RtcErrorType) -> Self {
match value {
sys_err::ffi::RtcErrorType::InvalidState => Self::InvalidState,
_ => Self::Internal,
}
}
}
impl From<sys_err::ffi::RtcError> for RtcError {
fn from(value: sys_err::ffi::RtcError) -> Self {
Self { error_type: value.error_type.into(), message: value.message }
}
}
impl From<MediaType> for sys_rtc::ffi::MediaType {
fn from(value: MediaType) -> Self {
match value {
MediaType::Audio => Self::Audio,
MediaType::Video => Self::Video,
MediaType::Data => Self::Data,
MediaType::Unsupported => Self::Unsupported,
}
}
}
@@ -0,0 +1,273 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Packet trailer support for end-to-end frame metadata propagation.
//!
//! This module provides functionality to embed user-supplied metadata
//! in encoded video frames as trailers. The timestamps/frameIDs are preserved
//! through the WebRTC pipeline and can be extracted on the receiver side.
//!
//! On the send side, user timestamps/frameIDs are stored in the handler's internal
//! map keyed by RTP timestamp. When the encoder produces a frame,
//! the transformer looks up the metadata via the frame's CaptureTime().
//!
//! On the receive side, extracted frame metadata is stored in an
//! internal map keyed by RTP timestamp. Decoded frames look up their
//! metadata via lookup_frame_metadata(rtp_timestamp).
use std::sync::Arc;
use cxx::SharedPtr;
use webrtc_sys::packet_trailer::ffi as sys_pt;
use crate::{
peer_connection_factory::PeerConnectionFactory, rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
};
/// Stage reached by a native local video frame in the publish pipeline.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublishTimingStage {
/// The adapted raw frame was handed to WebRTC's encoder path.
EncoderUpload,
/// WebRTC produced an encoded frame for packetization.
EncoderOutput,
/// The encoded frame was handed back to WebRTC's packetizer.
WebrtcPacketize,
}
/// Stage reached by a native remote video frame in the subscribe pipeline.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SubscribeTimingStage {
/// WebRTC produced an encoded frame after RTP depacketization.
WebrtcReceive,
/// The encoded frame was handed to WebRTC's decoder.
DecoderUpload,
/// WebRTC produced a decoded frame for the native video sink.
DecoderOutput,
}
/// Timestamped native local video publish pipeline event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PublishTimingEvent {
/// Publish pipeline stage reached by the frame.
pub stage: PublishTimingStage,
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
pub timestamp_us: u64,
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
pub capture_timestamp_us: u64,
/// Optional application frame ID associated with this frame.
pub frame_id: Option<u32>,
}
/// Timestamped native remote video subscribe pipeline event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SubscribeTimingEvent {
/// Subscribe pipeline stage reached by the frame.
pub stage: SubscribeTimingStage,
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
pub timestamp_us: u64,
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
pub capture_timestamp_us: u64,
/// Optional application frame ID associated with this frame.
pub frame_id: Option<u32>,
}
/// Callback invoked for native local video publish timing events.
pub type PublishTimingObserver = Arc<dyn Fn(PublishTimingEvent) + Send + Sync + 'static>;
/// Callback invoked for native remote video subscribe timing events.
pub type SubscribeTimingObserver = Arc<dyn Fn(SubscribeTimingEvent) + Send + Sync + 'static>;
impl From<sys_pt::VideoPublishTimingStage> for PublishTimingStage {
fn from(stage: sys_pt::VideoPublishTimingStage) -> Self {
match stage {
sys_pt::VideoPublishTimingStage::EncoderUpload => Self::EncoderUpload,
sys_pt::VideoPublishTimingStage::EncoderOutput => Self::EncoderOutput,
sys_pt::VideoPublishTimingStage::WebrtcPacketize => Self::WebrtcPacketize,
_ => Self::WebrtcPacketize,
}
}
}
impl From<sys_pt::VideoPublishTimingEvent> for PublishTimingEvent {
fn from(event: sys_pt::VideoPublishTimingEvent) -> Self {
Self {
stage: event.stage.into(),
timestamp_us: event.timestamp_us,
capture_timestamp_us: event.capture_timestamp_us,
frame_id: (event.frame_id != 0).then_some(event.frame_id),
}
}
}
impl From<sys_pt::VideoSubscribeTimingStage> for SubscribeTimingStage {
fn from(stage: sys_pt::VideoSubscribeTimingStage) -> Self {
match stage {
sys_pt::VideoSubscribeTimingStage::WebrtcReceive => Self::WebrtcReceive,
sys_pt::VideoSubscribeTimingStage::DecoderUpload => Self::DecoderUpload,
sys_pt::VideoSubscribeTimingStage::DecoderOutput => Self::DecoderOutput,
_ => Self::DecoderOutput,
}
}
}
impl From<sys_pt::VideoSubscribeTimingEvent> for SubscribeTimingEvent {
fn from(event: sys_pt::VideoSubscribeTimingEvent) -> Self {
Self {
stage: event.stage.into(),
timestamp_us: event.timestamp_us,
capture_timestamp_us: event.capture_timestamp_us,
frame_id: (event.frame_id != 0).then_some(event.frame_id),
}
}
}
/// Handler for packet trailer embedding/extraction on RTP streams.
///
/// For sender side: Stores frame metadata keyed by capture timestamp
/// and embeds them as binary payload trailers on encoded frames before they
/// are sent. Use `store_frame_metadata()` to associate metadata with
/// a captured frame.
///
/// For receiver side: Extracts frame metadata from received frames
/// and makes them available for retrieval via `lookup_frame_metadata()`.
#[derive(Clone)]
pub struct PacketTrailerHandler {
sys_handle: SharedPtr<sys_pt::PacketTrailerHandler>,
}
impl PacketTrailerHandler {
/// Enable or disable timestamp embedding/extraction.
pub fn set_enabled(&self, enabled: bool) {
self.sys_handle.set_enabled(enabled);
}
/// Check if timestamp embedding/extraction is enabled.
pub fn enabled(&self) -> bool {
self.sys_handle.enabled()
}
/// Lookup the frame metadata for a given RTP timestamp (receiver side).
/// Returns `Some((user_timestamp, frame_id))` if found, `None` otherwise.
/// The entry is removed from the map after a successful lookup.
pub fn lookup_frame_metadata(&self, rtp_timestamp: u32) -> Option<(u64, u32)> {
let ts = self.sys_handle.lookup_timestamp(rtp_timestamp);
if ts != u64::MAX {
let frame_id = self.sys_handle.last_lookup_frame_id();
Some((ts, frame_id))
} else {
None
}
}
/// Store frame metadata for a given capture timestamp (sender side).
///
/// The `capture_timestamp_us` must be the TimestampAligner-adjusted
/// timestamp (as produced by `VideoTrackSource::on_captured_frame`),
/// NOT the original `timestamp_us` from the VideoFrame. The transformer
/// looks up the metadata by the frame's `CaptureTime()` which is
/// derived from the aligned value.
///
/// In normal usage this is called automatically by the C++ layer --
/// callers should set `user_timestamp` and `frame_id` on the
/// `VideoFrame` and let `capture_frame` / `on_captured_frame` handle
/// the rest.
pub fn store_frame_metadata(
&self,
capture_timestamp_us: i64,
user_timestamp: u64,
frame_id: u32,
) {
self.sys_handle.store_frame_metadata(capture_timestamp_us, user_timestamp, frame_id);
}
pub(crate) fn sys_handle(&self) -> SharedPtr<sys_pt::PacketTrailerHandler> {
self.sys_handle.clone()
}
/// Set the callback receiving sender-side publish timing events.
pub fn set_publish_timing_observer(&self, observer: Option<PublishTimingObserver>) {
if let Some(observer) = observer {
self.sys_handle.set_publish_timing_observer(Box::new(
webrtc_sys::packet_trailer::VideoPublishTimingObserverWrapper::new(Box::new(
move |event| observer(event.into()),
)),
));
} else {
self.sys_handle.clear_publish_timing_observer();
}
}
/// Set the callback receiving receiver-side subscribe timing events.
pub fn set_subscribe_timing_observer(&self, observer: Option<SubscribeTimingObserver>) {
if let Some(observer) = observer {
self.sys_handle.set_subscribe_timing_observer(Box::new(
webrtc_sys::packet_trailer::VideoSubscribeTimingObserverWrapper::new(Box::new(
move |event| observer(event.into()),
)),
));
} else {
self.sys_handle.clear_subscribe_timing_observer();
}
}
pub(crate) fn emit_subscribe_timing(
&self,
stage: SubscribeTimingStage,
capture_timestamp_us: u64,
frame_id: u32,
) {
let stage = match stage {
SubscribeTimingStage::WebrtcReceive => sys_pt::VideoSubscribeTimingStage::WebrtcReceive,
SubscribeTimingStage::DecoderUpload => sys_pt::VideoSubscribeTimingStage::DecoderUpload,
SubscribeTimingStage::DecoderOutput => sys_pt::VideoSubscribeTimingStage::DecoderOutput,
};
self.sys_handle.emit_subscribe_timing(stage, capture_timestamp_us, frame_id);
}
}
/// Create a sender-side packet trailer handler.
///
/// This handler will embed frame metadata into encoded frames before
/// they are packetized and sent. Use `store_frame_metadata()` to
/// associate metadata with a captured frame's capture timestamp.
pub fn create_sender_handler(
peer_factory: &PeerConnectionFactory,
sender: &RtpSender,
) -> PacketTrailerHandler {
PacketTrailerHandler {
sys_handle: sys_pt::new_packet_trailer_sender(
peer_factory.handle.sys_handle.clone(),
sender.handle.sys_handle.clone(),
),
}
}
/// Create a receiver-side packet trailer handler.
///
/// This handler will extract frame metadata from received frames
/// and store them in a map keyed by RTP timestamp. Use
/// `lookup_frame_metadata(rtp_timestamp)` to retrieve the metadata
/// for a specific decoded frame.
pub fn create_receiver_handler(
peer_factory: &PeerConnectionFactory,
receiver: &RtpReceiver,
) -> PacketTrailerHandler {
PacketTrailerHandler {
sys_handle: sys_pt::new_packet_trailer_receiver(
peer_factory.handle.sys_handle.clone(),
receiver.handle.sys_handle.clone(),
),
}
}
@@ -0,0 +1,619 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use tokio::sync::{mpsc, oneshot};
use webrtc_sys::{
data_channel as sys_dc, jsep as sys_jsep, peer_connection as sys_pc,
peer_connection_factory as sys_pcf, rtc_error as sys_err,
};
use crate::{
data_channel::{DataChannel, DataChannelInit},
ice_candidate::IceCandidate,
imp::{
data_channel as imp_dc, ice_candidate as imp_ic, media_stream as imp_ms,
media_stream_track as imp_mst, rtp_receiver as imp_rr, rtp_sender as imp_rs,
rtp_transceiver as imp_rt, session_description as imp_sdp,
},
media_stream::MediaStream,
media_stream_track::MediaStreamTrack,
peer_connection::{
AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError,
OnIceConnectionChange, OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange,
OnTrack, PeerConnectionState, SignalingState, TrackEvent,
},
peer_connection_factory::{
ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
},
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
session_description::SessionDescription,
stats::RtcStats,
MediaType, RtcError, RtcErrorType,
};
impl From<OfferOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
fn from(options: OfferOptions) -> Self {
Self {
ice_restart: options.ice_restart,
offer_to_receive_audio: options.offer_to_receive_audio as i32,
offer_to_receive_video: options.offer_to_receive_video as i32,
..Default::default()
}
}
}
impl From<AnswerOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
fn from(_options: AnswerOptions) -> Self {
Self::default()
}
}
impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
match state {
sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
_ => panic!("unknown PeerConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
match state {
sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
IceConnectionState::Connected
}
sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
IceConnectionState::Completed
}
sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
IceConnectionState::Disconnected
}
sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
_ => panic!("unknown IceConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
match state {
sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
_ => panic!("unknown IceGatheringState"),
}
}
}
impl From<sys_pc::ffi::SignalingState> for SignalingState {
fn from(state: sys_pc::ffi::SignalingState) -> Self {
match state {
sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
_ => panic!("unknown SignalingState"),
}
}
}
impl From<IceServer> for sys_pc::ffi::IceServer {
fn from(value: IceServer) -> Self {
sys_pc::ffi::IceServer {
urls: value.urls,
username: value.username,
password: value.password,
}
}
}
impl From<ContinualGatheringPolicy> for sys_pc::ffi::ContinualGatheringPolicy {
fn from(value: ContinualGatheringPolicy) -> Self {
match value {
ContinualGatheringPolicy::GatherOnce => {
sys_pc::ffi::ContinualGatheringPolicy::GatherOnce
}
ContinualGatheringPolicy::GatherContinually => {
sys_pc::ffi::ContinualGatheringPolicy::GatherContinually
}
}
}
}
impl From<IceTransportsType> for sys_pc::ffi::IceTransportsType {
fn from(value: IceTransportsType) -> Self {
match value {
IceTransportsType::Relay => sys_pc::ffi::IceTransportsType::Relay,
IceTransportsType::NoHost => sys_pc::ffi::IceTransportsType::NoHost,
IceTransportsType::All => sys_pc::ffi::IceTransportsType::All,
}
}
}
impl From<RtcConfiguration> for sys_pc::ffi::RtcConfiguration {
fn from(value: RtcConfiguration) -> Self {
Self {
ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
continual_gathering_policy: value.continual_gathering_policy.into(),
ice_transport_type: value.ice_transport_type.into(),
}
}
}
#[derive(Clone)]
pub struct PeerConnection {
observer: Arc<PeerObserver>,
pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
}
impl PeerConnection {
pub fn configure(
sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
observer: Arc<PeerObserver>,
) -> Self {
Self { sys_handle, observer }
}
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
let res = self.sys_handle.set_configuration(config.into());
match res {
Ok(_) => Ok(()),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub async fn create_offer(
&self,
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
self.sys_handle.create_offer(
options.into(),
ctx,
|ctx, sdp| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Ok(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
}));
},
|ctx, error| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Err(error.into()));
},
);
rx.recv().await.unwrap()
}
pub async fn create_answer(
&self,
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
self.sys_handle.create_answer(
options.into(),
ctx,
|ctx, sdp| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Ok(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
}));
},
|ctx, error| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Err(error.into()));
},
);
rx.recv().await.unwrap()
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.set_local_description(desc.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.unwrap()
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.set_remote_description(desc.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "set_remote_description cancelled".to_owned(),
})?
}
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.add_ice_candidate(candidate.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "add_ice_candidate cancelled".to_owned(),
})?
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RtcError> {
let res = self.sys_handle.create_data_channel(label.to_string(), init.into());
match res {
Ok(sys_handle) => {
Ok(DataChannel { handle: imp_dc::DataChannel::configure(sys_handle) })
}
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
}
}
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
stream_ids: &[T],
) -> Result<RtpSender, RtcError> {
let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
match res {
Ok(sys_handle) => Ok(RtpSender { handle: imp_rs::RtpSender { sys_handle } }),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self.sys_handle.add_transceiver(track.sys_handle(), init.into());
match res {
Ok(sys_handle) => Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle } }),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self.sys_handle.add_transceiver_for_media(media_type.into(), init.into());
match res {
Ok(cxx_handle) => {
Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle: cxx_handle } })
}
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn restart_ice(&self) {
self.sys_handle.restart_ice();
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn connection_state(&self) -> PeerConnectionState {
self.sys_handle.connection_state().into()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.sys_handle.ice_connection_state().into()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.sys_handle.ice_gathering_state().into()
}
pub fn signaling_state(&self) -> SignalingState {
self.sys_handle.signaling_state().into()
}
pub fn current_local_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_local_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
}
pub fn current_remote_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_remote_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.sys_handle
.remove_track(sender.handle.sys_handle)
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn senders(&self) -> Vec<RtpSender> {
self.sys_handle
.get_senders()
.into_iter()
.map(|sender| RtpSender { handle: imp_rs::RtpSender { sys_handle: sender.ptr } })
.collect()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.sys_handle
.get_receivers()
.into_iter()
.map(|receiver| RtpReceiver {
handle: imp_rr::RtpReceiver { sys_handle: receiver.ptr },
})
.collect()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.sys_handle
.get_transceivers()
.into_iter()
.map(|transceiver| RtpTransceiver {
handle: imp_rt::RtpTransceiver { sys_handle: transceiver.ptr },
})
.collect()
}
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
*self.observer.connection_change_handler.lock() = f;
}
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
*self.observer.data_channel_handler.lock() = f;
}
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
*self.observer.ice_candidate_handler.lock() = f;
}
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
*self.observer.ice_candidate_error_handler.lock() = f;
}
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
*self.observer.ice_connection_change_handler.lock() = f;
}
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
*self.observer.ice_gathering_change_handler.lock() = f;
}
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
*self.observer.negotiation_needed_handler.lock() = f;
}
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
*self.observer.signaling_change_handler.lock() = f;
}
pub fn on_track(&self, f: Option<OnTrack>) {
*self.observer.track_handler.lock() = f;
}
}
#[derive(Default)]
pub struct PeerObserver {
pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
pub data_channel_handler: Mutex<Option<OnDataChannel>>,
pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
pub track_handler: Mutex<Option<OnTrack>>,
}
impl sys_pcf::PeerConnectionObserver for PeerObserver {
fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
if let Some(f) = self.signaling_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
if let Some(f) = self.data_channel_handler.lock().as_mut() {
f(DataChannel { handle: imp_dc::DataChannel::configure(data_channel) });
}
}
fn on_renegotiation_needed(&self) {}
fn on_negotiation_needed_event(&self, event: u32) {
if let Some(f) = self.negotiation_needed_handler.lock().as_mut() {
f(event);
}
}
fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
if let Some(f) = self.ice_connection_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
if let Some(f) = self.connection_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
if let Some(f) = self.ice_gathering_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
if let Some(f) = self.ice_candidate_handler.lock().as_mut() {
f(IceCandidate { handle: imp_ic::IceCandidate { sys_handle: candidate } });
}
}
fn on_ice_candidate_error(
&self,
address: String,
port: i32,
url: String,
error_code: i32,
error_text: String,
) {
if let Some(f) = self.ice_candidate_error_handler.lock().as_mut() {
f(IceCandidateError { address, port, url, error_code, error_text });
}
}
fn on_ice_candidates_removed(
&self,
_removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
) {
}
fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
fn on_ice_selected_candidate_pair_changed(
&self,
_event: sys_pcf::ffi::CandidatePairChangeEvent,
) {
}
fn on_add_track(
&self,
_receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
_streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
) {
}
fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
if let Some(f) = self.track_handler.lock().as_mut() {
let receiver = transceiver.receiver();
let streams = receiver.streams();
let track = receiver.track();
f(TrackEvent {
receiver: RtpReceiver { handle: imp_rr::RtpReceiver { sys_handle: receiver } },
streams: streams
.into_iter()
.map(|s| MediaStream { handle: imp_ms::MediaStream { sys_handle: s.ptr } })
.collect(),
track: imp_mst::new_media_stream_track(track),
transceiver: RtpTransceiver {
handle: imp_rt::RtpTransceiver { sys_handle: transceiver },
},
});
}
}
fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
fn on_interesting_usage(&self, _usage_pattern: i32) {}
}
@@ -0,0 +1,361 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::{SharedPtr, UniquePtr};
use lazy_static::lazy_static;
use parking_lot::Mutex;
use webrtc_sys::{peer_connection_factory as sys_pcf, rtc_error as sys_err, webrtc as sys_rtc};
use crate::{
audio_source::native::NativeAudioSource,
audio_track::RtcAudioTrack,
imp::{audio_track as imp_at, peer_connection as imp_pc, video_track as imp_vt},
peer_connection::PeerConnection,
peer_connection_factory::RtcConfiguration,
rtp_parameters::RtpCapabilities,
video_source::native::NativeVideoSource,
video_track::RtcVideoTrack,
MediaType, RtcError,
};
lazy_static! {
static ref LOG_SINK: Mutex<Option<UniquePtr<sys_rtc::ffi::LogSink>>> = Default::default();
}
#[derive(Clone)]
pub struct PeerConnectionFactory {
pub(crate) sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
}
impl Default for PeerConnectionFactory {
fn default() -> Self {
let mut log_sink = LOG_SINK.lock();
if log_sink.is_none() {
*log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, _| {
let msg = msg.strip_suffix("\r\n").or(msg.strip_suffix('\n')).unwrap_or(&msg);
log::debug!(target: "libwebrtc", "{}", msg);
}));
}
let sys_handle = sys_pcf::ffi::create_peer_connection_factory();
Self { sys_handle }
}
}
impl PeerConnectionFactory {
pub fn create_peer_connection(
&self,
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
let observer = Arc::new(imp_pc::PeerObserver::default());
let res = self.sys_handle.create_peer_connection(
config.into(),
Box::new(sys_pcf::PeerConnectionObserverWrapper::new(observer.clone())),
);
match res {
Ok(sys_handle) => Ok(PeerConnection {
handle: imp_pc::PeerConnection::configure(sys_handle, observer),
}),
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
}
}
pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
RtcVideoTrack {
handle: imp_vt::RtcVideoTrack::new(
self.sys_handle.create_video_track(label.to_string(), source.handle.sys_handle()),
),
}
}
pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
RtcAudioTrack {
handle: imp_at::RtcAudioTrack {
sys_handle: self
.sys_handle
.create_audio_track(label.to_string(), source.handle.sys_handle()),
},
}
}
/// Create an audio track that uses the Platform ADM for capture.
///
/// This requires that `enable_platform_adm()` was called first.
/// The track will capture audio from the selected recording device.
pub fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
RtcAudioTrack {
handle: imp_at::RtcAudioTrack {
sys_handle: self.sys_handle.create_device_audio_track(label.to_string()),
},
}
}
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle.rtp_sender_capabilities(media_type.into()).into()
}
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle.rtp_receiver_capabilities(media_type.into()).into()
}
// ===== Device Management Methods =====
/// Get the number of playout (output) devices
pub fn playout_devices(&self) -> i16 {
self.sys_handle.audio_device().playout_devices()
}
/// Get the number of recording (input) devices
pub fn recording_devices(&self) -> i16 {
self.sys_handle.audio_device().recording_devices()
}
/// Get the name of a playout device by index
pub fn playout_device_name(&self, index: u16) -> String {
self.sys_handle.audio_device().playout_device_name(index)
}
/// Get the name of a recording device by index
pub fn recording_device_name(&self, index: u16) -> String {
self.sys_handle.audio_device().recording_device_name(index)
}
/// Get the GUID of a playout device by index
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
pub fn playout_device_guid(&self, index: u16) -> String {
self.sys_handle.audio_device().playout_device_guid(index)
}
/// Get the GUID of a recording device by index
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
pub fn recording_device_guid(&self, index: u16) -> String {
self.sys_handle.audio_device().recording_device_guid(index)
}
/// Set the playout device by index
pub fn set_playout_device(&self, index: u16) -> bool {
self.sys_handle.audio_device().set_playout_device(index)
}
/// Set the recording device by index
pub fn set_recording_device(&self, index: u16) -> bool {
self.sys_handle.audio_device().set_recording_device(index)
}
/// Set the playout device by GUID
/// This is preferred over index as GUIDs are stable across device hot-plug events.
pub fn set_playout_device_by_guid(&self, guid: &str) -> bool {
self.sys_handle.audio_device().set_playout_device_by_guid(guid.to_string())
}
/// Set the recording device by GUID
/// This is preferred over index as GUIDs are stable across device hot-plug events.
pub fn set_recording_device_by_guid(&self, guid: &str) -> bool {
self.sys_handle.audio_device().set_recording_device_by_guid(guid.to_string())
}
/// Stop recording (clears initialized state, allowing device switch)
pub fn stop_recording(&self) -> bool {
self.sys_handle.audio_device().stop_recording()
}
/// Initialize recording
pub fn init_recording(&self) -> bool {
self.sys_handle.audio_device().init_recording()
}
/// Start recording
pub fn start_recording(&self) -> bool {
self.sys_handle.audio_device().start_recording()
}
/// Check if recording is initialized
pub fn recording_is_initialized(&self) -> bool {
self.sys_handle.audio_device().recording_is_initialized()
}
/// Stop playout (clears initialized state, allowing device switch)
pub fn stop_playout(&self) -> bool {
self.sys_handle.audio_device().stop_playout()
}
/// Initialize playout
pub fn init_playout(&self) -> bool {
self.sys_handle.audio_device().init_playout()
}
/// Start playout
pub fn start_playout(&self) -> bool {
self.sys_handle.audio_device().start_playout()
}
/// Check if playout is initialized
pub fn playout_is_initialized(&self) -> bool {
self.sys_handle.audio_device().playout_is_initialized()
}
// ===== Built-in Audio Processing Methods =====
// These control hardware AEC/AGC/NS on platforms that support it (iOS, some Android)
/// Check if built-in (hardware) AEC is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_aec_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_aec_is_available()
}
/// Check if built-in (hardware) AGC is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_agc_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_agc_is_available()
}
/// Check if built-in (hardware) NS is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_ns_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_ns_is_available()
}
/// Enable or disable built-in (hardware) AEC.
///
/// When disabled on platforms that support it, WebRTC's software AEC
/// will be used instead.
pub fn enable_builtin_aec(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_aec(enable)
}
/// Enable or disable built-in (hardware) AGC.
///
/// When disabled on platforms that support it, WebRTC's software AGC
/// will be used instead.
pub fn enable_builtin_agc(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_agc(enable)
}
/// Enable or disable built-in (hardware) NS.
///
/// When disabled on platforms that support it, WebRTC's software NS
/// will be used instead.
pub fn enable_builtin_ns(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_ns(enable)
}
/// Control whether ADM recording (microphone) is enabled.
///
/// When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops.
/// Use this when only using NativeAudioSource (no microphone capture needed).
/// This prevents the microphone from interfering with the audio pipeline.
pub fn set_adm_recording_enabled(&self, enabled: bool) {
self.sys_handle.audio_device().set_adm_recording_enabled(enabled)
}
/// Check if ADM recording (microphone) is enabled.
pub fn adm_recording_enabled(&self) -> bool {
self.sys_handle.audio_device().adm_recording_enabled()
}
/// Control whether ADM playout (speakers) is enabled.
///
/// When disabled (default), playout uses synthetic mode - remote audio is
/// delivered via FFI callbacks to the application (e.g., Unity AudioSource).
/// When enabled, remote audio plays through the platform speakers with AEC.
pub fn set_adm_playout_enabled(&self, enabled: bool) {
self.sys_handle.audio_device().set_adm_playout_enabled(enabled)
}
/// Check if ADM playout (speakers) is enabled.
pub fn adm_playout_enabled(&self) -> bool {
self.sys_handle.audio_device().adm_playout_enabled()
}
// ===== Platform ADM Lifecycle Management =====
/// Acquires a reference to the Platform ADM.
///
/// On first call, creates and initializes the Platform ADM. On subsequent
/// calls, just increments the reference count.
///
/// Returns true if Platform ADM is ready for use, false if initialization failed.
pub fn acquire_platform_adm(&self) -> bool {
self.sys_handle.audio_device().acquire_platform_adm()
}
/// Releases a reference to the Platform ADM.
///
/// When the reference count reaches zero, the Platform ADM is terminated
/// and the proxy returns to synthetic mode.
pub fn release_platform_adm(&self) {
self.sys_handle.audio_device().release_platform_adm()
}
/// Returns the current reference count for the Platform ADM.
pub fn platform_adm_ref_count(&self) -> i32 {
self.sys_handle.audio_device().platform_adm_ref_count()
}
/// Returns true if Platform ADM is currently active (ref_count > 0).
pub fn is_platform_adm_active(&self) -> bool {
self.sys_handle.audio_device().is_platform_adm_active()
}
/// Ensures the Platform ADM exists, retrying creation if an earlier
/// attempt failed (e.g. the OS audio stack was still starting up).
///
/// Returns true if the Platform ADM is available after the call.
pub fn ensure_platform_adm(&self) -> bool {
self.sys_handle.audio_device().ensure_platform_adm()
}
/// Returns true if the Platform ADM has been created and initialized.
/// Distinguishes "audio stack unavailable" from "zero audio devices".
pub fn platform_adm_available(&self) -> bool {
self.sys_handle.audio_device().platform_adm_available()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static TEST_MUTEX: Mutex<()> = Mutex::new(());
#[tokio::test]
async fn test_peer_connection_factory_and_audio_device_controller_bridge() {
let _guard = TEST_MUTEX.lock().expect("test mutex poisoned");
let _ = env_logger::builder().is_test(true).try_init();
let factory = PeerConnectionFactory::default();
let source = NativeVideoSource::default();
let _track = factory.create_video_track("test", source);
let recording_count = factory.recording_devices();
let playout_count = factory.playout_devices();
assert!(recording_count >= 0);
assert!(playout_count >= 0);
let initial_recording = factory.adm_recording_enabled();
factory.set_adm_recording_enabled(!initial_recording);
assert_eq!(factory.adm_recording_enabled(), !initial_recording);
factory.set_adm_recording_enabled(initial_recording);
}
}
@@ -0,0 +1,263 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use webrtc_sys::{rtp_parameters as sys_rp, webrtc as sys_webrtc};
use crate::rtp_parameters::*;
impl From<sys_webrtc::ffi::Priority> for Priority {
fn from(value: sys_webrtc::ffi::Priority) -> Self {
match value {
sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
sys_webrtc::ffi::Priority::Low => Self::Low,
sys_webrtc::ffi::Priority::Medium => Self::Medium,
sys_webrtc::ffi::Priority::High => Self::High,
_ => panic!("unknown Priority"),
}
}
}
impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
fn from(value: sys_rp::ffi::RtpExtension) -> Self {
Self { uri: value.uri, id: value.id, encrypted: value.encrypt }
}
}
impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
fn from(value: sys_rp::ffi::RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
rtcp: value.rtcp.into(),
}
}
}
impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
Self {
mime_type: value.mime_type,
payload_type: value.payload_type as u8,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
channels: value.has_num_channels.then_some(value.num_channels as u16),
}
}
}
impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
Self { cname: value.cname, reduced_size: value.reduced_size }
}
}
impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
Self {
active: value.active,
max_bitrate: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps as u64),
max_framerate: value.has_max_framerate.then_some(value.max_framerate),
priority: value.network_priority.into(),
rid: value.rid,
scale_resolution_down_by: value
.has_scale_resolution_down_by
.then_some(value.scale_resolution_down_by),
scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode),
}
}
}
impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
Self {
channels: value.has_num_channels.then_some(value.num_channels as u16),
mime_type: value.mime_type,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
sdp_fmtp_line: {
let parameters: Vec<String> = value
.parameters
.into_iter()
.map(|key_value| {
if !key_value.key.is_empty() {
format!("{}={}", key_value.key, key_value.value)
} else {
key_value.value
}
})
.collect();
if !parameters.is_empty() {
Some(parameters.join(";"))
} else {
None
}
},
}
}
}
impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
Self { direction: value.direction.into(), uri: value.uri }
}
}
impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
}
}
}
impl From<Priority> for sys_webrtc::ffi::Priority {
fn from(value: Priority) -> Self {
match value {
Priority::VeryLow => Self::VeryLow,
Priority::Low => Self::Low,
Priority::Medium => Self::Medium,
Priority::High => Self::High,
}
}
}
impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
fn from(value: RtpHeaderExtensionParameters) -> Self {
Self { uri: value.uri, id: value.id, encrypt: value.encrypted }
}
}
impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
fn from(value: RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
encodings: Vec::new(),
rtcp: value.rtcp.into(),
transaction_id: "".to_string(),
mid: "".to_string(),
has_degradation_preference: false,
degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
}
}
}
impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
fn from(value: RtpCodecParameters) -> Self {
Self {
payload_type: value.payload_type as i32,
mime_type: value.mime_type,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
name: "".to_string(),
kind: sys_rp::ffi::MediaType::Audio,
has_max_ptime: false,
max_ptime: 0,
has_ptime: false,
ptime: 0,
rtcp_feedback: Vec::new(),
parameters: Vec::new(),
}
}
}
impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
fn from(value: RtcpParameters) -> Self {
Self {
cname: value.cname,
reduced_size: value.reduced_size,
has_ssrc: false,
ssrc: 0,
mux: false,
}
}
}
impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
fn from(value: RtpEncodingParameters) -> Self {
Self {
active: value.active,
has_max_bitrate_bps: value.max_bitrate.is_some(),
max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
has_max_framerate: value.max_framerate.is_some(),
max_framerate: value.max_framerate.unwrap_or_default(),
network_priority: value.priority.into(),
rid: value.rid,
has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
adaptive_ptime: false,
bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
has_min_bitrate_bps: false,
min_bitrate_bps: 0,
has_num_temporal_layers: false,
num_temporal_layers: 0,
has_scalability_mode: value.scalability_mode.is_some(),
scalability_mode: value.scalability_mode.unwrap_or_default(),
has_ssrc: false,
ssrc: 0,
}
}
}
impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
fn from(value: RtpCodecCapability) -> Self {
let mime_type: Vec<&str> = value.mime_type.split('/').collect();
let kind = match mime_type[0] {
"audio" => sys_webrtc::ffi::MediaType::Audio,
"video" => sys_webrtc::ffi::MediaType::Video,
_ => panic!("invalid media type"),
};
let name = mime_type[1].to_string();
Self {
name,
kind,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
parameters: {
value
.sdp_fmtp_line
.map(|sdp_fmtp_line| {
sdp_fmtp_line
.split(';')
.map(|v| {
let key_value: Vec<&str> = v.split('=').collect();
if key_value.len() == 2 {
sys_rp::ffi::StringKeyValue {
key: key_value[0].to_string(),
value: key_value[1].to_string(),
}
} else {
sys_rp::ffi::StringKeyValue {
key: "".to_string(),
value: key_value[0].to_string(),
}
}
})
.collect()
})
.unwrap_or_default()
},
// Ignore
mime_type: String::default(), // !!
has_preferred_payload_type: false,
preferred_payload_type: 0,
rtcp_feedback: Vec::default(),
}
}
}
@@ -0,0 +1,65 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::rtp_receiver as sys_rr;
use crate::{
imp::media_stream_track::new_media_stream_track, media_stream_track::MediaStreamTrack,
rtp_parameters::RtpParameters, stats::RtcStats, RtcError, RtcErrorType,
};
#[derive(Clone)]
pub struct RtpReceiver {
pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
}
impl RtpReceiver {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_rr::ReceiverContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
}
@@ -0,0 +1,83 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::{rtc_error as sys_err, rtp_sender as sys_rs};
use super::media_stream_track::new_media_stream_track;
use crate::{
media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
RtcErrorType,
};
#[derive(Clone)]
pub struct RtpSender {
pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
}
impl RtpSender {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_rs::SenderContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
if !self.sys_handle.set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle())) {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "Failed to set track".to_string(),
});
}
Ok(())
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.sys_handle
.set_parameters(parameters.into())
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
}
@@ -0,0 +1,98 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::{rtc_error as sys_err, rtp_transceiver as sys_rt, webrtc as sys_webrtc};
use crate::{
imp::{rtp_receiver::RtpReceiver, rtp_sender::RtpSender},
rtp_parameters::RtpCodecCapability,
rtp_receiver, rtp_sender,
rtp_transceiver::{RtpTransceiverDirection, RtpTransceiverInit},
RtcError,
};
impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
match value {
sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
sys_webrtc::ffi::RtpTransceiverDirection::Stopped => Self::Stopped,
_ => panic!("unknown RtpTransceiverDirection"),
}
}
}
impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
fn from(value: RtpTransceiverDirection) -> Self {
match value {
RtpTransceiverDirection::SendRecv => Self::SendRecv,
RtpTransceiverDirection::SendOnly => Self::SendOnly,
RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
RtpTransceiverDirection::Inactive => Self::Inactive,
RtpTransceiverDirection::Stopped => Self::Stopped,
}
}
}
impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
fn from(value: RtpTransceiverInit) -> Self {
Self {
direction: value.direction.into(),
stream_ids: value.stream_ids,
send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
}
}
}
#[derive(Clone)]
pub struct RtpTransceiver {
pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
}
impl RtpTransceiver {
pub fn mid(&self) -> Option<String> {
self.sys_handle.mid().ok()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.sys_handle.current_direction().ok().map(Into::into)
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.sys_handle.direction().into()
}
pub fn sender(&self) -> rtp_sender::RtpSender {
rtp_sender::RtpSender { handle: RtpSender { sys_handle: self.sys_handle.sender() } }
}
pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
rtp_receiver::RtpReceiver { handle: RtpReceiver { sys_handle: self.sys_handle.receiver() } }
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.sys_handle
.set_codec_preferences(codecs.into_iter().map(Into::into).collect())
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
pub fn stop(&self) -> Result<(), RtcError> {
self.sys_handle
.stop_standard()
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
}
@@ -0,0 +1,82 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::jsep as sys_jsep;
use crate::session_description::{self, SdpParseError, SdpType};
impl From<sys_jsep::ffi::SdpType> for SdpType {
fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
match sdp_type {
sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
_ => panic!("unknown SdpType"),
}
}
}
impl From<SdpType> for sys_jsep::ffi::SdpType {
fn from(sdp_type: SdpType) -> Self {
match sdp_type {
SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
}
}
}
impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
Self { line: e.line, description: e.description }
}
}
pub struct SessionDescription {
pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
}
impl SessionDescription {
pub fn parse(
sdp: &str,
sdp_type: SdpType,
) -> Result<session_description::SessionDescription, SdpParseError> {
let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
match res {
Ok(sys_handle) => Ok(session_description::SessionDescription {
handle: SessionDescription { sys_handle },
}),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_type(&self) -> SdpType {
self.sys_handle.sdp_type().into()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
impl Clone for SessionDescription {
fn clone(&self) -> Self {
SessionDescription { sys_handle: self.sys_handle.clone() }
}
}
@@ -0,0 +1,956 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::slice;
use cxx::UniquePtr;
use webrtc_sys::{video_frame as vf_sys, video_frame_buffer as vfb_sys};
use super::yuv_helper;
use crate::video_frame::{self as vf, VideoFormatType, VideoRotation};
/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
/// because directly using platform specific types is not valid (e.g user callback)
/// All the types inside this module are only used internally. For public types, see the top level
/// video_frame.rs
pub fn new_video_frame_buffer(
mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
) -> Box<dyn vf::VideoBuffer + Send + Sync> {
unsafe {
match sys_handle.buffer_type() {
vfb_sys::ffi::VideoFrameBufferType::Native => {
Box::new(vf::native::NativeBuffer { handle: NativeBuffer { sys_handle } })
}
vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
handle: I420Buffer { sys_handle: sys_handle.pin_mut().get_i420() },
}),
vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
handle: I420ABuffer { sys_handle: sys_handle.pin_mut().get_i420a() },
}),
vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
handle: I422Buffer { sys_handle: sys_handle.pin_mut().get_i422() },
}),
vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
handle: I444Buffer { sys_handle: sys_handle.pin_mut().get_i444() },
}),
vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
handle: I010Buffer { sys_handle: sys_handle.pin_mut().get_i010() },
}),
vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
handle: NV12Buffer { sys_handle: sys_handle.pin_mut().get_nv12() },
}),
_ => unreachable!(),
}
}
}
impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
match rotation {
vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
_ => panic!("invalid VideoRotation"),
}
}
}
impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
fn from(rotation: VideoRotation) -> Self {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
}
}
}
macro_rules! recursive_cast {
($ptr:expr $(, $fnc:ident)*) => {
{
let ptr = $ptr;
$(
let ptr = vfb_sys::ffi::$fnc(ptr);
)*
ptr
}
};
}
pub struct NativeBuffer {
sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
}
pub struct I420Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
}
pub struct I420ABuffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
}
pub struct I422Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
}
pub struct I444Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
}
pub struct I010Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
}
pub struct NV12Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
}
macro_rules! impl_to_argb {
(I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
match $format {
$(
VideoFormatType::$variant => {
let (data_y, data_u, data_v) = $self.data();
yuv_helper::$fnc(
data_y,
$self.stride_y(),
data_u,
$self.stride_u(),
data_v,
$self.stride_v(),
$dst,
$dst_stride,
$dst_width,
$dst_height,
)
}
)+
}
};
(I420ABuffer) => {
todo!();
}
}
#[allow(unused_unsafe)]
impl NativeBuffer {
pub fn from_fluxer_d3d11_texture(
handle: u64,
width: u32,
height: u32,
dxgi_format: u32,
) -> Option<vf::native::NativeBuffer> {
let sys_handle = vfb_sys::ffi::new_fluxer_d3d11_texture_buffer(
handle,
width,
height,
dxgi_format,
);
if sys_handle.is_null() {
return None;
}
Some(vf::native::NativeBuffer {
handle: NativeBuffer { sys_handle },
})
}
#[allow(clippy::too_many_arguments)]
pub fn from_fluxer_dmabuf_texture(
fds: [i32; 4],
plane_count: u32,
width: u32,
height: u32,
drm_format: u32,
modifier: u64,
strides: [u32; 4],
offsets: [u32; 4],
device_uuid_hi: u64,
device_uuid_lo: u64,
) -> Option<vf::native::NativeBuffer> {
let sys_handle = vfb_sys::ffi::new_fluxer_dmabuf_texture_buffer(
fds[0],
fds[1],
fds[2],
fds[3],
plane_count,
width,
height,
drm_format,
modifier,
strides[0],
strides[1],
strides[2],
strides[3],
offsets[0],
offsets[1],
offsets[2],
offsets[3],
device_uuid_hi,
device_uuid_lo,
);
if sys_handle.is_null() {
return None;
}
Some(vf::native::NativeBuffer {
handle: NativeBuffer { sys_handle },
})
}
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub unsafe fn from_cv_pixel_buffer(
cv_pixel_buffer: *mut std::ffi::c_void,
) -> vf::native::NativeBuffer {
vf::native::NativeBuffer {
handle: NativeBuffer {
sys_handle: vfb_sys::ffi::new_native_buffer_from_platform_image_buffer(
cv_pixel_buffer as *mut _,
),
},
}
}
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
unsafe { vfb_sys::ffi::native_buffer_to_platform_image_buffer(&self.sys_handle) as *mut _ }
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
&self.sys_handle
}
pub fn width(&self) -> u32 {
self.sys_handle.width()
}
pub fn height(&self) -> u32 {
self.sys_handle.height()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer { sys_handle: unsafe { self.sys_handle.to_i420() } }
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
impl I420Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I420Buffer {
vf::I420Buffer {
handle: I420Buffer {
sys_handle: vfb_sys::ffi::new_i420_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
// We make a copy of the buffer because internally, when calling ToI420()
// if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
// for the new created type
let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
impl_to_argb!(
I420Buffer
[
ARGB: i420_to_argb,
BGRA: i420_to_bgra,
ABGR: i420_to_abgr,
RGBA: i420_to_rgba
],
format, self, dst, dst_stride, dst_width, dst_height
)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420Buffer {
vf::I420Buffer {
handle: I420Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I420ABuffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn stride_a(&self) -> u32 {
self.sys_handle.stride_a()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
let data_a = self.sys_handle.data_a();
let has_data_a = !data_a.is_null();
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
has_data_a.then_some(slice::from_raw_parts(
data_a,
(self.stride_a() * self.height()) as usize,
)),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420ABuffer {
vf::I420ABuffer {
handle: I420ABuffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I422Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I422Buffer {
vf::I422Buffer {
handle: I422Buffer {
sys_handle: vfb_sys::ffi::new_i422_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I422Buffer {
vf::I422Buffer {
handle: I422Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I444Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I444Buffer {
vf::I444Buffer {
handle: I444Buffer {
sys_handle: vfb_sys::ffi::new_i444_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I444Buffer {
vf::I444Buffer {
handle: I444Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I010Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I010Buffer {
vf::I010Buffer {
handle: I010Buffer {
sys_handle: vfb_sys::ffi::new_i010_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts(
(*ptr).data_y(),
(self.stride_y() * self.height()) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_u(),
(self.stride_u() * chroma_height) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_v(),
(self.stride_v() * chroma_height) as usize / 2,
),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I010Buffer {
vf::I010Buffer {
handle: I010Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl NV12Buffer {
pub fn new(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> vf::NV12Buffer {
vf::NV12Buffer {
handle: NV12Buffer {
sys_handle: vfb_sys::ffi::new_nv12_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_uv.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe {
&*recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb)
}
}
pub fn width(&self) -> u32 {
unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_y()
}
}
pub fn stride_uv(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_uv()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts(
(*ptr).data_uv(),
(self.stride_uv() * chroma_height) as usize,
),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::NV12Buffer {
vf::NV12Buffer {
handle: NV12Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
@@ -0,0 +1,147 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use cxx::SharedPtr;
use livekit_runtime::interval;
use parking_lot::Mutex;
use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_track as vt_sys};
use crate::{
native::packet_trailer::PacketTrailerHandler,
video_frame::{I420Buffer, VideoBuffer, VideoFrame},
video_source::VideoResolution,
};
impl From<vt_sys::ffi::VideoResolution> for VideoResolution {
fn from(res: vt_sys::ffi::VideoResolution) -> Self {
Self { width: res.width, height: res.height }
}
}
impl From<VideoResolution> for vt_sys::ffi::VideoResolution {
fn from(res: VideoResolution) -> Self {
Self { width: res.width, height: res.height }
}
}
#[derive(Clone)]
pub struct NativeVideoSource {
sys_handle: SharedPtr<vt_sys::ffi::VideoTrackSource>,
inner: Arc<Mutex<VideoSourceInner>>,
}
struct VideoSourceInner {
captured_frames: usize,
}
impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
let source = Self {
sys_handle: vt_sys::ffi::new_video_track_source(
&vt_sys::ffi::VideoResolution::from(resolution.clone()),
is_screencast,
),
inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
};
livekit_runtime::spawn({
let source = source.clone();
let i420 = I420Buffer::new(resolution.width, resolution.height);
async move {
let mut interval = interval(Duration::from_millis(100)); // 10 fps
loop {
interval.tick().await;
let inner = source.inner.lock();
if inner.captured_frames > 0 {
break;
}
let mut builder = vf_sys::ffi::new_video_frame_builder();
builder.pin_mut().set_rotation(VideoRotation::VideoRotation0);
builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle());
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
source.sys_handle.on_captured_frame(
&builder.pin_mut().build(),
&vt_sys::ffi::FrameMetadata {
has_packet_trailer: false,
user_timestamp: 0,
frame_id: 0,
},
);
}
}
});
source
}
pub fn sys_handle(&self) -> SharedPtr<vt_sys::ffi::VideoTrackSource> {
self.sys_handle.clone()
}
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
let mut builder = vf_sys::ffi::new_video_frame_builder();
builder.pin_mut().set_rotation(frame.rotation.into());
builder.pin_mut().set_video_frame_buffer(frame.buffer.as_ref().sys_handle());
let capture_ts = if frame.timestamp_us == 0 {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
now.as_micros() as i64
} else {
frame.timestamp_us
};
builder.pin_mut().set_timestamp_us(capture_ts);
let (has_trailer, user_ts, fid) = match frame.frame_metadata {
Some(meta) => (true, meta.user_timestamp.unwrap_or(0), meta.frame_id.unwrap_or(0)),
None => (false, 0, 0),
};
self.inner.lock().captured_frames += 1;
self.sys_handle.on_captured_frame(
&builder.pin_mut().build(),
&vt_sys::ffi::FrameMetadata {
has_packet_trailer: has_trailer,
user_timestamp: user_ts,
frame_id: fid,
},
);
}
/// Set the packet trailer handler used by this source.
///
/// When set, any frame captured with a `user_timestamp` value will
/// automatically have its timestamp stored in the handler so the
/// `PacketTrailerTransformer` can embed it into the encoded frame.
/// The handler is set on the C++ VideoTrackSource so it has access to
/// the TimestampAligner-adjusted capture timestamp for correct keying.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
}
pub fn video_resolution(&self) -> VideoResolution {
self.sys_handle.video_resolution().into()
}
}
@@ -0,0 +1,350 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
collections::VecDeque,
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
task::{Context, Poll, Waker},
};
use cxx::{SharedPtr, UniquePtr};
use livekit_runtime::Stream;
use parking_lot::Mutex;
use rtrb::{Consumer, Producer, PushError, RingBuffer};
use webrtc_sys::video_track as sys_vt;
use super::{packet_trailer::SubscribeTimingStage, video_frame::new_video_frame_buffer};
use crate::{
native::packet_trailer::PacketTrailerHandler,
video_frame::{BoxVideoFrame, FrameMetadata, VideoFrame},
video_track::RtcVideoTrack,
};
pub struct NativeVideoStream {
native_sink: SharedPtr<sys_vt::ffi::NativeVideoSink>,
observer: Arc<VideoTrackObserver>,
video_track: RtcVideoTrack,
frame_queue: Arc<VideoFrameQueue>,
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack, queue_size_frames: Option<usize>) -> Self {
let frame_queue = Arc::new(VideoFrameQueue::new(queue_size_frames));
// Auto-wire the packet trailer handler from the track if one is set.
let handler = video_track.handle.packet_trailer_handler();
let observer = Arc::new(VideoTrackObserver {
frame_queue: frame_queue.clone(),
packet_trailer_handler: parking_lot::Mutex::new(handler),
});
let native_sink = sys_vt::ffi::new_native_video_sink(Box::new(
sys_vt::VideoSinkWrapper::new(observer.clone()),
));
let video = unsafe { sys_vt::ffi::media_to_video(video_track.sys_handle()) };
video.add_sink(&native_sink);
Self { native_sink, observer, video_track, frame_queue }
}
/// Set the packet trailer handler for this stream.
///
/// When set, each frame produced by this stream will have its
/// `user_timestamp` field populated from the handler's receive
/// map (looked up by RTP timestamp).
///
/// Note: If the handler was already set on the `RtcVideoTrack` before
/// creating this stream, it is automatically wired up. This method is
/// only needed if you want to override or set the handler after
/// construction.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
*self.observer.packet_trailer_handler.lock() = Some(handler);
}
pub fn track(&self) -> RtcVideoTrack {
self.video_track.clone()
}
pub fn close(&mut self) {
let video = unsafe { sys_vt::ffi::media_to_video(self.video_track.sys_handle()) };
video.remove_sink(&self.native_sink);
self.frame_queue.close();
}
}
impl Drop for NativeVideoStream {
fn drop(&mut self) {
self.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.frame_queue.poll_recv(cx)
}
}
struct VideoTrackObserver {
frame_queue: Arc<VideoFrameQueue>,
packet_trailer_handler: parking_lot::Mutex<Option<PacketTrailerHandler>>,
}
impl sys_vt::VideoSink for VideoTrackObserver {
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
let rtp_timestamp = frame.timestamp();
let packet_trailer_handler = self.packet_trailer_handler.lock().clone();
let frame_metadata = packet_trailer_handler
.as_ref()
.and_then(|handler| {
handler.lookup_frame_metadata(rtp_timestamp).map(|(ts, fid)| {
handler.emit_subscribe_timing(SubscribeTimingStage::DecoderOutput, ts, fid);
(ts, fid)
})
})
.map(|(ts, fid)| FrameMetadata {
user_timestamp: Some(ts),
frame_id: if fid != 0 { Some(fid) } else { None },
});
self.frame_queue.push(VideoFrame {
rotation: frame.rotation().into(),
timestamp_us: frame.timestamp_us(),
frame_metadata,
buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
});
}
fn on_discarded_frame(&self) {}
fn on_constraints_changed(&self, _constraints: sys_vt::ffi::VideoTrackSourceConstraints) {}
}
struct VideoFrameQueue {
kind: VideoFrameQueueKind,
closed: AtomicBool,
dropped_frames: AtomicU64,
waker: Mutex<Option<Waker>>,
}
enum VideoFrameQueueKind {
Bounded(BoundedVideoFrameQueue),
Unbounded(UnboundedVideoFrameQueue),
}
struct BoundedVideoFrameQueue {
producer: Mutex<Producer<BoxVideoFrame>>,
consumer: Mutex<Consumer<BoxVideoFrame>>,
}
struct UnboundedVideoFrameQueue {
frames: Mutex<VecDeque<BoxVideoFrame>>,
}
impl VideoFrameQueue {
fn new(capacity: Option<usize>) -> Self {
let kind = match capacity.filter(|capacity| *capacity > 0) {
Some(capacity) => {
let (producer, consumer) = RingBuffer::new(capacity);
VideoFrameQueueKind::Bounded(BoundedVideoFrameQueue {
producer: Mutex::new(producer),
consumer: Mutex::new(consumer),
})
}
None => VideoFrameQueueKind::Unbounded(UnboundedVideoFrameQueue {
frames: Mutex::new(VecDeque::new()),
}),
};
Self {
kind,
closed: AtomicBool::new(false),
dropped_frames: AtomicU64::new(0),
waker: Mutex::new(None),
}
}
fn push(&self, frame: BoxVideoFrame) {
if self.closed.load(Ordering::Acquire) {
return;
}
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
VideoFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().push_back(frame);
}
}
self.wake_receiver();
}
fn push_bounded(&self, queue: &BoundedVideoFrameQueue, mut frame: BoxVideoFrame) {
loop {
let push_result = queue.producer.lock().push(frame);
match push_result {
Ok(()) => return,
Err(PushError::Full(returned_frame)) => {
frame = returned_frame;
let dropped = queue.consumer.lock().pop().is_ok();
if dropped {
self.record_drop();
} else {
return;
}
}
}
}
}
fn close(&self) {
self.closed.store(true, Ordering::Release);
self.wake_receiver();
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => {
let mut consumer = queue.consumer.lock();
while consumer.pop().is_ok() {}
}
VideoFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().clear();
}
}
}
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<BoxVideoFrame>> {
if let Some(frame) = self.try_pop() {
return Poll::Ready(Some(frame));
}
if self.closed.load(Ordering::Acquire) {
return Poll::Ready(None);
}
*self.waker.lock() = Some(cx.waker().clone());
if let Some(frame) = self.try_pop() {
self.waker.lock().take();
Poll::Ready(Some(frame))
} else if self.closed.load(Ordering::Acquire) {
Poll::Ready(None)
} else {
Poll::Pending
}
}
fn try_pop(&self) -> Option<BoxVideoFrame> {
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
VideoFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
}
}
fn wake_receiver(&self) {
let waker = self.waker.lock().take();
if let Some(waker) = waker {
waker.wake();
}
}
fn record_drop(&self) {
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
if dropped_frames == 1 || dropped_frames % 100 == 0 {
log::warn!(
"native video stream queue overflow; dropped {} queued frames",
dropped_frames
);
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::Ordering;
use super::VideoFrameQueue;
use crate::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation};
fn test_frame(timestamp_us: i64) -> BoxVideoFrame {
VideoFrame {
rotation: VideoRotation::VideoRotation0,
timestamp_us,
frame_metadata: None,
buffer: Box::new(I420Buffer::new(2, 2)),
}
}
fn pop_timestamp(queue: &VideoFrameQueue) -> Option<i64> {
queue.try_pop().map(|frame| frame.timestamp_us)
}
#[test]
fn bounded_queue_preserves_fifo_order_under_capacity() {
let queue = VideoFrameQueue::new(Some(3));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(pop_timestamp(&queue), Some(1));
assert_eq!(pop_timestamp(&queue), Some(2));
assert_eq!(pop_timestamp(&queue), Some(3));
assert_eq!(pop_timestamp(&queue), None);
}
#[test]
fn bounded_queue_drops_oldest_when_full() {
let queue = VideoFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
assert_eq!(pop_timestamp(&queue), Some(2));
assert_eq!(pop_timestamp(&queue), Some(3));
assert_eq!(pop_timestamp(&queue), None);
}
#[test]
fn unbounded_queue_retains_all_frames() {
let queue = VideoFrameQueue::new(None);
for timestamp_us in 1..=4 {
queue.push(test_frame(timestamp_us));
}
for timestamp_us in 1..=4 {
assert_eq!(pop_timestamp(&queue), Some(timestamp_us));
}
assert_eq!(pop_timestamp(&queue), None);
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
}
#[test]
fn close_clears_buffer_and_rejects_future_pushes() {
let queue = VideoFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.close();
queue.push(test_frame(2));
assert_eq!(pop_timestamp(&queue), None);
}
}
@@ -0,0 +1,56 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use sys_vt::ffi::video_to_media;
use webrtc_sys::video_track as sys_vt;
use super::media_stream_track::impl_media_stream_track;
use super::packet_trailer::PacketTrailerHandler;
use crate::media_stream_track::RtcTrackState;
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>,
packet_trailer_handler: Arc<Mutex<Option<PacketTrailerHandler>>>,
}
impl RtcVideoTrack {
impl_media_stream_track!(video_to_media);
pub(crate) fn new(sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>) -> Self {
Self { sys_handle, packet_trailer_handler: Arc::new(Mutex::new(None)) }
}
pub fn sys_handle(&self) -> SharedPtr<sys_vt::ffi::MediaStreamTrack> {
video_to_media(self.sys_handle.clone())
}
/// Set the packet trailer handler for this track.
///
/// When set, any `NativeVideoStream` created from this track will
/// automatically use this handler to populate `user_timestamp`
/// on each decoded frame.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.packet_trailer_handler.lock().replace(handler);
}
/// Get the packet trailer handler, if one has been set.
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
self.packet_trailer_handler.lock().clone()
}
}
@@ -0,0 +1,868 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(clippy::too_many_arguments)]
use webrtc_sys::yuv_helper as yuv_sys;
fn argb_assert_safety(src: &[u8], src_stride: u32, _width: i32, height: i32) {
let height_abs = height.unsigned_abs();
let min = (src_stride * height_abs) as usize;
assert!(src.len() >= min, "src isn't large enough");
}
fn i420_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let chroma_height = (height_abs + 1) / 2;
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * chroma_height) as usize;
let min_v = (src_stride_v * chroma_height) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn nv12_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let chroma_height = (height_abs + 1) / 2;
let min_y = (src_stride_y * height_abs) as usize;
let min_uv = (src_stride_uv * chroma_height) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_uv.len() >= min_uv, "src_uv isn't large enough");
}
fn i444_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * height_abs) as usize;
let min_v = (src_stride_v * height_abs) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn i422_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * height_abs) as usize;
let min_v = (src_stride_v * height_abs) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn i010_assert_safety(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs: u32 = height.unsigned_abs();
let chroma_height = height_abs / 2;
let min_y = (src_stride_y * height_abs) as usize / 2;
let min_u = (src_stride_u * chroma_height) as usize / 2;
let min_v = (src_stride_v * chroma_height) as usize / 2;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
macro_rules! i420_to_rgba {
($x:ident) => {
pub fn $x(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst: &mut [u8],
dst_stride: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst, dst_stride, width, height);
unsafe {
yuv_sys::ffi::$x(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst.as_mut_ptr(),
dst_stride as i32,
width,
height,
)
.unwrap();
}
}
};
}
macro_rules! rgba_to_i420 {
($x:ident) => {
pub fn $x(
src_argb: &[u8],
src_stride_argb: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
argb_assert_safety(src_argb, src_stride_argb, width, height);
unsafe {
yuv_sys::ffi::$x(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
};
}
pub fn argb_to_rgb24(
src_argb: &[u8],
src_stride_argb: u32,
dst_rgb24: &mut [u8],
dst_stride_rgb24: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_argb, src_stride_argb, width, height);
argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height);
unsafe {
yuv_sys::ffi::argb_to_rgb24(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_rgb24.as_mut_ptr(),
dst_stride_rgb24 as i32,
width,
height,
)
.unwrap();
}
}
// I420 <> RGB conversion
rgba_to_i420!(argb_to_i420);
rgba_to_i420!(abgr_to_i420);
i420_to_rgba!(i420_to_argb);
i420_to_rgba!(i420_to_bgra);
i420_to_rgba!(i420_to_abgr);
i420_to_rgba!(i420_to_rgba);
pub fn i420_to_nv12(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::i420_to_nv12(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap();
}
}
pub fn nv12_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::nv12_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i444_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
pub fn i422_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i422_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_i420(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i010_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap()
}
}
pub fn nv12_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::nv12_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn nv12_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::nv12_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i444_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i444_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn i422_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i422_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn i422_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i422_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_argb(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i010_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_abgr(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i010_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn abgr_to_nv12(
src_abgr: &[u8],
src_stride_abgr: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_abgr, src_stride_abgr, width, height);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::abgr_to_nv12(
src_abgr.as_ptr(),
src_stride_abgr as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap()
}
}
pub fn argb_to_nv12(
src_argb: &[u8],
src_stride_argb: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_argb, src_stride_argb, width, height);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::argb_to_nv12(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap()
}
}
@@ -0,0 +1,345 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
data_channel::{DataChannel, DataChannelInit},
ice_candidate::IceCandidate,
imp::peer_connection as imp_pc,
media_stream::MediaStream,
media_stream_track::MediaStreamTrack,
peer_connection_factory::RtcConfiguration,
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
session_description::SessionDescription,
stats::RtcStats,
MediaType, RtcError,
};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PeerConnectionState {
New,
Connecting,
Connected,
Disconnected,
Failed,
Closed,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceConnectionState {
New,
Checking,
Connected,
Completed,
Failed,
Disconnected,
Closed,
Max,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceGatheringState {
New,
Gathering,
Complete,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SignalingState {
Stable,
HaveLocalOffer,
HaveLocalPrAnswer,
HaveRemoteOffer,
HaveRemotePrAnswer,
Closed,
}
#[derive(Debug, Clone, Default)]
pub struct OfferOptions {
pub ice_restart: bool,
pub offer_to_receive_audio: bool,
pub offer_to_receive_video: bool,
}
#[derive(Debug, Clone, Default)]
pub struct AnswerOptions {}
#[derive(Debug, Clone)]
pub struct IceCandidateError {
pub address: String,
pub port: i32,
pub url: String,
pub error_code: i32,
pub error_text: String,
}
#[derive(Debug, Clone)]
pub struct TrackEvent {
pub receiver: RtpReceiver,
pub streams: Vec<MediaStream>,
pub track: MediaStreamTrack,
pub transceiver: RtpTransceiver,
}
pub type OnConnectionChange = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
pub type OnDataChannel = Box<dyn FnMut(DataChannel) + Send + Sync>;
pub type OnIceCandidate = Box<dyn FnMut(IceCandidate) + Send + Sync>;
pub type OnIceCandidateError = Box<dyn FnMut(IceCandidateError) + Send + Sync>;
pub type OnIceConnectionChange = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
pub type OnIceGatheringChange = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
pub type OnNegotiationNeeded = Box<dyn FnMut(u32) + Send + Sync>;
pub type OnSignalingChange = Box<dyn FnMut(SignalingState) + Send + Sync>;
pub type OnTrack = Box<dyn FnMut(TrackEvent) + Send + Sync>;
#[derive(Clone)]
pub struct PeerConnection {
pub(crate) handle: imp_pc::PeerConnection,
}
impl PeerConnection {
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
self.handle.set_configuration(config)
}
pub async fn create_offer(
&self,
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_offer(options).await
}
pub async fn create_answer(
&self,
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_answer(options).await
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_local_description(desc).await
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_remote_description(desc).await
}
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
self.handle.add_ice_candidate(candidate).await
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RtcError> {
self.handle.create_data_channel(label, init)
}
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
streams_ids: &[T],
) -> Result<RtpSender, RtcError> {
self.handle.add_track(track, streams_ids)
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.handle.remove_track(sender)
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver(track, init)
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver_for_media(media_type, init)
}
pub fn close(&self) {
self.handle.close()
}
pub fn restart_ice(&self) {
self.handle.restart_ice()
}
pub fn connection_state(&self) -> PeerConnectionState {
self.handle.connection_state()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.handle.ice_connection_state()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.handle.ice_gathering_state()
}
pub fn signaling_state(&self) -> SignalingState {
self.handle.signaling_state()
}
pub fn current_local_description(&self) -> Option<SessionDescription> {
self.handle.current_local_description()
}
pub fn current_remote_description(&self) -> Option<SessionDescription> {
self.handle.current_remote_description()
}
pub fn senders(&self) -> Vec<RtpSender> {
self.handle.senders()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.handle.receivers()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.handle.transceivers()
}
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
self.handle.on_connection_state_change(f)
}
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
self.handle.on_data_channel(f)
}
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
self.handle.on_ice_candidate(f)
}
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
self.handle.on_ice_candidate_error(f)
}
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
self.handle.on_ice_connection_state_change(f)
}
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
self.handle.on_ice_gathering_state_change(f)
}
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
self.handle.on_negotiation_needed(f)
}
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
self.handle.on_signaling_state_change(f)
}
pub fn on_track(&self, f: Option<OnTrack>) {
self.handle.on_track(f)
}
}
impl Debug for PeerConnection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PeerConnection")
.field("state", &self.connection_state())
.field("ice_state", &self.ice_connection_state())
.finish()
}
}
#[cfg(test)]
mod tests {
use log::trace;
use tokio::sync::mpsc;
use crate::{peer_connection::*, peer_connection_factory::*};
#[tokio::test]
async fn create_pc() {
let _ = env_logger::builder().is_test(true).try_init();
let factory = PeerConnectionFactory::default();
let config = RtcConfiguration {
ice_servers: vec![IceServer {
urls: vec!["stun:stun1.l.google.com:19302".to_string()],
username: "".into(),
password: "".into(),
}],
continual_gathering_policy: ContinualGatheringPolicy::GatherOnce,
ice_transport_type: IceTransportsType::All,
};
let bob = factory.create_peer_connection(config.clone()).unwrap();
let alice = factory.create_peer_connection(config.clone()).unwrap();
let (bob_ice_tx, mut bob_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
let (alice_ice_tx, mut alice_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
let (alice_dc_tx, mut alice_dc_rx) = mpsc::unbounded_channel::<DataChannel>();
bob.on_ice_candidate(Some(Box::new(move |candidate| {
bob_ice_tx.send(candidate).unwrap();
})));
alice.on_ice_candidate(Some(Box::new(move |candidate| {
alice_ice_tx.send(candidate).unwrap();
})));
alice.on_data_channel(Some(Box::new(move |dc| {
alice_dc_tx.send(dc).unwrap();
})));
let bob_dc = bob.create_data_channel("test_dc", DataChannelInit::default()).unwrap();
let offer = bob.create_offer(OfferOptions::default()).await.unwrap();
trace!("Bob offer: {:?}", offer);
bob.set_local_description(offer.clone()).await.unwrap();
alice.set_remote_description(offer).await.unwrap();
let answer = alice.create_answer(AnswerOptions::default()).await.unwrap();
trace!("Alice answer: {:?}", answer);
alice.set_local_description(answer.clone()).await.unwrap();
bob.set_remote_description(answer).await.unwrap();
let bob_ice = bob_ice_rx.recv().await.unwrap();
let alice_ice = alice_ice_rx.recv().await.unwrap();
bob.add_ice_candidate(alice_ice).await.unwrap();
alice.add_ice_candidate(bob_ice).await.unwrap();
let (data_tx, mut data_rx) = mpsc::unbounded_channel::<String>();
let alice_dc = alice_dc_rx.recv().await.unwrap();
alice_dc.on_message(Some(Box::new(move |buffer| {
data_tx.send(String::from_utf8_lossy(buffer.data).to_string()).unwrap();
})));
bob_dc.send(b"This is a test", true).unwrap();
assert_eq!(data_rx.recv().await.unwrap(), "This is a test");
alice.close();
bob.close();
}
}
@@ -0,0 +1,316 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::peer_connection_factory as imp_pcf, peer_connection::PeerConnection,
rtp_parameters::RtpCapabilities, MediaType, RtcError,
};
#[derive(Debug, Clone)]
pub struct IceServer {
pub urls: Vec<String>,
pub username: String,
pub password: String,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ContinualGatheringPolicy {
GatherOnce,
GatherContinually,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceTransportsType {
Relay,
NoHost,
All,
}
#[derive(Debug, Clone)]
pub struct RtcConfiguration {
pub ice_servers: Vec<IceServer>,
pub continual_gathering_policy: ContinualGatheringPolicy,
pub ice_transport_type: IceTransportsType,
}
impl Default for RtcConfiguration {
fn default() -> Self {
Self {
ice_servers: vec![],
continual_gathering_policy: ContinualGatheringPolicy::GatherContinually,
ice_transport_type: IceTransportsType::All,
}
}
}
#[derive(Clone, Default)]
pub struct PeerConnectionFactory {
pub(crate) handle: imp_pcf::PeerConnectionFactory,
}
impl Debug for PeerConnectionFactory {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("PeerConnectionFactory").finish()
}
}
impl PeerConnectionFactory {
pub fn create_peer_connection(
&self,
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
self.handle.create_peer_connection(config)
}
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_sender_capabilities(media_type)
}
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_receiver_capabilities(media_type)
}
}
pub mod native {
use super::PeerConnectionFactory;
use crate::{
audio_source::native::NativeAudioSource, audio_track::RtcAudioTrack,
video_source::native::NativeVideoSource, video_track::RtcVideoTrack,
};
pub trait PeerConnectionFactoryExt {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack;
/// Create an audio track that uses the Platform ADM for capture.
/// The track will capture audio from the selected recording device.
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack;
// Device enumeration
fn playout_devices(&self) -> i16;
fn recording_devices(&self) -> i16;
fn playout_device_name(&self, index: u16) -> String;
fn recording_device_name(&self, index: u16) -> String;
/// Get device GUID (platform-specific unique identifier, stable across hot-plug)
fn playout_device_guid(&self, index: u16) -> String;
fn recording_device_guid(&self, index: u16) -> String;
// Device selection by index
fn set_playout_device(&self, index: u16) -> bool;
fn set_recording_device(&self, index: u16) -> bool;
/// Device selection by GUID (preferred - stable across device changes)
fn set_playout_device_by_guid(&self, guid: &str) -> bool;
fn set_recording_device_by_guid(&self, guid: &str) -> bool;
// Recording control (for device switching while active)
fn stop_recording(&self) -> bool;
fn init_recording(&self) -> bool;
fn start_recording(&self) -> bool;
fn recording_is_initialized(&self) -> bool;
// Playout control (for device switching while active)
fn stop_playout(&self) -> bool;
fn init_playout(&self) -> bool;
fn start_playout(&self) -> bool;
fn playout_is_initialized(&self) -> bool;
// Built-in audio processing (hardware AEC/AGC/NS)
// Only available on iOS and some Android devices
fn builtin_aec_is_available(&self) -> bool;
fn builtin_agc_is_available(&self) -> bool;
fn builtin_ns_is_available(&self) -> bool;
fn enable_builtin_aec(&self, enable: bool) -> bool;
fn enable_builtin_agc(&self, enable: bool) -> bool;
fn enable_builtin_ns(&self, enable: bool) -> bool;
// ADM recording control
// Use this to disable microphone when only using NativeAudioSource
fn set_adm_recording_enabled(&self, enabled: bool);
fn adm_recording_enabled(&self) -> bool;
// ADM playout control
// When disabled (default), playout uses synthetic mode - remote audio is
// delivered via FFI callbacks. When enabled, plays through platform speakers.
fn set_adm_playout_enabled(&self, enabled: bool);
fn adm_playout_enabled(&self) -> bool;
// Platform ADM lifecycle management
// Call acquire_platform_adm when creating PlatformAudio.
// Call release_platform_adm when disposing PlatformAudio.
// The Platform ADM is only created when first acquired, and terminated
// when the last reference is released.
fn acquire_platform_adm(&self) -> bool;
fn release_platform_adm(&self);
fn platform_adm_ref_count(&self) -> i32;
fn is_platform_adm_active(&self) -> bool;
// Ensures the Platform ADM exists, retrying creation if an earlier
// attempt failed (e.g. the OS audio stack was still starting up).
fn ensure_platform_adm(&self) -> bool;
// Distinguishes "audio stack unavailable" from "zero audio devices".
fn platform_adm_available(&self) -> bool;
}
impl PeerConnectionFactoryExt for PeerConnectionFactory {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
self.handle.create_video_track(label, source)
}
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
self.handle.create_audio_track(label, source)
}
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
self.handle.create_device_audio_track(label)
}
fn playout_devices(&self) -> i16 {
self.handle.playout_devices()
}
fn recording_devices(&self) -> i16 {
self.handle.recording_devices()
}
fn playout_device_name(&self, index: u16) -> String {
self.handle.playout_device_name(index)
}
fn recording_device_name(&self, index: u16) -> String {
self.handle.recording_device_name(index)
}
fn playout_device_guid(&self, index: u16) -> String {
self.handle.playout_device_guid(index)
}
fn recording_device_guid(&self, index: u16) -> String {
self.handle.recording_device_guid(index)
}
fn set_playout_device(&self, index: u16) -> bool {
self.handle.set_playout_device(index)
}
fn set_recording_device(&self, index: u16) -> bool {
self.handle.set_recording_device(index)
}
fn set_playout_device_by_guid(&self, guid: &str) -> bool {
self.handle.set_playout_device_by_guid(guid)
}
fn set_recording_device_by_guid(&self, guid: &str) -> bool {
self.handle.set_recording_device_by_guid(guid)
}
fn stop_recording(&self) -> bool {
self.handle.stop_recording()
}
fn init_recording(&self) -> bool {
self.handle.init_recording()
}
fn start_recording(&self) -> bool {
self.handle.start_recording()
}
fn recording_is_initialized(&self) -> bool {
self.handle.recording_is_initialized()
}
fn stop_playout(&self) -> bool {
self.handle.stop_playout()
}
fn init_playout(&self) -> bool {
self.handle.init_playout()
}
fn start_playout(&self) -> bool {
self.handle.start_playout()
}
fn playout_is_initialized(&self) -> bool {
self.handle.playout_is_initialized()
}
fn builtin_aec_is_available(&self) -> bool {
self.handle.builtin_aec_is_available()
}
fn builtin_agc_is_available(&self) -> bool {
self.handle.builtin_agc_is_available()
}
fn builtin_ns_is_available(&self) -> bool {
self.handle.builtin_ns_is_available()
}
fn enable_builtin_aec(&self, enable: bool) -> bool {
self.handle.enable_builtin_aec(enable)
}
fn enable_builtin_agc(&self, enable: bool) -> bool {
self.handle.enable_builtin_agc(enable)
}
fn enable_builtin_ns(&self, enable: bool) -> bool {
self.handle.enable_builtin_ns(enable)
}
fn set_adm_recording_enabled(&self, enabled: bool) {
self.handle.set_adm_recording_enabled(enabled)
}
fn adm_recording_enabled(&self) -> bool {
self.handle.adm_recording_enabled()
}
fn set_adm_playout_enabled(&self, enabled: bool) {
self.handle.set_adm_playout_enabled(enabled)
}
fn adm_playout_enabled(&self) -> bool {
self.handle.adm_playout_enabled()
}
fn acquire_platform_adm(&self) -> bool {
self.handle.acquire_platform_adm()
}
fn release_platform_adm(&self) {
self.handle.release_platform_adm()
}
fn platform_adm_ref_count(&self) -> i32 {
self.handle.platform_adm_ref_count()
}
fn is_platform_adm_active(&self) -> bool {
self.handle.is_platform_adm_active()
}
fn ensure_platform_adm(&self) -> bool {
self.handle.ensure_platform_adm()
}
fn platform_adm_available(&self) -> bool {
self.handle.platform_adm_available()
}
}
}
@@ -0,0 +1,43 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub use crate::{
audio_frame::AudioFrame,
audio_source::{AudioSourceOptions, RtcAudioSource},
audio_track::RtcAudioTrack,
data_channel::{DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataChannelState},
ice_candidate::IceCandidate,
media_stream::MediaStream,
media_stream_track::{MediaStreamTrack, RtcTrackState},
peer_connection::{
AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection,
PeerConnectionState, SignalingState,
},
peer_connection_factory::{
ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory,
RtcConfiguration,
},
rtp_parameters::*,
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit},
session_description::{SdpType, SessionDescription},
video_frame::{
BoxVideoBuffer, BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer,
NV12Buffer, VideoBuffer, VideoBufferType, VideoFormatType, VideoFrame, VideoRotation,
},
video_source::{RtcVideoSource, VideoResolution},
video_track::RtcVideoTrack,
MediaType, RtcError, RtcErrorType,
};
@@ -0,0 +1,44 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use webrtc_sys::recorded_audio_tap::ffi as sys;
use webrtc_sys::recorded_audio_tap::RecordedAudioSinkWrapper;
/// Handle to an installed recorded-audio sink. Dropping it does not clear
/// the sink; call [`clear_recorded_audio_sink`] with this generation.
pub type RecordedAudioSinkGeneration = u64;
/// Installs a process-global tap on platform-ADM recorded microphone audio.
///
/// `callback` is invoked on the ADM capture thread with one 48kHz mono
/// 10ms frame (480 samples) per call: `(samples, sample_rate_hz,
/// num_channels, samples_per_channel)`. It must be wait-free: do no
/// allocation or blocking work, only hand the frame to a bounded queue.
/// Returns a generation token to pass to [`clear_recorded_audio_sink`].
pub fn set_recorded_audio_sink<F>(callback: F) -> RecordedAudioSinkGeneration
where
F: Fn(&[i16], i32, usize, usize) + Send + Sync + 'static,
{
sys::set_recorded_audio_sink(Box::new(RecordedAudioSinkWrapper::new(Box::new(callback))))
}
/// Removes the recorded-audio sink, but only if `generation` is still the
/// installed one. A stale token is a no-op, so a late teardown cannot
/// clobber a sink a newer caller installed.
pub fn clear_recorded_audio_sink(generation: RecordedAudioSinkGeneration) {
sys::clear_recorded_audio_sink(generation);
}
}
@@ -0,0 +1,98 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::rtp_transceiver::RtpTransceiverDirection;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Priority {
VeryLow,
Low,
Medium,
High,
}
#[derive(Debug, Clone)]
pub struct RtpHeaderExtensionParameters {
pub uri: String,
pub id: i32,
pub encrypted: bool,
}
#[derive(Debug, Clone, Default)]
pub struct RtpParameters {
pub codecs: Vec<RtpCodecParameters>,
pub header_extensions: Vec<RtpHeaderExtensionParameters>,
pub rtcp: RtcpParameters,
}
#[derive(Debug, Clone, Default)]
pub struct RtpCodecParameters {
pub payload_type: u8,
pub mime_type: String, // read-only
pub clock_rate: Option<u64>,
pub channels: Option<u16>,
}
#[derive(Debug, Clone, Default)]
pub struct RtcpParameters {
pub cname: String,
pub reduced_size: bool,
}
#[derive(Debug, Clone)]
pub struct RtpEncodingParameters {
pub active: bool,
pub max_bitrate: Option<u64>,
pub max_framerate: Option<f64>,
pub priority: Priority,
pub rid: String,
pub scale_resolution_down_by: Option<f64>,
/// RTP scalability mode (e.g. "L3T3_KEY"). Required to enable true
/// SVC for codecs that support it (VP9, AV1).
pub scalability_mode: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RtpCodecCapability {
pub channels: Option<u16>,
pub clock_rate: Option<u64>,
pub mime_type: String,
pub sdp_fmtp_line: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RtpHeaderExtensionCapability {
pub uri: String,
pub direction: RtpTransceiverDirection,
}
#[derive(Debug, Clone)]
pub struct RtpCapabilities {
pub codecs: Vec<RtpCodecCapability>,
pub header_extensions: Vec<RtpHeaderExtensionCapability>,
}
impl Default for RtpEncodingParameters {
fn default() -> Self {
Self {
active: true,
max_bitrate: None,
max_framerate: None,
priority: Priority::Low,
rid: String::default(),
scale_resolution_down_by: None,
scalability_mode: None,
}
}
}
@@ -0,0 +1,48 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_receiver as imp_rr, media_stream_track::MediaStreamTrack,
rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
};
#[derive(Clone)]
pub struct RtpReceiver {
pub(crate) handle: imp_rr::RtpReceiver,
}
impl RtpReceiver {
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn parameters(&self) -> RtpParameters {
self.handle.parameters()
}
}
impl Debug for RtpReceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver")
.field("track", &self.track())
.field("cname", &self.parameters().rtcp.cname)
.finish()
}
}
@@ -0,0 +1,53 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_sender as imp_rs, media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters,
stats::RtcStats, RtcError,
};
#[derive(Clone)]
pub struct RtpSender {
pub(crate) handle: imp_rs::RtpSender,
}
impl RtpSender {
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
self.handle.set_track(track)
}
pub fn parameters(&self) -> RtpParameters {
self.handle.parameters()
}
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.handle.set_parameters(parameters)
}
}
impl Debug for RtpSender {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver").field("cname", &self.parameters().rtcp.cname).finish()
}
}
@@ -0,0 +1,85 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_transceiver as imp_rt,
rtp_parameters::{RtpCodecCapability, RtpEncodingParameters},
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
RtcError,
};
#[derive(Debug, Clone)]
pub struct RtpTransceiverInit {
pub direction: RtpTransceiverDirection,
pub stream_ids: Vec<String>,
pub send_encodings: Vec<RtpEncodingParameters>,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtpTransceiverDirection {
SendRecv,
SendOnly,
RecvOnly,
Inactive,
Stopped,
}
#[derive(Clone)]
pub struct RtpTransceiver {
pub(crate) handle: imp_rt::RtpTransceiver,
}
impl RtpTransceiver {
pub fn mid(&self) -> Option<String> {
self.handle.mid()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.handle.current_direction()
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.handle.direction()
}
pub fn sender(&self) -> RtpSender {
self.handle.sender()
}
pub fn receiver(&self) -> RtpReceiver {
self.handle.receiver()
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.handle.set_codec_preferences(codecs)
}
pub fn stop(&self) -> Result<(), RtcError> {
self.handle.stop()
}
}
impl Debug for RtpTransceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpTransceiver")
.field("mid", &self.mid())
.field("direction", &self.direction())
.field("sender", &self.sender())
.field("receiver", &self.receiver())
.finish()
}
}
@@ -0,0 +1,90 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
fmt::{Debug, Display},
str::FromStr,
};
use thiserror::Error;
use crate::imp::session_description as sd_imp;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SdpType {
Offer,
PrAnswer,
Answer,
Rollback,
}
impl FromStr for SdpType {
type Err = &'static str;
fn from_str(sdp_type: &str) -> Result<Self, Self::Err> {
match sdp_type {
"offer" => Ok(Self::Offer),
"pranswer" => Ok(Self::PrAnswer),
"answer" => Ok(Self::Answer),
"rollback" => Ok(Self::Rollback),
_ => Err("invalid SdpType"),
}
}
}
impl Display for SdpType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let s = match self {
SdpType::Offer => "offer",
SdpType::PrAnswer => "pranswer",
SdpType::Answer => "answer",
SdpType::Rollback => "rollback",
};
write!(f, "{}", s)
}
}
#[derive(Clone)]
pub struct SessionDescription {
pub(crate) handle: sd_imp::SessionDescription,
}
#[derive(Clone, Error, Debug)]
#[error("Failed to parse sdp: {line} - {description}")]
pub struct SdpParseError {
pub line: String,
pub description: String,
}
impl SessionDescription {
pub fn parse(sdp: &str, sdp_type: SdpType) -> Result<Self, SdpParseError> {
sd_imp::SessionDescription::parse(sdp, sdp_type)
}
pub fn sdp_type(&self) -> SdpType {
self.handle.sdp_type()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for SessionDescription {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SessionDescription").field("sdp_type", &self.sdp_type()).finish()
}
}
@@ -0,0 +1,624 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use serde::Deserialize;
use crate::data_channel::DataChannelState;
/// Values from https://www.w3.org/TR/webrtc-stats/ (NOTE: Some of the structs are not in the SPEC
/// but inside libwebrtc)
/// serde will handle the magic of correctly deserializing the json into our structs.
/// The enums values are inside encapsulated inside option because we're not sure about their
/// default values (So we default to None instead of an arbitrary value)
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum RtcStats {
Codec(CodecStats),
InboundRtp(InboundRtpStats),
OutboundRtp(OutboundRtpStats),
RemoteInboundRtp(RemoteInboundRtpStats),
RemoteOutboundRtp(RemoteOutboundRtpStats),
MediaSource(MediaSourceStats),
MediaPlayout(MediaPlayoutStats),
PeerConnection(PeerConnectionStats),
DataChannel(DataChannelStats),
Transport(TransportStats),
CandidatePair(CandidatePairStats),
LocalCandidate(LocalCandidateStats),
RemoteCandidate(RemoteCandidateStats),
Certificate(CertificateStats),
Stream(StreamStats),
Track, // Deprecated
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum QualityLimitationReason {
#[default]
None,
Cpu,
Bandwidth,
Other,
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceRole {
#[default]
Unknown,
Controlling,
Controlled,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DtlsTransportState {
New,
Connecting,
Connected,
Closed,
Failed,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceTransportState {
New,
Checking,
Connected,
Completed,
Disconnected,
Failed,
Closed,
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DtlsRole {
Client,
Server,
#[default]
Unknown,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum IceCandidatePairState {
Frozen,
Waiting,
InProgress, // in-progress
Failed,
Succeeded,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceCandidateType {
Host,
Srflx,
Prflx,
Relay,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceServerTransportProtocol {
Udp,
Tcp,
Tls,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceTcpCandidateType {
Active,
Passive,
So,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CodecStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub codec: dictionaries::CodecStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct InboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub received: dictionaries::ReceivedRtpStreamStats,
#[serde(flatten)]
pub inbound: dictionaries::InboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct OutboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub sent: dictionaries::SentRtpStreamStats,
#[serde(flatten)]
pub outbound: dictionaries::OutboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteInboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub received: dictionaries::ReceivedRtpStreamStats,
#[serde(flatten)]
pub remote_inbound: dictionaries::RemoteInboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteOutboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub sent: dictionaries::SentRtpStreamStats,
#[serde(flatten)]
pub remote_outbound: dictionaries::RemoteOutboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct MediaSourceStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub source: dictionaries::MediaSourceStats,
#[serde(flatten)]
pub audio: dictionaries::AudioSourceStats,
#[serde(flatten)]
pub video: dictionaries::VideoSourceStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct MediaPlayoutStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub audio_playout: dictionaries::AudioPlayoutStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct PeerConnectionStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub pc: dictionaries::PeerConnectionStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DataChannelStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub dc: dictionaries::DataChannelStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TransportStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub transport: dictionaries::TransportStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CandidatePairStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub candidate_pair: dictionaries::CandidatePairStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct LocalCandidateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub local_candidate: dictionaries::IceCandidateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteCandidateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub remote_candidate: dictionaries::IceCandidateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CertificateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub certificate: dictionaries::CertificateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct StreamStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::StreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TrackStats {}
pub mod dictionaries {
use super::*;
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RtcStats {
pub id: String,
pub timestamp: i64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CodecStats {
pub payload_type: u32,
pub transport_id: String,
pub mime_type: String,
pub clock_rate: u32,
pub channels: u32,
pub sdp_fmtp_line: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RtpStreamStats {
pub ssrc: u32,
pub kind: String,
pub transport_id: String,
pub codec_id: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct ReceivedRtpStreamStats {
pub packets_received: u64,
pub packets_lost: i64,
pub jitter: f64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct InboundRtpStreamStats {
pub track_identifier: String,
pub mid: String,
pub remote_id: String,
pub frames_decoded: u32,
pub key_frames_decoded: u32,
pub frames_rendered: u32,
pub frames_dropped: u32,
pub frame_width: u32,
pub frame_height: u32,
pub frames_per_second: f64,
pub qp_sum: u64,
pub total_decode_time: f64,
pub total_inter_frame_delay: f64,
pub total_squared_inter_frame_delay: f64,
pub pause_count: u32,
pub total_pause_duration: f64,
pub freeze_count: u32,
pub total_freeze_duration: f64,
pub last_packet_received_timestamp: f64,
pub header_bytes_received: u64,
pub packets_discarded: u64,
pub fec_bytes_received: u64,
pub fec_packets_received: u64,
pub fec_packets_discarded: u64,
pub bytes_received: u64,
pub nack_count: u32,
pub fir_count: u32,
pub pli_count: u32,
pub total_processing_delay: f64,
pub estimated_playout_timestamp: f64,
pub jitter_buffer_delay: f64,
pub jitter_buffer_target_delay: f64,
pub jitter_buffer_emitted_count: u64,
pub jitter_buffer_minimum_delay: f64,
pub total_samples_received: u64,
pub concealed_samples: u64,
pub silent_concealed_samples: u64,
pub concealment_events: u64,
pub inserted_samples_for_deceleration: u64,
pub removed_samples_for_acceleration: u64,
pub audio_level: f64,
pub total_audio_energy: f64,
pub total_samples_duration: f64,
pub frames_received: u64,
pub decoder_implementation: String,
pub playout_id: String,
pub power_efficient_decoder: bool,
pub frames_assembled_from_multiple_packets: u64,
pub total_assembly_time: f64,
pub retransmitted_packets_received: u64,
pub retransmitted_bytes_received: u64,
pub rtx_ssrc: u32,
pub fec_ssrc: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct SentRtpStreamStats {
pub packets_sent: u64,
pub bytes_sent: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct OutboundRtpStreamStats {
pub mid: String,
pub media_source_id: String,
pub remote_id: String,
pub rid: String,
pub header_bytes_sent: u64,
pub retransmitted_packets_sent: u64,
pub retransmitted_bytes_sent: u64,
pub rtx_ssrc: u32,
pub target_bitrate: f64,
pub total_encoded_bytes_target: u64,
pub frame_width: u32,
pub frame_height: u32,
pub frames_per_second: f64,
pub frames_sent: u32,
pub huge_frames_sent: u32,
pub frames_encoded: u32,
pub key_frames_encoded: u32,
pub qp_sum: u64,
pub total_encode_time: f64,
pub total_packet_send_delay: f64,
pub quality_limitation_reason: QualityLimitationReason,
pub quality_limitation_durations: HashMap<String, f64>,
pub quality_limitation_resolution_changes: u32,
pub nack_count: u32,
pub fir_count: u32,
pub pli_count: u32,
pub encoder_implementation: String,
pub power_efficient_encoder: bool,
pub active: bool,
pub scalibility_mode: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RemoteInboundRtpStreamStats {
pub local_id: String,
pub round_trip_time: f64,
pub total_round_trip_time: f64,
pub fraction_lost: f64,
pub round_trip_time_measurements: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RemoteOutboundRtpStreamStats {
pub local_id: String,
pub remote_timestamp: f64,
pub reports_sent: u64,
pub round_trip_time: f64,
pub total_round_trip_time: f64,
pub round_trip_time_measurements: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct MediaSourceStats {
pub track_identifier: String,
pub kind: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct AudioSourceStats {
pub audio_level: f64,
pub total_audio_energy: f64,
pub total_samples_duration: f64,
pub echo_return_loss: f64,
pub echo_return_loss_enhancement: f64,
pub dropped_samples_duration: f64,
pub dropped_samples_events: u32,
pub total_capture_delay: f64,
pub total_samples_captured: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct VideoSourceStats {
pub width: u32,
pub height: u32,
pub frames: u32,
pub frames_per_second: f64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct AudioPlayoutStats {
pub kind: String,
pub synthesized_samples_duration: f64,
pub synthesized_samples_events: u32,
pub total_samples_duration: f64,
pub total_playout_delay: f64,
pub total_samples_count: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct PeerConnectionStats {
pub data_channels_opened: u32,
pub data_channels_closed: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct DataChannelStats {
pub label: String,
pub protocol: String,
pub data_channel_identifier: i32,
pub state: Option<DataChannelState>,
pub messages_sent: u32,
pub bytes_sent: u64,
pub messages_received: u32,
pub bytes_received: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct TransportStats {
pub packets_sent: u64,
pub packets_received: u64,
pub bytes_sent: u64,
pub bytes_received: u64,
pub ice_role: IceRole,
pub ice_local_username_fragment: String,
pub dtls_state: Option<DtlsTransportState>,
pub ice_state: Option<IceTransportState>,
pub selected_candidate_pair_id: String,
pub local_certificate_id: String,
pub remote_certificate_id: String,
pub tls_version: String,
pub dtls_cipher: String,
pub dtls_role: DtlsRole,
pub srtp_cipher: String,
pub selected_candidate_pair_changes: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CandidatePairStats {
pub transport_id: String,
pub local_candidate_id: String,
pub remote_candidate_id: String,
pub state: Option<IceCandidatePairState>,
pub nominated: bool,
pub packets_sent: u64,
pub packets_received: u64,
pub bytes_sent: u64,
pub bytes_received: u64,
pub last_packet_sent_timestamp: f64,
pub last_packet_received_timestamp: f64,
pub total_round_trip_time: f64,
pub current_round_trip_time: f64,
pub available_outgoing_bitrate: f64,
pub available_incoming_bitrate: f64,
pub requests_received: u64,
pub requests_sent: u64,
pub responses_received: u64,
pub responses_sent: u64,
pub consent_requests_sent: u64,
pub packets_discarded_on_send: u32,
pub bytes_discarded_on_send: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct IceCandidateStats {
pub transport_id: String,
pub address: String,
pub port: i32,
pub protocol: String,
pub candidate_type: Option<IceCandidateType>,
pub priority: i32,
pub url: String,
pub relay_protocol: Option<IceServerTransportProtocol>,
pub foundation: String,
pub related_address: String,
pub related_port: i32,
pub username_fragment: String,
pub tcp_type: Option<IceTcpCandidateType>,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CertificateStats {
pub fingerprint: String,
pub fingerprint_algorithm: String,
pub base64_certificate: String,
pub issuer_certificate_id: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct StreamStats {
pub id: String,
pub stream_identifier: String,
// pub timestamp: i64,
}
}
@@ -0,0 +1,596 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use thiserror::Error;
use crate::imp::video_frame as vf_imp;
#[derive(Debug, Error)]
pub enum SinkError {
#[error("platform error: {0}")]
Platform(String),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoRotation {
VideoRotation0 = 0,
VideoRotation90 = 90,
VideoRotation180 = 180,
VideoRotation270 = 270,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoFormatType {
ARGB,
BGRA,
ABGR,
RGBA,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum VideoBufferType {
Native,
I420,
I420A,
I422,
I444,
I010,
NV12,
}
/// Metadata carried alongside a video frame via the packet trailer mechanism.
///
/// Each field corresponds to an independently negotiable packet trailer feature
/// (`PTF_USER_TIMESTAMP`, `PTF_FRAME_ID`), so individual fields are `Option`.
#[derive(Debug, Clone, Copy)]
pub struct FrameMetadata {
/// Wall-clock capture time in microseconds, when `PTF_USER_TIMESTAMP` is enabled.
pub user_timestamp: Option<u64>,
/// Monotonically increasing frame identifier, when `PTF_FRAME_ID` is enabled.
pub frame_id: Option<u32>,
}
#[derive(Debug)]
pub struct VideoFrame<T>
where
T: AsRef<dyn VideoBuffer>,
{
pub rotation: VideoRotation,
pub timestamp_us: i64, // When the frame was captured in microseconds
/// Packet-trailer metadata, if any trailer features are active.
pub frame_metadata: Option<FrameMetadata>,
pub buffer: T,
}
impl<T: AsRef<dyn VideoBuffer>> VideoFrame<T> {
pub fn new(rotation: VideoRotation, buffer: T) -> Self {
Self { rotation, timestamp_us: 0, frame_metadata: None, buffer }
}
}
pub type BoxVideoBuffer = Box<dyn VideoBuffer>;
pub type BoxVideoFrame = VideoFrame<BoxVideoBuffer>;
pub(crate) mod internal {
use super::{I420Buffer, VideoFormatType};
pub trait BufferSealed: Send + Sync {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
);
}
}
pub trait VideoBuffer: internal::BufferSealed + Debug {
fn width(&self) -> u32;
fn height(&self) -> u32;
fn buffer_type(&self) -> VideoBufferType;
#[cfg(not(target_arch = "wasm32"))]
fn as_native(&self) -> Option<&native::NativeBuffer> {
None
}
fn as_i420(&self) -> Option<&I420Buffer> {
None
}
fn as_i420a(&self) -> Option<&I420ABuffer> {
None
}
fn as_i422(&self) -> Option<&I422Buffer> {
None
}
fn as_i444(&self) -> Option<&I444Buffer> {
None
}
fn as_i010(&self) -> Option<&I010Buffer> {
None
}
fn as_nv12(&self) -> Option<&NV12Buffer> {
None
}
}
macro_rules! new_buffer_type {
($type:ident, $variant:ident, $as:ident) => {
pub struct $type {
pub(crate) handle: vf_imp::$type,
}
impl $crate::video_frame::internal::BufferSealed for $type {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
self.handle.sys_handle()
}
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer {
I420Buffer { handle: self.handle.to_i420() }
}
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
stride: u32,
width: i32,
height: i32,
) {
self.handle.to_argb(format, dst, stride, width, height)
}
}
impl VideoBuffer for $type {
fn width(&self) -> u32 {
self.handle.width()
}
fn height(&self) -> u32 {
self.handle.height()
}
fn buffer_type(&self) -> VideoBufferType {
VideoBufferType::$variant
}
fn $as(&self) -> Option<&$type> {
Some(self)
}
}
impl Debug for $type {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!($type))
.field("width", &self.width())
.field("height", &self.height())
.finish()
}
}
impl AsRef<dyn VideoBuffer> for $type {
fn as_ref(&self) -> &(dyn VideoBuffer + 'static) {
self
}
}
};
}
new_buffer_type!(I420Buffer, I420, as_i420);
new_buffer_type!(I420ABuffer, I420A, as_i420a);
new_buffer_type!(I422Buffer, I422, as_i422);
new_buffer_type!(I444Buffer, I444, as_i444);
new_buffer_type!(I010Buffer, I010, as_i010);
new_buffer_type!(NV12Buffer, NV12, as_nv12);
impl I420Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I420Buffer {
vf_imp::I420Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I420Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I420ABuffer {
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32, u32) {
(
self.handle.stride_y(),
self.handle.stride_u(),
self.handle.stride_v(),
self.handle.stride_a(),
)
}
#[allow(clippy::type_complexity)]
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
self.handle.data()
}
#[allow(clippy::type_complexity)]
pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) {
let (data_y, data_u, data_v, data_a) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
data_a.map(|data_a| {
std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len())
}),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420ABuffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I422Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I422Buffer {
vf_imp::I422Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I422Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I422Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I444Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I444Buffer {
vf_imp::I444Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I444Buffer {
Self::with_strides(width, height, width, width, width)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I444Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I010Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I010Buffer {
vf_imp::I010Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I010Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I010Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl NV12Buffer {
pub fn with_strides(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> NV12Buffer {
vf_imp::NV12Buffer::new(width, height, stride_y, stride_uv)
}
pub fn new(width: u32, height: u32) -> NV12Buffer {
Self::with_strides(width, height, width, width + width % 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32) {
(self.handle.stride_y(), self.handle.stride_uv())
}
pub fn data(&self) -> (&[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) {
let (data_y, data_uv) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> NV12Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::Debug;
use super::{vf_imp, I420Buffer, VideoBuffer, VideoBufferType, VideoFormatType};
new_buffer_type!(NativeBuffer, Native, as_native);
impl NativeBuffer {
pub fn from_fluxer_d3d11_texture(
handle: u64,
width: u32,
height: u32,
dxgi_format: u32,
) -> Option<Self> {
vf_imp::NativeBuffer::from_fluxer_d3d11_texture(
handle,
width,
height,
dxgi_format,
)
}
#[allow(clippy::too_many_arguments)]
pub fn from_fluxer_dmabuf_texture(
fds: [i32; 4],
plane_count: u32,
width: u32,
height: u32,
drm_format: u32,
modifier: u64,
strides: [u32; 4],
offsets: [u32; 4],
device_uuid_hi: u64,
device_uuid_lo: u64,
) -> Option<Self> {
vf_imp::NativeBuffer::from_fluxer_dmabuf_texture(
fds,
plane_count,
width,
height,
drm_format,
modifier,
strides,
offsets,
device_uuid_hi,
device_uuid_lo,
)
}
/// Creates a `NativeBuffer` from a `CVPixelBufferRef` pointer.
///
/// This function does not bump the reference count of the pixel buffer.
///
/// Safety: The given pointer must be a valid `CVPixelBufferRef`.
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub unsafe fn from_cv_pixel_buffer(cv_pixel_buffer: *mut std::ffi::c_void) -> Self {
vf_imp::NativeBuffer::from_cv_pixel_buffer(cv_pixel_buffer)
}
/// Returns the `CVPixelBufferRef` that backs this buffer, or `null` if
/// this buffer is not currently backed by a `CVPixelBufferRef`.
///
/// This function does not bump the reference count of the pixel buffer.
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
self.handle.get_cv_pixel_buffer()
}
}
pub trait VideoFrameBufferExt: VideoBuffer {
fn to_i420(&self) -> I420Buffer;
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
);
}
impl<T: VideoBuffer> VideoFrameBufferExt for T {
fn to_i420(&self) -> I420Buffer {
self.to_i420()
}
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {
use super::VideoFrameBuffer;
#[derive(Debug)]
pub struct WebGlBuffer {}
impl VideoFrameBuffer for WebGlBuffer {}
}
@@ -0,0 +1,97 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{enum_dispatch, imp::video_source as vs_imp};
#[derive(Debug, Clone)]
pub struct VideoResolution {
pub width: u32,
pub height: u32,
}
impl Default for VideoResolution {
// Default to 720p
fn default() -> Self {
VideoResolution { width: 1280, height: 720 }
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcVideoSource {
// TODO(theomonnom): Web video sources (eq. to tracks on browsers?)
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeVideoSource),
}
// TODO(theomonnom): Support enum dispatch with conditional compilation?
impl RtcVideoSource {
enum_dispatch!(
[Native];
pub fn video_resolution(self: &Self) -> VideoResolution;
);
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::{Debug, Formatter};
use super::*;
use crate::native::packet_trailer::PacketTrailerHandler;
use crate::video_frame::{VideoBuffer, VideoFrame};
#[derive(Clone)]
pub struct NativeVideoSource {
pub(crate) handle: vs_imp::NativeVideoSource,
}
impl Debug for NativeVideoSource {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoSource").finish()
}
}
impl Default for NativeVideoSource {
fn default() -> Self {
Self::new(VideoResolution::default(), false)
}
}
impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self {
Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
}
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
self.handle.capture_frame(frame)
}
/// Set the packet trailer handler used by this source.
///
/// When set, any frame captured with a `user_timestamp` value will
/// automatically have its timestamp stored in the handler (keyed by
/// the TimestampAligner-adjusted capture timestamp) so the
/// `PacketTrailerTransformer` can embed it into the encoded frame.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler)
}
pub fn video_resolution(&self) -> VideoResolution {
self.handle.video_resolution()
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
@@ -0,0 +1,123 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::video_stream as stream_imp;
// There is no shared sink between native and web platforms.
// Each platform requires different configuration (e.g: WebGlContext, ..)
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::{
fmt::{Debug, Formatter},
pin::Pin,
task::{Context, Poll},
};
use super::stream_imp;
use crate::{
native::packet_trailer::PacketTrailerHandler, video_frame::BoxVideoFrame,
video_track::RtcVideoTrack,
};
use livekit_runtime::Stream;
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 1;
#[derive(Clone, Debug, Default)]
pub struct NativeVideoStreamOptions {
/// Maximum number of queued WebRTC sink frames after the video callback.
///
/// `None` uses the default bounded queue size of 1 frame. `Some(0)`
/// opts into unbounded buffering. Positive values bound the queue, and
/// the stream drops the oldest queued frames on overflow so render
/// latency stays bounded.
///
/// If your application consumes both audio and video, keep the queue
/// sizing strategy coordinated across both streams. Using a much larger
/// queue, or unbounded buffering, for only one of them can increase
/// end-to-end latency for that stream and cause audio/video drift.
pub queue_size_frames: Option<usize>,
}
pub struct NativeVideoStream {
pub(crate) handle: stream_imp::NativeVideoStream,
}
impl Debug for NativeVideoStream {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoStream").field("track", &self.track()).finish()
}
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack) -> Self {
Self {
handle: stream_imp::NativeVideoStream::new(
video_track,
Some(DEFAULT_QUEUE_SIZE_FRAMES),
),
}
}
pub fn with_options(video_track: RtcVideoTrack, options: NativeVideoStreamOptions) -> Self {
Self {
handle: stream_imp::NativeVideoStream::new(
video_track,
normalize_queue_size_frames(options.queue_size_frames),
),
}
}
/// Set the packet trailer handler for this stream.
///
/// When set, each frame produced by this stream will have its
/// `user_timestamp` field populated by looking up the user
/// timestamp for each frame's RTP timestamp.
///
/// Note: If the handler was already set on the `RtcVideoTrack`
/// before creating this stream, it is automatically wired up.
/// This method is only needed to override or set the handler
/// after construction.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler);
}
pub fn track(&self) -> RtcVideoTrack {
self.handle.track()
}
pub fn close(&mut self) {
self.handle.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.get_mut().handle).poll_next(cx)
}
}
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
match queue_size_frames {
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
Some(0) => None,
Some(value) => Some(value),
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
@@ -0,0 +1,58 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::video_track as imp_vt,
media_stream_track::{media_stream_track, RtcTrackState},
};
#[cfg(not(target_arch = "wasm32"))]
use crate::native::packet_trailer::PacketTrailerHandler;
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) handle: imp_vt::RtcVideoTrack,
}
impl RtcVideoTrack {
media_stream_track!();
/// Set the packet trailer handler for this track.
///
/// When set, any `NativeVideoStream` created from this track will
/// automatically use this handler to populate `user_timestamp`
/// on each decoded frame.
#[cfg(not(target_arch = "wasm32"))]
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler);
}
/// Get the packet trailer handler, if one has been set.
#[cfg(not(target_arch = "wasm32"))]
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
self.handle.packet_trailer_handler()
}
}
impl Debug for RtcVideoTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcVideoTrack")
.field("id", &self.id())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,117 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use core::str;
use std::cell::RefCell;
use std::rc::Rc;
use wasm_bindgen::prelude::*;
use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState};
use crate::data_channel::{
DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange,
};
impl From<RtcDataChannelState> for DataState {
fn from(value: RtcDataChannelState) -> Self {
match value {
RtcDataChannelState::Connecting => Self::Connecting,
RtcDataChannelState::Open => Self::Open,
RtcDataChannelState::Closing => Self::Closing,
RtcDataChannelState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
#[derive(Clone)]
pub struct DataChannel {
sys_handle: web_sys::RtcDataChannel,
on_closing: Rc<RefCell<Option<JsValue>>>,
}
impl DataChannelTrait for DataChannel {
fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if binary {
self.sys_handle
.send_with_u8_array(data)
.map_err(|_| DataChannelError::Send)
} else {
let utf8 = str::from_utf8(data)?;
self.sys_handle
.send_with_str(utf8)
.map_err(|_| DataChannelError::Send)
}
}
fn label(&self) -> String {
self.sys_handle.label()
}
fn state(&self) -> DataState {
self.sys_handle.ready_state().into()
}
fn close(&self) {
self.sys_handle.close();
}
fn on_state_change(&self, callback: Option<OnStateChange>) {
if let Some(mut callback) = callback {
let dc = self.clone();
let js_callback = Closure::new(move |_: RtcDataChannelEvent| {
callback(dc.state());
});
let js_callback = js_callback.into_js_value();
self.sys_handle
.set_onopen(Some(js_callback.unchecked_ref()));
self.sys_handle
.set_onclose(Some(js_callback.unchecked_ref()));
self.sys_handle
.add_event_listener_with_callback("closing", js_callback.unchecked_ref())
.unwrap();
self.on_closing.replace(Some(js_callback));
} else {
self.sys_handle.set_onopen(None);
self.sys_handle.set_onclose(None);
if let Some(on_closing) = self.on_closing.take() {
self.sys_handle
.remove_event_listener_with_callback("closing", on_closing.unchecked_ref())
.unwrap();
}
self.on_closing.replace(None);
}
}
fn on_message(&self, callback: Option<OnMessage>) {
let js_callback = callback.map(|mut callback| {
Closure::new(move |event: MessageEvent| {
if let Some(str) = event.as_string() {
callback(str.as_bytes(), false);
}
})
.into_js_value()
});
self.sys_handle.set_onmessage(
js_callback
.as_ref()
.map(|callback| callback.unchecked_ref()),
);
}
fn on_buffered_amount_change(&self, _callback: Option<OnBufferedAmountChange>) {
todo!("onbufferedamountlow instead?")
}
}
@@ -0,0 +1,372 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::VideoTrack;
use crate::{
media_stream::{
BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer,
PlanarYuvBuffer, VideoFrameBuffer,
},
video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer},
I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer,
};
use std::sync::mpsc;
use web_sys::{WebGlRenderingContext, WebGlTexture};
#[derive(Debug)]
pub struct WebGlVideoSink {
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
tex: WebGlTexture,
}
/// Create a new WebGL texture and update it inside requestAnimationFrame
impl WebGlVideoSink {
pub fn new(
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
) -> Result<(Self, mpsc::Receiver<VideoFrame<WebGlBuffer>>), SinkError> {
let (sender, receiver) = mpsc::channel();
let tex = gl_ctx.create_texture()?;
Ok((Self { track, gl_ctx, tex }, receiver))
}
}
#[derive(Debug, Clone)]
pub struct WebGlBuffer {
width: i32,
height: i32,
tex: WebGlTexture,
}
impl VideoFrameBuffer for WebGlBuffer {
fn width(&self) -> i32 {
self.width
}
fn height(&self) -> i32 {
self.height
}
}
/// The following types could be implemented if we want
/// to support VideoFrame with WebCodecs
#[derive(Debug)]
pub struct I420Buffer {}
#[derive(Debug)]
pub struct I420ABuffer {}
#[derive(Debug)]
pub struct I422Buffer {}
#[derive(Debug)]
pub struct I444Buffer {}
#[derive(Debug)]
pub struct I010Buffer {}
#[derive(Debug)]
pub struct NV12Buffer {}
impl VideoFrameBuffer for I420Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I420ABuffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I422Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I444Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I010Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for NV12Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420ABuffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I422Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I444Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I010Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420ABuffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I422Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I444Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv16BBuffer for I010Buffer {
fn data_y(&self) -> &[u16] {
unimplemented!()
}
fn data_u(&self) -> &[u16] {
unimplemented!()
}
fn data_v(&self) -> &[u16] {
unimplemented!()
}
}
impl BiplanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_uv(&self) -> i32 {
unimplemented!()
}
}
impl BiplanarYuv8Buffer for NV12Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_uv(&self) -> &[u8] {
unimplemented!()
}
}
@@ -0,0 +1,15 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
unimplemented!();
@@ -0,0 +1,148 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
name = "tract-linalg"
version = "0.19.16"
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
autobenches = false
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
readme = "README.md"
keywords = [
"TensorFlow",
"NeuralNetworks",
]
categories = ["science"]
license = "MIT/Apache-2.0"
repository = "https://github.com/snipsco/tract"
resolver = "1"
[[bench]]
name = "arm64"
bench = false
harness = false
[[bench]]
name = "mat_vec"
harness = false
[[bench]]
name = "mm_for_wavenet_hw"
harness = false
[[bench]]
name = "conv_for_wavenet_hw"
harness = false
[[bench]]
name = "mm_for_inception"
harness = false
[[bench]]
name = "mm_for_asr_am"
harness = false
[[bench]]
name = "sigmoid"
harness = false
[[bench]]
name = "arm64simd"
bench = false
harness = false
[[bench]]
name = "arm32neon"
bench = false
harness = false
[[bench]]
name = "packing"
bench = false
harness = false
[[bench]]
name = "virtual_im2col"
harness = false
[[bench]]
name = "x86_64"
bench = false
harness = false
[dependencies.derive-new]
version = "0.5.9"
[dependencies.downcast-rs]
version = "1.2.0"
[dependencies.dyn-clone]
version = "1.0.4"
[dependencies.lazy_static]
version = "1.4.0"
[dependencies.log]
version = "0.4.14"
[dependencies.num-traits]
version = "0.2.14"
[dependencies.paste]
version = "1.0.5"
[dependencies.scan_fmt]
version = "0.2.6"
[dependencies.tract-data]
version = "=0.19.16"
[dev-dependencies.criterion]
version = "0.4"
[dev-dependencies.nu-ansi-term]
version = "0.46"
[dev-dependencies.proptest]
version = "1.0.0"
[build-dependencies.cc]
version = "1.0.69"
[build-dependencies.half]
version = "2"
features = [
"std",
"num-traits",
]
[build-dependencies.liquid]
version = "0.26"
[build-dependencies.liquid-core]
version = "0.26"
[build-dependencies.smallvec]
version = "1.6.1"
[build-dependencies.unicode-normalization]
version = "0.1.19"
[build-dependencies.walkdir]
version = "2.3.2"
[features]
default = []
no_fp16 = []
[badges.maintenance]
status = "actively-developed"

Some files were not shown because too many files have changed in this diff Show More